1 # SPDX-FileCopyrightText: 2021 Intel Corporation.
3 # SPDX-License-Identifier: Apache-2.0
6 # Preflight: ALL checks must PASS
7 # Only assert issues (do NOT change anything)
9 # Manual run: 'ansible-playbook -i inventory.ini playbooks/preflight.yml --flush-cache'
12 # On Ansible Host (localhost):
13 # - Check Ansible version (match)
14 # - Check Python version (min)
15 # - Check Group Vars (exist)
16 # - Check CMK Hosts (valid targets)
17 # On All targets (k8s-cluster):
18 # - Check Linux Distro
19 # - Check Hostnames (match Inventory)
20 # - Check CMK Config (isolcpus defined)
21 # - Check isolcpus Total (not more than actual)
22 # - Check isolcpus IDs (valid on system)
23 # - Check isolcpus OS Reserved (not 0,1,etc)
24 # On Worker Nodes Only (kube-node):
25 # - Check DP Interfaces (is not empty)
26 # - Check DP Interfaces Name (optional)
27 # - Check DP Interfaces Bus Info (pciid)
28 # - Check QAT Devices Bus Info (pciid)
29 # - Check QAT SRIOV VFs (max)
30 # - Check SGX configuration
31 # - Check OVS DPDK Dependencies (for 1G Hugepages)
32 # - Check VPP Dependencies (for 2M Hugepages)
33 # - Check CNI Dependencies (for OVS DPDK or VPP and Hugepages)
34 # - Check SST (not on RHEL 8.2 or old OSs)
35 # - Warn BIOS VT-d (should be enabled)
36 # - Warn BIOS Hyper-Threading (should be enabled)
37 # - Warn collectd (kernel update needed on old OSs)
38 # - Check OVS DPDK Version Compatability (for OVS support)
40 # additional vars required:
41 # bmra_supported_ansible: # must be version
42 # bmra_supported_python: # min version
43 # bmra_supported_distros: [] # list
44 # bmra_supported_distros_versions: [] # list
47 ##################################
48 # Prerequisites for Ansible Host #
49 ##################################
53 bmra_supported_ansible: 2.9.20
54 bmra_supported_python: 2.7
58 - debug: msg="Ansible version is {{ ansible_version.string }}"
59 - name: Check Ansible Version
61 that: (ansible_version.full is version_compare(bmra_supported_ansible, '=='))
62 msg: "Ansible version must be {{ bmra_supported_ansible }}. Please update"
64 - debug: msg="Python version is {{ ansible_python_version }}"
65 - name: Check Python Version
67 that: (ansible_python_version is version_compare(bmra_supported_python, '>='))
68 msg: "Python version must be at least {{ bmra_supported_python }}. Please update"
70 - name: Read Group Vars
72 path: "{{ inventory_dir }}/group_vars/"
73 register: group_vars_details
75 - name: Check Group Vars
77 that: "group_vars_details.stat.exists and group_vars_details.stat.isdir"
78 msg: "File group_vars/all.yml does NOT exist. Must be created per Guide"
82 - cmk_enabled = {{ cmk_enabled }} (group_vars/all.yml)
83 - cmk_use_all_hosts = {{ cmk_use_all_hosts }} (group_vars/all.yml)
84 - cmk_hosts_list = {{ cmk_hosts_list | default('') }} (group_vars/all.yml)
85 - all targets = {{ groups['all'] }} (inventory.ini)
86 when: cmk_enabled is defined # CMK expected true for all profiles except basic
88 - name: Check Intel CMK Hosts
90 that: "item in groups['all']"
91 msg: "Hostname '{{ item }}' is NOT a valid target from inventory. Please correct the cmk_hosts_list or disable the CMK feature in group vars"
92 with_items: "{{ (cmk_hosts_list.split(',') if (cmk_hosts_list is defined and cmk_hosts_list | length > 0) else []) }}"
93 when: cmk_enabled is defined and cmk_enabled and not cmk_use_all_hosts
96 ##############################################
97 # Prerequisites for Control and Worker Nodes #
98 ##############################################
101 bmra_supported_distros: [CentOS, RedHat, Ubuntu]
102 bmra_supported_distros_versions: ['7.6', '7.8', '7.9', '8.2', '8.3', '8.4', '18.04', '20.04', '21.04']
104 isolcpus_discretes: []
108 - debug: msg="Linux distribution on target is {{ ansible_distribution }} {{ ansible_distribution_version }} {{ ansible_distribution_release }}"
109 - name: Check Linux Distro and Version
111 that: "ansible_distribution in bmra_supported_distros and ansible_distribution_version in bmra_supported_distros_versions"
113 - Linux distribution {{ ansible_distribution }} {{ ansible_distribution_version }} on target '{{ inventory_hostname }}' is NOT supported
114 - Must be one of {{ bmra_supported_distros }} and version {{ bmra_supported_distros_versions }}
116 # - name: Check Linux Across Cluster
117 # TODO ?? Linux OS must be the same on all targets (no mix-n-match)
119 - name: regather network facts in case hostname recently changed
121 gather_subset: network
122 - debug: msg="Inventory target '{{ inventory_hostname }}' has the actual system hostname '{{ ansible_hostname }}'"
123 - name: Check Inventory Hostnames
126 - "Target '{{ inventory_hostname }}' in inventory does NOT match the actual system hostname '{{ ansible_hostname }}'."
127 - "If it's done intentionally, please ignore this message."
129 - inventory_hostname != ansible_hostname
131 # Early check if SELinux is configured properly
133 - name: "Collect packages facts"
137 - "Current SELinux status:"
138 - "status: {{ ansible_selinux.status | default('') }}"
139 - "policy version: {{ ansible_selinux.policyvers | default('') }}"
140 - "type: {{ ansible_selinux.type | default('') }}"
141 - "mode: {{ ansible_selinux.mode | default('') }}"
142 - "config_mode: {{ ansible_selinux.config_mode | default('') }}"
144 - name: check selinux condition possibly causing system boot failure
147 - "Current SELinux setup might cause the system possibly will not boot up on next reboot."
148 - "Please, check SELinux settings and set it up according to the documentation."
150 - "'selinux-policy' not in ansible_facts.packages"
151 - "'selinux-policy-targeted' not in ansible_facts.packages"
153 - ansible_os_family == "RedHat"
155 # STORY: "cmk requires isolcpus to be configured"
159 - cmk_enabled = {{ cmk_enabled }} (group_vars/all.yml)
160 - cmk_use_all_hosts = {{ cmk_use_all_hosts }} (group_vars/all.yml)
161 - cmk_hosts_list = {{ cmk_hosts_list | default('') }} (group_vars/all.yml)
162 - cmk_shared_num_cores = {{ cmk_shared_num_cores }} (group_vars/all.yml)
163 - cmk_exclusive_num_cores = {{ cmk_exclusive_num_cores }} (group_vars/all.yml)
164 - isolcpus_enabled = {{ isolcpus_enabled }} (host_vars)
165 - isolcpus = {{ isolcpus }} (host_vars)
166 - ansible_processor_count = {{ ansible_processor_count }}
167 - ansible_processor_cores = {{ ansible_processor_cores }}
168 - ansible_processor_threads_per_core = {{ ansible_processor_threads_per_core }}
169 - ansible_processor_vcpus = {{ ansible_processor_vcpus }}
170 - CPUs Reserved for OS = 0...{{ ansible_processor_count - 1 }}
171 # - CPUs Reserved for OS = {{ lookup('sequence','0-{{ ansible_processor_count - 1 }}').split(',') }} # [E207] Nested jinja pattern
173 - name: Check Intel CMK Config
175 that: ({{ cmk_enabled }} and {{ isolcpus_enabled }} and "{{ isolcpus }}" | length > 0)
177 - Incorrect configuration pertaining Intel CMK. Conflicting or improper values detected
178 - When Intel CMK is enabled, CPUs isolation ('isolcpus') must be set according to the example file for host_vars. Please correct the configuration
180 - name: Split isolcpus Groups
182 isolcpus_groups: "{{ isolcpus.split(',') }}"
184 - debug: msg="isolcpus_groups = {{ isolcpus_groups }}"
186 - name: Filter isolcpus Ranges
188 isolcpus_ranges: "{{ isolcpus_ranges + [item] }}"
189 with_items: "{{ isolcpus_groups }}"
192 - debug: msg="isolcpus_ranges = {{ isolcpus_ranges }}"
194 - name: Filter isolcpus Discretes
196 isolcpus_discretes: "{{ isolcpus_discretes + [item] }}"
197 with_items: "{{ isolcpus_groups }}"
198 when: ("-" not in item)
200 - debug: msg="isolcpus_discretes = {{ isolcpus_discretes }}"
202 - name: Build isolcpus List
204 isolcpus_list: "{{ isolcpus_list | default([]) | union(isolcpus_discretes) | union([item]) }}"
205 with_sequence: "{{ isolcpus_ranges }}"
207 - debug: msg="isolcpus_list = {{ isolcpus_list }}"
209 - name: Check isolcpus Total
211 that: "{{ isolcpus_list | length }} <= ansible_processor_vcpus"
213 - Incorrect configuration pertaining isolcpus. Conflicting or improper values detected
214 - The number of isolcpus {{ isolcpus_list | length }}, exceeds total CPUs on target {{ ansible_processor_vcpus }}. Please correct the configuration
215 when: isolcpus is defined
217 - name: Check isolcpus IDs
219 that: "item | int <= ansible_processor_vcpus"
221 - Incorrect configuration pertaining isolcpus. Conflicting or improper values detected
222 - The CPU ID {{ item }} set for isolcpus is NOT actually present on target. Please correct the configuration
223 with_items: "{{ isolcpus_list }}"
224 when: isolcpus is defined
226 #TODO relationship between cmk shared/exclusive cores and isolcpus
229 - cmk_enabled is defined
230 - (not cmk_hosts_list is defined) or (inventory_hostname in cmk_hosts_list) #CMK expected true for all profiles except basic
231 # {% if not cmk_use_all_hosts %}
232 - "'kube-node' in group_names"
236 ####################################
237 # Prerequisites for Worker Node(s) #
238 ####################################
245 # STORY: "nic bus info specified is present on system"
247 msg: "Dataplane (DP) interface(s) defined in host_vars = {{ dataplane_interfaces }}"
248 when: dataplane_interfaces is defined
250 - name: Check DP Interfaces
252 that: "dataplane_interfaces != []"
253 msg: "Dataplane (DP) interface(s) on target '{{ ansible_hostname }}' must be set in host_vars. Please correct the configuration"
255 - dataplane_interfaces is defined
256 - (update_nic_drivers is defined and update_nic_drivers) or
257 (install_ddp_packages is defined and install_ddp_packages) or
258 (sriov_cni_enabled is defined and sriov_cni_enabled) or
259 (sriov_network_operator_enabled is defined and sriov_network_operator_enabled)
262 msg: "Network interfaces present on target '{{ ansible_hostname }}' = {{ ansible_interfaces }}"
264 - name: Read Physical NICs PCIIDs
266 phy_nics_pciids: "{{ phy_nics_pciids + [ ansible_facts[item]['pciid'] ] }}"
267 with_items: "{{ ansible_interfaces }}"
268 when: ansible_facts[item]['pciid'] is defined and ansible_facts[item]['type'] == "ether"
270 - debug: msg="PCI Slots for the NICs on target '{{ ansible_hostname }}' = {{ phy_nics_pciids }}"
272 - name: Check DP Interfaces Names
274 that: ("{{ item.name }}" in {{ ansible_interfaces }})
275 msg: "Dataplane interface '{{ item.name }}' defined in host_vars does NOT exist on target. Please correct the configuration"
276 with_items: "{{ dataplane_interfaces }}"
277 when: dataplane_interfaces is defined and dataplane_interfaces != []
280 - name: Check DP Interfaces Bus Info
282 that: ("{{ item.bus_info }}" in "{{ phy_nics_pciids }}")
283 msg: "Dataplane interface '{{ item.name }}' defined with PCI ID '{{ item.bus_info }}' does NOT exist on target. Please correct the configuration"
284 with_items: "{{ dataplane_interfaces }}"
285 when: dataplane_interfaces is defined and dataplane_interfaces != []
289 # QAT Devices list is okay to be left empty (default), but if was defined, device(s) must exist on target
291 msg: "QAT device(s) defined in host_vars = {{ qat_devices }}"
292 when: qat_devices is defined
294 - name: Read QAT PCIIDs
295 shell: lshw -businfo -numeric | grep -i quickassist
298 when: qat_devices is defined
301 msg: "QAT devices found on target = {{ lshw_qat.stdout }}"
302 when: qat_devices is defined
304 - name: Check QAT Devices' Bus Info
306 that: ("{{ item.qat_id }}" in """{{ lshw_qat.stdout }}""")
307 msg: "QAT device '{{ item.qat_dev }}' defined with PCI ID '{{ item.qat_id }}' does NOT exist on target. Please correct the configuration"
308 with_items: "{{ qat_devices }}"
309 when: qat_devices is defined and qat_devices != []
312 # STORY: "qat_sriov_numvfs should not exceed max supported (16) per each dev_ID"
315 - qat_sriov_numvfs for {{ item.qat_id }} = {{ item.qat_sriov_numvfs }} (host_vars)
316 - update_qat_drivers = {{ update_qat_drivers }} (host_vars)
317 with_items: "{{ qat_devices }}"
318 when: qat_devices is defined and qat_devices != [] # update_qat_drivers expected as 'true' for all profiles except basic
320 - name: Check QAT SRIOV VFs
322 that: ({{ item.qat_sriov_numvfs }} <= 16)
324 - Incorrect configuration pertaining QAT SRIOV. Conflicting or improper values detected
325 - When SRIOV VFs are set for QAT, max value is 16 for each ID (max 48 total per card). Please correct the configuration
326 with_items: "{{ qat_devices }}"
328 - update_qat_drivers is defined and update_qat_drivers
329 - qat_devices is defined and qat_devices != []
330 # OpenSSL & OpenSSL*Engine must only be configured / installed when update_qat_drivers is set to 'true' and qat_devices is defined in host vars
331 - name: check OpenSSL and OpenSSL*Engine requirements
335 - qat_devices is defined and qat_devices != []
336 fail_msg: "OpenSSL & OpenSSL*Engine will only configured if update_qat_drivers is set to 'true' & qat_devices is defined in host vars"
337 success_msg: "OpenSSL & OpenSSL*Engine verification completed"
338 when: openssl_install is defined and openssl_install
340 - name: check KMRA requirements
344 fail_msg: "KMRA installation requires sgx_dp_enabled set to 'true'"
345 success_msg: "KMRA requirements verified"
346 when: kmra_enabled is defined and kmra_enabled
348 - name: check SGX configuration
352 fail_msg: "SGX drivers installation requires sgx_enabled set to 'true'"
353 success_msg: "SGX configuration verified"
355 - sgx_dp_enabled is defined and sgx_dp_enabled
356 - (ansible_distribution == 'Ubuntu' and ansible_distribution_version != '21.04')
357 or (ansible_os_family == 'RedHat' and ansible_distribution_version != '8.4')
359 - name: check NFD configuration
363 fail_msg: "SGX DP requires nfd_enabled set to 'true'"
364 success_msg: "NFD configuration verified"
365 when: sgx_dp_enabled is defined and sgx_dp_enabled
367 - name: check kmra_pccs_api_key presence
370 - kmra_pccs_api_key is defined
372 - "kmra_pccs_api_key is not defined"
373 success_msg: "kmra_pccs_api_key presence is verified"
375 - kmra_enabled is defined and kmra_enabled
377 - name: check PCCS API key length
380 - kmra_pccs_api_key | length == 32
381 fail_msg: "PCCS API Key should be 32 bytes long"
382 success_msg: "PCCS API key length verified"
384 - kmra_enabled is defined and kmra_enabled
386 - name: check PCCS API key is not a placeholder
389 - kmra_pccs_api_key is defined
390 - kmra_pccs_api_key != "ffffffffffffffffffffffffffffffff"
392 - "Please, visit https://api.portal.trustedservices.intel.com/provisioning-certification and click on 'Subscribe'"
393 - "to generate PCCS API key."
394 - "PCCS API key is essential for KMRA deployment and usage."
395 success_msg: "PCCS API key verified"
397 - kmra_enabled is defined and kmra_enabled
399 # STORY: "vpp/ovsdpdk require hugepage enabled and configured"
402 - vpp_enabled = {{ vpp_enabled }} (host_vars)
403 - example_net_attach_defs = {{ example_net_attach_defs }} (group_vars/all.yml)
404 - userspace_ovs_dpdk = {{ example_net_attach_defs['userspace_ovs_dpdk'] }} (group_vars/all.yml)
405 - userspace_vpp = {{ example_net_attach_defs['userspace_vpp'] }} (group_vars/all.yml)
406 - sriov_net_dp = {{ example_net_attach_defs['sriov_net_dp'] }} (group_vars/all.yml)
407 - userspace_cni_enabled = {{ userspace_cni_enabled }} (host_vars)
408 - sriov_cni_enabled = {{ sriov_cni_enabled }} (host_vars)
409 - sriov_network_operator_enabled = {{ sriov_network_operator_enabled }} (host_vars)
410 - bond_cni_enabled = {{ bond_cni_enabled }} (host_vars)
411 - ovs_dpdk_enabled = {{ ovs_dpdk_enabled }} (host_vars)
412 - userspace_cni_enabled = {{ userspace_cni_enabled }} (host_vars)
413 - hugepages_enabled = {{ hugepages_enabled }} (host_vars)
414 - default_hugepage_size = {{ default_hugepage_size }} (host_vars)
415 - number_of_hugepages = {{ number_of_hugepages }} (host_vars)
416 when: vpp_enabled is defined #host_vars
418 - name: Check OVS DPDK Dependencies
421 ({{ ovs_dpdk_enabled }} and not {{ vpp_enabled }} and {{ hugepages_enabled }} and
422 "{{ default_hugepage_size }}" == "1G" and {{ number_of_hugepages }} >= 0)
425 - Incorrect configuration pertaining OVS DPDK. Conflicting or improper values detected
426 - When OVS DPDK is enabled, VPP must be disabled and Hugepages must be set to 1G according to host_vars example. Please correct the configuration
427 when: ovs_dpdk_enabled is defined and ovs_dpdk_enabled
429 - name: Check VPP Dependencies
432 ({{ vpp_enabled }} and not {{ ovs_dpdk_enabled }} and {{ hugepages_enabled }} and
433 "{{ default_hugepage_size }}" == "2M" and {{ number_of_hugepages }} >= 0)
434 or {{ ovs_dpdk_enabled }}
436 - Incorrect configuration pertaining VPP. Conflicting or improper values detected
437 - When VPP is enabled, OVS DPDK must be disabled and Hugepages must be set to 2M according to host_vars example. Please correct the configuration
438 when: vpp_enabled is defined and vpp_enabled
441 # STORY: "cnis require net-attach-defs to be enabled"
442 - name: Check CNI Config
445 ({{ userspace_cni_enabled }} and {{ ovs_dpdk_enabled }} and {{ example_net_attach_defs['userspace_ovs_dpdk'] }} and not {{ vpp_enabled }} and
446 not {{ example_net_attach_defs['userspace_vpp'] }} and {{ hugepages_enabled }} and
447 "{{ default_hugepage_size }}" == "1G" and {{ number_of_hugepages }} >= 0)
448 or ({{ userspace_cni_enabled }} and not {{ ovs_dpdk_enabled }} and not {{ example_net_attach_defs['userspace_ovs_dpdk'] }} and {{ vpp_enabled }}
449 and {{ example_net_attach_defs['userspace_vpp'] }} and {{ hugepages_enabled }} and
450 "{{ default_hugepage_size }}" == "2M" and {{ number_of_hugepages }} >= 0)
452 - Incorrect configuration pertaining CNI. Conflicting or improper values detected.
453 - When CNI is enabled, either OVS DPDK either VPP must be enabled and Hugepages must be according to example files. Please correct the configuration
454 when: userspace_cni_enabled is defined and userspace_cni_enabled
457 # STORY: "If SST enabled, confirm minimum kernel or kernel_update specified"
458 - name: Check SST # see Jira NPF-1545
460 that: (not sst_bf_configuration_enabled)
461 msg: "SST-BF is NOT supported on {{ ansible_distribution }} {{ ansible_distribution_version }}. Please use a different OS or disable this feature"
463 - sst_bf_configuration_enabled is defined
464 - (ansible_distribution == "RedHat" and ansible_distribution_version == '8.2') or ansible_distribution_version in ['7.6', '7.8', '7.9', '18.04']
467 # STORY: Intel VT-d should be enabled in BIOS
468 - name: Check Intel VT-d
469 shell: dmesg | grep DMAR | grep remapping
470 register: dmesg_dmar_remap
474 - debug: msg="dmesg >> {{ dmesg_dmar_remap.stdout }}"
476 - name: Warn about Intel VT-d
478 msg: "Warning: Intel VT-d appears DISABLED on target. Please check BIOS under 'Advanced > Integrated IO Configuration' and Enable if necessary"
479 when: dmesg_dmar_remap.stdout|length == 0
483 # STORY: CPU Hyper-Threading should be enabled in BIOS
484 - name: Warn about Hyper-Threading
486 msg: "Warning: CPU Hyper-Threading is DISABLED on target. Please check BIOS under 'Advanced > Processor Configuration' and Enable if necessary"
487 when: ansible_processor_threads_per_core != 2
491 # STORY: "check for collectd. See Jira NPF-1687"
492 - name: Warn about collectd
494 msg: "Warning: On {{ ansible_distribution }} {{ ansible_distribution_version }} collectd won't work unless 'update_kernel' is enabled in group_vars"
495 when: ansible_distribution_version in ['7.6', '18.04']
499 # STORY: TEMPORARY: "ovs dpdk version requirements"
502 - install_dpdk = {{ install_dpdk }} (host_vars)
503 - dpdk_version = {{ dpdk_version }} (host_vars)
504 - ovs_dpdk_enabled = {{ ovs_dpdk_enabled }} (host_vars)
505 - ovs_version = {{ ovs_version }} (host_vars)
507 - install_dpdk is defined #host_vars
508 - dpdk_version is defined #host_vars
509 - ovs_version is defined #host_vars
510 - ovs_dpdk_enabled is defined and ovs_dpdk_enabled #host_vars
512 - name: Check OVS DPDK compatibility
515 "{{ ovs_version }} == \"v2.15.0\" and {{ dpdk_version }} >= \"20.11\""
516 or "{{ ovs_version }} == \"v2.14.2\" and {{ dpdk_version }} == \"19.11.6\""
517 or "{{ ovs_version }} == \"v2.14.1\" and {{ dpdk_version }} == \"19.11.6\""
518 or "{{ ovs_version }} == \"v2.14.0\" and {{ dpdk_version }} == \"19.11.6\""
519 or "{{ ovs_version }} == \"v2.13.3\" and {{ dpdk_version }} == \"19.11.6\""
520 or "{{ ovs_version }} == \"v2.13.2\" and {{ dpdk_version }} == \"19.11.6\""
521 or "{{ ovs_version }} == \"v2.13.1\" and {{ dpdk_version }} == \"19.11.6\""
522 or "{{ ovs_version }} == \"v2.13.0\" and {{ dpdk_version }} == \"19.11.6\""
523 msg: "OVS {{ ovs_version }} does not build with DPDK version {{ dpdk_version }}. Please correct the host_vars configuration"
525 - dpdk_version is defined #host_vars
526 - ovs_version is defined #host_vars
527 - ovs_dpdk_enabled is defined and ovs_dpdk_enabled #host_vars
532 # - name: Print all variables/facts known for a host
533 # ansible.builtin.debug:
534 # var: hostvars[inventory_hostname]