From 92ff19c63d31deb2612800828ab09e5749e12476 Mon Sep 17 00:00:00 2001 From: "Michael S. Pedersen" Date: Fri, 23 Apr 2021 10:04:20 +0000 Subject: [PATCH] Update BMRA to v21.03 Update Kuberef to use most recent release of BMRA. Also updates configuration with minor changes and additions. Signed-off-by: Michael S. Pedersen Change-Id: Ifc4ef114564a2085000c23347390089227c0ebd9 Reviewed-on: https://gerrit.opnfv.org/gerrit/c/kuberef/+/72429 Tested-by: jenkins-ci Reviewed-by: Rihab Banday --- deploy.env | 1 + functions.sh | 9 +- hw_config/equinix-metal/idf.yaml | 26 ++- hw_config/ericsson-pod1/idf.yaml | 26 ++- hw_config/intel/idf.yaml | 26 ++- inventory/group_vars/all/global.yaml | 3 + playbooks/roles/bmra-config/templates/all.j2 | 54 +++-- playbooks/roles/bmra-config/templates/inventory.j2 | 4 +- playbooks/roles/bmra-config/templates/kube-node.j2 | 122 +++++++++-- sw_config/bmra/ansible.cfg | 15 ++ sw_config/bmra/patched_rhel_packages.yml | 226 +++++++++++++++++++++ sw_config/bmra/patched_vfio.yml | 38 +++- 12 files changed, 488 insertions(+), 62 deletions(-) create mode 100644 sw_config/bmra/ansible.cfg create mode 100644 sw_config/bmra/patched_rhel_packages.yml diff --git a/deploy.env b/deploy.env index 85641ce..967bf90 100644 --- a/deploy.env +++ b/deploy.env @@ -1,6 +1,7 @@ # Define environment variables export VENDOR=${VENDOR:-intel} +export DISTRO=${DISTRO:-centos7} export INSTALLER=bmra export BMRA_PROFILE=$(yq r $CURRENTPATH/hw_config/$VENDOR/idf.yaml bmra.profile) diff --git a/functions.sh b/functions.sh index fce71f6..1a07ec0 100755 --- a/functions.sh +++ b/functions.sh @@ -210,7 +210,7 @@ provision_k8s() { ansible_cmd="/bin/bash -c '" if [[ "$DEPLOYMENT" == "k8s" ]]; then ansible-playbook -i "$CURRENTPATH"/sw_config/bmra/inventory.ini "$CURRENTPATH"/playbooks/pre-install.yaml - ansible_cmd+="yum -y remove python-netaddr; pip install --upgrade pip; pip install ansible==2.9.6; ansible-playbook -i /bmra/inventory.ini /bmra/playbooks/k8s/patch_kubespray.yml;" + ansible_cmd+="yum -y remove python-netaddr; pip install --upgrade pip; pip install ansible==2.9.17; ansible-playbook -i /bmra/inventory.ini /bmra/playbooks/k8s/patch_kubespray.yml;" fi ansible_cmd+="ansible-playbook -i /bmra/inventory.ini /bmra/playbooks/${BMRA_PROFILE}.yml'" @@ -226,7 +226,7 @@ if ! command -v docker; then done fi if [ ! -d "${PROJECT_ROOT}/container-experience-kits" ]; then - git clone --recurse-submodules --depth 1 https://github.com/intel/container-experience-kits.git -b v2.1.0 ${PROJECT_ROOT}/container-experience-kits/ + git clone --recurse-submodules --depth 1 https://github.com/intel/container-experience-kits.git -b v21.03 ${PROJECT_ROOT}/container-experience-kits/ cp -r ${PROJECT_ROOT}/container-experience-kits/examples/${BMRA_PROFILE}/group_vars ${PROJECT_ROOT}/container-experience-kits/ # NOTE The following condition/workaround will be removed once the reported issue https://github.com/intel/container-experience-kits/issues/68 # is fixed upstream @@ -245,6 +245,11 @@ cp ${PROJECT_ROOT}/${INSTALLER}/patched_cmk_build.yml \ ${PROJECT_ROOT}/container-experience-kits/roles/cmk_install/tasks/main.yml cp ${PROJECT_ROOT}/${INSTALLER}/patched_vfio.yml \ ${PROJECT_ROOT}/container-experience-kits/roles/sriov_nic_init/tasks/bind_vf_driver.yml +cp ${PROJECT_ROOT}/${INSTALLER}/ansible.cfg \ + ${PROJECT_ROOT}/container-experience-kits/ansible.cfg +cp ${PROJECT_ROOT}/${INSTALLER}/patched_rhel_packages.yml \ + ${PROJECT_ROOT}/container-experience-kits/roles/bootstrap/install_packages/tasks/rhel.yml + sudo docker run --rm \ -e ANSIBLE_CONFIG=/bmra/ansible.cfg \ -e PROFILE=${BMRA_PROFILE} \ diff --git a/hw_config/equinix-metal/idf.yaml b/hw_config/equinix-metal/idf.yaml index d085c7c..7b6c07e 100644 --- a/hw_config/equinix-metal/idf.yaml +++ b/hw_config/equinix-metal/idf.yaml @@ -44,14 +44,21 @@ bmra: network_roles: sriov: - name: eno2 - bus_info: "19:00.1" - device_info: "8086:1572:0200" - driver: iavf + pci: "19:00.1" + pf_driver: i40e + vf_driver: iavf sriov_dpdk: - name: eno4 - bus_info: "19:00.3" - device_info: "8086:1572:0200" - driver: vfio-pci + pci: "19:00.3" + pf_driver: i40e + vf_driver: vfio-pci + device_roles: +# qat: +# - name: crypto01 +# pci: "0000:ab:00.0" +# mod_type: qat_c62x +# pci_type: c6xx +# vfs: 4 features: sriov: enable: true @@ -66,7 +73,8 @@ bmra: hugepages_2M: 10240 isolcpus: enable: true - cpus: "8-27,36-55" + autogenerate: true + cpus: "8-27,36-55" # Not used when autogenerate is true nfd: true cmk: enable: true @@ -78,4 +86,8 @@ bmra: tas: enable: true demo_policy: false + bond_cni: true psp: true + qat: + enable: false + update_drivers: false diff --git a/hw_config/ericsson-pod1/idf.yaml b/hw_config/ericsson-pod1/idf.yaml index a160e0a..10e2433 100644 --- a/hw_config/ericsson-pod1/idf.yaml +++ b/hw_config/ericsson-pod1/idf.yaml @@ -73,14 +73,21 @@ bmra: network_roles: sriov: - name: eth2 - bus_info: "05:00.0" - device_info: "8086:10f8:0200" - driver: vfio-pci + pci: "05:00.0" + pf_driver: i40e + vf_driver: vfio-pci sriov_dpdk: - name: eth3 - bus_info: "05:00.1" - device_info: "8086:10f8:0200" - driver: vfio-pci + pci: "05:00.1" + pf_driver: i40e + vf_driver: vfio-pci + device_roles: +# qat: +# - name: crypto01 +# pci: "0000:ab:00.0" +# mod_type: qat_c62x +# pci_type: c6xx +# vfs: 4 features: sriov: enable: false @@ -95,7 +102,8 @@ bmra: hugepages_2M: 10240 isolcpus: enable: true - cpus: "4-9,14-19,24-29,34-39" + autogenerate: true + cpus: "4-9,14-19,24-29,34-39" # Not used when autogenerate is true nfd: true cmk: enable: true @@ -107,4 +115,8 @@ bmra: tas: enable: true demo_policy: false + bond_cni: true psp: true + qat: + enable: false + update_drivers: false diff --git a/hw_config/intel/idf.yaml b/hw_config/intel/idf.yaml index 07c45cc..4ce7c47 100644 --- a/hw_config/intel/idf.yaml +++ b/hw_config/intel/idf.yaml @@ -73,14 +73,21 @@ bmra: network_roles: sriov: - name: eth0 - bus_info: "18:00.0" - device_info: "8086:158b:0200" - driver: vfio-pci + pci: "18:00.0" + pf_driver: i40e + vf_driver: vfio-pci sriov_dpdk: - name: eth1 - bus_info: "18:00.1" - device_info: "8086:158b:0200" - driver: iavf + pci: "18:00.1" + pf_driver: i40e + vf_driver: iavf + device_roles: +# qat: +# - name: crypto01 +# pci: "0000:ab:00.0" +# mod_type: qat_c62x +# pci_type: c6xx +# vfs: 4 features: sriov: enable: true @@ -95,7 +102,8 @@ bmra: hugepages_2M: 10240 isolcpus: enable: true - cpus: "4-19,24-39,44-59,64-79" + autogenerate: true + cpus: "4-19,24-39,44-59,64-79" # Not used when autogenerate is true nfd: true cmk: enable: true @@ -107,4 +115,8 @@ bmra: tas: enable: true demo_policy: false + bond_cni: true psp: true + qat: + enable: false + update_drivers: false diff --git a/inventory/group_vars/all/global.yaml b/inventory/group_vars/all/global.yaml index c88d794..8b2adf9 100644 --- a/inventory/group_vars/all/global.yaml +++ b/inventory/group_vars/all/global.yaml @@ -25,3 +25,6 @@ pub_key: "{{ lookup('env', 'HOME') }}/.ssh/id_rsa.pub" # deployment type for kuberef deployment_type: "{{ lookup('env', 'DEPLOYMENT') }}" + +# OS distro from DISTRO from environment variables +os_distro: "{{ lookup('env', 'DISTRO') }}" diff --git a/playbooks/roles/bmra-config/templates/all.j2 b/playbooks/roles/bmra-config/templates/all.j2 index 6dc074e..1dbabe2 100644 --- a/playbooks/roles/bmra-config/templates/all.j2 +++ b/playbooks/roles/bmra-config/templates/all.j2 @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020 Intel Corporation. +## Copyright (c) 2020-2021 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -18,14 +18,15 @@ # Kubernetes version kubernetes: true -kube_version: v1.18.8 -#kube_version: v1.17.11 -#kube_version: v1.16.14 +#kube_version: v1.20.4 +kube_version: v1.19.8 +#kube_version: v1.18.16 # Run system-wide package update (apt dist-upgrade, yum update, ...) # Note: enabling this may lead to unexpected results # Tip: you can set this per host using host_vars update_all_packages: false +update_kernel: true # Node Feature Discovery nfd_enabled: {{ bmra.features.nfd }} @@ -33,7 +34,7 @@ nfd_build_image_locally: false nfd_namespace: kube-system nfd_sleep_interval: 60s -# Intel CPU Manager for Kubernetes +# Intel CPU Manager for Kubernetes (CMK) cmk_enabled: {{ bmra.features.cmk.enable }} cmk_namespace: kube-system cmk_use_all_hosts: false # 'true' will deploy CMK on the controller nodes too @@ -42,19 +43,13 @@ cmk_shared_num_cores: {{ bmra.features.cmk.num_shared_cores }} # number of CPU c cmk_exclusive_num_cores: {{ bmra.features.cmk.num_exclusive_cores }} # number of CPU cores to be assigned to the "exclusive" pool on each of the nodes # cmk_shared_mode: packed # choose between: packed, spread, default: packed # cmk_exclusive_mode: packed # choose between: packed, spread, default: packed +autogenerate_isolcpus: {{ bmra.features.isolcpus.autogenerate }} # Native CPU Manager (Kubernetes built-in) -# Note: Enabling CMK and built-in CPU Manager is not recommended. +# Note: Enabling CMK and built-in Native CPU Manager is NOT recommended. # Setting this option as "true" enables the "static" policy, otherwise the default "none" policy is used. +# The reserved CPU cores settings are individual per each worker node, and therefore are available to configure in the host_vars file native_cpu_manager_enabled: false -# Amount of CPU cores that will be reserved for the housekeeping (2000m = 2000 millicores = 2 cores) -native_cpu_manager_system_reserved_cpus: 2000m -# Amount of CPU cores that will be reserved for Kubelet -native_cpu_manager_kube_reserved_cpus: 1000m -# Explicit list of the CPUs reserved from pods scheduling. -# Note: Supported only with kube_version 1.17 and newer, overrides native_cpu_manager_system_reserved_cpus and native_cpu_manager_kube_reserved_cpus. -#native_cpu_manager_reserved_cpus: "0,1,2" -# Note: All reamining unreserved CPU cores will be consumed by the workloads. # Enable Kubernetes built-in Topology Manager topology_manager_enabled: {{ bmra.features.topology_manager.enable }} @@ -93,17 +88,37 @@ sriovdp_config_data: | "devices": ["1889"], "drivers": ["vfio-pci"] } + }, + { + "resourceName": "intel_fpga", + "deviceType": "accelerator", + "selectors": { + "vendors": ["8086"], + "devices": ["0d90"] + } } ] } # Intel QAT Device Plugin for Kubernetes -qat_dp_enabled: false +qat_dp_enabled: {{ bmra.features.qat.enable }} qat_dp_namespace: kube-system +qat_dp_build_image_locally: true # Intel GPU Device Plugin for Kubernetes gpu_dp_enabled: false gpu_dp_namespace: kube-system +gpu_dp_build_image_locally: true + +# Intel SGX Device Plugin for Kubernetes +sgx_dp_enabled: false +sgx_dp_build_image_locally: true +# ProvisionLimit is a number of containers that can share +# the same SGX provision device. +sgx_dp_provision_limit: 20 +# EnclaveLimit is a number of containers that can share the +# same SGX enclave device. +sgx_dp_enclave_limit: 20 # Intel Telemetry Aware Scheduling tas_enabled: {{ bmra.features.tas.enable }} @@ -120,7 +135,7 @@ example_net_attach_defs: ## Proxy configuration ## #http_proxy: "http://proxy.example.com:1080" #https_proxy: "http://proxy.example.com:1080" -#additional_no_proxy: ".example.com" +#additional_no_proxy: ".example.com,mirror_ip" # (Ubuntu only) disables DNS stub listener which may cause issues on Ubuntu dns_disable_stub_listener: false @@ -138,9 +153,16 @@ kube_pods_subnet: 10.244.0.0/16 kube_service_addresses: 10.233.0.0/18 kube_proxy_mode: iptables +# comment this line out if you want to expose k8s services of type nodePort externally. +kube_proxy_nodeport_addresses_cidr: 127.0.0.0/8 + # please leave it set to "true", otherwise Intel BMRA features deployed as Helm charts won't be installed helm_enabled: true +# local Docker Hub mirror, if it exists +#docker_registry_mirrors: +# - http://mirror_ip:mirror_port + # Docker registry running on the cluster allows us to store images not avaialble on Docker Hub, e.g. CMK registry_local_address: "localhost:30500" diff --git a/playbooks/roles/bmra-config/templates/inventory.j2 b/playbooks/roles/bmra-config/templates/inventory.j2 index 6008179..7f6cde0 100644 --- a/playbooks/roles/bmra-config/templates/inventory.j2 +++ b/playbooks/roles/bmra-config/templates/inventory.j2 @@ -2,7 +2,7 @@ {% for node in nodes %} {{ idf.kubespray.hostnames[node.name] }} ansible_host={{ node.interfaces[idf.net_config[engine.pxe_network].interface].address }} ip={{ node.interfaces[idf.net_config[engine.pxe_network].interface].address }} {% endfor %} -localhost ansible_python_interpreter=/usr/bin/python2 +localhost ansible_connection=local ansible_python_interpreter=/usr/bin/python2 {% for role in ['kube-master', 'etcd', 'kube-node'] %} [{{ role }}] @@ -19,7 +19,7 @@ kube-node [calico-rr] -{% if deployment_type == 'k8s' %} +{% if os_distro|lower != 'centos7' %} [all:vars] ansible_python_interpreter=/usr/bin/python3 {% endif %} diff --git a/playbooks/roles/bmra-config/templates/kube-node.j2 b/playbooks/roles/bmra-config/templates/kube-node.j2 index f32fbdb..09a6958 100644 --- a/playbooks/roles/bmra-config/templates/kube-node.j2 +++ b/playbooks/roles/bmra-config/templates/kube-node.j2 @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020 Intel Corporation. +## Copyright (c) 2020-2021 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -20,33 +20,44 @@ iommu_enabled: {{ bmra.features.sriov.enable }} # dataplane interface configuration list +{% if bmra.network_roles.sriov is defined or bmra.network_roles.sriov_dpdk is defined %} dataplane_interfaces: {% for intf in bmra.network_roles.sriov %} - name: {{ intf.name }} - bus_info: "{{ intf.bus_info }}" - device_info: "{{ intf.device_info }}" + bus_info: "{{ intf.pci }}" + pf_driver: {{ intf.pf_driver }} + default_vf_driver: "{{ intf.vf_driver }}" sriov_numvfs: {{ bmra.features.sriov.sriov_vfs_per_port }} - vf_driver: {{ intf.driver }} {% endfor %} {% for intf in bmra.network_roles.sriov_dpdk %} - name: {{ intf.name }} - bus_info: "{{ intf.bus_info }}" - device_info: "{{ intf.device_info }}" + bus_info: "{{ intf.pci }}" + pf_driver: {{ intf.pf_driver }} + default_vf_driver: "{{ intf.vf_driver }}" sriov_numvfs: {{ bmra.features.sriov.sriov_dpdk_vfs_per_port }} - vf_driver: {{ intf.driver }} {% endfor %} +{% else %} +dataplane_interface: [] +{% endif %} sriov_cni_enabled: {{ bmra.features.sriov_cni }} # Bond CNI -bond_cni_enabled: false +bond_cni_enabled: {{ bmra.features.bond_cni }} # Install DPDK (required for SR-IOV networking) install_dpdk: true +# DPDK version +dpdk_version: "19.11.6" + +# Custom DPDK patches local path +# dpdk_local_patches_dir: "/tmp/patches/dpdk-19.11.6" + # Userspace networking userspace_cni_enabled: false ovs_dpdk_enabled: false # Should be enabled with Userspace CNI, when VPP is set to "false"; 1G hugepages required +ovs_version: "v2.13.0" # CPU mask for OVS-DPDK PMD threads ovs_dpdk_lcore_mask: 0x1 # Huge memory pages allocated by OVS-DPDK per NUMA node in megabytes @@ -69,9 +80,41 @@ firmware_update_nics: [] # install Intel x700 & x800 series NICs DDP packages install_ddp_packages: false - -# set how many VFs per single QAT device PF should be created -qat_sriov_numvfs: 16 +# set 'true' to enable custom ddp package to be loaded after reboot +enable_ice_systemd_service: false +# Enabling this feature will install QAT drivers + services +update_qat_drivers: {{ bmra.features.qat.update_drivers }} + +# qat interface configuration list +{% if bmra.device_roles.qat is defined %} +qat_devices: +{% for dev in bmra.device_roles.qat %} + - qat_dev: {{ dev.name }} + qat_id: "{{ dev.pci }}" + qat_module_type: {{ dev.mod_type }} + qat_pci_type: {{ dev.pci_type }} + qat_sriov_numvfs: {{ dev.vfs }} +{% endfor %} +{% else %} +qat_devices: [] +{% endif %} +# - qat_dev: crypto01 # Device name as separate QAT Symmetric Crypto devices on which qat_sriov_numvfs will be distributed +# qat_id: "0000:ab:00.0" # Device QAT id one using DPDK compatible driver for VF devices to be used by vfio-pci kernel driver, replace as required +# qat_module_type: qat_c62x # QAT Crypto Poll Mode Kernel Module supported are qat_dh895xcc,qat_c62x,qat_c3xxx,qat_200xx,qat_c4xxx,qat_d15xx +# qat_pci_type: c6xx # QAT Crypto Poll Mode Pci Driver id supported are dh895xcc,c6xx,c3xxx,d15xx,200xx & c4xxx +# qat_sriov_numvfs: 12 # set how many VFs per qat_id to be created such as c6xxvf support 32 so per device will be 10+10+12=32, replace as required + # Note: If don't want to assign VFs to id leave it as 0 such as qat_sriov_numvfs: 0 +# - qat_dev: crypto02 +# qat_id: "0000:xy:00.0" +# qat_module_type: qat_c62x +# qat_pci_type: c6xx +# qat_sriov_numvfs: 10 + +# - qat_dev: crypto03 +# qat_id: "0000:yz:00.0" +# qat_module_type: qat_c62x +# qat_pci_type: c6xx +# qat_sriov_numvfs: 10 # Enables hugepages support hugepages_enabled: {{ bmra.features.hugepages.enable }} @@ -85,10 +128,19 @@ hugepages_2M: {{ bmra.features.hugepages.hugepages_2M }} # CPU isolation from Linux scheduler isolcpus_enabled: {{ bmra.features.isolcpus.enable }} -isolcpus: "{{ bmra.features.isolcpus.cpus }}" # Update to match group_vars requested exclusive/shared cores +# Disable CMKs autogenerate_isolcpus in group_vars to set custom isolcpus range; Otherwise this range will be automatically generated +# If defining range and using CMK you must account group_vars requested exclusive/shared cores +isolcpus: "{{ bmra.features.isolcpus.cpus }}" -# Max number of processors to support (physical & logical cores) -cpu_count: 144 +# Native CPU Manager (Kubernetes built-in) + +native_cpu_manager_system_reserved_cpus: 2000m +# Amount of CPU cores that will be reserved for Kubelet +native_cpu_manager_kube_reserved_cpus: 1000m +# Explicit list of the CPUs reserved from pods scheduling. +# Note: Supported only with kube_version 1.17 and newer, overrides native_cpu_manager_system_reserved_cpus and native_cpu_manager_kube_reserved_cpus. +#native_cpu_manager_reserved_cpus: "0,1,2" +# Note: All reamining unreserved CPU cores will be consumed by the workloads. # Enable/Disable Intel PState scaling driver intel_pstate_enabled: true @@ -109,9 +161,51 @@ sst_bf_configuration_enabled: false # [r] Revert cores to min/Turbo (set min/max to 800/3900) clx_sst_bf_mode: s +# Intel Speed Select Base-Frequency configuration for Ice Lake (ICX) Platforms. +# [true] Enable Intel Speed Select Base Frequency (SST-BF) +# [false] Disable Intel Speed Select Base Frequency (SST-BF) +# Requires `sst_bf_configuration_enabled` variable to be 'true' +icx_sst_bf_enabled: false +# Prioritze (SST-CP) power flow to high frequency cores in case of CPU power constraints. +icx_sst_bf_with_core_priority: false + +# SST CP config +# Variables are only examples. +# For more information, please visit: +# https://www.kernel.org/doc/html/latest/admin-guide/pm/intel-speed-select.html#enable-clos-based-prioritization +# Enabling this configuration overrides `icx_sst_bf_with_core_priority`. +sst_cp_configuration_enabled: false +sst_cp_priority_type: 0 # 0 - Proportional, 1 - Ordered +sst_cp_clos_groups: # configure up to 4 CLOS groups + - id: 0 + frequency_weight: 0 # used only with Proportional type + min_MHz: 0 + max_MHz: 25500 + - id: 1 + frequency_weight: 0 # used only with Proportional type + min_MHz: 0 + max_MHz: 25500 + - id: 2 + frequency_weight: 0 # used only with Proportional type + min_MHz: 0 + max_MHz: 25500 + - id: 3 + frequency_weight: 0 # used only with Proportional type + min_MHz: 0 + max_MHz: 25500 +sst_cp_cpu_clos: + - clos: 0 + cpus: 1,2,4..6,8-10 + - clos: 1 + cpus: 3,7 + + # (CentOS 7 only) install real time kernel and related packages for flexran install_real_time_package: false +# Intel Software Guard Extensions (SGX) +sgx_enabled: false + # Telemetry configuration # intel_pmu plugin collects information provided by Linux perf interface. enable_intel_pmu_plugin: false diff --git a/sw_config/bmra/ansible.cfg b/sw_config/bmra/ansible.cfg new file mode 100644 index 0000000..1808b58 --- /dev/null +++ b/sw_config/bmra/ansible.cfg @@ -0,0 +1,15 @@ +[ssh_connection] +pipelining=True +ssh_args = -o ControlMaster=auto -o ControlPersist=30m -o ConnectionAttempts=100 -o UserKnownHostsFile=/dev/null -o ServerAliveInterval=60 -o ServerAliveCountMax=10 + +[defaults] +force_valid_group_names = ignore +display_skipped_hosts = no +host_key_checking = False +gathering = smart + +fact_caching = jsonfile +fact_caching_connection = /tmp +fact_caching_timeout = 7200 + +action_plugins=./action_plugins:~/.ansible/plugins/action:/usr/share/ansible/plugins/action diff --git a/sw_config/bmra/patched_rhel_packages.yml b/sw_config/bmra/patched_rhel_packages.yml new file mode 100644 index 0000000..7e65484 --- /dev/null +++ b/sw_config/bmra/patched_rhel_packages.yml @@ -0,0 +1,226 @@ +## +## Copyright (c) 2020-2021 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +- name: enable PowerTools repository on CentOS >= 8 and < 8.3 +# noqa 303 - yum is called intenionallly here + command: yum config-manager --set-enabled PowerTools + when: + - ansible_distribution == "CentOS" + - ansible_distribution_version >= '8' and ansible_distribution_version < '8.3' + +- name: enable PowerTools repository on CentOS >= 8.3 +# noqa 303 - yum is called intenionallly here + command: yum config-manager --set-enabled powertools + when: + - ansible_distribution == "CentOS" + - ansible_distribution_version >= '8.3' + +- name: enable CodeReady Linux Builder repository on RHEL 8 + rhsm_repository: + name: codeready-builder-for-rhel-8-x86_64-rpms + when: + - ansible_distribution == "RedHat" + - ansible_distribution_version >= '8' + +- name: install epel-release on CentOS + package: + name: epel-release + when: + - ansible_distribution == "CentOS" + +- name: obtain EPEL GPG key on RHEL8 + rpm_key: + state: present + key: https://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-8 + when: + - ansible_distribution == "RedHat" + - ansible_distribution_version >= '8' + +- name: install epel-release on RHEL8 + package: + name: https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm + when: + - ansible_distribution == "RedHat" + - ansible_distribution_version >= '8' + +- name: get full distribution versions + command: cat /etc/redhat-release + register: release + changed_when: true + +- name: set full distribution version + set_fact: + full_dist_version: "{{ release.stdout | regex_replace('.*(\\d+.\\d+.\\d\\d\\d\\d).*', '\\1') }}" + +- name: update CentOS Vault yum repository on CentOS 7 + yum_repository: + name: C{{ full_dist_version }}-base + description: CentOS-{{ full_dist_version }} - Base + file: CentOS-Vault + baseurl: http://vault.centos.org/{{ full_dist_version }}/os/$basearch/ + gpgcheck: yes + gpgkey: file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-{{ ansible_distribution_major_version }} + enabled: yes + when: + - ansible_distribution == "CentOS" + - ansible_distribution_version < '7.9' + - not update_kernel + +#- name: update CentOS Vault yum repository on CentOS 8 +# yum_repository: +# name: C{{ full_dist_version }}-base +# description: CentOS-{{ full_dist_version }} - Base +# file: CentOS-Vault +# baseurl: http://vault.centos.org/{{ full_dist_version }}/BaseOS/$basearch/os/ +# baseurl: http://vault.centos.org/{{ full_dist_version }}/BaseOS/Source/ +# gpgcheck: yes +# gpgkey: file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial +# enabled: yes +# when: +# - ansible_distribution == "CentOS" +# - ansible_distribution_version >= '8' and ansible_distribution_version < '8.3' +# - not update_kernel + +# CentOS-Vault repo not working for CentOS 8, so install kernel headers directly +- name: pull matching kernel headers on CentOS 8.2 + package: + name: "{{ item }}" + state: present + register: source_status + with_items: + - "https://vault.centos.org/8.2.2004/BaseOS/x86_64/os/Packages/kernel-headers-4.18.0-193.el8.x86_64.rpm" + - "https://vault.centos.org/8.2.2004/BaseOS/x86_64/os/Packages/kernel-devel-4.18.0-193.el8.x86_64.rpm" + when: + - ansible_distribution == "CentOS" + - ansible_distribution_version == '8.2' + - not update_kernel + +- name: pull matching kernel headers on CentOS 8.3 + package: + name: "{{ item }}" + state: present + register: source_status + with_items: + - "http://mirror.centos.org/centos/8/BaseOS/x86_64/os/Packages/kernel-headers-4.18.0-240.el8.x86_64.rpm" + - "http://mirror.centos.org/centos/8/BaseOS/x86_64/os/Packages/kernel-devel-4.18.0-240.el8.x86_64.rpm" +# - "https://vault.centos.org/8.3.2011/BaseOS/x86_64/os/Packages/kernel-headers-4.18.0-240.el8.x86_64.rpm" +# - "https://vault.centos.org/8.3.2011/BaseOS/x86_64/os/Packages/kernel-devel-4.18.0-240.el8.x86_64.rpm" + when: + - ansible_distribution == "CentOS" + - ansible_distribution_version == '8.3' + - not update_kernel + +# pull the matching kernel headers if kernel is not updated +- name: pull matching kernel headers from configured repos +# noqa 503 - more than one condition, can't be a handler + package: + name: + - kernel-headers-{{ ansible_kernel }} + - kernel-devel-{{ ansible_kernel }} + register: kernel_source + retries: 3 + until: kernel_source is success + when: + - not source_status.changed + - ansible_os_family == "RedHat" + - not update_kernel + +- name: install the 'Development tools' package group + package: + name: "@Development tools" + when: + - ansible_os_family == "RedHat" + +- name: install pip + package: + name: python-pip + when: + - ansible_distribution in ["RedHat", "CentOS"] + - ansible_distribution_version < '8' + +- name: install pip + package: + name: python3-pip + when: + - ansible_distribution in ["RedHat", "CentOS"] + - ansible_distribution_version >= '8' + +- name: update all packages + package: + name: '*' + state: latest # noqa 403 + exclude: kernel* + when: + - ansible_os_family == "RedHat" + - update_all_packages | default(false) + +- name: update to the latest kernel and kernel headers on the Red Hat OS family + package: + name: + - kernel + - kernel-devel + state: latest # noqa 403 + notify: + - reboot server + when: + - ansible_os_family == "RedHat" + - update_kernel | default(false) + +#note(przemeklal): fixes issue with missing selinux in packet.net CentOS 7 images +- name: ensure selinux is installed on CentOS/RHEL 7 + package: + name: + - policycoreutils + - policycoreutils-python + - selinux-policy + - selinux-policy-targeted + - libselinux-utils + - setools + - setools-console + - shtool + - lshw + state: present + when: + - ansible_distribution in ["RedHat", "CentOS"] + - ansible_distribution_version < '8' + +# Workaround for Equinix Metal CentOS 7 +- name: set selinux to permissive + lineinfile: + path: "/etc/sysconfig/selinux" + regexp: '^SELINUX=enforcing' + line: 'SELINUX=permissive' + when: + - ansible_distribution in ["RedHat", "CentOS"] + - ansible_distribution_version < '8' + +- name: Set python is python3 + alternatives: + name: python + path: /usr/bin/python3 + link: /usr/bin/python + when: + - ansible_distribution == 'CentOS' or ansible_distribution == 'RedHat' + - ansible_distribution_version >= '8' + +- name: install command line tools to collect hardware details + package: + name: + - hwinfo + - inxi + - jq + state: present + when: ansible_distribution in ["RedHat", "CentOS"] diff --git a/sw_config/bmra/patched_vfio.yml b/sw_config/bmra/patched_vfio.yml index c0a6e25..81d4ab5 100644 --- a/sw_config/bmra/patched_vfio.yml +++ b/sw_config/bmra/patched_vfio.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020 Intel Corporation. +## Copyright (c) 2020-2021 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -18,17 +18,41 @@ - name: Check that selected driver module is available # if modinfo fails, lookup loaded modules as modinfo might return error # for igb_uio and potentially other modules not included with the kernel - shell: "modinfo {{ vf_driver }} || grep {{ vf_driver }} /proc/modules || grep {{ vf_driver }} /lib/modules/$(uname -r)/modules.builtin" + shell: "modinfo {{ vf_driver.value }} || grep {{ vf_driver.value }} /proc/modules || grep {{ vf_driver.value }} /lib/modules/$(uname -r)/modules.builtin" register: shell_result ignore_errors: yes failed_when: no changed_when: no + with_dict: "{{ item.sriov_vfs | default({}) | combine({'default': item.default_vf_driver}) }}" + loop_control: + loop_var: vf_driver + +- name: pre-create empty dict for VFs + set_fact: + vfs_acc: {} + +- name: populate VFs dict with values + set_fact: + vfs_acc: "{{ vfs_acc | combine({idx : item.default_vf_driver}) }}" + loop: "{{ range(item.sriov_numvfs | default(0) | int) | list }}" + loop_control: + index_var: idx + loop_var: vf_default + +- name: update VFs dict with default drivers + set_fact: + vfs_acc: "{{ vfs_acc | combine({vf.key | regex_replace('.*_(\\d*)', '\\1') | int : vf.value}) }}" + loop: "{{ item.sriov_vfs | default({}) | dict2items | sort(attribute='key') }}" + loop_control: + loop_var: vf + extended: yes + when: ansible_loop.index < (item.sriov_numvfs | default(0) | int ) # get a list of VFs PCI addresses and save the configuration - name: attach VFs driver block: - name: fetch VFs pci addresses for a PF - shell: "for vf in /sys/class/net/{{ pfname }}/device/virtfn*;do basename $(readlink -f $vf);done" + shell: "for vf in /sys/class/net/{{ item.name }}/device/virtfn*;do basename $(readlink -f $vf);done | sort" register: vf_pciids args: executable: /bin/bash @@ -37,16 +61,16 @@ - name: save VF driver binding lineinfile: path: "{{ sriov_config_path }}/bmra_interfaces" - line: "{{ this_item }} {{ vf_driver }}" - regexp: "^{{ this_item }}" + line: "{{ this_item[0] }} {{ this_item[1].value }}" + regexp: "^{{ this_item[0] }}" create: yes owner: root group: root mode: '0600' - loop: "{{ vf_pciids.stdout_lines }}" + loop: "{{ vf_pciids.stdout_lines | zip(vfs_acc | dict2items) | list }}" loop_control: loop_var: this_item when: - vf_pciids.stderr|length == 0 - vf_pciids.stdout_lines|length > 0 - when: shell_result.rc == 0 + when: shell_result.results | sum(attribute='rc') == 0 -- 2.16.6