Updated Kuberef to use BMRA v2.1 instead of v2.0.
Removed a few workarounds used for BMRA v2.0.
Updated a few dependencies in deployment container.
Updated Kuberef templates for BMRA configuration.
Signed-off-by: Michael S. Pedersen <michaelx.pedersen@intel.com>
Change-Id: I091da09277ca9ecc0d3d7574dc3c4f000e73432d
Reviewed-on: https://gerrit.opnfv.org/gerrit/c/kuberef/+/72064
Tested-by: jenkins-ci <jenkins-opnfv-ci@opnfv.org>
Reviewed-by: Rihab Banday <rihab.banday@ericsson.com>
ansible_cmd="/bin/bash -c '"
if [[ "$DEPLOYMENT" == "k8s" ]]; then
ansible-playbook -i "$CURRENTPATH"/sw_config/bmra/inventory.ini "$CURRENTPATH"/playbooks/pre-install.yaml
- ansible_cmd+="pip install --upgrade pip==9.0.3; pip install ansible==2.9.6; ansible-playbook -i /bmra/inventory.ini /bmra/playbooks/k8s/patch_kubespray.yml;"
+ ansible_cmd+="yum -y remove python-netaddr; pip install --upgrade pip; pip install ansible==2.9.6; ansible-playbook -i /bmra/inventory.ini /bmra/playbooks/k8s/patch_kubespray.yml;"
fi
ansible_cmd+="ansible-playbook -i /bmra/inventory.ini /bmra/playbooks/${BMRA_PROFILE}.yml'"
done
fi
if [ ! -d "${PROJECT_ROOT}/container-experience-kits" ]; then
- git clone --recurse-submodules --depth 1 https://github.com/intel/container-experience-kits.git -b v2.0.0 ${PROJECT_ROOT}/container-experience-kits/
+ git clone --recurse-submodules --depth 1 https://github.com/intel/container-experience-kits.git -b v2.1.0 ${PROJECT_ROOT}/container-experience-kits/
cp -r ${PROJECT_ROOT}/container-experience-kits/examples/${BMRA_PROFILE}/group_vars ${PROJECT_ROOT}/container-experience-kits/
# NOTE The following condition/workaround will be removed once the reported issue https://github.com/intel/container-experience-kits/issues/68
# is fixed upstream
${PROJECT_ROOT}/container-experience-kits/
cp ${PROJECT_ROOT}/${INSTALLER}/{all.yml,kube-node.yml} \
${PROJECT_ROOT}/container-experience-kits/group_vars/
-cp ${PROJECT_ROOT}/${INSTALLER}/dpdk_patch.yml \
- ${PROJECT_ROOT}/container-experience-kits/roles/install_dpdk/tasks/main.yml
-cp ${PROJECT_ROOT}/${INSTALLER}/patched_tas.yml \
- ${PROJECT_ROOT}/container-experience-kits/roles/tas_install/tasks/tas.yml
-cp ${PROJECT_ROOT}/${INSTALLER}/patched_cmk.yml \
- ${PROJECT_ROOT}/container-experience-kits/roles/cmk_install/charts/cpu-manager-for-kubernetes/templates/job.yml
cp ${PROJECT_ROOT}/${INSTALLER}/patched_cmk_build.yml \
${PROJECT_ROOT}/container-experience-kits/roles/cmk_install/tasks/main.yml
sudo docker run --rm \
policy: "best-effort"
tas:
enable: true
- demo_policy: true
+ demo_policy: false
psp: true
policy: "best-effort"
tas:
enable: true
- demo_policy: true
+ demo_policy: false
psp: true
sriov_net_dp_enabled: {{ bmra.features.sriov_net_dp }}
sriov_net_dp_namespace: kube-system
# whether to build and store image locally or use one from public external registry
-sriov_net_dp_build_image_locally: false
+sriov_net_dp_build_image_locally: true
# SR-IOV network device plugin configuration.
# For more information on supported configuration refer to: https://github.com/intel/sriov-network-device-plugin#configurations
sriovdp_config_data: |
{% for node in nodes %}
{{ idf.kubespray.hostnames[node.name] }} ansible_host={{ node.interfaces[idf.net_config[engine.pxe_network].interface].address }} ip={{ node.interfaces[idf.net_config[engine.pxe_network].interface].address }}
{% endfor %}
-localhost
+localhost ansible_python_interpreter=/usr/bin/python2
{% for role in ['kube-master', 'etcd', 'kube-node'] %}
[{{ role }}]
---
# Kubernetes node configuration
-# Enable SR-IOV networking related setup
-sriov_enabled: {{ bmra.features.sriov.enable }}
+# Enable IOMMU (required for SR-IOV networking)
+iommu_enabled: {{ bmra.features.sriov.enable }}
-# SR-IOV PF specific configuration list
-sriov_nics:
+# dataplane interface configuration list
+dataplane_interfaces:
{% for intf in bmra.network_roles.sriov %}
- name: {{ intf.name }}
bus_info: "{{ intf.bus_info }}"
# Bond CNI
bond_cni_enabled: false
-# install DPDK
-install_dpdk: true # DPDK installation is required for sriov_enabled:true; default to false
+# Install DPDK (required for SR-IOV networking)
+install_dpdk: true
# Userspace networking
userspace_cni_enabled: false
update_nic_firmware: false
# Additional list of NIC interfaces that the FW update will be executed on
-# NOTE: FW update will be also executed on all NICs listed in "sriov_nics[*].name"
-firmware_update_nics:
- - intf1
- - intf2
+# NOTE: FW update will be also executed on all NICs listed in "dataplane_interfaces[*].name"
+firmware_update_nics: []
+# - enp24s0f0
+# - enp24s0f1
# install Intel x700 & x800 series NICs DDP packages
install_ddp_packages: false
# Enable/Disable Intel PState scaling driver
intel_pstate_enabled: true
# Config options for intel_pstate: disable, passive, force, no_hwp, hwp_only, support_acpi_ppc, per_cpu_perf_limits
-intel_pstate: disable
+intel_pstate: hwp_only
# Enable/Disable Intel Turbo Boost PState attribute
turbo_boost_enabled: false
+++ /dev/null
-##
-## Copyright (c) 2020 Intel Corporation.
-##
-## Licensed under the Apache License, Version 2.0 (the "License");
-## you may not use this file except in compliance with the License.
-## You may obtain a copy of the License at
-##
-## http://www.apache.org/licenses/LICENSE-2.0
-##
-## Unless required by applicable law or agreed to in writing, software
-## distributed under the License is distributed on an "AS IS" BASIS,
-## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-## See the License for the specific language governing permissions and
-## limitations under the License.
-##
----
-- name: install dependencies
- include_role:
- name: install_dependencies
-
-- name: download DPDK
- unarchive:
- src: "{{ dpdk_url }}"
- dest: "/usr/src"
- remote_src: yes
- list_files: yes
- mode: 0755
- register: dpdk_download
-
-- name: set local dpdk directory path
- set_fact:
- dpdk_dir: "{{ dpdk_download.dest }}/{{ dpdk_download.files[0] }}"
-
-- name: run make config
- make:
- chdir: "{{ dpdk_dir }}"
- target: config
- params:
- T: "{{ dpdk_target }}"
-
-- name: update ansible_kernel fact
- setup:
- filter: 'ansible_kernel'
-
-- name: patch DPDK (kni)
- lineinfile:
- path: "{{ dpdk_dir }}/kernel/linux/kni/kni_net.c"
- regexp: '^\s*kni_net_tx_timeout'
- line: 'kni_net_tx_timeout(struct net_device *dev, unsigned int txqueue)'
- when:
- - ansible_distribution == "CentOS"
- - ansible_distribution_major_version == '8'
- - ansible_kernel is version('4.18.0-240','>=')
-
-- name: enable virtio-user support
- lineinfile:
- path: "{{ dpdk_dir }}/build/.config"
- regexp: '^CONFIG_RTE_VIRTIO_USER'
- line: 'CONFIG_RTE_VIRTIO_USER=y'
- mode: 0600
-
-- name: enable PCAP PMD support
- lineinfile:
- path: "{{ dpdk_dir }}/build/.config"
- regexp: '^CONFIG_RTE_LIBRTE_PMD_PCAP'
- line: 'CONFIG_RTE_LIBRTE_PMD_PCAP=y'
- mode: 0600
-
-- name: build DPDK
- make:
- target: install
- chdir: "{{ dpdk_dir }}"
- params:
- T: "{{ dpdk_target }}"
- DESTDIR: install
- prefix: "/usr"
-
-- name: find dpdk tools
- find:
- path: "{{ dpdk_dir }}"
- patterns: "dpdk-devbind.py"
- recurse: yes
- register: dpdk_tools_dir
-
-- name: set path to dpdk usertools directory
- set_fact:
- dpdk_tools: "{{ dpdk_tools_dir.files[0].path }}"
-
-- name: load userspace modules
- modprobe:
- name: "{{ item }}"
- state: present
- with_items:
- - vfio-pci
- - uio
-
-- name: install dpdk-devbind.py in /usr/local/bin
- copy:
- remote_src: yes
- src: "{{ dpdk_tools }}"
- dest: "/usr/local/bin/dpdk-devbind.py"
- mode: 0700
- owner: root
- group: root
- become: yes
-
-- name: load intel module
- command: "insmod {{ dpdk_dir }}/{{ dpdk_target }}/kmod/igb_uio.ko"
- register: result
- failed_when: "'No such file or directory' in result.stderr"
- changed_when: "'already bound' not in result.stderr"
+++ /dev/null
-{{- $fullName := include "cmk.fullname" . -}}
-{{- $exclusiveMode := .Values.exclusiveMode -}}
-{{- $numExclusiveCores := .Values.numExclusiveCores -}}
-{{- $sharedMode := .Values.sharedMode -}}
-{{- $numSharedCores := .Values.numSharedCores -}}
-{{- $pullPolicy := .Values.image.pullPolicy -}}
-{{- $image := .Values.image.repository -}}
-{{- $tag := .Values.image.tag -}}
-{{- $noTaint := .Values.noTaint }}
-{{ range splitList "," .Values.hosts.list }}
----
-apiVersion: batch/v1
-kind: Job
-metadata:
- annotations:
- helm.sh/hook: pre-install,pre-upgrade,pre-rollback
- helm.sh/hook-weight: "10"
- labels:
- app: {{ $fullName }}-init-discover-{{ . }}
- name: {{ $fullName }}-init-discover-{{ . }}
-spec:
- template:
- spec:
- serviceAccountName: {{ $fullName }}
- restartPolicy: Never
- tolerations:
- - key: cmk
- operator: Exists
- containers:
- - name: install
- image: {{ $image }}:{{ $tag }}
- imagePullPolicy: {{ $pullPolicy }}
- command: ["/bin/bash", "-c"]
- args:
- - "/cmk/cmk.py install"
- volumeMounts:
- - mountPath: /opt/bin
- name: cmk-install-dir
- - name: init
- image: {{ $image }}:{{ $tag }}
- imagePullPolicy: {{ $pullPolicy }}
- env:
- - name: CMK_PROC_FS
- value: "/host/proc"
- - name: NODE_NAME
- valueFrom:
- fieldRef:
- apiVersion: v1
- fieldPath: spec.nodeName
- command: ["/bin/bash", "-c"]
- args:
- - "/cmk/cmk.py init --exclusive-mode={{ $exclusiveMode }} --num-exclusive-cores={{ $numExclusiveCores }} --shared-mode={{ $sharedMode }} --num-shared-cores={{ $numSharedCores }}"
- volumeMounts:
- - mountPath: /host/proc
- name: host-proc
- readOnly: true
- - name: discover
- image: {{ $image }}:{{ $tag }}
- imagePullPolicy: {{ $pullPolicy }}
- env:
- - name: CMK_PROC_FS
- value: /host/proc
- - name: NODE_NAME
- valueFrom:
- fieldRef:
- apiVersion: v1
- fieldPath: spec.nodeName
- command: ["/bin/bash", "-c"]
- args:
- - "/cmk/cmk.py discover {{ if $noTaint }}--no-taint{{ end }}"
- volumeMounts:
- - mountPath: /host/proc
- name: host-proc
- readOnly: true
- volumes:
- - hostPath:
- path: /proc
- type: Directory
- name: host-proc
- - hostPath:
- path: /opt/bin
- type: DirectoryOrCreate
- name: cmk-install-dir
-{{ end }}
+++ /dev/null
-##
-## Copyright (c) 2020 Intel Corporation.
-##
-## Licensed under the Apache License, Version 2.0 (the "License");
-## you may not use this file except in compliance with the License.
-## You may obtain a copy of the License at
-##
-## http://www.apache.org/licenses/LICENSE-2.0
-##
-## Unless required by applicable law or agreed to in writing, software
-## distributed under the License is distributed on an "AS IS" BASIS,
-## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-## See the License for the specific language governing permissions and
-## limitations under the License.
-##
----
-- name : install dependencies
- include_role:
- name: install_dependencies
-
-- name: update to git2 on RHEL 7 based distros
- include_role:
- name: git2_install
- when:
- - ansible_distribution == 'RedHat' or ansible_distribution == 'CentOS'
- - ansible_distribution_version < '8'
-
-- name: check if stable repo has already been added
- command: helm repo list
- register: helm_repo_list
- failed_when: false
- changed_when: false
-
-- name: add Helm stable repo
- command: helm repo add stable https://charts.helm.sh/stable
- when: '"https://charts.helm.sh/stable" not in helm_repo_list.stdout'
- register: helm_add_result
- changed_when: '"has been added to your repositories" in helm_add_result.stdout'
-
-- name: update Helm repo before installation of public charts
- command: helm repo update
- register: helm_update_result
- changed_when: '"Successfully got an update" in helm_update_result.stdout'
-
-
-- name: create Helm charts directory if needed
- file:
- path: /usr/src/charts
- state: directory
- mode: 0755
-
-- name: generate cert and key
- include_role:
- name: create_signed_k8s_certs
- vars:
- secret_name: "{{ tas_extender_secret_name }}"
- service_name: tas-telemetry-aware-scheduling
- key_pair_name: tas
- host_secrets_folder: "{{ tas_ssl_mount_path }}"
- k8s_namespace: "{{ tas_namespace }}"
- csr_cluster_name: "{{ cluster_name | default('cluster.local') }}"
- when: tas_tls_enabled
-
-- name: clone TAS repository
- git:
- repo: "{{ tas_git_url }}"
- version: "{{ tas_git_version }}"
- dest: "{{ tas_dir }}"
- force: yes
-
-- name: make build and make image - TAS
- make:
- target: "{{ item }}"
- chdir: "{{ tas_dir }}"
- loop:
- - build
- - image
-
-- name: tag TAS-controller and TAS-extender
- # TAS Makefile always creates ":latest" version images
- command: docker tag {{ item }}:latest {{ registry_local_address }}/{{ item }}:{{ tas_version }}
- loop:
- - tas-controller
- - tas-extender
- changed_when: false
-
-- name: push TAS-controller and TAS-extender image to local registry
- command: docker push {{ registry_local_address }}/{{ item }}:{{ tas_version }}
- loop:
- - tas-controller
- - tas-extender
- changed_when: true
-
-- name: create descheduler directory if needed
- file:
- path: "{{ sigs_k8s_io_dir }}"
- state: directory
- mode: 0755
-
-- name: clone Descheduler for Kubernetes
- git:
- repo: "{{ descheduler_git_url }}"
- dest: "{{ descheduler_dir }}"
- force: yes
- version: "{{ descheduler_git_version }}"
-
-- name: install descheduler
- make:
- chdir: "{{ descheduler_dir }}"
-
-- name: copy Helm chart resource definition to controller node
- copy:
- src: "{{ role_path }}/charts/{{ item }}"
- dest: "/usr/src/charts/"
- mode: preserve
- loop:
- - telemetry-aware-scheduling
- - tas-policy-crd.yml
-
-- name: populate tas Helm chart values template and push to controller node
- template:
- src: "tas-values.yml.j2"
- dest: "/usr/src/charts/tas-values.yml"
- force: yes
- mode: preserve
-
-- name: create TASPolicy resource
- command: kubectl apply -f tas-policy-crd.yml
- args:
- chdir: "/usr/src/charts"
- changed_when: true
-
-- name: install TAS helm chart
- command: helm upgrade --install --namespace {{ tas_namespace }} {{ tas_name }} -f tas-values.yml telemetry-aware-scheduling/
- args:
- chdir: "/usr/src/charts"
- retries: 5
- delay: 5
- register: result
- until: result.rc == 0
- changed_when: true
-
-- name: Configure arguments from Kubernetes Scheduler file if they exist - dnsPolicy
- lineinfile:
- path: /etc/kubernetes/manifests/kube-scheduler.yaml
- insertafter: "spec:"
- line: " dnsPolicy: ClusterFirstWithHostNet"
- regexp: " dnsPolicy: "
- state: present
- mode: 0600
-
-- name: Configure arguments to our kube-scheduler manifest - configmap
- lineinfile:
- path: /etc/kubernetes/manifests/kube-scheduler.yaml
- insertafter: " - kube-scheduler"
- line: "{{ item.arg }}={{ item.value }}"
- regexp: "{{ item.arg }}"
- state: present
- mode: 0600
- with_items:
- - { arg: " - --policy-configmap", value: "{{ tas_name }}-telemetry-aware-scheduling-scheduler-extender-policy" }
- - { arg: " - --policy-configmap-namespace", value: "{{ tas_namespace }}" }
-
-# TAS Demo Policy
-- name: template TAS demo policy
- template:
- src: "tas-demo-policy.yml.j2"
- dest: "/usr/src/charts/tas-demo-policy.yml"
- force: yes
- mode: preserve
- when:
- - tas_enable_demo_policy
-
-- name: create TAS demo policy resource
- command: kubectl apply -f tas-demo-policy.yml
- args:
- chdir: "/usr/src/charts"
- when:
- - tas_enable_demo_policy
Code Review / kuberef.git / commitdiff