3 # SPDX-FileCopyrightText: 2021 Ericsson AB and others
5 # SPDX-License-Identifier: Apache-2.0
7 OS_ID=$(grep '^ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g')
10 _print_msg "INFO" "$1"
14 _print_msg "ERROR" "$1"
19 echo "$(date +%H:%M:%S) - $1: $2"
27 if [ "${DEBUG:-false}" == "true" ]; then
31 check_prerequisites() {
32 info "Check prerequisites"
34 #-------------------------------------------------------------------------------
35 # Check for DEPLOYMENT type
36 #-------------------------------------------------------------------------------
37 if ! [[ "$DEPLOYMENT" =~ ^(full|k8s)$ ]]; then
38 error "Unsupported value for DEPLOYMENT ($DEPLOYMENT)"
41 #-------------------------------------------------------------------------------
42 # We shouldn't be running as root
43 #-------------------------------------------------------------------------------
44 if [[ "$(whoami)" == "root" ]] && [[ "$DEPLOYMENT" != "k8s" ]]; then
45 error "This script must not be run as root! Please switch to a regular user before running the script."
48 #-------------------------------------------------------------------------------
49 # Check for passwordless sudo
50 #-------------------------------------------------------------------------------
51 if ! sudo -n "true"; then
52 error "passwordless sudo is needed for '$(id -nu)' user."
55 #-------------------------------------------------------------------------------
56 # Check if SSH key exists
57 #-------------------------------------------------------------------------------
58 if [[ ! -f "$HOME/.ssh/id_rsa" ]]; then
59 error "You must have SSH keypair in order to run this script!"
62 #-------------------------------------------------------------------------------
63 # We are using sudo so we need to make sure that env_reset is not present
64 #-------------------------------------------------------------------------------
65 sudo sed -i "s/^Defaults.*env_reset/#&/" /etc/sudoers
67 #-------------------------------------------------------------------------------
68 # Installing prerequisites
69 #-------------------------------------------------------------------------------
70 if [ "$OS_ID" == "ubuntu" ]; then
75 if [ $RESULT -ne 0 ]; then
76 sudo apt-add-repository --yes --update ppa:ansible/ansible
77 sudo apt-get install -y ansible
82 if [ $RESULT -ne 0 ]; then
83 sudo wget https://github.com/mikefarah/yq/releases/download/3.4.1/yq_linux_amd64 -O /usr/bin/yq
84 sudo chmod +x /usr/bin/yq
89 if [ $RESULT -ne 0 ]; then
90 sudo apt-get install -y virsh
95 if [ $RESULT -ne 0 ]; then
96 sudo apt-get install -y jq
101 if [ $RESULT -ne 0 ]; then
102 sudo apt-get install -y virtualenv
106 if [ $RESULT -ne 0 ]; then
107 sudo apt-get install -y pip
110 elif [ "$OS_ID" == "centos" ]; then
115 if [ $RESULT -ne 0 ]; then
116 sudo dnf install epel-release
117 sudo dnf install ansible
122 if [ $RESULT -ne 0 ]; then
123 sudo wget https://github.com/mikefarah/yq/releases/download/3.4.1/yq_linux_amd64 -O /usr/bin/yq
124 sudo chmod +x /usr/bin/yq
129 if [ $RESULT -ne 0 ]; then
130 sudo yum install -y virsh
135 if [ $RESULT -ne 0 ]; then
136 sudo yum install -y jq
141 if [ $RESULT -ne 0 ]; then
142 sudo yum install -y virtualenv
146 if [ $RESULT -ne 0 ]; then
147 sudo yum install -y pip
151 #-------------------------------------------------------------------------------
152 # Check if necessary tools are installed
153 #-------------------------------------------------------------------------------
154 for tool in ansible yq virsh jq docker virtualenv pip; do
155 if ! command -v "$tool" &> /dev/null; then
156 error "$tool not found. Please install."
160 #-------------------------------------------------------------------------------
161 # Check if user belongs to libvirt's group
162 #-------------------------------------------------------------------------------
163 libvirt_group="libvirt"
164 # shellcheck disable=SC1091
165 source /etc/os-release || source /usr/lib/os-release
166 if [ "${ID,,}" == "ubuntu" ] && [ "$VERSION_ID" == "16.04" ]; then
169 if ! groups | grep "$libvirt_group"; then
170 error "$(id -nu) user doesn't belong to $libvirt_group group."
174 # Get jumphost VM PXE IP
181 assert_non_empty "$host" "get_ip - host parameter not provided"
183 PXE_NETWORK=$(yq r "$CURRENTPATH"/hw_config/"$VENDOR"/idf.yaml engine.pxe_network)
184 assert_non_empty "$PXE_NETWORK" "PXE network for jump VM not defined in IDF."
186 PXE_IF_INDEX=$(yq r "$CURRENTPATH"/hw_config/"${VENDOR}"/idf.yaml idf.net_config."$PXE_NETWORK".interface)
187 assert_non_empty "$PXE_IF_INDEX" "Index of PXE interface not found in IDF."
189 PXE_IF_IP=$(yq r "$CURRENTPATH"/hw_config/"${VENDOR}"/pdf.yaml "$host".interfaces["$PXE_IF_INDEX"].address)
190 assert_non_empty "$PXE_IF_IP" "IP of PXE interface not found in PDF."
195 # Get public MAC for VM
202 assert_non_empty "$host" "get_mac - host parameter not provided"
204 PUB_NETWORK=$(yq r "$CURRENTPATH"/hw_config/"$VENDOR"/idf.yaml engine.public_network)
205 assert_non_empty "$PUB_NETWORK" "Public network for jump VM not defined in IDF."
207 PUB_IF_INDEX=$(yq r "$CURRENTPATH"/hw_config/"${VENDOR}"/idf.yaml idf.net_config."$PUB_NETWORK".interface)
208 assert_non_empty "$PUB_IF_INDEX" "Index of public interface not found in IDF."
210 PUB_IF_MAC=$(yq r "$CURRENTPATH"/hw_config/"${VENDOR}"/pdf.yaml "$host".interfaces["$PUB_IF_INDEX"].mac_address)
211 assert_non_empty "$PUB_IF_MAC" "MAC of public interface not found in PDF."
217 if [[ "$DEPLOYMENT" == "full" ]]; then
218 ip=$(get_host_pxe_ip "jumphost")
220 mac=$(get_host_pub_mac "jumphost")
221 JUMPHOST_NAME=$(yq r "$CURRENTPATH"/hw_config/"$VENDOR"/pdf.yaml jumphost.name)
222 ipblock=$(virsh domifaddr "$JUMPHOST_NAME" --full | grep "$mac" | awk '{print $4}' | tail -n 1)
223 assert_non_empty "$ipblock" "IP subnet for VM not available."
229 # Copy files needed by Infra engine & BMRA in the jumphost VM
232 docker_config="/opt/kuberef/docker_config"
233 scp -r -o StrictHostKeyChecking=no \
234 "$CURRENTPATH"/{hw_config/"$VENDOR"/,sw_config/"$INSTALLER"/} \
235 "$USERNAME@${vm_ip}:$PROJECT_ROOT"
236 if [[ "$DEPLOYMENT" != "full" ]]; then
237 scp -r -o StrictHostKeyChecking=no \
239 "$USERNAME@${vm_ip}:.ssh/id_rsa"
241 if [ -f "$docker_config" ]; then
242 scp -r -o StrictHostKeyChecking=no \
243 "$docker_config" "$USERNAME@${vm_ip}:$PROJECT_ROOT"
248 provision_hosts_baremetal() {
249 CMD="./deploy.sh -s ironic -d ${DISTRO} -p file:///${PROJECT_ROOT}/engine/engine/pdf.yaml -i file:///${PROJECT_ROOT}/engine/engine/idf.yaml"
250 if [ "${DEBUG:-false}" == "true" ]; then
254 # shellcheck disable=SC2087
255 ssh -o StrictHostKeyChecking=no -tT "$USERNAME"@"$(get_vm_ip)" << EOF
256 # Install and run cloud-infra
257 if [ ! -d "${PROJECT_ROOT}/engine" ]; then
258 ssh-keygen -t rsa -N "" -f "${PROJECT_ROOT}"/.ssh/id_rsa
259 git clone https://gerrit.nordix.org/infra/engine.git
261 cp "${PROJECT_ROOT}"/"${VENDOR}"/{pdf.yaml,idf.yaml} \
262 "${PROJECT_ROOT}"/engine/engine
263 cd "${PROJECT_ROOT}"/engine/engine || return
268 provision_hosts_vms() {
269 # shellcheck disable=SC2087
270 # Install and run cloud-infra
271 if [ ! -d "${CURRENTPATH}/engine" ]; then
272 git clone https://gerrit.nordix.org/infra/engine.git "${CURRENTPATH}"/engine
274 cp "${CURRENTPATH}"/hw_config/"${VENDOR}"/{pdf.yaml,idf.yaml} "${CURRENTPATH}"/engine/engine
275 cd "${CURRENTPATH}"/engine/engine || return
276 CMD="./deploy.sh -s ironic -p file:///${CURRENTPATH}/engine/engine/pdf.yaml -i file:///${CURRENTPATH}/engine/engine/idf.yaml"
277 if [ "${DEBUG:-false}" == "true" ]; then
284 # Setup networking on provisioned hosts (Adapt setup_network.sh according to your network setup)
286 # Set Upper limit of number nodes in RI2 cluster (starting from 0)
287 NODE_MAX_ID=$(($(yq r "$CURRENTPATH"/hw_config/"$VENDOR"/idf.yaml --length idf.kubespray.hostnames)-1))
289 for idx in $(seq 0 "$NODE_MAX_ID"); do
290 NODE_IP=$(get_host_pxe_ip "nodes[${idx}]")
292 # shellcheck disable=SC2087
293 ssh -o StrictHostKeyChecking=no -tT "$USERNAME"@"$(get_vm_ip)" << EOF
294 ssh -o StrictHostKeyChecking=no root@"${NODE_IP}" \
295 'bash -s' < "${PROJECT_ROOT}"/"${VENDOR}"/setup_network.sh
300 # k8s Provisioning (currently BMRA)
301 provision_k8s_baremetal() {
302 ansible_cmd="/bin/bash -c '"
303 if [[ "$DEPLOYMENT" == "k8s" ]]; then
304 ansible-playbook -i "$CURRENTPATH"/sw_config/bmra/inventory.ini "$CURRENTPATH"/playbooks/pre-install.yaml
305 ansible_cmd+="yum -y remove python-netaddr; ansible-playbook -i /bmra/inventory.ini /bmra/playbooks/k8s/patch_kubespray.yml;"
307 ansible_cmd+="ansible-playbook -i /bmra/inventory.ini /bmra/playbooks/${BMRA_PROFILE}.yml'"
309 # shellcheck disable=SC2087
310 ssh -o StrictHostKeyChecking=no -tT "$USERNAME"@"$(get_vm_ip)" << EOF
312 if ! command -v docker; then
313 curl -fsSL https://get.docker.com/ | sh
314 printf "Waiting for docker service..."
315 until sudo docker info; do
320 if [ ! -d "${PROJECT_ROOT}/container-experience-kits" ]; then
321 git clone --recurse-submodules --depth 1 https://github.com/intel/container-experience-kits.git -b v21.08 "${PROJECT_ROOT}"/container-experience-kits/
322 cp -r "${PROJECT_ROOT}"/container-experience-kits/examples/"${BMRA_PROFILE}"/group_vars "${PROJECT_ROOT}"/container-experience-kits/
324 if [ -f "${PROJECT_ROOT}/docker_config" ]; then
325 cp "${PROJECT_ROOT}"/docker_config \
326 "${PROJECT_ROOT}"/"${INSTALLER}"/dockerhub_credentials/vars/main.yml
327 cp -r "${PROJECT_ROOT}"/"${INSTALLER}"/dockerhub_credentials \
328 "${PROJECT_ROOT}"/container-experience-kits/roles/
329 cp "${PROJECT_ROOT}"/"${INSTALLER}"/patched_k8s.yml \
330 "${PROJECT_ROOT}"/container-experience-kits/playbooks/k8s/k8s.yml
332 cp "${PROJECT_ROOT}"/"${INSTALLER}"/{inventory.ini,ansible.cfg} \
333 "${PROJECT_ROOT}"/container-experience-kits/
334 cp "${PROJECT_ROOT}"/"${INSTALLER}"/{all.yml,kube-node.yml} \
335 "${PROJECT_ROOT}"/container-experience-kits/group_vars/
336 cp "${PROJECT_ROOT}"/"${INSTALLER}"/patched_cmk_build.yml \
337 "${PROJECT_ROOT}"/container-experience-kits/roles/cmk_install/tasks/main.yml
338 cp "${PROJECT_ROOT}"/"${INSTALLER}"/patched_vfio.yml \
339 "${PROJECT_ROOT}"/container-experience-kits/roles/sriov_nic_init/tasks/bind_vf_driver.yml
340 cp "${PROJECT_ROOT}"/"${INSTALLER}"/patched_rhel_packages.yml \
341 "${PROJECT_ROOT}"/container-experience-kits/roles/bootstrap/install_packages/tasks/rhel.yml
342 cp "${PROJECT_ROOT}"/"${INSTALLER}"/patched_packages.yml \
343 "${PROJECT_ROOT}"/container-experience-kits/roles/bootstrap/install_packages/tasks/main.yml
344 cp "${PROJECT_ROOT}"/"${INSTALLER}"/patched_kubespray_requirements.txt \
345 "${PROJECT_ROOT}"/container-experience-kits/playbooks/k8s/kubespray/requirements.txt
346 cp "${PROJECT_ROOT}"/"${INSTALLER}"/patched_preflight.yml \
347 "${PROJECT_ROOT}"/container-experience-kits/playbooks/preflight.yml
348 cp "${PROJECT_ROOT}"/"${INSTALLER}"/patched_sriov_cni_install.yml \
349 "${PROJECT_ROOT}"/container-experience-kits/roles/sriov_cni_install/tasks/main.yml
350 cp "${PROJECT_ROOT}"/"${INSTALLER}"/patched_install_dpdk_meson.yml \
351 "${PROJECT_ROOT}"/container-experience-kits/roles/install_dpdk/tasks/install_dpdk_meson.yml
353 sudo docker run --rm \
354 -e ANSIBLE_CONFIG=/bmra/ansible.cfg \
355 -e PROFILE="${BMRA_PROFILE}" \
356 -v "${PROJECT_ROOT}"/container-experience-kits:/bmra \
357 -v ~/.ssh/:/root/.ssh/ rihabbanday/bmra21.08-install:centos \
362 provision_k8s_vms() {
363 # shellcheck disable=SC2087
365 if [ ! -d "${CURRENTPATH}/container-experience-kits" ]; then
366 git clone --recurse-submodules --depth 1 https://github.com/intel/container-experience-kits.git -b v21.08 "${CURRENTPATH}"/container-experience-kits/
367 cp -r "${CURRENTPATH}"/container-experience-kits/examples/"${BMRA_PROFILE}"/group_vars "${CURRENTPATH}"/container-experience-kits/
369 cp "${CURRENTPATH}"/sw_config/bmra/{inventory.ini,ansible.cfg} \
370 "${CURRENTPATH}"/container-experience-kits/
371 cp "${CURRENTPATH}"/sw_config/bmra/{all.yml,kube-node.yml} \
372 "${CURRENTPATH}"/container-experience-kits/group_vars/
373 cp "${CURRENTPATH}"/sw_config/bmra/patched_cmk_build.yml \
374 "${CURRENTPATH}"/container-experience-kits/roles/cmk_install/tasks/main.yml
375 cp "${CURRENTPATH}"/sw_config/bmra/patched_vfio.yml \
376 "${CURRENTPATH}"/container-experience-kits/roles/sriov_nic_init/tasks/bind_vf_driver.yml
377 cp "${CURRENTPATH}"/sw_config/bmra/patched_rhel_packages.yml \
378 "${CURRENTPATH}"/container-experience-kits/roles/bootstrap/install_packages/tasks/rhel.yml
379 cp "${CURRENTPATH}"/sw_config/bmra/patched_packages.yml \
380 "${CURRENTPATH}"/container-experience-kits/roles/bootstrap/install_packages/tasks/main.yml
381 cp "${CURRENTPATH}"/sw_config/"${INSTALLER}"/patched_kubespray_requirements.txt \
382 "${CURRENTPATH}"/container-experience-kits/playbooks/k8s/kubespray/requirements.txt
383 cp "${CURRENTPATH}"/sw_config/"${INSTALLER}"/patched_preflight.yml \
384 "${CURRENTPATH}"/container-experience-kits/playbooks/preflight.yml
385 cp "${CURRENTPATH}"/sw_config/"${INSTALLER}"/patched_sriov_cni_install.yml \
386 "${CURRENTPATH}"/container-experience-kits/roles/sriov_cni_install/tasks/main.yml
387 cp "${CURRENTPATH}"/sw_config/"${INSTALLER}"/patched_install_dpdk_meson.yml \
388 "${CURRENTPATH}"/container-experience-kits/roles/install_dpdk/tasks/install_dpdk_meson.yml
390 ansible-playbook -i "$CURRENTPATH"/sw_config/bmra/inventory.ini "$CURRENTPATH"/playbooks/pre-install.yaml
392 # Ansible upgrade below can be removed once image is updated
393 sudo docker run --rm \
394 -e ANSIBLE_CONFIG=/bmra/ansible.cfg \
395 -e PROFILE="${BMRA_PROFILE}" \
396 -v "${CURRENTPATH}"/container-experience-kits:/bmra \
397 -v ~/.ssh/:/root/.ssh/ rihabbanday/bmra21.08-install:centos \
398 ansible-playbook -i /bmra/inventory.ini /bmra/playbooks/"${BMRA_PROFILE}".yml
401 # Copy kubeconfig to the appropriate location needed by functest containers
403 # TODO Use Kubespray variables in BMRA to simplify this
404 MASTER_IP=$(get_host_pxe_ip "nodes[0]")
405 # shellcheck disable=SC2087
406 ssh -o StrictHostKeyChecking=no -tT "$USERNAME"@"$(get_vm_ip)" << EOF
407 scp -o StrictHostKeyChecking=no -q root@"$MASTER_IP":/root/.kube/config "${PROJECT_ROOT}"/kubeconfig
408 sed -i 's/127.0.0.1/$MASTER_IP/g' "${PROJECT_ROOT}"/kubeconfig
411 # Copy kubeconfig from Jump VM to appropriate location in Jump Host
412 # Direct scp to the specified location doesn't work due to permission/ssh-keys
413 scp -o StrictHostKeyChecking=no "$USERNAME"@"$(get_vm_ip)":"${PROJECT_ROOT}"/kubeconfig kubeconfig
414 if [ -d "/home/opnfv/functest-kubernetes" ]; then
415 sudo cp kubeconfig /home/opnfv/functest-kubernetes/config
419 # Creates a python virtual environment
420 creates_virtualenv() {
421 if [ ! -d "$CURRENTPATH/.venv" ]; then
422 virtualenv "$CURRENTPATH/.venv"
424 # shellcheck disable=SC1090
425 source "$CURRENTPATH/.venv/bin/activate"
426 pip install -r "$CURRENTPATH/requirements.txt"
429 # Executes a specific Ansible playbook
431 ansible_cmd="$(command -v ansible-playbook) -i $CURRENTPATH/inventory/localhost.ini -e ansible_python_interpreter=$(command -v python)"
432 if [ "${DEBUG:-false}" == "true" ]; then
435 eval "$ansible_cmd $CURRENTPATH/playbooks/${1}.yaml"