3 # SPDX-FileCopyrightText: 2021 Ericsson AB and others
5 # SPDX-License-Identifier: Apache-2.0
12 _print_msg "ERROR" "$1"
17 echo "$(date +%H:%M:%S) - $1: $2"
25 if [ "${DEBUG:-false}" == "true" ]; then
29 check_prerequisites() {
30 info "Check prerequisites"
32 #-------------------------------------------------------------------------------
33 # Check for DEPLOYMENT type
34 #-------------------------------------------------------------------------------
35 if ! [[ "$DEPLOYMENT" =~ ^(full|k8s)$ ]]; then
36 error "Unsupported value for DEPLOYMENT ($DEPLOYMENT)"
39 #-------------------------------------------------------------------------------
40 # We shouldn't be running as root
41 #-------------------------------------------------------------------------------
42 if [[ "$(whoami)" == "root" ]] && [[ "$DEPLOYMENT" != "k8s" ]]; then
43 error "This script must not be run as root! Please switch to a regular user before running the script."
46 #-------------------------------------------------------------------------------
47 # Check for passwordless sudo
48 #-------------------------------------------------------------------------------
49 if ! sudo -n "true"; then
50 error "passwordless sudo is needed for '$(id -nu)' user."
53 #-------------------------------------------------------------------------------
54 # Check if SSH key exists
55 #-------------------------------------------------------------------------------
56 if [[ ! -f "$HOME/.ssh/id_rsa" ]]; then
57 error "You must have SSH keypair in order to run this script!"
60 #-------------------------------------------------------------------------------
61 # We are using sudo so we need to make sure that env_reset is not present
62 #-------------------------------------------------------------------------------
63 sudo sed -i "s/^Defaults.*env_reset/#&/" /etc/sudoers
65 #-------------------------------------------------------------------------------
66 # Check if necessary tools are installed
67 #-------------------------------------------------------------------------------
68 for tool in ansible yq virsh jq docker virtualenv pip; do
69 if ! command -v "$tool" &> /dev/null; then
70 error "$tool not found. Please install."
74 #-------------------------------------------------------------------------------
75 # Check if user belongs to libvirt's group
76 #-------------------------------------------------------------------------------
77 libvirt_group="libvirt"
78 # shellcheck disable=SC1091
79 source /etc/os-release || source /usr/lib/os-release
80 if [ "${ID,,}" == "ubuntu" ] && [ "$VERSION_ID" == "16.04" ]; then
83 if ! groups | grep "$libvirt_group"; then
84 error "$(id -nu) user doesn't belong to $libvirt_group group."
88 # Get jumphost VM PXE IP
95 assert_non_empty "$host" "get_ip - host parameter not provided"
97 PXE_NETWORK=$(yq r "$CURRENTPATH"/hw_config/"$VENDOR"/idf.yaml engine.pxe_network)
98 assert_non_empty "$PXE_NETWORK" "PXE network for jump VM not defined in IDF."
100 PXE_IF_INDEX=$(yq r "$CURRENTPATH"/hw_config/"${VENDOR}"/idf.yaml idf.net_config."$PXE_NETWORK".interface)
101 assert_non_empty "$PXE_IF_INDEX" "Index of PXE interface not found in IDF."
103 PXE_IF_IP=$(yq r "$CURRENTPATH"/hw_config/"${VENDOR}"/pdf.yaml "$host".interfaces["$PXE_IF_INDEX"].address)
104 assert_non_empty "$PXE_IF_IP" "IP of PXE interface not found in PDF."
109 # Get public MAC for VM
116 assert_non_empty "$host" "get_mac - host parameter not provided"
118 PUB_NETWORK=$(yq r "$CURRENTPATH"/hw_config/"$VENDOR"/idf.yaml engine.public_network)
119 assert_non_empty "$PUB_NETWORK" "Public network for jump VM not defined in IDF."
121 PUB_IF_INDEX=$(yq r "$CURRENTPATH"/hw_config/"${VENDOR}"/idf.yaml idf.net_config."$PUB_NETWORK".interface)
122 assert_non_empty "$PUB_IF_INDEX" "Index of public interface not found in IDF."
124 PUB_IF_MAC=$(yq r "$CURRENTPATH"/hw_config/"${VENDOR}"/pdf.yaml "$host".interfaces["$PUB_IF_INDEX"].mac_address)
125 assert_non_empty "$PUB_IF_MAC" "MAC of public interface not found in PDF."
131 if [[ "$DEPLOYMENT" == "full" ]]; then
132 ip=$(get_host_pxe_ip "jumphost")
134 mac=$(get_host_pub_mac "jumphost")
135 JUMPHOST_NAME=$(yq r "$CURRENTPATH"/hw_config/"$VENDOR"/pdf.yaml jumphost.name)
136 ipblock=$(virsh domifaddr "$JUMPHOST_NAME" --full | grep "$mac" | awk '{print $4}' | tail -n 1)
137 assert_non_empty "$ipblock" "IP subnet for VM not available."
143 # Copy files needed by Infra engine & BMRA in the jumphost VM
146 docker_config="/opt/kuberef/docker_config"
147 scp -r -o StrictHostKeyChecking=no \
148 "$CURRENTPATH"/{hw_config/"$VENDOR"/,sw_config/"$INSTALLER"/} \
149 "$USERNAME@${vm_ip}:$PROJECT_ROOT"
150 if [[ "$DEPLOYMENT" != "full" ]]; then
151 scp -r -o StrictHostKeyChecking=no \
153 "$USERNAME@${vm_ip}:.ssh/id_rsa"
155 if [ -f "$docker_config" ]; then
156 scp -r -o StrictHostKeyChecking=no \
157 "$docker_config" "$USERNAME@${vm_ip}:$PROJECT_ROOT"
162 provision_hosts_baremetal() {
163 if [ "${DEBUG:-false}" == "true" ]; then
169 # shellcheck disable=SC2087
170 ssh -o StrictHostKeyChecking=no -tT "$USERNAME"@"$(get_vm_ip)" << EOF
171 # Install and run cloud-infra
172 if [ ! -d "${PROJECT_ROOT}/engine" ]; then
173 ssh-keygen -t rsa -N "" -f "${PROJECT_ROOT}"/.ssh/id_rsa
174 git clone https://gerrit.nordix.org/infra/engine.git
176 cp "${PROJECT_ROOT}"/"${VENDOR}"/{pdf.yaml,idf.yaml} \
177 "${PROJECT_ROOT}"/engine/engine
178 cd "${PROJECT_ROOT}"/engine/engine || return
179 ./deploy.sh -s ironic "${DEBUG_FLAG}" -d "${DISTRO}" \
180 -p file:///"${PROJECT_ROOT}"/engine/engine/pdf.yaml \
181 -i file:///"${PROJECT_ROOT}"/engine/engine/idf.yaml
185 provision_hosts_vms() {
186 if [ "${DEBUG:-false}" == "true" ]; then
192 # shellcheck disable=SC2087
193 # Install and run cloud-infra
194 if [ ! -d "${CURRENTPATH}/engine" ]; then
195 git clone https://gerrit.nordix.org/infra/engine.git "${CURRENTPATH}"/engine
197 cp "${CURRENTPATH}"/hw_config/"${VENDOR}"/{pdf.yaml,idf.yaml} "${CURRENTPATH}"/engine/engine
198 cd "${CURRENTPATH}"/engine/engine || return
199 ./deploy.sh -s ironic "${DEBUG_FLAG}" \
200 -p file:///"${CURRENTPATH}"/engine/engine/pdf.yaml \
201 -i file:///"${CURRENTPATH}"/engine/engine/idf.yaml
204 # Setup networking on provisioned hosts (Adapt setup_network.sh according to your network setup)
206 # Set Upper limit of number nodes in RI2 cluster (starting from 0)
207 NODE_MAX_ID=$(($(yq r "$CURRENTPATH"/hw_config/"$VENDOR"/idf.yaml --length idf.kubespray.hostnames)-1))
209 for idx in $(seq 0 "$NODE_MAX_ID"); do
210 NODE_IP=$(get_host_pxe_ip "nodes[${idx}]")
212 # shellcheck disable=SC2087
213 ssh -o StrictHostKeyChecking=no -tT "$USERNAME"@"$(get_vm_ip)" << EOF
214 ssh -o StrictHostKeyChecking=no root@"${NODE_IP}" \
215 'bash -s' < "${PROJECT_ROOT}"/"${VENDOR}"/setup_network.sh
220 # k8s Provisioning (currently BMRA)
221 provision_k8s_baremetal() {
222 ansible_cmd="/bin/bash -c '"
223 if [[ "$DEPLOYMENT" == "k8s" ]]; then
224 ansible-playbook -i "$CURRENTPATH"/sw_config/bmra/inventory.ini "$CURRENTPATH"/playbooks/pre-install.yaml
225 ansible_cmd+="yum -y remove python-netaddr; ansible-playbook -i /bmra/inventory.ini /bmra/playbooks/k8s/patch_kubespray.yml;"
227 ansible_cmd+="ansible-playbook -i /bmra/inventory.ini /bmra/playbooks/${BMRA_PROFILE}.yml'"
229 # shellcheck disable=SC2087
230 ssh -o StrictHostKeyChecking=no -tT "$USERNAME"@"$(get_vm_ip)" << EOF
232 if ! command -v docker; then
233 curl -fsSL https://get.docker.com/ | sh
234 printf "Waiting for docker service..."
235 until sudo docker info; do
240 if [ ! -d "${PROJECT_ROOT}/container-experience-kits" ]; then
241 git clone --recurse-submodules --depth 1 https://github.com/intel/container-experience-kits.git -b v21.08 "${PROJECT_ROOT}"/container-experience-kits/
242 cp -r "${PROJECT_ROOT}"/container-experience-kits/examples/"${BMRA_PROFILE}"/group_vars "${PROJECT_ROOT}"/container-experience-kits/
244 if [ -f "${PROJECT_ROOT}/docker_config" ]; then
245 cp "${PROJECT_ROOT}"/docker_config \
246 "${PROJECT_ROOT}"/"${INSTALLER}"/dockerhub_credentials/vars/main.yml
247 cp -r "${PROJECT_ROOT}"/"${INSTALLER}"/dockerhub_credentials \
248 "${PROJECT_ROOT}"/container-experience-kits/roles/
249 cp "${PROJECT_ROOT}"/"${INSTALLER}"/patched_k8s.yml \
250 "${PROJECT_ROOT}"/container-experience-kits/playbooks/k8s/k8s.yml
252 cp "${PROJECT_ROOT}"/"${INSTALLER}"/{inventory.ini,ansible.cfg} \
253 "${PROJECT_ROOT}"/container-experience-kits/
254 cp "${PROJECT_ROOT}"/"${INSTALLER}"/{all.yml,kube-node.yml} \
255 "${PROJECT_ROOT}"/container-experience-kits/group_vars/
256 cp "${PROJECT_ROOT}"/"${INSTALLER}"/patched_cmk_build.yml \
257 "${PROJECT_ROOT}"/container-experience-kits/roles/cmk_install/tasks/main.yml
258 cp "${PROJECT_ROOT}"/"${INSTALLER}"/patched_vfio.yml \
259 "${PROJECT_ROOT}"/container-experience-kits/roles/sriov_nic_init/tasks/bind_vf_driver.yml
260 cp "${PROJECT_ROOT}"/"${INSTALLER}"/patched_rhel_packages.yml \
261 "${PROJECT_ROOT}"/container-experience-kits/roles/bootstrap/install_packages/tasks/rhel.yml
262 cp "${PROJECT_ROOT}"/"${INSTALLER}"/patched_packages.yml \
263 "${PROJECT_ROOT}"/container-experience-kits/roles/bootstrap/install_packages/tasks/main.yml
264 cp "${PROJECT_ROOT}"/"${INSTALLER}"/patched_kubespray_requirements.txt \
265 "${PROJECT_ROOT}"/container-experience-kits/playbooks/k8s/kubespray/requirements.txt
266 cp "${PROJECT_ROOT}"/"${INSTALLER}"/patched_preflight.yml \
267 "${PROJECT_ROOT}"/container-experience-kits/playbooks/preflight.yml
268 cp "${PROJECT_ROOT}"/"${INSTALLER}"/patched_sriov_cni_install.yml \
269 "${PROJECT_ROOT}"/container-experience-kits/roles/sriov_cni_install/tasks/main.yml
270 cp "${PROJECT_ROOT}"/"${INSTALLER}"/patched_install_dpdk_meson.yml \
271 "${PROJECT_ROOT}"/container-experience-kits/roles/install_dpdk/tasks/install_dpdk_meson.yml
273 sudo docker run --rm \
274 -e ANSIBLE_CONFIG=/bmra/ansible.cfg \
275 -e PROFILE="${BMRA_PROFILE}" \
276 -v "${PROJECT_ROOT}"/container-experience-kits:/bmra \
277 -v ~/.ssh/:/root/.ssh/ rihabbanday/bmra21.08-install:centos \
282 provision_k8s_vms() {
283 # shellcheck disable=SC2087
285 if [ ! -d "${CURRENTPATH}/container-experience-kits" ]; then
286 git clone --recurse-submodules --depth 1 https://github.com/intel/container-experience-kits.git -b v21.08 "${CURRENTPATH}"/container-experience-kits/
287 cp -r "${CURRENTPATH}"/container-experience-kits/examples/"${BMRA_PROFILE}"/group_vars "${CURRENTPATH}"/container-experience-kits/
289 cp "${CURRENTPATH}"/sw_config/bmra/{inventory.ini,ansible.cfg} \
290 "${CURRENTPATH}"/container-experience-kits/
291 cp "${CURRENTPATH}"/sw_config/bmra/{all.yml,kube-node.yml} \
292 "${CURRENTPATH}"/container-experience-kits/group_vars/
293 cp "${CURRENTPATH}"/sw_config/bmra/patched_cmk_build.yml \
294 "${CURRENTPATH}"/container-experience-kits/roles/cmk_install/tasks/main.yml
295 cp "${CURRENTPATH}"/sw_config/bmra/patched_vfio.yml \
296 "${CURRENTPATH}"/container-experience-kits/roles/sriov_nic_init/tasks/bind_vf_driver.yml
297 cp "${CURRENTPATH}"/sw_config/bmra/patched_rhel_packages.yml \
298 "${CURRENTPATH}"/container-experience-kits/roles/bootstrap/install_packages/tasks/rhel.yml
299 cp "${CURRENTPATH}"/sw_config/bmra/patched_packages.yml \
300 "${CURRENTPATH}"/container-experience-kits/roles/bootstrap/install_packages/tasks/main.yml
301 cp "${CURRENTPATH}"/sw_config/"${INSTALLER}"/patched_kubespray_requirements.txt \
302 "${CURRENTPATH}"/container-experience-kits/playbooks/k8s/kubespray/requirements.txt
303 cp "${CURRENTPATH}"/sw_config/"${INSTALLER}"/patched_preflight.yml \
304 "${CURRENTPATH}"/container-experience-kits/playbooks/preflight.yml
305 cp "${CURRENTPATH}"/sw_config/"${INSTALLER}"/patched_sriov_cni_install.yml \
306 "${CURRENTPATH}"/container-experience-kits/roles/sriov_cni_install/tasks/main.yml
307 cp "${CURRENTPATH}"/sw_config/"${INSTALLER}"/patched_install_dpdk_meson.yml \
308 "${CURRENTPATH}"/container-experience-kits/roles/install_dpdk/tasks/install_dpdk_meson.yml
310 ansible-playbook -i "$CURRENTPATH"/sw_config/bmra/inventory.ini "$CURRENTPATH"/playbooks/pre-install.yaml
312 # Ansible upgrade below can be removed once image is updated
313 sudo docker run --rm \
314 -e ANSIBLE_CONFIG=/bmra/ansible.cfg \
315 -e PROFILE="${BMRA_PROFILE}" \
316 -v "${CURRENTPATH}"/container-experience-kits:/bmra \
317 -v ~/.ssh/:/root/.ssh/ rihabbanday/bmra21.08-install:centos \
318 ansible-playbook -i /bmra/inventory.ini /bmra/playbooks/"${BMRA_PROFILE}".yml
321 # Copy kubeconfig to the appropriate location needed by functest containers
323 # TODO Use Kubespray variables in BMRA to simplify this
324 MASTER_IP=$(get_host_pxe_ip "nodes[0]")
325 # shellcheck disable=SC2087
326 ssh -o StrictHostKeyChecking=no -tT "$USERNAME"@"$(get_vm_ip)" << EOF
327 scp -o StrictHostKeyChecking=no -q root@"$MASTER_IP":/root/.kube/config "${PROJECT_ROOT}"/kubeconfig
328 sed -i 's/127.0.0.1/$MASTER_IP/g' "${PROJECT_ROOT}"/kubeconfig
331 # Copy kubeconfig from Jump VM to appropriate location in Jump Host
332 # Direct scp to the specified location doesn't work due to permission/ssh-keys
333 scp -o StrictHostKeyChecking=no "$USERNAME"@"$(get_vm_ip)":"${PROJECT_ROOT}"/kubeconfig kubeconfig
334 if [ -d "/home/opnfv/functest-kubernetes" ]; then
335 sudo cp kubeconfig /home/opnfv/functest-kubernetes/config
339 # Creates a python virtual environment
340 creates_virtualenv() {
341 if [ ! -d "$CURRENTPATH/.venv" ]; then
342 virtualenv "$CURRENTPATH/.venv"
344 # shellcheck disable=SC1090
345 source "$CURRENTPATH/.venv/bin/activate"
346 pip install -r "$CURRENTPATH/requirements.txt"
349 # Executes a specific Ansible playbook
351 ansible_cmd="$(command -v ansible-playbook) -i $CURRENTPATH/inventory/localhost.ini -e ansible_python_interpreter=$(command -v python)"
352 if [ "${DEBUG:-false}" == "true" ]; then
355 eval "$ansible_cmd $CURRENTPATH/playbooks/${1}.yaml"