functions.sh

   1 #!/bin/bash
   2
   3 # SPDX-FileCopyrightText: 2021 Ericsson AB and others
   4 #
   5 # SPDX-License-Identifier: Apache-2.0
   6
   7 info() {
   8     _print_msg "INFO" "$1"
   9 }
  10
  11 error() {
  12     _print_msg "ERROR" "$1"
  13     exit 1
  14 }
  15
  16 _print_msg() {
  17     echo "$(date +%H:%M:%S) - $1: $2"
  18 }
  19
  20 assert_non_empty() {
  21     if [ -z "$1" ]; then
  22         error "$2"
  23     fi
  24 }
  25 if [ "${DEBUG:-false}" == "true" ]; then
  26     set -o xtrace
  27 fi
  28
  29 check_prerequisites() {
  30     info "Check prerequisites"
  31
  32     #-------------------------------------------------------------------------------
  33     # Check for DEPLOYMENT type
  34     #-------------------------------------------------------------------------------
  35     if ! [[ "$DEPLOYMENT" =~ ^(full|k8s)$ ]]; then
  36         error "Unsupported value for DEPLOYMENT ($DEPLOYMENT)"
  37     fi
  38
  39     #-------------------------------------------------------------------------------
  40     # We shouldn't be running as root
  41     #-------------------------------------------------------------------------------
  42     if [[ "$(whoami)" == "root" ]] && [[ "$DEPLOYMENT" != "k8s" ]]; then
  43         error "This script must not be run as root! Please switch to a regular user before running the script."
  44     fi
  45
  46     #-------------------------------------------------------------------------------
  47     # Check for passwordless sudo
  48     #-------------------------------------------------------------------------------
  49     if ! sudo -n "true"; then
  50         error "passwordless sudo is needed for '$(id -nu)' user."
  51     fi
  52
  53     #-------------------------------------------------------------------------------
  54     # Check if SSH key exists
  55     #-------------------------------------------------------------------------------
  56     if [[ ! -f "$HOME/.ssh/id_rsa" ]]; then
  57         error "You must have SSH keypair in order to run this script!"
  58     fi
  59
  60     #-------------------------------------------------------------------------------
  61     # We are using sudo so we need to make sure that env_reset is not present
  62     #-------------------------------------------------------------------------------
  63     sudo sed -i "s/^Defaults.*env_reset/#&/" /etc/sudoers
  64
  65     #-------------------------------------------------------------------------------
  66     # Check if necessary tools are installed
  67     #-------------------------------------------------------------------------------
  68     for tool in ansible yq virsh jq docker virtualenv pip; do
  69         if ! command -v "$tool" &> /dev/null; then
  70             error "$tool not found. Please install."
  71         fi
  72     done
  73
  74     #-------------------------------------------------------------------------------
  75     # Check if user belongs to libvirt's group
  76     #-------------------------------------------------------------------------------
  77     libvirt_group="libvirt"
  78     # shellcheck disable=SC1091
  79     source /etc/os-release || source /usr/lib/os-release
  80     if [ "${ID,,}" == "ubuntu" ] && [ "$VERSION_ID" == "16.04" ]; then
  81         libvirt_group+="d"
  82     fi
  83     if ! groups | grep "$libvirt_group"; then
  84         error "$(id -nu) user doesn't belong to $libvirt_group group."
  85     fi
  86 }
  87
  88 # Get jumphost VM PXE IP
  89 get_host_pxe_ip() {
  90     local PXE_NETWORK
  91     local PXE_IF_INDEX
  92     local PXE_IF_IP
  93
  94     host=$1
  95     assert_non_empty "$host" "get_ip - host parameter not provided"
  96
  97     PXE_NETWORK=$(yq r "$CURRENTPATH"/hw_config/"$VENDOR"/idf.yaml engine.pxe_network)
  98     assert_non_empty "$PXE_NETWORK" "PXE network for jump VM not defined in IDF."
  99
 100     PXE_IF_INDEX=$(yq r "$CURRENTPATH"/hw_config/"${VENDOR}"/idf.yaml idf.net_config."$PXE_NETWORK".interface)
 101     assert_non_empty "$PXE_IF_INDEX" "Index of PXE interface not found in IDF."
 102
 103     PXE_IF_IP=$(yq r "$CURRENTPATH"/hw_config/"${VENDOR}"/pdf.yaml "$host".interfaces["$PXE_IF_INDEX"].address)
 104     assert_non_empty "$PXE_IF_IP" "IP of PXE interface not found in PDF."
 105
 106     echo "$PXE_IF_IP"
 107 }
 108
 109 # Get public MAC for VM
 110 get_host_pub_mac() {
 111     local PUB_NETWORK
 112     local PUB_IF_INDEX
 113     local PUB_IF_MAC
 114
 115     host=$1
 116     assert_non_empty "$host" "get_mac - host parameter not provided"
 117
 118     PUB_NETWORK=$(yq r "$CURRENTPATH"/hw_config/"$VENDOR"/idf.yaml  engine.public_network)
 119     assert_non_empty "$PUB_NETWORK" "Public network for jump VM not defined in IDF."
 120
 121     PUB_IF_INDEX=$(yq r "$CURRENTPATH"/hw_config/"${VENDOR}"/idf.yaml idf.net_config."$PUB_NETWORK".interface)
 122     assert_non_empty "$PUB_IF_INDEX" "Index of public interface not found in IDF."
 123
 124     PUB_IF_MAC=$(yq r "$CURRENTPATH"/hw_config/"${VENDOR}"/pdf.yaml "$host".interfaces["$PUB_IF_INDEX"].mac_address)
 125     assert_non_empty "$PUB_IF_MAC" "MAC of public interface not found in PDF."
 126     echo "$PUB_IF_MAC"
 127 }
 128
 129 # Get jumphost VM IP
 130 get_vm_ip() {
 131     if [[ "$DEPLOYMENT" == "full" ]]; then
 132         ip=$(get_host_pxe_ip "jumphost")
 133     else
 134         mac=$(get_host_pub_mac "jumphost")
 135         JUMPHOST_NAME=$(yq r "$CURRENTPATH"/hw_config/"$VENDOR"/pdf.yaml jumphost.name)
 136         ipblock=$(virsh domifaddr "$JUMPHOST_NAME" --full | grep "$mac" | awk '{print $4}' | tail -n 1)
 137         assert_non_empty "$ipblock" "IP subnet for VM not available."
 138         ip="${ipblock%/*}"
 139     fi
 140     echo "$ip"
 141 }
 142
 143 # Copy files needed by Infra engine & BMRA in the jumphost VM
 144 copy_files_jump() {
 145     vm_ip="$(get_vm_ip)"
 146     docker_config="/opt/kuberef/docker_config"
 147     scp -r -o StrictHostKeyChecking=no \
 148     "$CURRENTPATH"/{hw_config/"$VENDOR"/,sw_config/"$INSTALLER"/} \
 149     "$USERNAME@${vm_ip}:$PROJECT_ROOT"
 150     if [[ "$DEPLOYMENT" != "full" ]]; then
 151         scp -r -o StrictHostKeyChecking=no \
 152         ~/.ssh/id_rsa \
 153         "$USERNAME@${vm_ip}:.ssh/id_rsa"
 154     fi
 155     if [ -f "$docker_config" ]; then
 156         scp -r -o StrictHostKeyChecking=no \
 157         "$docker_config" "$USERNAME@${vm_ip}:$PROJECT_ROOT"
 158     fi
 159 }
 160
 161 # Host Provisioning
 162 provision_hosts_baremetal() {
 163     CMD="./deploy.sh -s ironic -d ${DISTRO} -p file:///${PROJECT_ROOT}/engine/engine/pdf.yaml -i file:///${PROJECT_ROOT}/engine/engine/idf.yaml"
 164     if [ "${DEBUG:-false}" == "true" ]; then
 165         CMD+=" -v"
 166     fi
 167
 168     # shellcheck disable=SC2087
 169     ssh -o StrictHostKeyChecking=no -tT "$USERNAME"@"$(get_vm_ip)" << EOF
 170 # Install and run cloud-infra
 171 if [ ! -d "${PROJECT_ROOT}/engine" ]; then
 172     ssh-keygen -t rsa -N "" -f "${PROJECT_ROOT}"/.ssh/id_rsa
 173     git clone https://gerrit.nordix.org/infra/engine.git
 174 fi
 175 cp "${PROJECT_ROOT}"/"${VENDOR}"/{pdf.yaml,idf.yaml} \
 176 "${PROJECT_ROOT}"/engine/engine
 177 cd "${PROJECT_ROOT}"/engine/engine || return
 178 ${CMD}
 179 EOF
 180 }
 181
 182 provision_hosts_vms() {
 183     # shellcheck disable=SC2087
 184     # Install and run cloud-infra
 185     if [ ! -d "${CURRENTPATH}/engine" ]; then
 186         git clone https://gerrit.nordix.org/infra/engine.git "${CURRENTPATH}"/engine
 187     fi
 188     cp "${CURRENTPATH}"/hw_config/"${VENDOR}"/{pdf.yaml,idf.yaml} "${CURRENTPATH}"/engine/engine
 189     cd "${CURRENTPATH}"/engine/engine || return
 190     CMD="./deploy.sh -s ironic -p file:///${CURRENTPATH}/engine/engine/pdf.yaml -i file:///${CURRENTPATH}/engine/engine/idf.yaml"
 191     if [ "${DEBUG:-false}" == "true" ]; then
 192         CMD+=" -v"
 193     fi
 194
 195     ${CMD}
 196 }
 197
 198 # Setup networking on provisioned hosts (Adapt setup_network.sh according to your network setup)
 199 setup_network() {
 200     # Set Upper limit of number nodes in RI2 cluster (starting from 0)
 201     NODE_MAX_ID=$(($(yq r "$CURRENTPATH"/hw_config/"$VENDOR"/idf.yaml --length idf.kubespray.hostnames)-1))
 202
 203     for idx in $(seq 0 "$NODE_MAX_ID"); do
 204         NODE_IP=$(get_host_pxe_ip "nodes[${idx}]")
 205         # SSH to jumphost
 206         # shellcheck disable=SC2087
 207         ssh -o StrictHostKeyChecking=no -tT "$USERNAME"@"$(get_vm_ip)" << EOF
 208 ssh -o StrictHostKeyChecking=no root@"${NODE_IP}" \
 209     'bash -s' <  "${PROJECT_ROOT}"/"${VENDOR}"/setup_network.sh
 210 EOF
 211     done
 212 }
 213
 214 # k8s Provisioning (currently BMRA)
 215 provision_k8s_baremetal() {
 216     ansible_cmd="/bin/bash -c '"
 217     if [[ "$DEPLOYMENT" == "k8s" ]]; then
 218         ansible-playbook -i "$CURRENTPATH"/sw_config/bmra/inventory.ini "$CURRENTPATH"/playbooks/pre-install.yaml
 219         ansible_cmd+="yum -y remove python-netaddr; ansible-playbook -i /bmra/inventory.ini /bmra/playbooks/k8s/patch_kubespray.yml;"
 220     fi
 221     ansible_cmd+="ansible-playbook -i /bmra/inventory.ini /bmra/playbooks/${BMRA_PROFILE}.yml'"
 222
 223     # shellcheck disable=SC2087
 224     ssh -o StrictHostKeyChecking=no -tT "$USERNAME"@"$(get_vm_ip)" << EOF
 225 # Install BMRA
 226 if ! command -v docker; then
 227     curl -fsSL https://get.docker.com/ | sh
 228     printf "Waiting for docker service..."
 229     until sudo docker info; do
 230         printf "."
 231         sleep 2
 232     done
 233 fi
 234 if [ ! -d "${PROJECT_ROOT}/container-experience-kits" ]; then
 235     git clone --recurse-submodules --depth 1 https://github.com/intel/container-experience-kits.git -b v21.08 "${PROJECT_ROOT}"/container-experience-kits/
 236     cp -r "${PROJECT_ROOT}"/container-experience-kits/examples/"${BMRA_PROFILE}"/group_vars "${PROJECT_ROOT}"/container-experience-kits/
 237 fi
 238 if [ -f "${PROJECT_ROOT}/docker_config" ]; then
 239     cp "${PROJECT_ROOT}"/docker_config \
 240         "${PROJECT_ROOT}"/"${INSTALLER}"/dockerhub_credentials/vars/main.yml
 241     cp -r "${PROJECT_ROOT}"/"${INSTALLER}"/dockerhub_credentials \
 242         "${PROJECT_ROOT}"/container-experience-kits/roles/
 243     cp "${PROJECT_ROOT}"/"${INSTALLER}"/patched_k8s.yml \
 244         "${PROJECT_ROOT}"/container-experience-kits/playbooks/k8s/k8s.yml
 245 fi
 246 cp "${PROJECT_ROOT}"/"${INSTALLER}"/{inventory.ini,ansible.cfg} \
 247     "${PROJECT_ROOT}"/container-experience-kits/
 248 cp "${PROJECT_ROOT}"/"${INSTALLER}"/{all.yml,kube-node.yml} \
 249     "${PROJECT_ROOT}"/container-experience-kits/group_vars/
 250 cp "${PROJECT_ROOT}"/"${INSTALLER}"/patched_cmk_build.yml \
 251     "${PROJECT_ROOT}"/container-experience-kits/roles/cmk_install/tasks/main.yml
 252 cp "${PROJECT_ROOT}"/"${INSTALLER}"/patched_vfio.yml \
 253     "${PROJECT_ROOT}"/container-experience-kits/roles/sriov_nic_init/tasks/bind_vf_driver.yml
 254 cp "${PROJECT_ROOT}"/"${INSTALLER}"/patched_rhel_packages.yml \
 255     "${PROJECT_ROOT}"/container-experience-kits/roles/bootstrap/install_packages/tasks/rhel.yml
 256 cp "${PROJECT_ROOT}"/"${INSTALLER}"/patched_packages.yml \
 257     "${PROJECT_ROOT}"/container-experience-kits/roles/bootstrap/install_packages/tasks/main.yml
 258 cp "${PROJECT_ROOT}"/"${INSTALLER}"/patched_kubespray_requirements.txt \
 259     "${PROJECT_ROOT}"/container-experience-kits/playbooks/k8s/kubespray/requirements.txt
 260 cp "${PROJECT_ROOT}"/"${INSTALLER}"/patched_preflight.yml \
 261     "${PROJECT_ROOT}"/container-experience-kits/playbooks/preflight.yml
 262 cp "${PROJECT_ROOT}"/"${INSTALLER}"/patched_sriov_cni_install.yml \
 263     "${PROJECT_ROOT}"/container-experience-kits/roles/sriov_cni_install/tasks/main.yml
 264 cp "${PROJECT_ROOT}"/"${INSTALLER}"/patched_install_dpdk_meson.yml \
 265     "${PROJECT_ROOT}"/container-experience-kits/roles/install_dpdk/tasks/install_dpdk_meson.yml
 266
 267 sudo docker run --rm \
 268 -e ANSIBLE_CONFIG=/bmra/ansible.cfg \
 269 -e PROFILE="${BMRA_PROFILE}" \
 270 -v "${PROJECT_ROOT}"/container-experience-kits:/bmra \
 271 -v ~/.ssh/:/root/.ssh/ rihabbanday/bmra21.08-install:centos \
 272 ${ansible_cmd}
 273 EOF
 274 }
 275
 276 provision_k8s_vms() {
 277     # shellcheck disable=SC2087
 278 # Install BMRA
 279 if [ ! -d "${CURRENTPATH}/container-experience-kits" ]; then
 280     git clone --recurse-submodules --depth 1 https://github.com/intel/container-experience-kits.git -b v21.08 "${CURRENTPATH}"/container-experience-kits/
 281     cp -r "${CURRENTPATH}"/container-experience-kits/examples/"${BMRA_PROFILE}"/group_vars "${CURRENTPATH}"/container-experience-kits/
 282 fi
 283 cp "${CURRENTPATH}"/sw_config/bmra/{inventory.ini,ansible.cfg} \
 284     "${CURRENTPATH}"/container-experience-kits/
 285 cp "${CURRENTPATH}"/sw_config/bmra/{all.yml,kube-node.yml} \
 286     "${CURRENTPATH}"/container-experience-kits/group_vars/
 287 cp "${CURRENTPATH}"/sw_config/bmra/patched_cmk_build.yml \
 288     "${CURRENTPATH}"/container-experience-kits/roles/cmk_install/tasks/main.yml
 289 cp "${CURRENTPATH}"/sw_config/bmra/patched_vfio.yml \
 290    "${CURRENTPATH}"/container-experience-kits/roles/sriov_nic_init/tasks/bind_vf_driver.yml
 291 cp "${CURRENTPATH}"/sw_config/bmra/patched_rhel_packages.yml \
 292     "${CURRENTPATH}"/container-experience-kits/roles/bootstrap/install_packages/tasks/rhel.yml
 293 cp "${CURRENTPATH}"/sw_config/bmra/patched_packages.yml \
 294     "${CURRENTPATH}"/container-experience-kits/roles/bootstrap/install_packages/tasks/main.yml
 295 cp "${CURRENTPATH}"/sw_config/"${INSTALLER}"/patched_kubespray_requirements.txt \
 296     "${CURRENTPATH}"/container-experience-kits/playbooks/k8s/kubespray/requirements.txt
 297 cp "${CURRENTPATH}"/sw_config/"${INSTALLER}"/patched_preflight.yml \
 298     "${CURRENTPATH}"/container-experience-kits/playbooks/preflight.yml
 299 cp "${CURRENTPATH}"/sw_config/"${INSTALLER}"/patched_sriov_cni_install.yml \
 300     "${CURRENTPATH}"/container-experience-kits/roles/sriov_cni_install/tasks/main.yml
 301 cp "${CURRENTPATH}"/sw_config/"${INSTALLER}"/patched_install_dpdk_meson.yml \
 302     "${CURRENTPATH}"/container-experience-kits/roles/install_dpdk/tasks/install_dpdk_meson.yml
 303
 304 ansible-playbook -i "$CURRENTPATH"/sw_config/bmra/inventory.ini "$CURRENTPATH"/playbooks/pre-install.yaml
 305
 306 # Ansible upgrade below can be removed once image is updated
 307 sudo docker run --rm \
 308 -e ANSIBLE_CONFIG=/bmra/ansible.cfg \
 309 -e PROFILE="${BMRA_PROFILE}" \
 310 -v "${CURRENTPATH}"/container-experience-kits:/bmra \
 311 -v ~/.ssh/:/root/.ssh/ rihabbanday/bmra21.08-install:centos \
 312 ansible-playbook -i /bmra/inventory.ini /bmra/playbooks/"${BMRA_PROFILE}".yml
 313 }
 314
 315 # Copy kubeconfig to the appropriate location needed by functest containers
 316 copy_k8s_config() {
 317 # TODO Use Kubespray variables in BMRA to simplify this
 318     MASTER_IP=$(get_host_pxe_ip "nodes[0]")
 319     # shellcheck disable=SC2087
 320     ssh -o StrictHostKeyChecking=no -tT "$USERNAME"@"$(get_vm_ip)" << EOF
 321 scp -o StrictHostKeyChecking=no -q root@"$MASTER_IP":/root/.kube/config "${PROJECT_ROOT}"/kubeconfig
 322 sed -i 's/127.0.0.1/$MASTER_IP/g' "${PROJECT_ROOT}"/kubeconfig
 323 EOF
 324
 325 # Copy kubeconfig from Jump VM to appropriate location in Jump Host
 326 # Direct scp to the specified location doesn't work due to permission/ssh-keys
 327     scp  -o StrictHostKeyChecking=no "$USERNAME"@"$(get_vm_ip)":"${PROJECT_ROOT}"/kubeconfig kubeconfig
 328     if [ -d "/home/opnfv/functest-kubernetes" ]; then
 329         sudo cp kubeconfig /home/opnfv/functest-kubernetes/config
 330     fi
 331 }
 332
 333 # Creates a python virtual environment
 334 creates_virtualenv() {
 335     if [  ! -d "$CURRENTPATH/.venv" ]; then
 336         virtualenv "$CURRENTPATH/.venv"
 337     fi
 338     # shellcheck disable=SC1090
 339     source "$CURRENTPATH/.venv/bin/activate"
 340     pip install -r "$CURRENTPATH/requirements.txt"
 341 }
 342
 343 # Executes a specific Ansible playbook
 344 run_playbook() {
 345     ansible_cmd="$(command -v ansible-playbook) -i $CURRENTPATH/inventory/localhost.ini -e ansible_python_interpreter=$(command -v python)"
 346     if [ "${DEBUG:-false}" == "true" ]; then
 347         ansible_cmd+=" -vvv"
 348     fi
 349     eval "$ansible_cmd $CURRENTPATH/playbooks/${1}.yaml"
 350 }