X-Git-Url: https://gerrit.opnfv.org/gerrit/gitweb?a=blobdiff_plain;f=mcp%2Fscripts%2Flib.sh;h=293262e89ae3724513f169798348e672a4393927;hb=7099ffcba4c7a84dfd2ac53fd59149948665f884;hp=298590309c154af66548f5bf8a46540944e40787;hpb=d8cf4a9b8fb25af7fed10b8b9e69f70b601ccbce;p=fuel.git diff --git a/mcp/scripts/lib.sh b/mcp/scripts/lib.sh index 298590309..293262e89 100644 --- a/mcp/scripts/lib.sh +++ b/mcp/scripts/lib.sh @@ -1,7 +1,7 @@ #!/bin/bash -e -# shellcheck disable=SC2155,SC1001 +# shellcheck disable=SC2155,SC1001,SC2015,SC2128 ############################################################################## -# Copyright (c) 2017 Mirantis Inc., Enea AB and others. +# Copyright (c) 2018 Mirantis Inc., Enea AB and others. # All rights reserved. This program and the accompanying materials # are made available under the terms of the Apache License, Version 2.0 # which accompanies this distribution, and is available at @@ -32,16 +32,17 @@ function get_base_image { local image_dir=$2 mkdir -p "${image_dir}" - wget -P "${image_dir}" -N "${base_image}" + wget --progress=dot:giga -P "${image_dir}" -N "${base_image}" } function __kernel_modules { # Load mandatory kernel modules: loop, nbd local image_dir=$1 - sudo modprobe loop + test -e /dev/loop-control || sudo modprobe loop if sudo modprobe nbd max_part=8 || sudo modprobe -f nbd max_part=8; then return 0 fi + if [ -e /dev/nbd0 ]; then return 0; fi # nbd might be inbuilt # CentOS (or RHEL family in general) do not provide 'nbd' out of the box echo "[WARN] 'nbd' kernel module cannot be loaded!" if [ ! -e /etc/redhat-release ]; then @@ -120,7 +121,7 @@ function mount_image { break fi done - OPNFV_LOOP_DEV=$(losetup -f) + OPNFV_LOOP_DEV=$(sudo losetup -f) OPNFV_MAP_DEV=/dev/mapper/$(basename "${OPNFV_NBD_DEV}")p1 export OPNFV_MNT_DIR OPNFV_LOOP_DEV [ -n "${OPNFV_NBD_DEV}" ] && [ -n "${OPNFV_LOOP_DEV}" ] || exit 1 @@ -213,7 +214,7 @@ function cleanup_mounts { fi fi if [ -n "${OPNFV_LOOP_DEV}" ] && \ - losetup "${OPNFV_LOOP_DEV}" 1>&2 > /dev/null; then + sudo losetup "${OPNFV_LOOP_DEV}" 1>&2 > /dev/null; then sudo losetup -d "${OPNFV_LOOP_DEV}" fi if [ -n "${OPNFV_NBD_DEV}" ]; then @@ -225,6 +226,7 @@ function cleanup_mounts { function cleanup_uefi { # Clean up Ubuntu boot entry if cfg01, kvm nodes online from previous deploy local cmd_str="ssh ${SSH_OPTS} ${SSH_SALT}" + ping -c 1 -w 1 "${SALT_MASTER}" || return 0 [ ! "$(hostname)" = 'cfg01' ] || cmd_str='eval' ${cmd_str} "sudo salt -C 'kvm* or cmp*' cmd.run \ \"which efibootmgr > /dev/null 2>&1 && \ @@ -241,7 +243,8 @@ function cleanup_vms { for node in $(virsh list --name --all | grep -P '\w{3}\d{2}'); do virsh domblklist "${node}" | awk '/^.da/ {print $2}' | \ xargs --no-run-if-empty -I{} sudo rm -f {} - virsh undefine "${node}" --remove-all-storage --nvram + virsh undefine "${node}" --remove-all-storage --nvram || \ + virsh undefine "${node}" --remove-all-storage done } @@ -282,14 +285,16 @@ function prepare_vms { ln -sf "${image_dir}/${_tmp}" "${image_dir}/${image}" fi - envsubst < user-data.template > user-data.sh # CWD should be - # Create config ISO and resize OS disk image for each foundation node VM for node in "${vnodes[@]}"; do - ./create-config-drive.sh -k "$(basename "${SSH_KEY}").pub" -u user-data.sh \ - -h "${node}" "${image_dir}/mcp_${node}.iso" + ./create-config-drive.sh -k "$(basename "${SSH_KEY}").pub" \ + -u 'user-data.sh' -h "${node}" "${image_dir}/mcp_${node}.iso" cp "${image_dir}/${image}" "${image_dir}/mcp_${node}.qcow2" qemu-img resize "${image_dir}/mcp_${node}.qcow2" 100G + # Prepare dedicated drive for cinder on cmp nodes + if [[ "${node}" =~ ^(cmp) ]]; then + qemu-img create "${image_dir}/mcp_${node}_storage.qcow2" 100G + fi done # VCP VMs base image specific changes @@ -310,31 +315,93 @@ function prepare_vms { fi } +function jumpserver_pkg_install { + local req_type=$1 + if [ -n "$(command -v apt-get)" ]; then + pkg_type='deb'; pkg_cmd='sudo apt-get install -y' + else + pkg_type='rpm'; pkg_cmd='sudo yum install -y --skip-broken' + fi + eval "$(parse_yaml "./requirements_${pkg_type}.yaml")" + for section in 'common' "$(uname -i)"; do + section_var="${req_type}_${section}[*]" + pkg_list+=" ${!section_var}" + done + # shellcheck disable=SC2086 + ${pkg_cmd} ${pkg_list} +} + +function jumpserver_check_requirements { + # shellcheck disable=SC2178 + local vnodes=$1; shift + local br=("$@") + local err_br_not_found='Linux bridge not found!' + local err_br_virsh_net='is a virtual network, Linux bridge expected!' + local warn_br_endpoint="Endpoints might be inaccessible from external hosts!" + # MaaS requires a Linux bridge for PXE/admin + if [[ "${vnodes}" =~ mas01 ]]; then + if ! brctl showmacs "${br[0]}" >/dev/null 2>&1; then + notify_e "[ERROR] PXE/admin (${br[0]}) ${err_br_not_found}" + fi + # Assume virsh network name matches bridge name (true if created by us) + if virsh net-info "${br[0]}" >/dev/null 2>&1; then + notify_e "[ERROR] ${br[0]} ${err_br_virsh_net}" + fi + fi + # If virtual nodes are present, public should be a Linux bridge + if [ "$(echo "${vnodes}" | wc -w)" -gt 2 ]; then + if ! brctl showmacs "${br[3]}" >/dev/null 2>&1; then + if [[ "${vnodes}" =~ mas01 ]]; then + # Baremetal nodes *require* a proper public network + notify_e "[ERROR] Public (${br[3]}) ${err_br_not_found}" + else + notify_n "[WARN] Public (${br[3]}) ${err_br_not_found}" 3 + notify_n "[WARN] ${warn_br_endpoint}" 3 + fi + fi + if virsh net-info "${br[3]}" >/dev/null 2>&1; then + if [[ "${vnodes}" =~ mas01 ]]; then + notify_e "[ERROR] ${br[3]} ${err_br_virsh_net}" + else + notify_n "[WARN] ${br[3]} ${err_br_virsh_net}" 3 + notify_n "[WARN] ${warn_br_endpoint}" 3 + fi + fi + fi +} + function create_networks { - local vnode_networks=("$@") + local all_vnode_networks=("mcpcontrol" "$@") # create required networks, including constant "mcpcontrol" - # FIXME(alav): since we renamed "pxe" to "mcpcontrol", we need to make sure - # we delete the old "pxe" virtual network, or it would cause IP conflicts. - for net in "pxe" "mcpcontrol" "${vnode_networks[@]}"; do + for net in "${all_vnode_networks[@]}"; do if virsh net-info "${net}" >/dev/null 2>&1; then virsh net-destroy "${net}" || true virsh net-undefine "${net}" fi # in case of custom network, host should already have the bridge in place - if [ -f "net_${net}.xml" ] && [ ! -d "/sys/class/net/${net}/bridge" ]; then - virsh net-define "net_${net}.xml" + if [ -f "virsh_net/net_${net}.xml" ] && \ + [ ! -d "/sys/class/net/${net}/bridge" ]; then + virsh net-define "virsh_net/net_${net}.xml" virsh net-autostart "${net}" virsh net-start "${net}" fi done + # create veth pairs for relevant networks (mcpcontrol, pxebr, mgmt) + for i in $(seq 0 2 4); do + sudo ip link del "veth_mcp$i" || true + sudo ip link add "veth_mcp$i" type veth peer name "veth_mcp$((i+1))" + sudo ip link set "veth_mcp$i" up mtu 9000 + sudo ip link set "veth_mcp$((i+1))" up mtu 9000 + sudo brctl addif "${all_vnode_networks[$((i/2))]}" "veth_mcp$i" + done } function create_vms { local image_dir=$1; shift # vnode data should be serialized with the following format: - # ',,|,,[...]' + # ,,[,,,[,,, + # ,,,]]|,...' IFS='|' read -r -a vnodes <<< "$1"; shift - local vnode_networks=("$@") # AArch64: prepare arch specific arguments local virt_extra_args="" @@ -345,23 +412,49 @@ function create_vms { # create vms with specified options for serialized_vnode_data in "${vnodes[@]}"; do + if [ -z "${serialized_vnode_data}" ]; then continue; fi IFS=',' read -r -a vnode_data <<< "${serialized_vnode_data}" + # prepare VM CPU model, count, topology (optional), NUMA cells (optional, requires topo) + local virt_cpu_args=' --cpu host-passthrough' + local idx=6 # cell0.name index in serialized data + while [ -n "${vnode_data[${idx}]}" ]; do + virt_cpu_args+=",${vnode_data[${idx}]}.memory=${vnode_data[$((idx + 1))]}" + virt_cpu_args+=",${vnode_data[${idx}]}.cpus=${vnode_data[$((idx + 2))]}" + idx=$((idx + 3)) + done + virt_cpu_args+=" --vcpus vcpus=${vnode_data[2]}" + if [ -n "${vnode_data[5]}" ]; then + virt_cpu_args+=",sockets=${vnode_data[3]},cores=${vnode_data[4]},threads=${vnode_data[5]}" + fi + # prepare network args - net_args=" --network network=mcpcontrol,model=virtio" - if [ "${DEPLOY_TYPE:-}" = 'baremetal' ]; then + local vnode_networks=("$@") + if [[ "${vnode_data[0]}" =~ ^(cfg01|mas01) ]]; then + net_args=" --network network=mcpcontrol,model=virtio" # 3rd interface gets connected to PXE/Admin Bridge (cfg01, mas01) vnode_networks[2]="${vnode_networks[0]}" + else + net_args=" --network bridge=${vnode_networks[0]},model=virtio" fi for net in "${vnode_networks[@]:1}"; do net_args="${net_args} --network bridge=${net},model=virtio" done + # dedicated storage drive for cinder on cmp nodes + virt_extra_storage= + if [[ "${vnode_data[0]}" =~ ^(cmp) ]]; then + virt_extra_storage="--disk path=${image_dir}/mcp_${vnode_data[0]}_storage.qcow2,format=qcow2,bus=virtio,cache=none,io=native" + fi + + [ ! -e "${image_dir}/virt-manager" ] || VIRT_PREFIX="${image_dir}/virt-manager/" # shellcheck disable=SC2086 - virt-install --name "${vnode_data[0]}" \ - --ram "${vnode_data[1]}" --vcpus "${vnode_data[2]}" \ - --cpu host-passthrough --accelerate ${net_args} \ + ${VIRT_PREFIX}virt-install --name "${vnode_data[0]}" \ + ${virt_cpu_args} --accelerate \ + ${net_args} \ + --ram "${vnode_data[1]}" \ --disk path="${image_dir}/mcp_${vnode_data[0]}.qcow2",format=qcow2,bus=virtio,cache=none,io=native \ + ${virt_extra_storage} \ --os-type linux --os-variant none \ --boot hd --vnc --console pty --autostart --noreboot \ --disk path="${image_dir}/mcp_${vnode_data[0]}.iso",device=cdrom \ @@ -372,14 +465,28 @@ function create_vms { function update_mcpcontrol_network { # set static ip address for salt master node, MaaS node - local cmac=$(virsh domiflist cfg01 2>&1| awk '/mcpcontrol/ {print $5; exit}') local amac=$(virsh domiflist mas01 2>&1| awk '/mcpcontrol/ {print $5; exit}') - virsh net-update "mcpcontrol" add ip-dhcp-host \ - "" --live --config [ -z "${amac}" ] || virsh net-update "mcpcontrol" add ip-dhcp-host \ "" --live --config } +function reset_vms { + local vnodes=("$@") + local cmd_str="ssh ${SSH_OPTS} ${SSH_SALT}" + + # reset non-infrastructure vms, wait for them to come back online + for node in "${vnodes[@]}"; do + if [[ ! "${node}" =~ (cfg01|mas01) ]]; then + virsh reset "${node}" + fi + done + for node in "${vnodes[@]}"; do + if [[ ! "${node}" =~ (cfg01|mas01) ]]; then + wait_for 20.0 "${cmd_str} sudo salt -C '${node}*' saltutil.sync_all" + fi + done +} + function start_vms { local vnodes=("$@") @@ -390,6 +497,28 @@ function start_vms { done } +function prepare_containers { + local image_dir=$1 + [ -n "${image_dir}" ] || exit 1 + [ -n "${MCP_REPO_ROOT_PATH}" ] || exit 1 + docker-compose --version > /dev/null 2>&1 || COMPOSE_PREFIX="${image_dir}/" + + "${COMPOSE_PREFIX}docker-compose" -f docker-compose/docker-compose.yaml down + if [[ ! "${MCP_DOCKER_TAG}" =~ 'verify' ]]; then + "${COMPOSE_PREFIX}docker-compose" -f docker-compose/docker-compose.yaml pull + fi + sudo rm -rf "${image_dir}/"{salt,hosts,pki} "${image_dir}/nodes/"* + mkdir -p "${image_dir}/salt/"{master.d,minion.d} + touch "${image_dir}/hosts" +} + +function start_containers { + local image_dir=$1 + [ -n "${image_dir}" ] || exit 1 + docker-compose --version > /dev/null 2>&1 || COMPOSE_PREFIX="${image_dir}/" + "${COMPOSE_PREFIX}docker-compose" -f docker-compose/docker-compose.yaml up -d +} + function check_connection { local total_attempts=60 local sleep_time=5 @@ -443,12 +572,11 @@ function wait_for { for attempt in $(seq "${total_attempts}"); do echo "[wait_for] Attempt ${attempt}/${total_attempts%.*} for: ${cmdstr}" if [ "${total_attempts%.*}" = "${total_attempts}" ]; then - # shellcheck disable=SC2015 eval "${cmdstr}" && echo "[wait_for] OK: ${cmdstr}" && return 0 || true else - !(eval "${cmdstr}" || echo __fuel_wf_failure__) |& tee /dev/stderr | \ - grep -Eq '(Not connected|No response|__fuel_wf_failure__)' && \ - echo "[wait_for] OK: ${cmdstr}" && return 0 || true + ! (eval "${cmdstr}" || echo 'No response') |& tee /dev/stderr | \ + grep -Eq '(Not connected|No response)' && \ + echo "[wait_for] OK: ${cmdstr}" && return 0 || true fi sleep "${sleep_time}" done @@ -457,9 +585,74 @@ function wait_for { ) } +function do_udev_cfg { + local _conf='/etc/udev/rules.d/99-opnfv-fuel-vnet-mtu.rules' + # http://linuxaleph.blogspot.com/2013/01/how-to-network-jumbo-frames-to-kvm-guest.html + echo 'SUBSYSTEM=="net", ACTION=="add|change", KERNEL=="vnet*", RUN+="/bin/sh -c '"'/bin/sleep 1; /sbin/ip link set %k mtu 9000'\"" |& sudo tee "${_conf}" + echo 'SUBSYSTEM=="net", ACTION=="add|change", KERNEL=="*-nic", RUN+="/bin/sh -c '"'/bin/sleep 1; /sbin/ip link set %k mtu 9000'\"" |& sudo tee -a "${_conf}" + sudo udevadm control --reload + sudo udevadm trigger +} + +function do_sysctl_cfg { + local _conf='/etc/sysctl.d/99-opnfv-fuel-bridge.conf' + # https://wiki.libvirt.org/page/Net.bridge.bridge-nf-call_and_sysctl.conf + if modprobe br_netfilter bridge; then + echo 'net.bridge.bridge-nf-call-arptables = 0' |& sudo tee "${_conf}" + echo 'net.bridge.bridge-nf-call-iptables = 0' |& sudo tee -a "${_conf}" + echo 'net.bridge.bridge-nf-call-ip6tables = 0' |& sudo tee -a "${_conf}" + # Some distros / sysadmins explicitly blacklist br_netfilter + sudo sysctl -q -p "${_conf}" || true + fi +} + function get_nova_compute_pillar_data { local value=$(salt -C 'I@nova:compute and *01*' pillar.get _param:"${1}" --out yaml | cut -d ' ' -f2) if [ "${value}" != "''" ]; then - echo ${value} + echo "${value}" + fi +} + +function docker_install { + local image_dir=$1 + # Mininum effort attempt at installing Docker if missing + if ! docker --version; then + curl -fsSL https://get.docker.com -o get-docker.sh + sudo sh get-docker.sh + rm get-docker.sh + # On RHEL distros, the Docker service should be explicitly started + sudo systemctl start docker + else + DOCKER_VER=$(docker version --format '{{.Server.Version}}') + if [ "${DOCKER_VER%%.*}" -lt 2 ]; then + notify_e "[ERROR] Docker version ${DOCKER_VER} is too old, please upgrade it." + fi + fi + # Distro-provided docker-compose might be simply broken (Ubuntu 16.04, CentOS 7) + if ! docker-compose --version > /dev/null 2>&1; then + COMPOSE_BIN="${image_dir}/docker-compose" + COMPOSE_VERSION='1.22.0' + notify_n "[WARN] Using docker-compose ${COMPOSE_VERSION} in ${COMPOSE_BIN}" 3 + if [ ! -e "${COMPOSE_BIN}" ]; then + COMPOSE_URL="https://github.com/docker/compose/releases/download/${COMPOSE_VERSION}" + sudo curl -L "${COMPOSE_URL}/docker-compose-$(uname -s)-$(uname -m)" -o "${COMPOSE_BIN}" + sudo chmod +x "${COMPOSE_BIN}" + fi + fi +} + +function virtinst_install { + local image_dir=$1 + VIRT_VER=$(virt-install --version 2>&1) + if [ "${VIRT_VER//./}" -lt 140 ]; then + VIRT_TGZ="${image_dir}/virt-manager.tar.gz" + VIRT_VER='1.4.3' + VIRT_URL="https://github.com/virt-manager/virt-manager/archive/v${VIRT_VER}.tar.gz" + notify_n "[WARN] Using virt-install ${VIRT_VER} from ${VIRT_TGZ}" 3 + if [ ! -e "${VIRT_TGZ}" ]; then + curl -L "${VIRT_URL}" -o "${VIRT_TGZ}" + mkdir -p "${image_dir}/virt-manager" + tar xzf "${VIRT_TGZ}" -C "${image_dir}/virt-manager" --strip-components=1 + fi fi }