#!/bin/bash -e
-# shellcheck disable=SC2155,SC1001
+# shellcheck disable=SC2155,SC1001,SC2015,SC2128
##############################################################################
-# Copyright (c) 2017 Mirantis Inc., Enea AB and others.
+# Copyright (c) 2018 Mirantis Inc., Enea AB and others.
# All rights reserved. This program and the accompanying materials
# are made available under the terms of the Apache License, Version 2.0
# which accompanies this distribution, and is available at
function __kernel_modules {
# Load mandatory kernel modules: loop, nbd
local image_dir=$1
- sudo modprobe loop
+ test -e /dev/loop-control || sudo modprobe loop
if sudo modprobe nbd max_part=8 || sudo modprobe -f nbd max_part=8; then
return 0
fi
+ if [ -e /dev/nbd0 ]; then return 0; fi # nbd might be inbuilt
# CentOS (or RHEL family in general) do not provide 'nbd' out of the box
echo "[WARN] 'nbd' kernel module cannot be loaded!"
if [ ! -e /etc/redhat-release ]; then
break
fi
done
- OPNFV_LOOP_DEV=$(losetup -f)
+ OPNFV_LOOP_DEV=$(sudo losetup -f)
OPNFV_MAP_DEV=/dev/mapper/$(basename "${OPNFV_NBD_DEV}")p1
export OPNFV_MNT_DIR OPNFV_LOOP_DEV
[ -n "${OPNFV_NBD_DEV}" ] && [ -n "${OPNFV_LOOP_DEV}" ] || exit 1
fi
fi
if [ -n "${OPNFV_LOOP_DEV}" ] && \
- losetup "${OPNFV_LOOP_DEV}" 1>&2 > /dev/null; then
+ sudo losetup "${OPNFV_LOOP_DEV}" 1>&2 > /dev/null; then
sudo losetup -d "${OPNFV_LOOP_DEV}"
fi
if [ -n "${OPNFV_NBD_DEV}" ]; then
function cleanup_uefi {
# Clean up Ubuntu boot entry if cfg01, kvm nodes online from previous deploy
local cmd_str="ssh ${SSH_OPTS} ${SSH_SALT}"
+ ping -c 1 -w 1 "${SALT_MASTER}" || return 0
[ ! "$(hostname)" = 'cfg01' ] || cmd_str='eval'
${cmd_str} "sudo salt -C 'kvm* or cmp*' cmd.run \
\"which efibootmgr > /dev/null 2>&1 && \
for node in $(virsh list --name --all | grep -P '\w{3}\d{2}'); do
virsh domblklist "${node}" | awk '/^.da/ {print $2}' | \
xargs --no-run-if-empty -I{} sudo rm -f {}
- virsh undefine "${node}" --remove-all-storage --nvram
+ virsh undefine "${node}" --remove-all-storage --nvram || \
+ virsh undefine "${node}" --remove-all-storage
done
}
-u "${user_data}" -h "${node}" "${image_dir}/mcp_${node}.iso"
cp "${image_dir}/${image}" "${image_dir}/mcp_${node}.qcow2"
qemu-img resize "${image_dir}/mcp_${node}.qcow2" 100G
+ # Prepare dedicated drive for cinder on cmp nodes
+ if [[ "${node}" =~ ^(cmp) ]]; then
+ qemu-img create "${image_dir}/mcp_${node}_storage.qcow2" 100G
+ fi
done
# VCP VMs base image specific changes
fi
}
+function jumpserver_pkg_install {
+ local req_type=$1
+ if [ -n "$(command -v apt-get)" ]; then
+ pkg_type='deb'; pkg_cmd='sudo apt-get install -y'
+ else
+ pkg_type='rpm'; pkg_cmd='sudo yum install -y --skip-broken'
+ fi
+ eval "$(parse_yaml "./requirements_${pkg_type}.yaml")"
+ for section in 'common' "$(uname -i)"; do
+ section_var="${req_type}_${section}[*]"
+ pkg_list+=" ${!section_var}"
+ done
+ # shellcheck disable=SC2086
+ ${pkg_cmd} ${pkg_list}
+}
+
+function jumpserver_check_requirements {
+ # shellcheck disable=SC2178
+ local vnodes=$1; shift
+ local br=("$@")
+ local err_br_not_found='Linux bridge not found!'
+ local err_br_virsh_net='is a virtual network, Linux bridge expected!'
+ local warn_br_endpoint="Endpoints might be inaccessible from external hosts!"
+ # MaaS requires a Linux bridge for PXE/admin
+ if [[ "${vnodes}" =~ mas01 ]]; then
+ if ! brctl showmacs "${br[0]}" >/dev/null 2>&1; then
+ notify_e "[ERROR] PXE/admin (${br[0]}) ${err_br_not_found}"
+ fi
+ # Assume virsh network name matches bridge name (true if created by us)
+ if virsh net-info "${br[0]}" >/dev/null 2>&1; then
+ notify_e "[ERROR] ${br[0]} ${err_br_virsh_net}"
+ fi
+ fi
+ # If virtual nodes are present, public should be a Linux bridge
+ if [ "$(echo "${vnodes}" | wc -w)" -gt 2 ]; then
+ if ! brctl showmacs "${br[3]}" >/dev/null 2>&1; then
+ if [[ "${vnodes}" =~ mas01 ]]; then
+ # Baremetal nodes *require* a proper public network
+ notify_e "[ERROR] Public (${br[3]}) ${err_br_not_found}"
+ else
+ notify_n "[WARN] Public (${br[3]}) ${err_br_not_found}" 3
+ notify_n "[WARN] ${warn_br_endpoint}" 3
+ fi
+ fi
+ if virsh net-info "${br[3]}" >/dev/null 2>&1; then
+ if [[ "${vnodes}" =~ mas01 ]]; then
+ notify_e "[ERROR] ${br[3]} ${err_br_virsh_net}"
+ else
+ notify_n "[WARN] ${br[3]} ${err_br_virsh_net}" 3
+ notify_n "[WARN] ${warn_br_endpoint}" 3
+ fi
+ fi
+ fi
+}
+
function create_networks {
- local vnode_networks=("$@")
+ local all_vnode_networks=("mcpcontrol" "$@")
# create required networks, including constant "mcpcontrol"
- # FIXME(alav): since we renamed "pxe" to "mcpcontrol", we need to make sure
- # we delete the old "pxe" virtual network, or it would cause IP conflicts.
- for net in "pxe" "mcpcontrol" "${vnode_networks[@]}"; do
+ for net in "${all_vnode_networks[@]}"; do
if virsh net-info "${net}" >/dev/null 2>&1; then
virsh net-destroy "${net}" || true
virsh net-undefine "${net}"
virsh net-start "${net}"
fi
done
+ # create veth pairs for relevant networks (mcpcontrol, pxebr, mgmt)
+ for i in $(seq 0 2 4); do
+ sudo ip link del "veth_mcp$i" || true
+ sudo ip link add "veth_mcp$i" type veth peer name "veth_mcp$((i+1))"
+ sudo ip link set "veth_mcp$i" up mtu 9000
+ sudo ip link set "veth_mcp$((i+1))" up mtu 9000
+ sudo brctl addif "${all_vnode_networks[$((i/2))]}" "veth_mcp$i"
+ done
}
function create_vms {
# create vms with specified options
for serialized_vnode_data in "${vnodes[@]}"; do
+ if [ -z "${serialized_vnode_data}" ]; then continue; fi
IFS=',' read -r -a vnode_data <<< "${serialized_vnode_data}"
# prepare network args
net_args="${net_args} --network bridge=${net},model=virtio"
done
+ # dedicated storage drive for cinder on cmp nodes
+ virt_extra_storage=
+ if [[ "${vnode_data[0]}" =~ ^(cmp) ]]; then
+ virt_extra_storage="--disk path=${image_dir}/mcp_${vnode_data[0]}_storage.qcow2,format=qcow2,bus=virtio,cache=none,io=native"
+ fi
+
# shellcheck disable=SC2086
virt-install --name "${vnode_data[0]}" \
--ram "${vnode_data[1]}" --vcpus "${vnode_data[2]}" \
--cpu host-passthrough --accelerate ${net_args} \
--disk path="${image_dir}/mcp_${vnode_data[0]}.qcow2",format=qcow2,bus=virtio,cache=none,io=native \
+ ${virt_extra_storage} \
--os-type linux --os-variant none \
--boot hd --vnc --console pty --autostart --noreboot \
--disk path="${image_dir}/mcp_${vnode_data[0]}.iso",device=cdrom \
function update_mcpcontrol_network {
# set static ip address for salt master node, MaaS node
- local cmac=$(virsh domiflist cfg01 2>&1| awk '/mcpcontrol/ {print $5; exit}')
local amac=$(virsh domiflist mas01 2>&1| awk '/mcpcontrol/ {print $5; exit}')
- virsh net-update "mcpcontrol" add ip-dhcp-host \
- "<host mac='${cmac}' name='cfg01' ip='${SALT_MASTER}'/>" --live --config
[ -z "${amac}" ] || virsh net-update "mcpcontrol" add ip-dhcp-host \
"<host mac='${amac}' name='mas01' ip='${MAAS_IP}'/>" --live --config
}
+function reset_vms {
+ local vnodes=("$@")
+ local cmd_str="ssh ${SSH_OPTS} ${SSH_SALT}"
+
+ # reset non-infrastructure vms, wait for them to come back online
+ for node in "${vnodes[@]}"; do
+ if [[ ! "${node}" =~ (cfg01|mas01) ]]; then
+ virsh reset "${node}"
+ fi
+ done
+ for node in "${vnodes[@]}"; do
+ if [[ ! "${node}" =~ (cfg01|mas01) ]]; then
+ wait_for 20.0 "${cmd_str} sudo salt -C '${node}*' saltutil.sync_all"
+ fi
+ done
+}
+
function start_vms {
local vnodes=("$@")
done
}
+function prepare_containers {
+ local image_dir=$1
+ [ -n "${image_dir}" ] || exit 1
+ [ -n "${MCP_REPO_ROOT_PATH}" ] || exit 1
+
+ "${image_dir}/docker-compose" -f docker-compose/docker-compose.yaml down
+ sudo rm -rf "${image_dir}/salt" "${image_dir}/nodes/"*
+ mkdir -p "${image_dir}/salt/"{master.d,minion.d}
+ # salt state does not properly configure file_roots in master.conf, hard set it
+ sed -e 's/user: salt/user: root\nfile_recv: True/' -e 's/auto_accept:/open_mode:/' \
+ "${MCP_REPO_ROOT_PATH}/docker/files/salt/master.conf" > \
+ "${image_dir}/salt/master.d/opnfv.conf"
+ echo 'master: localhost' > "${image_dir}/salt/minion.d/opnfv.conf"
+ cp "${MCP_REPO_ROOT_PATH}/mcp/scripts/docker-compose/files/hosts" \
+ "${image_dir}/hosts"
+}
+
+function start_containers {
+ local image_dir=$1
+ [ -n "${image_dir}" ] || exit 1
+ "${image_dir}/docker-compose" -f docker-compose/docker-compose.yaml up --quiet-pull -d
+}
+
function check_connection {
local total_attempts=60
local sleep_time=5
for attempt in $(seq "${total_attempts}"); do
echo "[wait_for] Attempt ${attempt}/${total_attempts%.*} for: ${cmdstr}"
if [ "${total_attempts%.*}" = "${total_attempts}" ]; then
- # shellcheck disable=SC2015
eval "${cmdstr}" && echo "[wait_for] OK: ${cmdstr}" && return 0 || true
else
- !(eval "${cmdstr}" || echo __fuel_wf_failure__) |& tee /dev/stderr | \
- grep -Eq '(Not connected|No response|__fuel_wf_failure__)' && \
- echo "[wait_for] OK: ${cmdstr}" && return 0 || true
+ ! (eval "${cmdstr}" || echo 'No response') |& tee /dev/stderr | \
+ grep -Eq '(Not connected|No response)' && \
+ echo "[wait_for] OK: ${cmdstr}" && return 0 || true
fi
sleep "${sleep_time}"
done
)
}
+function do_udev_cfg {
+ local _conf='/etc/udev/rules.d/99-opnfv-fuel-vnet-mtu.rules'
+ # http://linuxaleph.blogspot.com/2013/01/how-to-network-jumbo-frames-to-kvm-guest.html
+ echo 'SUBSYSTEM=="net", ACTION=="add|change", KERNEL=="vnet*", RUN+="/bin/sh -c '"'/bin/sleep 1; /sbin/ip link set %k mtu 9000'\"" |& sudo tee "${_conf}"
+ echo 'SUBSYSTEM=="net", ACTION=="add|change", KERNEL=="*-nic", RUN+="/bin/sh -c '"'/bin/sleep 1; /sbin/ip link set %k mtu 9000'\"" |& sudo tee -a "${_conf}"
+ sudo udevadm control --reload
+ sudo udevadm trigger
+}
+
function do_sysctl_cfg {
local _conf='/etc/sysctl.d/99-opnfv-fuel-bridge.conf'
# https://wiki.libvirt.org/page/Net.bridge.bridge-nf-call_and_sysctl.conf
fi
}
-function do_templates() {
- local git_repo_root=$1; shift
- local image_dir=$1; shift
- local target_lab=$1; shift
- local target_pod=$1; shift
- local lab_config_uri=$1; shift
- local scenario_dir=${1:-}
-
- RECLASS_CLUSTER_DIR=$(cd "${git_repo_root}/mcp/reclass/classes/cluster"; pwd)
- PHAROS_GEN_CFG="./pharos/config/utils/generate_config.py"
- PHAROS_INSTALLER_ADAPTER="./pharos/config/installers/fuel/pod_config.yml.j2"
- PHAROS_INSTALLER_NETMAP="$(dirname "${PHAROS_INSTALLER_ADAPTER}")/net_map.j2"
- BASE_CONFIG_PDF="${lab_config_uri}/labs/${target_lab}/${target_pod}.yaml"
- BASE_CONFIG_IDF="${lab_config_uri}/labs/${target_lab}/idf-${target_pod}.yaml"
- LOCAL_PDF="${image_dir}/$(basename "${BASE_CONFIG_PDF}")"
- LOCAL_IDF="${image_dir}/$(basename "${BASE_CONFIG_IDF}")"
- LOCAL_PDF_RECLASS="${image_dir}/pod_config.yml"
-
- # Two-stage expansion, first stage handles pod_config and scenarios only
- if [ -n "${scenario_dir}" ]; then
- # Make sample PDF/IDF available via default lab-config (pharos submodule)
- ln -sf "$(readlink -f "../config/labs/local")" "./pharos/labs/"
-
- # Expand scenario file and main reclass input (pod_config.yaml) based on PDF
- if ! curl --create-dirs -o "${LOCAL_PDF}" "${BASE_CONFIG_PDF}"; then
- notify_e "[ERROR] Could not retrieve PDF (Pod Descriptor File)!"
- elif ! curl -o "${LOCAL_IDF}" "${BASE_CONFIG_IDF}"; then
- notify_e "[ERROR] Could not retrieve IDF (Installer Descriptor File)!"
- elif ! "${PHAROS_GEN_CFG}" -y "${LOCAL_PDF}" \
- -j "${PHAROS_INSTALLER_ADAPTER}" > "${LOCAL_PDF_RECLASS}"; then
- notify_e "[ERROR] Could not convert PDF+IDF to reclass model input!"
- fi
- template_dirs="${scenario_dir}"
- template_err_str='Could not convert j2 scenario definitions!'
+function docker_install {
+ local image_dir=$1
+ # Mininum effort attempt at installing Docker if missing
+ if ! docker --version; then
+ curl -fsSL https://get.docker.com -o get-docker.sh
+ sudo sh get-docker.sh
+ rm get-docker.sh
+ # On RHEL distros, the Docker service should be explicitly started
+ sudo systemctl start docker
else
- # Expand reclass and virsh network templates based on PDF + IDF
- printenv | \
- awk '/^(SALT|MCP|MAAS|CLUSTER).*=/ { gsub(/=/,": "); print }' >> "${LOCAL_PDF}"
- template_dirs="${RECLASS_CLUSTER_DIR} virsh_net ./*j2"
- template_err_str='Could not convert PDF to network definitions!'
- fi
- # shellcheck disable=SC2086
- find ${template_dirs} -name '*.j2' | while read -r tp; do
- # Jinja2 import does not allow '..' directory traversal
- ln -sf "$(readlink -f "${PHAROS_INSTALLER_NETMAP}")" "$(dirname "${tp}")"
- if ! "${PHAROS_GEN_CFG}" -y "${LOCAL_PDF}" -j "${tp}" > "${tp%.j2}"; then
- notify_e "[ERROR] ${template_err_str}"
+ DOCKER_VER=$(docker version --format '{{.Server.Version}}')
+ if [ "${DOCKER_VER%%.*}" -lt 2 ]; then
+ notify_e "[ERROR] Docker version ${DOCKER_VER} is too old, please upgrade it."
fi
- rm -f "$(dirname "${tp}")/$(basename "${PHAROS_INSTALLER_NETMAP}")"
- done
+ fi
+ # Distro-provided docker-compose might be simply broken (Ubuntu 16.04, CentOS 7)
+ COMPOSE_BIN="${image_dir}/docker-compose"
+ COMPOSE_VERSION='1.22.0'
+ notify_n "[WARN] Using docker-compose ${COMPOSE_VERSION} in ${COMPOSE_BIN}" 3
+ if [ ! -e "${COMPOSE_BIN}" ]; then
+ COMPOSE_URL="https://github.com/docker/compose/releases/download/${COMPOSE_VERSION}"
+ sudo curl -L "${COMPOSE_URL}/docker-compose-$(uname -s)-$(uname -m)" -o "${COMPOSE_BIN}"
+ sudo chmod +x "${COMPOSE_BIN}"
+ fi
}
Code Review / fuel.git / blobdiff