From ab18375a629010525ac15bc11ce2d4e4cf393fe9 Mon Sep 17 00:00:00 2001 From: Alexandru Avadanii Date: Sat, 18 Aug 2018 03:46:13 +0200 Subject: [PATCH] [docker] Switch to containerized Salt Master * Refactor OPNFV salt-formulas mechanism to resemble upstream git structure: - git submodules: add new submodule for each formula we patch; - create salt-formula-x directories for OPNFV formulas; - move mcp/metadata/service contents to their each formula subdir; - use `make patches-import` for patches previously handled by patch.sh; - retire patch.sh * states: add virtual_init: - mostly based on old salt.sh, which is now obsolete; - exclude salt-master service restart (it would kill the container); * scenarios: cleanup (rm cfg01 virtual node def), adopt virtual_init; * reclass: align our model with prebuilt container's Salt config: - drop linux:network pillar data (handled by Docker); - stop applying linux.system state on cfg01; - align salt user homedir; - drop salt-formula packages (preprovisioned); * minor plumbing in deploy.sh and lib.sh; JIRA: FUEL-383 Change-Id: I28708a9b399d3f19012212c71966ebda9d6fc0ac Signed-off-by: Alexandru Avadanii --- .gitmodules | 16 +++ ci/deploy.sh | 11 +- mcp/config/scenario/os-nosdn-nofeature-ha.yaml.j2 | 5 +- .../scenario/os-nosdn-nofeature-noha.yaml.j2 | 2 +- mcp/config/scenario/os-nosdn-ovs-ha.yaml.j2 | 5 +- mcp/config/scenario/os-nosdn-ovs-noha.yaml.j2 | 2 +- mcp/config/scenario/os-odl-nofeature-ha.yaml.j2 | 5 +- mcp/config/scenario/os-odl-nofeature-noha.yaml.j2 | 2 +- mcp/config/scenario/os-odl-ovs-noha.yaml.j2 | 2 +- mcp/config/scenario/os-ovn-nofeature-ha.yaml.j2 | 5 +- mcp/config/scenario/os-ovn-nofeature-noha.yaml.j2 | 2 +- mcp/config/states/virtual_init | 39 ++++++ mcp/patches/patch.sh | 19 --- mcp/patches/patches.list | 12 -- .../0001-Handle-extra-environment-variables.patch} | 3 + ...tem.repo-Debian-Use-proxy-for-keyservers.patch} | 2 +- .../0002-Set-ovs-bridges-as-L3-interfaces.patch} | 14 ++- ...0001-maas-region-skip-credentials-update.patch} | 6 +- .../0002-maas-region-allow-timeout-override.patch} | 10 +- .../0003-Add-machines.delete-co-pxe_nat-sls.patch | 137 +++++++++++++++++++++ .../all-mcp-arch-common/infra/config_pdf.yml.j2 | 9 +- .../cluster/mcp-common-ha/infra/config.yml.j2 | 31 ----- .../cluster/mcp-common-noha/infra/config.yml.j2 | 31 ----- mcp/salt-formulas/maas/machines/delete.sls | 20 --- .../maas/machines/mark_broken_fixed.sls | 20 --- .../maas/machines/override_failed_testing.sls | 20 --- mcp/salt-formulas/maas/pxe_nat.sls | 37 ------ mcp/salt-formulas/salt-formula-keystone | 1 + mcp/salt-formulas/salt-formula-linux | 1 + mcp/salt-formulas/salt-formula-maas | 1 + mcp/salt-formulas/salt-formula-nova | 1 + .../metadata/service}/server/single.yml | 0 .../metadata/service}/support.yml | 0 .../opendaylight/files/jetty.xml | 0 .../opendaylight/files/setenv.shell | 0 .../opendaylight/init.sls | 0 .../opendaylight/map.jinja | 0 .../opendaylight/server.sls | 0 .../metadata/service}/server/single.yml | 0 .../tacker/files/tacker.conf | 0 .../tacker/files/tacker.systemd | 0 .../{ => salt-formula-tacker}/tacker/init.sls | 0 .../{ => salt-formula-tacker}/tacker/map.jinja | 0 .../{ => salt-formula-tacker}/tacker/server.sls | 0 mcp/scripts/lib.sh | 43 ++++++- mcp/scripts/salt.sh | 132 -------------------- mcp/scripts/xdf_data.sh.j2 | 4 +- 47 files changed, 281 insertions(+), 369 deletions(-) create mode 100755 mcp/config/states/virtual_init delete mode 100755 mcp/patches/patch.sh delete mode 100644 mcp/patches/patches.list rename mcp/patches/{0008-Handle-extra-environment-variables.patch => salt-formula-keystone/0001-Handle-extra-environment-variables.patch} (93%) rename mcp/patches/{0011-system.repo-Debian-Add-keyserver-proxy-support.patch => salt-formula-linux/0001-system.repo-Debian-Use-proxy-for-keyservers.patch} (99%) rename mcp/patches/{0015-Set-ovs-bridges-as-L3-interfaces.patch => salt-formula-linux/0002-Set-ovs-bridges-as-L3-interfaces.patch} (91%) rename mcp/patches/{0002-maas-region-skip-credentials-update.patch => salt-formula-maas/0001-maas-region-skip-credentials-update.patch} (89%) rename mcp/patches/{0010-maas-region-allow-timeout-override.patch => salt-formula-maas/0002-maas-region-allow-timeout-override.patch} (93%) create mode 100644 mcp/patches/salt-formula-maas/0003-Add-machines.delete-co-pxe_nat-sls.patch delete mode 100644 mcp/salt-formulas/maas/machines/delete.sls delete mode 100644 mcp/salt-formulas/maas/machines/mark_broken_fixed.sls delete mode 100644 mcp/salt-formulas/maas/machines/override_failed_testing.sls delete mode 100644 mcp/salt-formulas/maas/pxe_nat.sls create mode 160000 mcp/salt-formulas/salt-formula-keystone create mode 160000 mcp/salt-formulas/salt-formula-linux create mode 160000 mcp/salt-formulas/salt-formula-maas create mode 160000 mcp/salt-formulas/salt-formula-nova rename mcp/{metadata/service/opendaylight => salt-formulas/salt-formula-opendaylight/metadata/service}/server/single.yml (100%) rename mcp/{metadata/service/opendaylight => salt-formulas/salt-formula-opendaylight/metadata/service}/support.yml (100%) rename mcp/salt-formulas/{ => salt-formula-opendaylight}/opendaylight/files/jetty.xml (100%) rename mcp/salt-formulas/{ => salt-formula-opendaylight}/opendaylight/files/setenv.shell (100%) rename mcp/salt-formulas/{ => salt-formula-opendaylight}/opendaylight/init.sls (100%) rename mcp/salt-formulas/{ => salt-formula-opendaylight}/opendaylight/map.jinja (100%) rename mcp/salt-formulas/{ => salt-formula-opendaylight}/opendaylight/server.sls (100%) rename mcp/{metadata/service/tacker => salt-formulas/salt-formula-tacker/metadata/service}/server/single.yml (100%) rename mcp/salt-formulas/{ => salt-formula-tacker}/tacker/files/tacker.conf (100%) rename mcp/salt-formulas/{ => salt-formula-tacker}/tacker/files/tacker.systemd (100%) rename mcp/salt-formulas/{ => salt-formula-tacker}/tacker/init.sls (100%) rename mcp/salt-formulas/{ => salt-formula-tacker}/tacker/map.jinja (100%) rename mcp/salt-formulas/{ => salt-formula-tacker}/tacker/server.sls (100%) delete mode 100755 mcp/scripts/salt.sh diff --git a/.gitmodules b/.gitmodules index d3501c3ec..767cf14df 100644 --- a/.gitmodules +++ b/.gitmodules @@ -14,3 +14,19 @@ path = docker url = https://github.com/epcim/docker-salt-formulas branch = master +[submodule "salt-formula-linux"] + path = mcp/salt-formulas/salt-formula-linux + url = https://github.com/salt-formulas/salt-formula-linux + branch = master +[submodule "salt-formula-nova"] + path = mcp/salt-formulas/salt-formula-nova + url = https://github.com/salt-formulas/salt-formula-nova + branch = master +[submodule "salt-formula-keystone"] + path = mcp/salt-formulas/salt-formula-keystone + url = https://github.com/salt-formulas/salt-formula-keystone + branch = master +[submodule "salt-formula-maas"] + path = mcp/salt-formulas/salt-formula-maas + url = https://github.com/salt-formulas/salt-formula-maas + branch = master diff --git a/ci/deploy.sh b/ci/deploy.sh index 47662f796..d3882a834 100755 --- a/ci/deploy.sh +++ b/ci/deploy.sh @@ -241,7 +241,7 @@ if [ ${USE_EXISTING_PKGS} -eq 1 ]; then else notify "[NOTE] Installing required distro pkgs" 2 jumpserver_pkg_install 'deploy' - docker_install + docker_install "${MCP_STORAGE_DIR}" fi if ! virsh list >/dev/null 2>&1; then @@ -289,22 +289,21 @@ if [ ${DRY_RUN} -eq 1 ]; then exit 0 elif [ ${USE_EXISTING_INFRA} -gt 0 ]; then notify "[NOTE] Use existing infra" 2 - check_connection else prepare_vms "${base_image}" "${MCP_STORAGE_DIR}" "${virtual_repos_pkgs}" \ "${virtual_nodes[@]}" + prepare_containers "${MCP_STORAGE_DIR}" create_networks "${OPNFV_BRIDGES[@]}" do_sysctl_cfg do_udev_cfg create_vms "${MCP_STORAGE_DIR}" "${virtual_nodes_data}" "${OPNFV_BRIDGES[@]}" update_mcpcontrol_network start_vms "${virtual_nodes[@]}" - check_connection -fi -if [ ${USE_EXISTING_INFRA} -lt 2 ]; then - wait_for 5 "./salt.sh ${MCP_STORAGE_DIR}/pod_config.yml ${virtual_nodes[*]}" fi +start_containers "${MCP_STORAGE_DIR}" +check_connection + # Openstack cluster setup set +x if [ ${INFRA_CREATION_ONLY} -eq 1 ] || [ ${NO_DEPLOY_ENVIRONMENT} -eq 1 ]; then diff --git a/mcp/config/scenario/os-nosdn-nofeature-ha.yaml.j2 b/mcp/config/scenario/os-nosdn-nofeature-ha.yaml.j2 index 7eb604e65..55da8f74b 100644 --- a/mcp/config/scenario/os-nosdn-nofeature-ha.yaml.j2 +++ b/mcp/config/scenario/os-nosdn-nofeature-ha.yaml.j2 @@ -10,6 +10,7 @@ cluster: domain: mcp-ovs-ha.local states: + - virtual_init {%- if nm.cluster.has_baremetal_nodes %} - maas - baremetal_init @@ -22,7 +23,6 @@ cluster: virtual: nodes: infra: - - cfg01 {%- if nm.cluster.has_baremetal_nodes %} - mas01 {%- endif %} @@ -34,9 +34,6 @@ virtual: - cmp001 - cmp002 # Below values are only used for each node if said node is virtual - cfg01: - vcpus: 4 - ram: 6144 mas01: vcpus: 4 ram: 6144 diff --git a/mcp/config/scenario/os-nosdn-nofeature-noha.yaml.j2 b/mcp/config/scenario/os-nosdn-nofeature-noha.yaml.j2 index a0ad96b87..cd214a5cd 100644 --- a/mcp/config/scenario/os-nosdn-nofeature-noha.yaml.j2 +++ b/mcp/config/scenario/os-nosdn-nofeature-noha.yaml.j2 @@ -10,6 +10,7 @@ cluster: domain: mcp-ovs-noha.local states: + - virtual_init {%- if nm.cluster.has_baremetal_nodes %} - maas - baremetal_init @@ -20,7 +21,6 @@ cluster: virtual: nodes: infra: - - cfg01 {%- if nm.cluster.has_baremetal_nodes %} - mas01 {%- endif %} diff --git a/mcp/config/scenario/os-nosdn-ovs-ha.yaml.j2 b/mcp/config/scenario/os-nosdn-ovs-ha.yaml.j2 index da9938ce0..0f7a7bc34 100644 --- a/mcp/config/scenario/os-nosdn-ovs-ha.yaml.j2 +++ b/mcp/config/scenario/os-nosdn-ovs-ha.yaml.j2 @@ -10,6 +10,7 @@ cluster: domain: mcp-ovs-dpdk-ha.local states: + - virtual_init {%- if nm.cluster.has_baremetal_nodes %} - maas - baremetal_init @@ -23,7 +24,6 @@ cluster: virtual: nodes: infra: - - cfg01 {%- if nm.cluster.has_baremetal_nodes %} - mas01 {%- endif %} @@ -35,9 +35,6 @@ virtual: - cmp001 - cmp002 # Below values are only used for each node if said node is virtual - cfg01: - vcpus: 4 - ram: 6144 mas01: vcpus: 4 ram: 6144 diff --git a/mcp/config/scenario/os-nosdn-ovs-noha.yaml.j2 b/mcp/config/scenario/os-nosdn-ovs-noha.yaml.j2 index bee0e926f..bf37f93ea 100644 --- a/mcp/config/scenario/os-nosdn-ovs-noha.yaml.j2 +++ b/mcp/config/scenario/os-nosdn-ovs-noha.yaml.j2 @@ -10,6 +10,7 @@ cluster: domain: mcp-ovs-dpdk-noha.local states: + - virtual_init {%- if nm.cluster.has_baremetal_nodes %} - maas - baremetal_init @@ -21,7 +22,6 @@ cluster: virtual: nodes: infra: - - cfg01 {%- if nm.cluster.has_baremetal_nodes %} - mas01 {%- endif %} diff --git a/mcp/config/scenario/os-odl-nofeature-ha.yaml.j2 b/mcp/config/scenario/os-odl-nofeature-ha.yaml.j2 index fbedbc6b9..b31dc136e 100644 --- a/mcp/config/scenario/os-odl-nofeature-ha.yaml.j2 +++ b/mcp/config/scenario/os-odl-nofeature-ha.yaml.j2 @@ -10,6 +10,7 @@ cluster: domain: mcp-odl-ha.local states: + - virtual_init {%- if nm.cluster.has_baremetal_nodes %} - maas - baremetal_init @@ -23,7 +24,6 @@ cluster: virtual: nodes: infra: - - cfg01 {%- if nm.cluster.has_baremetal_nodes %} - mas01 {%- endif %} @@ -35,9 +35,6 @@ virtual: - cmp001 - cmp002 # Below values are only used for each node if said node is virtual - cfg01: - vcpus: 4 - ram: 6144 mas01: vcpus: 4 ram: 6144 diff --git a/mcp/config/scenario/os-odl-nofeature-noha.yaml.j2 b/mcp/config/scenario/os-odl-nofeature-noha.yaml.j2 index 7df28c2b3..2a10a3163 100644 --- a/mcp/config/scenario/os-odl-nofeature-noha.yaml.j2 +++ b/mcp/config/scenario/os-odl-nofeature-noha.yaml.j2 @@ -10,6 +10,7 @@ cluster: domain: mcp-odl-noha.local states: + - virtual_init {%- if nm.cluster.has_baremetal_nodes %} - maas - baremetal_init @@ -21,7 +22,6 @@ cluster: virtual: nodes: infra: - - cfg01 {%- if nm.cluster.has_baremetal_nodes %} - mas01 {%- endif %} diff --git a/mcp/config/scenario/os-odl-ovs-noha.yaml.j2 b/mcp/config/scenario/os-odl-ovs-noha.yaml.j2 index c81d35c51..62c120b26 100644 --- a/mcp/config/scenario/os-odl-ovs-noha.yaml.j2 +++ b/mcp/config/scenario/os-odl-ovs-noha.yaml.j2 @@ -10,6 +10,7 @@ cluster: domain: mcp-odl-noha.local states: + - virtual_init {%- if nm.cluster.has_baremetal_nodes %} - maas - baremetal_init @@ -22,7 +23,6 @@ cluster: virtual: nodes: infra: - - cfg01 {%- if nm.cluster.has_baremetal_nodes %} - mas01 {%- endif %} diff --git a/mcp/config/scenario/os-ovn-nofeature-ha.yaml.j2 b/mcp/config/scenario/os-ovn-nofeature-ha.yaml.j2 index f81779b8a..021548f63 100644 --- a/mcp/config/scenario/os-ovn-nofeature-ha.yaml.j2 +++ b/mcp/config/scenario/os-ovn-nofeature-ha.yaml.j2 @@ -10,6 +10,7 @@ cluster: domain: mcp-ovn-ha.local states: + - virtual_init {%- if nm.cluster.has_baremetal_nodes %} - maas - baremetal_init @@ -22,7 +23,6 @@ cluster: virtual: nodes: infra: - - cfg01 {%- if nm.cluster.has_baremetal_nodes %} - mas01 {%- endif %} @@ -34,9 +34,6 @@ virtual: - cmp001 - cmp002 # Below values are only used for each node if said node is virtual - cfg01: - vcpus: 4 - ram: 6144 mas01: vcpus: 4 ram: 6144 diff --git a/mcp/config/scenario/os-ovn-nofeature-noha.yaml.j2 b/mcp/config/scenario/os-ovn-nofeature-noha.yaml.j2 index fee47d7eb..635106355 100644 --- a/mcp/config/scenario/os-ovn-nofeature-noha.yaml.j2 +++ b/mcp/config/scenario/os-ovn-nofeature-noha.yaml.j2 @@ -10,6 +10,7 @@ cluster: domain: mcp-ovn-noha.local states: + - virtual_init {%- if nm.cluster.has_baremetal_nodes %} - maas - baremetal_init @@ -19,7 +20,6 @@ cluster: virtual: nodes: infra: - - cfg01 {%- if nm.cluster.has_baremetal_nodes %} - mas01 {%- endif %} diff --git a/mcp/config/states/virtual_init b/mcp/config/states/virtual_init new file mode 100755 index 000000000..147e6b05e --- /dev/null +++ b/mcp/config/states/virtual_init @@ -0,0 +1,39 @@ +#!/bin/bash -e +# shellcheck disable=SC1090 +############################################################################## +# Copyright (c) 2018 Mirantis Inc., Enea AB and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +CI_DEBUG=${CI_DEBUG:-0}; [[ "${CI_DEBUG}" =~ (false|0) ]] || set -x + +# shellcheck disable=SC1090 +source "$(dirname "${BASH_SOURCE[0]}")/../../scripts/lib.sh" +source "$(dirname "${BASH_SOURCE[0]}")/../../scripts/xdf_data.sh" + +CI_DEBUG=${CI_DEBUG:-0}; [[ "${CI_DEBUG}" =~ (false|0) ]] || set -x +# shellcheck disable=SC2154,SC2086,SC2116 +LOCAL_VIRT_NODES=$(echo ${virtual_nodes[*]}) # unquoted to filter space +NODE_MASK="${LOCAL_VIRT_NODES// /|}" + +# wait_for 3.0 "salt-call state.apply salt exclude='[{id: salt_master_service}]'" +wait_for 5.0 "salt-call state.sls reclass,linux.network" +wait_for 3.0 "salt -C '*' saltutil.refresh_pillar" + +# NOTE: domain name changes are not yet supported without a clean redeploy + +# Init specific to VMs on FN (all for virtual, mas for baremetal) +wait_for 3.0 "(for n in ${LOCAL_VIRT_NODES}; do salt -C \${n}.* test.ping || exit; done)" + +wait_for 3.0 "salt -C 'E@^(${NODE_MASK}).*' state.sls linux.system,linux.storage" +wait_for 2.0 "salt -C 'E@^(${NODE_MASK}).*' state.sls linux.network" +salt -C "E@^(${NODE_MASK}).*" system.reboot +wait_for 90.0 "salt -C 'E@^(${NODE_MASK}).*' test.ping" +wait_for 3.0 "salt -C 'E@^(${NODE_MASK}).*' pkg.upgrade refresh=False dist_upgrade=True" +wait_for 3.0 "salt -C 'E@^(${NODE_MASK}).*' saltutil.sync_all" +wait_for 3.0 "salt -C 'E@^(${NODE_MASK}).*' state.apply salt" + +wait_for 3.0 "salt -C 'E@^(${NODE_MASK}).*' state.sls ntp" diff --git a/mcp/patches/patch.sh b/mcp/patches/patch.sh deleted file mode 100755 index bb48dcd07..000000000 --- a/mcp/patches/patch.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash -e -############################################################################## -# Copyright (c) 2017 Mirantis Inc., Enea AB and others. -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -CI_DEBUG=${CI_DEBUG:-0}; [[ "${CI_DEBUG}" =~ (false|0) ]] || set -x - -if [ -r "$1" ]; then - while IFS=': ' read -r p_dest p_file; do - if ! patch --dry-run -Rd "${p_dest}" -r - -s -p1 < \ - "/root/fuel/mcp/patches/${p_file}" > /dev/null; then - patch -d "${p_dest}" -p1 < "/root/fuel/mcp/patches/${p_file}" - fi - done < <(grep -vE '^#' "${1}" | grep -E "^.*${2}.*: ") -fi diff --git a/mcp/patches/patches.list b/mcp/patches/patches.list deleted file mode 100644 index baa15d79d..000000000 --- a/mcp/patches/patches.list +++ /dev/null @@ -1,12 +0,0 @@ -############################################################################## -# Copyright (c) 2018 Mirantis Inc., Enea AB and others. -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## -/usr/share/salt-formulas/env: 0002-maas-region-skip-credentials-update.patch -/usr/share/salt-formulas/env: 0008-Handle-extra-environment-variables.patch -/usr/share/salt-formulas/env: 0010-maas-region-allow-timeout-override.patch -/usr/share/salt-formulas/env: 0011-system.repo-Debian-Add-keyserver-proxy-support.patch -/usr/share/salt-formulas/env: 0015-Set-ovs-bridges-as-L3-interfaces.patch diff --git a/mcp/patches/0008-Handle-extra-environment-variables.patch b/mcp/patches/salt-formula-keystone/0001-Handle-extra-environment-variables.patch similarity index 93% rename from mcp/patches/0008-Handle-extra-environment-variables.patch rename to mcp/patches/salt-formula-keystone/0001-Handle-extra-environment-variables.patch index 8df3227f2..2108587b1 100644 --- a/mcp/patches/0008-Handle-extra-environment-variables.patch +++ b/mcp/patches/salt-formula-keystone/0001-Handle-extra-environment-variables.patch @@ -11,6 +11,9 @@ Date: Mon, 12 Mar 2018 17:43:09 +0400 Subject: [PATCH] Handle extra environment variables Change-Id: Ieae46ac65041630759c82238a8a5ce0535c454b2 +--- + keystone/files/keystonercv3 | 3 +++ + 1 file changed, 3 insertions(+) diff --git a/keystone/files/keystonercv3 b/keystone/files/keystonercv3 index 1b7f378..984c8a2 100644 diff --git a/mcp/patches/0011-system.repo-Debian-Add-keyserver-proxy-support.patch b/mcp/patches/salt-formula-linux/0001-system.repo-Debian-Use-proxy-for-keyservers.patch similarity index 99% rename from mcp/patches/0011-system.repo-Debian-Add-keyserver-proxy-support.patch rename to mcp/patches/salt-formula-linux/0001-system.repo-Debian-Use-proxy-for-keyservers.patch index dec29e6a7..4ad51e3e0 100644 --- a/mcp/patches/0011-system.repo-Debian-Add-keyserver-proxy-support.patch +++ b/mcp/patches/salt-formula-linux/0001-system.repo-Debian-Use-proxy-for-keyservers.patch @@ -37,7 +37,7 @@ Signed-off-by: Alexandru Avadanii 1 file changed, 47 insertions(+), 8 deletions(-) diff --git a/linux/system/repo.sls b/linux/system/repo.sls -index 9664129..d9f567e 100644 +index 0c0b026..82661f0 100644 --- a/linux/system/repo.sls +++ b/linux/system/repo.sls @@ -7,18 +7,27 @@ linux_repo_prereq_pkgs: diff --git a/mcp/patches/0015-Set-ovs-bridges-as-L3-interfaces.patch b/mcp/patches/salt-formula-linux/0002-Set-ovs-bridges-as-L3-interfaces.patch similarity index 91% rename from mcp/patches/0015-Set-ovs-bridges-as-L3-interfaces.patch rename to mcp/patches/salt-formula-linux/0002-Set-ovs-bridges-as-L3-interfaces.patch index a7b366b94..ee82474b1 100644 --- a/mcp/patches/0015-Set-ovs-bridges-as-L3-interfaces.patch +++ b/mcp/patches/salt-formula-linux/0002-Set-ovs-bridges-as-L3-interfaces.patch @@ -11,10 +11,16 @@ Date: Wed, 28 Feb 2018 17:54:28 +0400 Subject: [PATCH] Set ovs bridges as L3 interfaces Change-Id: I1e83129cc184cf481bea21d7aa452bf60d9e0499 +--- + linux/files/ovs_bridge | 19 +++++++++++++++++++ + linux/files/ovs_port | 7 ++++++- + linux/network/interface.sls | 28 ++++++++++++++++++++++++++++ + 3 files changed, 53 insertions(+), 1 deletion(-) + create mode 100644 linux/files/ovs_bridge diff --git a/linux/files/ovs_bridge b/linux/files/ovs_bridge new file mode 100644 -index 0000000..c609e45 +index 0000000..073c91c --- /dev/null +++ b/linux/files/ovs_bridge @@ -0,0 +1,19 @@ @@ -38,14 +44,14 @@ index 0000000..c609e45 + dns-nameservers {{ bridge.name_servers | join(' ') }} + {%- endif %} diff --git a/linux/files/ovs_port b/linux/files/ovs_port -index 222ca8e..efb0307 100644 +index 222ca8e..03072cd 100644 --- a/linux/files/ovs_port +++ b/linux/files/ovs_port @@ -1,6 +1,11 @@ +-auto {{ port_name }} +# With systemd, adding OVS bridges as 'auto' can cause race conditions +# https://github.com/openvswitch/ovs/blob/master/debian/openvswitch-switch.README.Debian +# auto {{ port_name }} --auto {{ port_name }} allow-{{ port.bridge }} {{ port_name }} iface {{ port_name }} inet {{ port.get('proto', 'manual') }} +{%- if '.' in port_name %} @@ -55,7 +61,7 @@ index 222ca8e..efb0307 100644 mtu {{ port.get('mtu', '1500') }} ovs_bridge {{ port.bridge }} diff --git a/linux/network/interface.sls b/linux/network/interface.sls -index 180f912..dcb295b 100644 +index c2d2a23..01131fa 100644 --- a/linux/network/interface.sls +++ b/linux/network/interface.sls @@ -91,6 +91,34 @@ add_int_{{ int_name }}_to_ovs_dpdk_bridge_{{ interface_name }}: diff --git a/mcp/patches/0002-maas-region-skip-credentials-update.patch b/mcp/patches/salt-formula-maas/0001-maas-region-skip-credentials-update.patch similarity index 89% rename from mcp/patches/0002-maas-region-skip-credentials-update.patch rename to mcp/patches/salt-formula-maas/0001-maas-region-skip-credentials-update.patch index 1d226e4cd..718f25b4e 100644 --- a/mcp/patches/0002-maas-region-skip-credentials-update.patch +++ b/mcp/patches/salt-formula-maas/0001-maas-region-skip-credentials-update.patch @@ -17,12 +17,14 @@ updating credentials. Signed-off-by: Alexandru Avadanii --- + maas/region.sls | 7 +++---- + 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/maas/region.sls b/maas/region.sls -index d3227ca..8a2243d 100644 +index 684fda4..5844f22 100644 --- a/maas/region.sls +++ b/maas/region.sls -@@ -6,10 +6,9 @@ +@@ -6,10 +6,9 @@ maas_region_packages: - names: {{ region.pkgs }} /etc/maas/regiond.conf: diff --git a/mcp/patches/0010-maas-region-allow-timeout-override.patch b/mcp/patches/salt-formula-maas/0002-maas-region-allow-timeout-override.patch similarity index 93% rename from mcp/patches/0010-maas-region-allow-timeout-override.patch rename to mcp/patches/salt-formula-maas/0002-maas-region-allow-timeout-override.patch index c6f9e3a52..aef087d92 100644 --- a/mcp/patches/0010-maas-region-allow-timeout-override.patch +++ b/mcp/patches/salt-formula-maas/0002-maas-region-allow-timeout-override.patch @@ -35,18 +35,20 @@ node's PXE physical interface, but that overcomplicates things. blocksize to be 1008. We can force it to be 1464 and gain some performance due to MTU beeing 1500 (i.e. allow bigger packets). - JIRA: FUEL-316 Signed-off-by: Alexandru Avadanii --- + maas/region.sls | 40 ++++++++++++++++++++++++++++++++++++++++ + 1 file changed, 40 insertions(+) diff --git a/maas/region.sls b/maas/region.sls +index 5844f22..d844bb4 100644 --- a/maas/region.sls +++ b/maas/region.sls -@@ -19,6 +19,46 @@ - - require: - - pkg: maas_region_packages +@@ -38,6 +38,46 @@ restore_maas_database_{{ region.database.name }}: + + {%- endif %} +maas_timeout_commissioning: + file.replace: diff --git a/mcp/patches/salt-formula-maas/0003-Add-machines.delete-co-pxe_nat-sls.patch b/mcp/patches/salt-formula-maas/0003-Add-machines.delete-co-pxe_nat-sls.patch new file mode 100644 index 000000000..9f13c4616 --- /dev/null +++ b/mcp/patches/salt-formula-maas/0003-Add-machines.delete-co-pxe_nat-sls.patch @@ -0,0 +1,137 @@ +From: Alexandru Avadanii +Date: Sun, 19 Aug 2018 05:38:27 +0200 +Subject: [PATCH] Add machines.delete & co, pxe_nat sls + +Signed-off-by: Alexandru Avadanii +--- + maas/machines/delete.sls | 20 ++++++++++++ + maas/machines/mark_broken_fixed.sls | 20 ++++++++++++ + maas/machines/override_failed_testing.sls | 20 ++++++++++++ + maas/pxe_nat.sls | 37 +++++++++++++++++++++++ + 4 files changed, 97 insertions(+) + create mode 100644 maas/machines/delete.sls + create mode 100644 maas/machines/mark_broken_fixed.sls + create mode 100644 maas/machines/override_failed_testing.sls + create mode 100644 maas/pxe_nat.sls + +diff --git a/maas/machines/delete.sls b/maas/machines/delete.sls +new file mode 100644 +index 0000000..2903f92 +--- /dev/null ++++ b/maas/machines/delete.sls +@@ -0,0 +1,20 @@ ++############################################################################## ++# Copyright (c) 2017 Mirantis Inc., Enea AB and others. ++# All rights reserved. This program and the accompanying materials ++# are made available under the terms of the Apache License, Version 2.0 ++# which accompanies this distribution, and is available at ++# http://www.apache.org/licenses/LICENSE-2.0 ++############################################################################## ++{%- from "maas/map.jinja" import region with context %} ++ ++maas_login_admin: ++ cmd.run: ++ - name: "maas-region apikey --username {{ region.admin.username }} > /var/lib/maas/.maas_credentials" ++ - unless: 'test -e /var/lib/maas/.maas_credentials' ++ ++# TODO: implement delete_machine via _modules/maas.py ++delete_machine: ++ cmd.run: ++ - name: "maas login {{ region.admin.username }} http://{{ region.bind.host }}:5240/MAAS/api/2.0 - < /var/lib/maas/.maas_credentials && maas opnfv machine delete {{ pillar['system_id'] }}" ++ - require: ++ - cmd: maas_login_admin +diff --git a/maas/machines/mark_broken_fixed.sls b/maas/machines/mark_broken_fixed.sls +new file mode 100644 +index 0000000..46691bb +--- /dev/null ++++ b/maas/machines/mark_broken_fixed.sls +@@ -0,0 +1,20 @@ ++############################################################################## ++# Copyright (c) 2017 Mirantis Inc., Enea AB and others. ++# All rights reserved. This program and the accompanying materials ++# are made available under the terms of the Apache License, Version 2.0 ++# which accompanies this distribution, and is available at ++# http://www.apache.org/licenses/LICENSE-2.0 ++############################################################################## ++{%- from "maas/map.jinja" import region with context %} ++ ++maas_login_admin: ++ cmd.run: ++ - name: "maas-region apikey --username {{ region.admin.username }} > /var/lib/maas/.maas_credentials" ++ - unless: 'test -e /var/lib/maas/.maas_credentials' ++ ++# TODO: implement mark_broken_fixed_machine via _modules/maas.py ++mark_broken_fixed_machine: ++ cmd.run: ++ - name: "maas login {{ region.admin.username }} http://{{ region.bind.host }}:5240/MAAS/api/2.0 - < /var/lib/maas/.maas_credentials && maas opnfv machine mark-broken {{ pillar['system_id'] }} && sleep 10 && maas opnfv machine mark-fixed {{ pillar['system_id'] }} && maas opnfv machine test {{ pillar['system_id'] }} testing_scripts=fio" ++ - require: ++ - cmd: maas_login_admin +diff --git a/maas/machines/override_failed_testing.sls b/maas/machines/override_failed_testing.sls +new file mode 100644 +index 0000000..e7fe1d2 +--- /dev/null ++++ b/maas/machines/override_failed_testing.sls +@@ -0,0 +1,20 @@ ++############################################################################## ++# Copyright (c) 2018 Mirantis Inc., Enea AB and others. ++# All rights reserved. This program and the accompanying materials ++# are made available under the terms of the Apache License, Version 2.0 ++# which accompanies this distribution, and is available at ++# http://www.apache.org/licenses/LICENSE-2.0 ++############################################################################## ++{%- from "maas/map.jinja" import region with context %} ++ ++maas_login_admin: ++ cmd.run: ++ - name: "maas-region apikey --username {{ region.admin.username }} > /var/lib/maas/.maas_credentials" ++ - unless: 'test -e /var/lib/maas/.maas_credentials' ++ ++# TODO: implement override_failed_testing via _modules/maas.py ++mark_broken_fixed_machine: ++ cmd.run: ++ - name: "maas login {{ region.admin.username }} http://{{ region.bind.host }}:5240/MAAS/api/2.0 - < /var/lib/maas/.maas_credentials && maas opnfv machine override-failed-testing {{ pillar['system_id'] }}" ++ - require: ++ - cmd: maas_login_admin +diff --git a/maas/pxe_nat.sls b/maas/pxe_nat.sls +new file mode 100644 +index 0000000..8a03c4f +--- /dev/null ++++ b/maas/pxe_nat.sls +@@ -0,0 +1,37 @@ ++############################################################################## ++# Copyright (c) 2017 Mirantis Inc., Enea AB and others. ++# All rights reserved. This program and the accompanying materials ++# are made available under the terms of the Apache License, Version 2.0 ++# which accompanies this distribution, and is available at ++# http://www.apache.org/licenses/LICENSE-2.0 ++############################################################################## ++net.ipv4.ip_forward: ++ sysctl.present: ++ - value: 1 ++ ++iptables_pxe_nat: ++ iptables.append: ++ - table: nat ++ - chain: POSTROUTING ++ - jump: MASQUERADE ++ - destination: 0/0 ++ - source: {{ salt['pillar.get']('_param:single_address') }}/{{ salt['pillar.get']('_param:opnfv_net_admin_mask') }} ++ - save: True ++ ++iptables_pxe_source: ++ iptables.append: ++ - table: filter ++ - chain: INPUT ++ - jump: ACCEPT ++ - destination: 0/0 ++ - source: {{ salt['pillar.get']('_param:single_address') }}/{{ salt['pillar.get']('_param:opnfv_net_admin_mask') }} ++ - save: True ++ ++iptables_pxe_destination: ++ iptables.append: ++ - table: filter ++ - chain: INPUT ++ - jump: ACCEPT ++ - destination: {{ salt['pillar.get']('_param:single_address') }}/{{ salt['pillar.get']('_param:opnfv_net_admin_mask') }} ++ - source: 0/0 ++ - save: True diff --git a/mcp/reclass/classes/cluster/all-mcp-arch-common/infra/config_pdf.yml.j2 b/mcp/reclass/classes/cluster/all-mcp-arch-common/infra/config_pdf.yml.j2 index a12e15663..5fa4ca486 100644 --- a/mcp/reclass/classes/cluster/all-mcp-arch-common/infra/config_pdf.yml.j2 +++ b/mcp/reclass/classes/cluster/all-mcp-arch-common/infra/config_pdf.yml.j2 @@ -8,14 +8,12 @@ {%- import 'net_map.j2' as nm with context %} --- classes: - - service.git.client - - system.linux.system.repo.mcp.salt - system.reclass.storage.salt {%- if nm.cluster.has_baremetal_nodes %} - system.reclass.storage.system.infra_maas_single {%- endif %} - system.salt.master.api - - system.salt.master.pkg + - system.salt.master.single - system.salt.minion.ca.salt_master parameters: _param: @@ -25,6 +23,11 @@ parameters: salt_master_base_environment: prd # yamllint disable-line rule:line-length salt_api_password_hash: "$6$sGnRlxGf$al5jMCetLP.vfI/fTl3Z0N7Za1aeiexL487jAtyRABVfT3NlwZxQGVhO7S1N8OwS/34VHYwZQA8lkXwKMN/GS1" + linux: + system: + user: + salt: + home: /home/salt salt: master: accept_policy: open_mode diff --git a/mcp/reclass/classes/cluster/mcp-common-ha/infra/config.yml.j2 b/mcp/reclass/classes/cluster/mcp-common-ha/infra/config.yml.j2 index 0e880c558..ee849a6cd 100644 --- a/mcp/reclass/classes/cluster/mcp-common-ha/infra/config.yml.j2 +++ b/mcp/reclass/classes/cluster/mcp-common-ha/infra/config.yml.j2 @@ -29,37 +29,6 @@ parameters: mcpcontrol_nic: ${_param:opnfv_fn_vm_primary_interface} single_nic: ${_param:opnfv_fn_vm_secondary_interface} pxe_admin_nic: ${_param:opnfv_fn_vm_tertiary_interface} - linux: - network: - interface: - mcpcontrol_int: - enabled: true - type: eth - proto: dhcp - name: ${_param:mcpcontrol_nic} - single: - enabled: true - type: eth - name: ${_param:single_nic} -{%- if conf.idf.fuel.jumphost.get('trunks', {}).get('mgmt', False) and (nm.vlan_mgmt | int > 0) %} - proto: manual - single_vlan: - enabled: true - type: vlan - name: ${_param:single_nic}.{{ nm.vlan_mgmt }} - use_interfaces: - - ${_param:single_nic} -{%- endif %} - proto: static - address: ${_param:single_address} - netmask: ${_param:opnfv_net_mgmt_mask} - pxe_admin_int: - enabled: true - type: eth - proto: static - name: ${_param:pxe_admin_nic} - address: ${_param:pxe_admin_address} - netmask: ${_param:opnfv_net_admin_mask} salt: master: accept_policy: open_mode diff --git a/mcp/reclass/classes/cluster/mcp-common-noha/infra/config.yml.j2 b/mcp/reclass/classes/cluster/mcp-common-noha/infra/config.yml.j2 index 36a2c27ec..a6bf5cda4 100644 --- a/mcp/reclass/classes/cluster/mcp-common-noha/infra/config.yml.j2 +++ b/mcp/reclass/classes/cluster/mcp-common-noha/infra/config.yml.j2 @@ -16,37 +16,6 @@ parameters: single_address: ${_param:opnfv_infra_config_address} salt_master_host: 127.0.0.1 salt_minion_ca_host: ${linux:network:fqdn} - linux: - network: - interface: - mcpcontrol_int: - enabled: true - type: eth - proto: dhcp - name: ${_param:opnfv_fn_vm_primary_interface} - single_int: - enabled: true - name: ${_param:opnfv_fn_vm_secondary_interface} - type: eth -{%- if conf.idf.fuel.jumphost.get('trunks', {}).get('mgmt', False) and (nm.vlan_mgmt | int > 0) %} - proto: manual - single_int_vlan: - enabled: true - type: vlan - name: ${_param:opnfv_fn_vm_secondary_interface}.{{ nm.vlan_mgmt }} - use_interfaces: - - ${_param:opnfv_fn_vm_secondary_interface} -{%- endif %} - proto: static - address: ${_param:single_address} - netmask: ${_param:opnfv_net_mgmt_mask} - pxe_admin_int: - enabled: true - type: eth - proto: static - name: ${_param:opnfv_fn_vm_tertiary_interface} - address: ${_param:opnfv_infra_config_pxe_admin_address} - netmask: ${_param:opnfv_net_admin_mask} salt: master: file_recv: true diff --git a/mcp/salt-formulas/maas/machines/delete.sls b/mcp/salt-formulas/maas/machines/delete.sls deleted file mode 100644 index 2903f9226..000000000 --- a/mcp/salt-formulas/maas/machines/delete.sls +++ /dev/null @@ -1,20 +0,0 @@ -############################################################################## -# Copyright (c) 2017 Mirantis Inc., Enea AB and others. -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## -{%- from "maas/map.jinja" import region with context %} - -maas_login_admin: - cmd.run: - - name: "maas-region apikey --username {{ region.admin.username }} > /var/lib/maas/.maas_credentials" - - unless: 'test -e /var/lib/maas/.maas_credentials' - -# TODO: implement delete_machine via _modules/maas.py -delete_machine: - cmd.run: - - name: "maas login {{ region.admin.username }} http://{{ region.bind.host }}:5240/MAAS/api/2.0 - < /var/lib/maas/.maas_credentials && maas opnfv machine delete {{ pillar['system_id'] }}" - - require: - - cmd: maas_login_admin diff --git a/mcp/salt-formulas/maas/machines/mark_broken_fixed.sls b/mcp/salt-formulas/maas/machines/mark_broken_fixed.sls deleted file mode 100644 index 46691bb09..000000000 --- a/mcp/salt-formulas/maas/machines/mark_broken_fixed.sls +++ /dev/null @@ -1,20 +0,0 @@ -############################################################################## -# Copyright (c) 2017 Mirantis Inc., Enea AB and others. -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## -{%- from "maas/map.jinja" import region with context %} - -maas_login_admin: - cmd.run: - - name: "maas-region apikey --username {{ region.admin.username }} > /var/lib/maas/.maas_credentials" - - unless: 'test -e /var/lib/maas/.maas_credentials' - -# TODO: implement mark_broken_fixed_machine via _modules/maas.py -mark_broken_fixed_machine: - cmd.run: - - name: "maas login {{ region.admin.username }} http://{{ region.bind.host }}:5240/MAAS/api/2.0 - < /var/lib/maas/.maas_credentials && maas opnfv machine mark-broken {{ pillar['system_id'] }} && sleep 10 && maas opnfv machine mark-fixed {{ pillar['system_id'] }} && maas opnfv machine test {{ pillar['system_id'] }} testing_scripts=fio" - - require: - - cmd: maas_login_admin diff --git a/mcp/salt-formulas/maas/machines/override_failed_testing.sls b/mcp/salt-formulas/maas/machines/override_failed_testing.sls deleted file mode 100644 index e7fe1d267..000000000 --- a/mcp/salt-formulas/maas/machines/override_failed_testing.sls +++ /dev/null @@ -1,20 +0,0 @@ -############################################################################## -# Copyright (c) 2018 Mirantis Inc., Enea AB and others. -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## -{%- from "maas/map.jinja" import region with context %} - -maas_login_admin: - cmd.run: - - name: "maas-region apikey --username {{ region.admin.username }} > /var/lib/maas/.maas_credentials" - - unless: 'test -e /var/lib/maas/.maas_credentials' - -# TODO: implement override_failed_testing via _modules/maas.py -mark_broken_fixed_machine: - cmd.run: - - name: "maas login {{ region.admin.username }} http://{{ region.bind.host }}:5240/MAAS/api/2.0 - < /var/lib/maas/.maas_credentials && maas opnfv machine override-failed-testing {{ pillar['system_id'] }}" - - require: - - cmd: maas_login_admin diff --git a/mcp/salt-formulas/maas/pxe_nat.sls b/mcp/salt-formulas/maas/pxe_nat.sls deleted file mode 100644 index 8a03c4fdb..000000000 --- a/mcp/salt-formulas/maas/pxe_nat.sls +++ /dev/null @@ -1,37 +0,0 @@ -############################################################################## -# Copyright (c) 2017 Mirantis Inc., Enea AB and others. -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## -net.ipv4.ip_forward: - sysctl.present: - - value: 1 - -iptables_pxe_nat: - iptables.append: - - table: nat - - chain: POSTROUTING - - jump: MASQUERADE - - destination: 0/0 - - source: {{ salt['pillar.get']('_param:single_address') }}/{{ salt['pillar.get']('_param:opnfv_net_admin_mask') }} - - save: True - -iptables_pxe_source: - iptables.append: - - table: filter - - chain: INPUT - - jump: ACCEPT - - destination: 0/0 - - source: {{ salt['pillar.get']('_param:single_address') }}/{{ salt['pillar.get']('_param:opnfv_net_admin_mask') }} - - save: True - -iptables_pxe_destination: - iptables.append: - - table: filter - - chain: INPUT - - jump: ACCEPT - - destination: {{ salt['pillar.get']('_param:single_address') }}/{{ salt['pillar.get']('_param:opnfv_net_admin_mask') }} - - source: 0/0 - - save: True diff --git a/mcp/salt-formulas/salt-formula-keystone b/mcp/salt-formulas/salt-formula-keystone new file mode 160000 index 000000000..df75142aa --- /dev/null +++ b/mcp/salt-formulas/salt-formula-keystone @@ -0,0 +1 @@ +Subproject commit df75142aaf6487c3af9e2868f4f12ef18228ca99 diff --git a/mcp/salt-formulas/salt-formula-linux b/mcp/salt-formulas/salt-formula-linux new file mode 160000 index 000000000..cf1b5b322 --- /dev/null +++ b/mcp/salt-formulas/salt-formula-linux @@ -0,0 +1 @@ +Subproject commit cf1b5b322a077dc8d6f9bf5b36a7a0a71a2b3a06 diff --git a/mcp/salt-formulas/salt-formula-maas b/mcp/salt-formulas/salt-formula-maas new file mode 160000 index 000000000..c6a12dea5 --- /dev/null +++ b/mcp/salt-formulas/salt-formula-maas @@ -0,0 +1 @@ +Subproject commit c6a12dea5f8d8c7c700f596dbabb805113f1d341 diff --git a/mcp/salt-formulas/salt-formula-nova b/mcp/salt-formulas/salt-formula-nova new file mode 160000 index 000000000..539e9938e --- /dev/null +++ b/mcp/salt-formulas/salt-formula-nova @@ -0,0 +1 @@ +Subproject commit 539e9938e74f8a1ad19c2c1a653761df53bc24b7 diff --git a/mcp/metadata/service/opendaylight/server/single.yml b/mcp/salt-formulas/salt-formula-opendaylight/metadata/service/server/single.yml similarity index 100% rename from mcp/metadata/service/opendaylight/server/single.yml rename to mcp/salt-formulas/salt-formula-opendaylight/metadata/service/server/single.yml diff --git a/mcp/metadata/service/opendaylight/support.yml b/mcp/salt-formulas/salt-formula-opendaylight/metadata/service/support.yml similarity index 100% rename from mcp/metadata/service/opendaylight/support.yml rename to mcp/salt-formulas/salt-formula-opendaylight/metadata/service/support.yml diff --git a/mcp/salt-formulas/opendaylight/files/jetty.xml b/mcp/salt-formulas/salt-formula-opendaylight/opendaylight/files/jetty.xml similarity index 100% rename from mcp/salt-formulas/opendaylight/files/jetty.xml rename to mcp/salt-formulas/salt-formula-opendaylight/opendaylight/files/jetty.xml diff --git a/mcp/salt-formulas/opendaylight/files/setenv.shell b/mcp/salt-formulas/salt-formula-opendaylight/opendaylight/files/setenv.shell similarity index 100% rename from mcp/salt-formulas/opendaylight/files/setenv.shell rename to mcp/salt-formulas/salt-formula-opendaylight/opendaylight/files/setenv.shell diff --git a/mcp/salt-formulas/opendaylight/init.sls b/mcp/salt-formulas/salt-formula-opendaylight/opendaylight/init.sls similarity index 100% rename from mcp/salt-formulas/opendaylight/init.sls rename to mcp/salt-formulas/salt-formula-opendaylight/opendaylight/init.sls diff --git a/mcp/salt-formulas/opendaylight/map.jinja b/mcp/salt-formulas/salt-formula-opendaylight/opendaylight/map.jinja similarity index 100% rename from mcp/salt-formulas/opendaylight/map.jinja rename to mcp/salt-formulas/salt-formula-opendaylight/opendaylight/map.jinja diff --git a/mcp/salt-formulas/opendaylight/server.sls b/mcp/salt-formulas/salt-formula-opendaylight/opendaylight/server.sls similarity index 100% rename from mcp/salt-formulas/opendaylight/server.sls rename to mcp/salt-formulas/salt-formula-opendaylight/opendaylight/server.sls diff --git a/mcp/metadata/service/tacker/server/single.yml b/mcp/salt-formulas/salt-formula-tacker/metadata/service/server/single.yml similarity index 100% rename from mcp/metadata/service/tacker/server/single.yml rename to mcp/salt-formulas/salt-formula-tacker/metadata/service/server/single.yml diff --git a/mcp/salt-formulas/tacker/files/tacker.conf b/mcp/salt-formulas/salt-formula-tacker/tacker/files/tacker.conf similarity index 100% rename from mcp/salt-formulas/tacker/files/tacker.conf rename to mcp/salt-formulas/salt-formula-tacker/tacker/files/tacker.conf diff --git a/mcp/salt-formulas/tacker/files/tacker.systemd b/mcp/salt-formulas/salt-formula-tacker/tacker/files/tacker.systemd similarity index 100% rename from mcp/salt-formulas/tacker/files/tacker.systemd rename to mcp/salt-formulas/salt-formula-tacker/tacker/files/tacker.systemd diff --git a/mcp/salt-formulas/tacker/init.sls b/mcp/salt-formulas/salt-formula-tacker/tacker/init.sls similarity index 100% rename from mcp/salt-formulas/tacker/init.sls rename to mcp/salt-formulas/salt-formula-tacker/tacker/init.sls diff --git a/mcp/salt-formulas/tacker/map.jinja b/mcp/salt-formulas/salt-formula-tacker/tacker/map.jinja similarity index 100% rename from mcp/salt-formulas/tacker/map.jinja rename to mcp/salt-formulas/salt-formula-tacker/tacker/map.jinja diff --git a/mcp/salt-formulas/tacker/server.sls b/mcp/salt-formulas/salt-formula-tacker/tacker/server.sls similarity index 100% rename from mcp/salt-formulas/tacker/server.sls rename to mcp/salt-formulas/salt-formula-tacker/tacker/server.sls diff --git a/mcp/scripts/lib.sh b/mcp/scripts/lib.sh index f5affb0d3..8f7cccd26 100644 --- a/mcp/scripts/lib.sh +++ b/mcp/scripts/lib.sh @@ -454,10 +454,7 @@ function create_vms { function update_mcpcontrol_network { # set static ip address for salt master node, MaaS node - local cmac=$(virsh domiflist cfg01 2>&1| awk '/mcpcontrol/ {print $5; exit}') local amac=$(virsh domiflist mas01 2>&1| awk '/mcpcontrol/ {print $5; exit}') - virsh net-update "mcpcontrol" add ip-dhcp-host \ - "" --live --config [ -z "${amac}" ] || virsh net-update "mcpcontrol" add ip-dhcp-host \ "" --live --config } @@ -489,6 +486,29 @@ function start_vms { done } +function prepare_containers { + local image_dir=$1 + [ -n "${image_dir}" ] || exit 1 + [ -n "${MCP_REPO_ROOT_PATH}" ] || exit 1 + + "${image_dir}/docker-compose" -f docker-compose/docker-compose.yaml down + sudo rm -rf "${image_dir}/salt" "${image_dir}/nodes/"* + mkdir -p "${image_dir}/salt/"{master.d,minion.d} + # salt state does not properly configure file_roots in master.conf, hard set it + sed -e 's/user: salt/user: root\nfile_recv: True/' -e 's/auto_accept:/open_mode:/' \ + "${MCP_REPO_ROOT_PATH}/docker/files/salt/master.conf" > \ + "${image_dir}/salt/master.d/opnfv.conf" + echo 'master: localhost' > "${image_dir}/salt/minion.d/opnfv.conf" + cp "${MCP_REPO_ROOT_PATH}/mcp/scripts/docker-compose/files/hosts" \ + "${image_dir}/hosts" +} + +function start_containers { + local image_dir=$1 + [ -n "${image_dir}" ] || exit 1 + "${image_dir}/docker-compose" -f docker-compose/docker-compose.yaml up --quiet-pull -d +} + function check_connection { local total_attempts=60 local sleep_time=5 @@ -584,12 +604,27 @@ function get_nova_compute_pillar_data { } function docker_install { + local image_dir=$1 # Mininum effort attempt at installing Docker if missing - if ! which docker; then + if ! docker --version; then curl -fsSL https://get.docker.com -o get-docker.sh sudo sh get-docker.sh rm get-docker.sh # On RHEL distros, the Docker service should be explicitly started sudo systemctl start docker + else + DOCKER_VER=$(docker version --format '{{.Server.Version}}') + if [ "${DOCKER_VER%%.*}" -lt 2 ]; then + notify_e "[ERROR] Docker version ${DOCKER_VER} is too old, please upgrade it." + fi + fi + # Distro-provided docker-compose might be simply broken (Ubuntu 16.04, CentOS 7) + COMPOSE_BIN="${image_dir}/docker-compose" + COMPOSE_VERSION='1.22.0' + notify_n "[WARN] Using docker-compose ${COMPOSE_VERSION} in ${COMPOSE_BIN}" 3 + if [ ! -e "${COMPOSE_BIN}" ]; then + COMPOSE_URL="https://github.com/docker/compose/releases/download/${COMPOSE_VERSION}" + sudo curl -L "${COMPOSE_URL}/docker-compose-$(uname -s)-$(uname -m)" -o "${COMPOSE_BIN}" + sudo chmod +x "${COMPOSE_BIN}" fi } diff --git a/mcp/scripts/salt.sh b/mcp/scripts/salt.sh deleted file mode 100755 index 46ef6ffbf..000000000 --- a/mcp/scripts/salt.sh +++ /dev/null @@ -1,132 +0,0 @@ -#!/bin/bash -e -############################################################################## -# Copyright (c) 2017 Mirantis Inc., Enea AB and others. -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## -# -# Deploy Salt Master -# - -CI_DEBUG=${CI_DEBUG:-0}; [[ "${CI_DEBUG}" =~ (false|0) ]] || set -x -F_GIT_ROOT=$(git rev-parse --show-toplevel) -F_GIT_DIR=$(cd "${F_GIT_ROOT}/mcp" && git rev-parse --git-dir) -F_GIT_SUBD=${F_GIT_ROOT#${F_GIT_DIR%%/.git*}} -OPNFV_TMP_DIR="/home/${SALT_MASTER_USER}/opnfv" -OPNFV_GIT_DIR="/root/opnfv" -OPNFV_FUEL_DIR="/root/fuel" # Should be in sync with patch.sh, scripts patches -OPNFV_RDIR="reclass/classes/cluster/all-mcp-arch-common" -OPNFV_VCP_IMG="mcp/scripts/base_image_opnfv_fuel_vcp.img" -OPNFV_VCP_DIR="/srv/salt/env/prd/salt/files/control/images" -LOCAL_GIT_DIR="${F_GIT_ROOT%${F_GIT_SUBD}}" -LOCAL_PDF_RECLASS=$1; shift -# shellcheck disable=SC2116,SC2086 -LOCAL_VIRT_NODES=$(echo ${*//cfg01/}) # unquoted to filter space -NODE_MASK="${LOCAL_VIRT_NODES// /|}" - -# push to cfg01 current git repo first (including submodules), at ~ubuntu/opnfv -# later we move it to ~root/opnfv (and ln as ~root/fuel); delete the temp clone -remote_tmp="${SSH_SALT}:$(basename "${OPNFV_TMP_DIR}")" -STORAGE_DIR=$(dirname "${LOCAL_PDF_RECLASS}") -REL_STORAGE_DIR_PATH=${STORAGE_DIR#${LOCAL_GIT_DIR}} -if [[ "${REL_STORAGE_DIR_PATH}" == "${STORAGE_DIR}" ]] -then - REL_STORAGE_DIR_PATH="" -fi -rsync -Erl --delete -e "ssh ${SSH_OPTS}" \ - --exclude={.gitignore,"$REL_STORAGE_DIR_PATH"} \ - "${LOCAL_GIT_DIR}/" "${remote_tmp}/" -if [ -n "${LOCAL_PDF_RECLASS}" ] && [ -f "${LOCAL_PDF_RECLASS}" ]; then - rsync -e "ssh ${SSH_OPTS}" "${LOCAL_PDF_RECLASS}" \ - "${remote_tmp}${F_GIT_SUBD}/mcp/${OPNFV_RDIR}/opnfv/" -fi -local_vcp_img=$(dirname "${LOCAL_PDF_RECLASS}")/$(basename "${OPNFV_VCP_IMG}") -if [ -e "${local_vcp_img}" ]; then - rsync -L -e "ssh ${SSH_OPTS}" "${local_vcp_img}" \ - "${remote_tmp}${F_GIT_SUBD}/${OPNFV_VCP_IMG}" -fi - -# ssh to cfg01 -# shellcheck disable=SC2086,2087 -ssh ${SSH_OPTS} "${SSH_SALT}" bash -s -e << SALT_INSTALL_END - sudo -i - set -e - export TERM=${TERM} - export CI_DEBUG=${CI_DEBUG}; [[ "${CI_DEBUG}" =~ (false|0) ]] || set -x - - echo -n 'Checking out cloud-init has finished running ...' - while [ ! -f /var/lib/cloud/instance/boot-finished ]; do echo -n '.'; sleep 1; done - echo ' done' - - mkdir -p /srv/salt /usr/share/salt-formulas/reclass - rm -rf ${OPNFV_GIT_DIR} ${OPNFV_FUEL_DIR} - mv ${OPNFV_TMP_DIR} ${OPNFV_GIT_DIR} && chown -R root.root ${OPNFV_GIT_DIR} - find ${OPNFV_GIT_DIR} -name '.git' -type f | while read f_git; do - sed -i 's@${LOCAL_GIT_DIR}@${OPNFV_GIT_DIR}@g' \$f_git - done - ln -sf ${OPNFV_GIT_DIR}${F_GIT_SUBD} ${OPNFV_FUEL_DIR} - ln -sf ${OPNFV_FUEL_DIR}/mcp/reclass /srv/salt - ln -sf ${OPNFV_FUEL_DIR}/mcp/deploy/scripts /srv/salt - ln -sf ${OPNFV_FUEL_DIR}/mcp/scripts/mcp.rsa $(dirname "${OPNFV_FUEL_DIR}") - - cp -r ${OPNFV_FUEL_DIR}/mcp/metadata/service /usr/share/salt-formulas/reclass - cd /srv/salt/reclass/classes/service && \ - ln -sf /usr/share/salt-formulas/reclass/service/opendaylight - - # Armband APT-MK nightly/extra repo for forked & extended reclass - wget -qO - https://linux.enea.com/apt-mk/public.gpg | apt-key add - - echo 'deb http://linux.enea.com/apt-mk/xenial nightly extra' > \ - '/etc/apt/sources.list.d/armband_mcp_extra.list' - apt-get update - - cd /srv/salt/scripts - export DEBIAN_FRONTEND=noninteractive - echo 'Dpkg::Use-Pty "0";' > /etc/apt/apt.conf.d/90silence-dpkg - OLD_DOMAIN=\$(grep -sPzo "id: cfg01\.\K(\S*)" /etc/salt/minion.d/minion.conf) || true - BOOTSTRAP_SALTSTACK_OPTS=" -r -dX stable 2017.7 " \ - MASTER_HOSTNAME=cfg01.${CLUSTER_DOMAIN} DISTRIB_REVISION=nightly \ - EXTRA_FORMULAS="nfs panko gnocchi oslo-templates" \ - ./salt-master-init.sh - salt-key -Ay - - cp -r ${OPNFV_FUEL_DIR}/mcp/salt-formulas/* /usr/share/salt-formulas/env - cd ${OPNFV_FUEL_DIR}/mcp/patches && ./patch.sh patches.list formulas - cd ${OPNFV_FUEL_DIR}/mcp/patches && ./patch.sh patches.list reclass - - source ${OPNFV_FUEL_DIR}/mcp/scripts/lib.sh - wait_for 3.0 "salt-call state.apply salt" - - # In case scenario changed (and implicitly domain name), re-register minions - if [ -n "\${OLD_DOMAIN}" ] && [ "\${OLD_DOMAIN}" != "${CLUSTER_DOMAIN}" ]; then - salt "*.\${OLD_DOMAIN}" cmd.run "grep \${OLD_DOMAIN} -sRl /etc/salt | \ - xargs --no-run-if-empty sed -i 's/\${OLD_DOMAIN}/${CLUSTER_DOMAIN}/g'; \ - service salt-minion restart" || true - salt-key -yd "*.\${OLD_DOMAIN}" - salt-key -Ay - fi - - # Init specific to VMs on FN (all for virtual, cfg|mas for baremetal) - wait_for 3.0 'salt -C "cfg01*" state.apply linux' - if [[ "${LOCAL_VIRT_NODES}" =~ mas ]]; then - wait_for 3.0 'salt -C "mas*" test.ping' - else - wait_for 3.0 '(for n in ${LOCAL_VIRT_NODES}; do salt -C \${n}.* test.ping || exit; done)' - fi - wait_for 3.0 'salt -C "E@^(${NODE_MASK}|cfg01).*" saltutil.sync_all' - wait_for 3.0 'salt -C "E@^(${NODE_MASK}|cfg01).*" state.apply salt' - - wait_for 3.0 'salt -C "E@^(${NODE_MASK}).*" state.sls linux.system,linux.storage' - wait_for 2.0 'salt -C "E@^(${NODE_MASK}).*" state.sls linux.network' - salt -C "E@^(${NODE_MASK}).*" system.reboot - wait_for 90.0 'salt -C "E@^(${NODE_MASK}).*" test.ping' - wait_for 3.0 'salt -C "E@^(${NODE_MASK}).*" pkg.upgrade refresh=False dist_upgrade=True' - - wait_for 3.0 'salt -C "E@^(${NODE_MASK}|cfg01).*" state.sls ntp' - - if [ -f "${OPNFV_FUEL_DIR}/${OPNFV_VCP_IMG}" ]; then - mkdir -p "${OPNFV_VCP_DIR}" - mv "${OPNFV_FUEL_DIR}/${OPNFV_VCP_IMG}" "${OPNFV_VCP_DIR}/" - fi -SALT_INSTALL_END diff --git a/mcp/scripts/xdf_data.sh.j2 b/mcp/scripts/xdf_data.sh.j2 index c0629fec4..b8c32135c 100644 --- a/mcp/scripts/xdf_data.sh.j2 +++ b/mcp/scripts/xdf_data.sh.j2 @@ -33,7 +33,7 @@ {%- macro serialize_vnodes() -%} {%- set arr = [] -%} {%- for section in section_map -%} - {%- for n in V.nodes[section] -%} + {%- for n in V.nodes[section] or [] -%} {%- if section_map[section] < 0 or conf.nodes[section_map[section] + loop.index0].node.type == 'virtual' -%} {%- if n not in V -%}{%- do V.update({n: {}}) -%}{%- endif -%} {%- do arr.append(pack([n, V[n].ram or arch.default.ram, @@ -48,7 +48,7 @@ {%- macro filter_nodes(type, output_as_query = False, sections = section_map) -%} {%- set arr = [] -%} {%- for section in sections -%} - {%- for n in V.nodes[section] -%} + {%- for n in V.nodes[section] or [] -%} {%- if ( (section_map[section] < 0 and type == 'virtual') or (section_map[section] >= 0 and conf.nodes[section_map[section] + loop.index0].node.type in type) ) -%} -- 2.16.6