From 1ea4713f79a2c1e7e57da73b3bf86eced559f201 Mon Sep 17 00:00:00 2001 From: Alexandru Avadanii Date: Thu, 8 Feb 2018 20:35:44 +0100 Subject: [PATCH] [j2 parsing] Adopt new generate_config features - bump Pharos git submodule for j2 'do' extension + batch mode; - adopt j2 'do' in our templates; - use int filter for 'native' vlan check; - lib.sh: adopt `-i` to remove `ln` hack for net_map.j2; - lib.sh: adopt `-b` to speedup template parsing; NOTE: Bumping Pharos will also bring in the latest changes in pod_config.yml.j2, which include massive IP shifts and updates. JIRA: FUEL-335 Change-Id: I7d3a997b3d8659d5f09f867870fb3a148c1ec6df Signed-off-by: Alexandru Avadanii --- .../scenario/os-nosdn-nofeature-novcp-ha.yaml.j2 | 2 +- .../mcp-pike-common-ha/infra/kvm_pdf.yml.j2 | 8 +++---- .../openstack_compute_pdf.yml.j2 | 6 ++--- .../mcp-pike-odl-ha/openstack/compute_pdf.yml.j2 | 4 ++-- .../openstack/compute_pdf.yml.j2 | 2 +- .../mcp-pike-ovs-ha/openstack/compute_pdf.yml.j2 | 4 ++-- mcp/scripts/lib.sh | 28 ++++++++++------------ mcp/scripts/pharos | 2 +- mcp/scripts/virsh_net/net_public.xml.j2 | 2 +- 9 files changed, 28 insertions(+), 30 deletions(-) diff --git a/mcp/config/scenario/os-nosdn-nofeature-novcp-ha.yaml.j2 b/mcp/config/scenario/os-nosdn-nofeature-novcp-ha.yaml.j2 index 2dfea5f60..8a366e9f9 100644 --- a/mcp/config/scenario/os-nosdn-nofeature-novcp-ha.yaml.j2 +++ b/mcp/config/scenario/os-nosdn-nofeature-novcp-ha.yaml.j2 @@ -9,7 +9,7 @@ {%- set cluster = {'has_virtual_nodes': False} %} {%- for node in conf.nodes %} {%- if not cluster.has_virtual_nodes and node.node.type == 'baremetal' %} - {%- set _ = cluster.update({'has_baremetal_nodes': True}) %} + {%- do cluster.update({'has_baremetal_nodes': True}) %} {%- endif %} {%- endfor %} --- diff --git a/mcp/reclass/classes/cluster/mcp-pike-common-ha/infra/kvm_pdf.yml.j2 b/mcp/reclass/classes/cluster/mcp-pike-common-ha/infra/kvm_pdf.yml.j2 index 120b6fb7e..97166c62c 100644 --- a/mcp/reclass/classes/cluster/mcp-pike-common-ha/infra/kvm_pdf.yml.j2 +++ b/mcp/reclass/classes/cluster/mcp-pike-common-ha/infra/kvm_pdf.yml.j2 @@ -27,7 +27,7 @@ parameters: {%- endfor %} {%- for vlan in vlans %} - {%- if vlan and vlan != 'native' %} + {%- if vlan | int > 0 %} {{ vlans[vlan] }}.{{ vlan }}: enabled: true proto: manual @@ -44,7 +44,7 @@ parameters: proto: dhcp type: bridge use_interfaces: - - {{ nm.ctl01.nic_admin }}{% if nm.vlan_admin and nm.vlan_admin != 'native' %}.{{ nm.vlan_admin }}{% endif %} + - {{ nm.ctl01.nic_admin }}{% if nm.vlan_admin | int > 0 %}.{{ nm.vlan_admin }}{% endif %} noifupdown: true br-ctl: enabled: true @@ -53,7 +53,7 @@ parameters: address: ${_param:single_address} netmask: 255.255.255.0 use_interfaces: - - {{ nm.ctl01.nic_mgmt }}{% if nm.vlan_mgmt and nm.vlan_mgmt != 'native' %}.{{ nm.vlan_mgmt }}{% endif %} + - {{ nm.ctl01.nic_mgmt }}{% if nm.vlan_mgmt | int > 0 %}.{{ nm.vlan_mgmt }}{% endif %} noifupdown: true br-ex: enabled: true @@ -62,5 +62,5 @@ parameters: netmask: ${_param:opnfv_net_public_mask} type: bridge use_interfaces: - - {{ nm.ctl01.nic_public }}{% if nm.vlan_public and nm.vlan_public != 'native' %}.{{ nm.vlan_public }}{% endif %} + - {{ nm.ctl01.nic_public }}{% if nm.vlan_public | int > 0 %}.{{ nm.vlan_public }}{% endif %} noifupdown: true diff --git a/mcp/reclass/classes/cluster/mcp-pike-common-ha/openstack_compute_pdf.yml.j2 b/mcp/reclass/classes/cluster/mcp-pike-common-ha/openstack_compute_pdf.yml.j2 index 48b9a9286..aa8b45f30 100644 --- a/mcp/reclass/classes/cluster/mcp-pike-common-ha/openstack_compute_pdf.yml.j2 +++ b/mcp/reclass/classes/cluster/mcp-pike-common-ha/openstack_compute_pdf.yml.j2 @@ -48,7 +48,7 @@ parameters: {%- endfor %} {%- for vlan in vlans %} - {%- if vlan and vlan != 'native' %} + {%- if vlan | int > 0 %} {{ vlans[vlan] }}.{{ vlan }}: enabled: true proto: manual @@ -66,7 +66,7 @@ parameters: address: ${_param:single_address} netmask: 255.255.255.0 use_interfaces: - - {{ nm.cmp001.nic_mgmt }}{% if nm.vlan_mgmt and nm.vlan_mgmt != 'native' %}.{{ nm.vlan_mgmt }}{% endif %} + - {{ nm.cmp001.nic_mgmt }}{% if nm.vlan_mgmt | int > 0 %}.{{ nm.vlan_mgmt }}{% endif %} br-floating: enabled: true type: ovs_bridge @@ -81,7 +81,7 @@ parameters: gateway: ${_param:opnfv_net_public_gw} ovs_port_{{ nm.cmp001.nic_public }}: enabled: true - name: {{ nm.cmp001.nic_public }}{% if nm.vlan_public and nm.vlan_public != 'native' %}.{{ nm.vlan_public }}{% endif %} + name: {{ nm.cmp001.nic_public }}{% if nm.vlan_public | int > 0 %}.{{ nm.vlan_public }}{% endif %} proto: manual ovs_port_type: OVSPort type: ovs_port diff --git a/mcp/reclass/classes/cluster/mcp-pike-odl-ha/openstack/compute_pdf.yml.j2 b/mcp/reclass/classes/cluster/mcp-pike-odl-ha/openstack/compute_pdf.yml.j2 index 4e0f7d147..a3a86723f 100644 --- a/mcp/reclass/classes/cluster/mcp-pike-odl-ha/openstack/compute_pdf.yml.j2 +++ b/mcp/reclass/classes/cluster/mcp-pike-odl-ha/openstack/compute_pdf.yml.j2 @@ -27,7 +27,7 @@ parameters: {%- endfor %} {%- for vlan in vlans %} - {%- if vlan and vlan != 'native' %} + {%- if vlan | int > 0 %} {{ vlans[vlan] }}.{{ vlan }}: enabled: true proto: manual @@ -45,4 +45,4 @@ parameters: netmask: 255.255.255.0 mtu: 1500 use_interfaces: - - {{ nm.cmp001.nic_private }}{% if nm.vlan_private and nm.vlan_private != 'native' %}.{{ nm.vlan_private }}{% endif %} + - {{ nm.cmp001.nic_private }}{% if nm.vlan_private | int > 0 %}.{{ nm.vlan_private }}{% endif %} diff --git a/mcp/reclass/classes/cluster/mcp-pike-ovs-dpdk-ha/openstack/compute_pdf.yml.j2 b/mcp/reclass/classes/cluster/mcp-pike-ovs-dpdk-ha/openstack/compute_pdf.yml.j2 index 925f895a7..de779bb78 100644 --- a/mcp/reclass/classes/cluster/mcp-pike-ovs-dpdk-ha/openstack/compute_pdf.yml.j2 +++ b/mcp/reclass/classes/cluster/mcp-pike-ovs-dpdk-ha/openstack/compute_pdf.yml.j2 @@ -24,6 +24,6 @@ parameters: type: dpdk_ovs_bridge address: ${_param:tenant_address} netmask: 255.255.255.0 - {%- if nm.vlan_private and nm.vlan_private != 'native' %} + {%- if nm.vlan_private | int > 0 %} tag: {{ nm.vlan_private }} {%- endif %} diff --git a/mcp/reclass/classes/cluster/mcp-pike-ovs-ha/openstack/compute_pdf.yml.j2 b/mcp/reclass/classes/cluster/mcp-pike-ovs-ha/openstack/compute_pdf.yml.j2 index 4e0f7d147..a3a86723f 100644 --- a/mcp/reclass/classes/cluster/mcp-pike-ovs-ha/openstack/compute_pdf.yml.j2 +++ b/mcp/reclass/classes/cluster/mcp-pike-ovs-ha/openstack/compute_pdf.yml.j2 @@ -27,7 +27,7 @@ parameters: {%- endfor %} {%- for vlan in vlans %} - {%- if vlan and vlan != 'native' %} + {%- if vlan | int > 0 %} {{ vlans[vlan] }}.{{ vlan }}: enabled: true proto: manual @@ -45,4 +45,4 @@ parameters: netmask: 255.255.255.0 mtu: 1500 use_interfaces: - - {{ nm.cmp001.nic_private }}{% if nm.vlan_private and nm.vlan_private != 'native' %}.{{ nm.vlan_private }}{% endif %} + - {{ nm.cmp001.nic_private }}{% if nm.vlan_private | int > 0 %}.{{ nm.vlan_private }}{% endif %} diff --git a/mcp/scripts/lib.sh b/mcp/scripts/lib.sh index c9159fff6..e522fd610 100644 --- a/mcp/scripts/lib.sh +++ b/mcp/scripts/lib.sh @@ -1,5 +1,5 @@ #!/bin/bash -e -# shellcheck disable=SC2155,SC1001,SC2015 +# shellcheck disable=SC2155,SC1001,SC2015,SC2128 ############################################################################## # Copyright (c) 2017 Mirantis Inc., Enea AB and others. # All rights reserved. This program and the accompanying materials @@ -318,6 +318,7 @@ function prepare_vms { } function jumpserver_check_requirements { + # shellcheck disable=SC2178 local vnodes=$1; shift local br=("$@") local err_br_not_found='Linux bridge not found!' @@ -538,8 +539,7 @@ function do_templates() { RECLASS_CLUSTER_DIR=$(cd "${git_repo_root}/mcp/reclass/classes/cluster"; pwd) PHAROS_GEN_CFG="./pharos/config/utils/generate_config.py" - PHAROS_INSTALLER_ADAPTER="./pharos/config/installers/fuel/pod_config.yml.j2" - PHAROS_INSTALLER_NETMAP="$(dirname "${PHAROS_INSTALLER_ADAPTER}")/net_map.j2" + PHAROS_IA=$(readlink -f "./pharos/config/installers/fuel/pod_config.yml.j2") PHAROS_VALIDATE_SCHEMA_SCRIPT="./pharos/config/utils/validate_schema.py" PHAROS_SCHEMA_PDF="./pharos/config/pdf/pod1.schema.yaml" PHAROS_SCHEMA_IDF="./pharos/config/pdf/idf-pod1.schema.yaml" @@ -563,14 +563,14 @@ function do_templates() { if [[ ! "$target_pod" =~ "virtual" ]]; then if ! "${PHAROS_VALIDATE_SCHEMA_SCRIPT}" -y "${LOCAL_PDF}" \ -s "${PHAROS_SCHEMA_PDF}"; then - notify_e "[ERROR] PDF does not match yaml schema!\n" 1>&2 + notify_e "[ERROR] PDF does not match yaml schema!" elif ! "${PHAROS_VALIDATE_SCHEMA_SCRIPT}" -y "${LOCAL_IDF}" \ -s "${PHAROS_SCHEMA_IDF}"; then - notify_e "[ERROR] IDF does not match yaml schema!\n" 1>&2 + notify_e "[ERROR] IDF does not match yaml schema!" fi fi if ! "${PHAROS_GEN_CFG}" -y "${LOCAL_PDF}" \ - -j "${PHAROS_INSTALLER_ADAPTER}" > "${image_dir}/pod_config.yml"; then + -j "${PHAROS_IA}" -v > "${image_dir}/pod_config.yml"; then notify_e "[ERROR] Could not convert PDF+IDF to reclass model input!" fi template_dirs="${scenario_dir}" @@ -579,16 +579,14 @@ function do_templates() { # Expand reclass and virsh network templates based on PDF + IDF printenv | \ awk '/^(SALT|MCP|MAAS|CLUSTER).*=/ { gsub(/=/,": "); print }' >> "${LOCAL_PDF}" - template_dirs="${RECLASS_CLUSTER_DIR} virsh_net ./*j2" + template_dirs="${RECLASS_CLUSTER_DIR} $(readlink -f virsh_net) $(readlink -f ./*j2)" template_err_str='Could not convert PDF to network definitions!' fi # shellcheck disable=SC2086 - find ${template_dirs} -name '*.j2' | while read -r tp; do - # Jinja2 import does not allow '..' directory traversal - ln -sf "$(readlink -f "${PHAROS_INSTALLER_NETMAP}")" "$(dirname "${tp}")" - if ! "${PHAROS_GEN_CFG}" -y "${LOCAL_PDF}" -j "${tp}" > "${tp%.j2}"; then - notify_e "[ERROR] ${template_err_str}" - fi - rm -f "$(dirname "${tp}")/$(basename "${PHAROS_INSTALLER_NETMAP}")" - done + j2args=$(find $template_dirs -name '*.j2' -exec echo -j {} \;) + # shellcheck disable=SC2086 + if ! "${PHAROS_GEN_CFG}" -y "${LOCAL_PDF}" ${j2args} -b -v \ + -i "$(dirname "${PHAROS_IA}")"; then + notify_e "[ERROR] ${template_err_str}" + fi } diff --git a/mcp/scripts/pharos b/mcp/scripts/pharos index fb55d5ab7..31435de50 160000 --- a/mcp/scripts/pharos +++ b/mcp/scripts/pharos @@ -1 +1 @@ -Subproject commit fb55d5ab735d6941ecfea9b0bcfe3a23eb8117ac +Subproject commit 31435de5085f1a5bb9cf965cdbce005fc04781d1 diff --git a/mcp/scripts/virsh_net/net_public.xml.j2 b/mcp/scripts/virsh_net/net_public.xml.j2 index bc9d64dc1..737b638b3 100644 --- a/mcp/scripts/virsh_net/net_public.xml.j2 +++ b/mcp/scripts/virsh_net/net_public.xml.j2 @@ -9,7 +9,7 @@ {%- set cluster = {'has_virtual_nodes': False} %} {%- for node in conf.nodes %} {%- if not cluster.has_virtual_nodes and node.node.type == 'virtual' %} - {%- set _ = cluster.update({'has_virtual_nodes': True}) %} + {%- do cluster.update({'has_virtual_nodes': True}) %} {%- endif %} {%- endfor %} {%- if conf.idf.net_config.public is defined %} -- 2.16.6