[j2 parsing] Adopt new generate_config features

- bump Pharos git submodule for j2 'do' extension + batch mode;
- adopt j2 'do' in our templates;
- use int filter for 'native' vlan check;
- lib.sh: adopt `-i` to remove `ln` hack for net_map.j2;
- lib.sh: adopt `-b` to speedup template parsing;

NOTE: Bumping Pharos will also bring in the latest changes in
pod_config.yml.j2, which include massive IP shifts and updates.

JIRA: FUEL-335

Change-Id: I7d3a997b3d8659d5f09f867870fb3a148c1ec6df
Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>

diff --git a/mcp/config/scenario/os-nosdn-nofeature-novcp-ha.yaml.j2 b/mcp/config/scenario/os-nosdn-nofeature-novcp-ha.yaml.j2
index 2dfea5f..8a366e9 100644 (file)
--- a/mcp/config/scenario/os-nosdn-nofeature-novcp-ha.yaml.j2
+++ b/mcp/config/scenario/os-nosdn-nofeature-novcp-ha.yaml.j2
@@ -9,7 +9,7 @@
 {%- set cluster = {'has_virtual_nodes': False} %}
 {%- for node in conf.nodes %}
   {%- if not cluster.has_virtual_nodes and node.node.type == 'baremetal' %}
-    {%- set _ = cluster.update({'has_baremetal_nodes': True}) %}
+    {%- do cluster.update({'has_baremetal_nodes': True}) %}
   {%- endif %}
 {%- endfor %}
 ---

diff --git a/mcp/reclass/classes/cluster/mcp-pike-common-ha/infra/kvm_pdf.yml.j2 b/mcp/reclass/classes/cluster/mcp-pike-common-ha/infra/kvm_pdf.yml.j2
index 120b6fb..97166c6 100644 (file)
--- a/mcp/reclass/classes/cluster/mcp-pike-common-ha/infra/kvm_pdf.yml.j2
+++ b/mcp/reclass/classes/cluster/mcp-pike-common-ha/infra/kvm_pdf.yml.j2
@@ -27,7 +27,7 @@ parameters:
 {%- endfor %}
 
 {%- for vlan in vlans %}
-    {%- if vlan and vlan != 'native' %}
+    {%- if vlan | int > 0 %}
         {{ vlans[vlan] }}.{{ vlan }}:
           enabled: true
           proto: manual
@@ -44,7 +44,7 @@ parameters:
           proto: dhcp
           type: bridge
           use_interfaces:
-            - {{ nm.ctl01.nic_admin }}{% if nm.vlan_admin and nm.vlan_admin != 'native' %}.{{ nm.vlan_admin }}{% endif %}
+            - {{ nm.ctl01.nic_admin }}{% if nm.vlan_admin | int > 0 %}.{{ nm.vlan_admin }}{% endif %}
           noifupdown: true
         br-ctl:
           enabled: true
@@ -53,7 +53,7 @@ parameters:
           address: ${_param:single_address}
           netmask: 255.255.255.0
           use_interfaces:
-            - {{ nm.ctl01.nic_mgmt }}{% if nm.vlan_mgmt and nm.vlan_mgmt != 'native' %}.{{ nm.vlan_mgmt }}{% endif %}
+            - {{ nm.ctl01.nic_mgmt }}{% if nm.vlan_mgmt | int > 0 %}.{{ nm.vlan_mgmt }}{% endif %}
           noifupdown: true
         br-ex:
           enabled: true
@@ -62,5 +62,5 @@ parameters:
           netmask: ${_param:opnfv_net_public_mask}
           type: bridge
           use_interfaces:
-            - {{ nm.ctl01.nic_public }}{% if nm.vlan_public and nm.vlan_public != 'native' %}.{{ nm.vlan_public }}{% endif %}
+            - {{ nm.ctl01.nic_public }}{% if nm.vlan_public | int > 0 %}.{{ nm.vlan_public }}{% endif %}
           noifupdown: true

diff --git a/mcp/reclass/classes/cluster/mcp-pike-common-ha/openstack_compute_pdf.yml.j2 b/mcp/reclass/classes/cluster/mcp-pike-common-ha/openstack_compute_pdf.yml.j2
index 48b9a92..aa8b45f 100644 (file)
--- a/mcp/reclass/classes/cluster/mcp-pike-common-ha/openstack_compute_pdf.yml.j2
+++ b/mcp/reclass/classes/cluster/mcp-pike-common-ha/openstack_compute_pdf.yml.j2
@@ -48,7 +48,7 @@ parameters:
 {%- endfor %}
 
 {%- for vlan in vlans %}
-    {%- if vlan and vlan != 'native' %}
+    {%- if vlan | int > 0 %}
         {{ vlans[vlan] }}.{{ vlan }}:
           enabled: true
           proto: manual
@@ -66,7 +66,7 @@ parameters:
           address: ${_param:single_address}
           netmask: 255.255.255.0
           use_interfaces:
-            - {{ nm.cmp001.nic_mgmt }}{% if nm.vlan_mgmt and nm.vlan_mgmt != 'native' %}.{{ nm.vlan_mgmt }}{% endif %}
+            - {{ nm.cmp001.nic_mgmt }}{% if nm.vlan_mgmt | int > 0 %}.{{ nm.vlan_mgmt }}{% endif %}
         br-floating:
           enabled: true
           type: ovs_bridge
@@ -81,7 +81,7 @@ parameters:
               gateway: ${_param:opnfv_net_public_gw}
         ovs_port_{{ nm.cmp001.nic_public }}:
           enabled: true
-          name: {{ nm.cmp001.nic_public }}{% if nm.vlan_public and nm.vlan_public != 'native' %}.{{ nm.vlan_public }}{% endif %}
+          name: {{ nm.cmp001.nic_public }}{% if nm.vlan_public | int > 0 %}.{{ nm.vlan_public }}{% endif %}
           proto: manual
           ovs_port_type: OVSPort
           type: ovs_port

diff --git a/mcp/reclass/classes/cluster/mcp-pike-odl-ha/openstack/compute_pdf.yml.j2 b/mcp/reclass/classes/cluster/mcp-pike-odl-ha/openstack/compute_pdf.yml.j2
index 4e0f7d1..a3a8672 100644 (file)
--- a/mcp/reclass/classes/cluster/mcp-pike-odl-ha/openstack/compute_pdf.yml.j2
+++ b/mcp/reclass/classes/cluster/mcp-pike-odl-ha/openstack/compute_pdf.yml.j2
@@ -27,7 +27,7 @@ parameters:
 {%- endfor %}
 
 {%- for vlan in vlans %}
-    {%- if vlan and vlan != 'native' %}
+    {%- if vlan | int > 0 %}
         {{ vlans[vlan] }}.{{ vlan }}:
           enabled: true
           proto: manual
@@ -45,4 +45,4 @@ parameters:
           netmask: 255.255.255.0
           mtu: 1500
           use_interfaces:
-            - {{ nm.cmp001.nic_private }}{% if nm.vlan_private and nm.vlan_private != 'native' %}.{{ nm.vlan_private }}{% endif %}
+            - {{ nm.cmp001.nic_private }}{% if nm.vlan_private | int > 0 %}.{{ nm.vlan_private }}{% endif %}

diff --git a/mcp/reclass/classes/cluster/mcp-pike-ovs-dpdk-ha/openstack/compute_pdf.yml.j2 b/mcp/reclass/classes/cluster/mcp-pike-ovs-dpdk-ha/openstack/compute_pdf.yml.j2
index 925f895..de779bb 100644 (file)
--- a/mcp/reclass/classes/cluster/mcp-pike-ovs-dpdk-ha/openstack/compute_pdf.yml.j2
+++ b/mcp/reclass/classes/cluster/mcp-pike-ovs-dpdk-ha/openstack/compute_pdf.yml.j2
@@ -24,6 +24,6 @@ parameters:
           type: dpdk_ovs_bridge
           address: ${_param:tenant_address}
           netmask: 255.255.255.0
-          {%- if nm.vlan_private and nm.vlan_private != 'native' %}
+          {%- if nm.vlan_private | int > 0 %}
           tag: {{ nm.vlan_private }}
           {%- endif %}

diff --git a/mcp/reclass/classes/cluster/mcp-pike-ovs-ha/openstack/compute_pdf.yml.j2 b/mcp/reclass/classes/cluster/mcp-pike-ovs-ha/openstack/compute_pdf.yml.j2
index 4e0f7d1..a3a8672 100644 (file)
--- a/mcp/reclass/classes/cluster/mcp-pike-ovs-ha/openstack/compute_pdf.yml.j2
+++ b/mcp/reclass/classes/cluster/mcp-pike-ovs-ha/openstack/compute_pdf.yml.j2
@@ -27,7 +27,7 @@ parameters:
 {%- endfor %}
 
 {%- for vlan in vlans %}
-    {%- if vlan and vlan != 'native' %}
+    {%- if vlan | int > 0 %}
         {{ vlans[vlan] }}.{{ vlan }}:
           enabled: true
           proto: manual
@@ -45,4 +45,4 @@ parameters:
           netmask: 255.255.255.0
           mtu: 1500
           use_interfaces:
-            - {{ nm.cmp001.nic_private }}{% if nm.vlan_private and nm.vlan_private != 'native' %}.{{ nm.vlan_private }}{% endif %}
+            - {{ nm.cmp001.nic_private }}{% if nm.vlan_private | int > 0 %}.{{ nm.vlan_private }}{% endif %}

diff --git a/mcp/scripts/lib.sh b/mcp/scripts/lib.sh
index c9159ff..e522fd6 100644 (file)
--- a/mcp/scripts/lib.sh
+++ b/mcp/scripts/lib.sh
@@ -1,5 +1,5 @@
 #!/bin/bash -e
-# shellcheck disable=SC2155,SC1001,SC2015
+# shellcheck disable=SC2155,SC1001,SC2015,SC2128
 ##############################################################################
 # Copyright (c) 2017 Mirantis Inc., Enea AB and others.
 # All rights reserved. This program and the accompanying materials
@@ -318,6 +318,7 @@ function prepare_vms {
 }
 
 function jumpserver_check_requirements {
+  # shellcheck disable=SC2178
   local vnodes=$1; shift
   local br=("$@")
   local err_br_not_found='Linux bridge not found!'
@@ -538,8 +539,7 @@ function do_templates() {
 
   RECLASS_CLUSTER_DIR=$(cd "${git_repo_root}/mcp/reclass/classes/cluster"; pwd)
   PHAROS_GEN_CFG="./pharos/config/utils/generate_config.py"
-  PHAROS_INSTALLER_ADAPTER="./pharos/config/installers/fuel/pod_config.yml.j2"
-  PHAROS_INSTALLER_NETMAP="$(dirname "${PHAROS_INSTALLER_ADAPTER}")/net_map.j2"
+  PHAROS_IA=$(readlink -f "./pharos/config/installers/fuel/pod_config.yml.j2")
   PHAROS_VALIDATE_SCHEMA_SCRIPT="./pharos/config/utils/validate_schema.py"
   PHAROS_SCHEMA_PDF="./pharos/config/pdf/pod1.schema.yaml"
   PHAROS_SCHEMA_IDF="./pharos/config/pdf/idf-pod1.schema.yaml"
@@ -563,14 +563,14 @@ function do_templates() {
     if [[ ! "$target_pod" =~ "virtual" ]]; then
       if ! "${PHAROS_VALIDATE_SCHEMA_SCRIPT}" -y "${LOCAL_PDF}" \
         -s "${PHAROS_SCHEMA_PDF}"; then
-        notify_e "[ERROR] PDF does not match yaml schema!\n" 1>&2
+        notify_e "[ERROR] PDF does not match yaml schema!"
       elif ! "${PHAROS_VALIDATE_SCHEMA_SCRIPT}" -y "${LOCAL_IDF}" \
         -s "${PHAROS_SCHEMA_IDF}"; then
-        notify_e "[ERROR] IDF does not match yaml schema!\n" 1>&2
+        notify_e "[ERROR] IDF does not match yaml schema!"
       fi
     fi
     if ! "${PHAROS_GEN_CFG}" -y "${LOCAL_PDF}" \
-        -j "${PHAROS_INSTALLER_ADAPTER}" > "${image_dir}/pod_config.yml"; then
+      -j "${PHAROS_IA}" -v > "${image_dir}/pod_config.yml"; then
       notify_e "[ERROR] Could not convert PDF+IDF to reclass model input!"
     fi
     template_dirs="${scenario_dir}"
@@ -579,16 +579,14 @@ function do_templates() {
     # Expand reclass and virsh network templates based on PDF + IDF
     printenv | \
       awk '/^(SALT|MCP|MAAS|CLUSTER).*=/ { gsub(/=/,": "); print }' >> "${LOCAL_PDF}"
-    template_dirs="${RECLASS_CLUSTER_DIR} virsh_net ./*j2"
+    template_dirs="${RECLASS_CLUSTER_DIR} $(readlink -f virsh_net) $(readlink -f ./*j2)"
     template_err_str='Could not convert PDF to network definitions!'
   fi
   # shellcheck disable=SC2086
-  find ${template_dirs} -name '*.j2' | while read -r tp; do
-    # Jinja2 import does not allow '..' directory traversal
-    ln -sf "$(readlink -f "${PHAROS_INSTALLER_NETMAP}")" "$(dirname "${tp}")"
-    if ! "${PHAROS_GEN_CFG}" -y "${LOCAL_PDF}" -j "${tp}" > "${tp%.j2}"; then
-      notify_e "[ERROR] ${template_err_str}"
-    fi
-    rm -f "$(dirname "${tp}")/$(basename "${PHAROS_INSTALLER_NETMAP}")"
-  done
+  j2args=$(find $template_dirs -name '*.j2' -exec echo -j {} \;)
+  # shellcheck disable=SC2086
+  if ! "${PHAROS_GEN_CFG}" -y "${LOCAL_PDF}" ${j2args} -b -v \
+    -i "$(dirname "${PHAROS_IA}")"; then
+    notify_e "[ERROR] ${template_err_str}"
+  fi
 }

diff --git a/mcp/scripts/pharos b/mcp/scripts/pharos
index fb55d5a..31435de 160000 (submodule)
--- a/mcp/scripts/pharos
+++ b/mcp/scripts/pharos
@@ -1 +1 @@
-Subproject commit fb55d5ab735d6941ecfea9b0bcfe3a23eb8117ac
+Subproject commit 31435de5085f1a5bb9cf965cdbce005fc04781d1

diff --git a/mcp/scripts/virsh_net/net_public.xml.j2 b/mcp/scripts/virsh_net/net_public.xml.j2
index bc9d64d..737b638 100644 (file)
--- a/mcp/scripts/virsh_net/net_public.xml.j2
+++ b/mcp/scripts/virsh_net/net_public.xml.j2
@@ -9,7 +9,7 @@
 {%- set cluster = {'has_virtual_nodes': False} %}
 {%- for node in conf.nodes %}
   {%- if not cluster.has_virtual_nodes and node.node.type == 'virtual' %}
-    {%- set _ = cluster.update({'has_virtual_nodes': True}) %}
+    {%- do cluster.update({'has_virtual_nodes': True}) %}
   {%- endif %}
 {%- endfor %}
 {%- if conf.idf.net_config.public is defined %}