docker build, deploy: Switch tooling to python3

Python2.7 is deprecated and packages are starting to enforce py3
usage (e.g. dockermake recently started supporting only 3.6).

Switch pipenv to python3, but allow pyhton3.5 by pinning dockermake
to v0.8 since Ubuntu Xenial does not have python3.6 easily available.

While at it, switch deploy tooling (PDF/IDF configuration parsing)
from python2 to pyhton3 too and fix some jumphost package requirements.

Change-Id: Id66d08d0f51a1bc35c1d78c1956df832a5536bde
Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
(cherry picked from commit fccf558f0c55c0b26448961a97d8edae3dd7db4d)

diff --git a/ci/build.sh b/ci/build.sh
index 8fab2ca..6119472 100755 (executable)
--- a/ci/build.sh
+++ b/ci/build.sh
@@ -56,26 +56,25 @@ trap do_exit SIGINT SIGTERM EXIT
 # Set no restrictive umask so that Jenkins can remove any residuals
 umask 0000
 
-# Clone git submodules and apply our patches
-make -C "${MCP_REPO_ROOT_PATH}/mcp/patches" deepclean patches-import
-
 pushd "${DEPLOY_DIR}" > /dev/null
 
 # Install distro packages and pip-managed prerequisites
-PYTHON_BIN_PATH="$(python -m site --user-base)/bin"
-PATH="$PATH:$PYTHON_BIN_PATH"
 notify "[NOTE] Installing required build-time distro and pip pkgs" 2
 jumpserver_pkg_install 'build'
-python -m pip install --upgrade pipenv --user
+PYTHON_BIN_PATH="$(python3 -m site --user-base)/bin"
+PATH="$PATH:$PYTHON_BIN_PATH"
+# Clone git submodules and apply our patches
+make -C "${MCP_REPO_ROOT_PATH}/mcp/patches" deepclean patches-import
+python3 -m pip install --upgrade pipenv --user
 docker_install
 
 popd > /dev/null
 pushd "${DOCKER_DIR}" > /dev/null
 
-env PIPENV_HIDE_EMOJIS=1 VIRTUALENV_ALWAYS_COPY=1 python -m pipenv --two install
-env PIPENV_HIDE_EMOJIS=1 VIRTUALENV_ALWAYS_COPY=1 python -m pipenv install invoke
+env PIPENV_HIDE_EMOJIS=1 VIRTUALENV_ALWAYS_COPY=1 python3 -m pipenv --three install
+env PIPENV_HIDE_EMOJIS=1 VIRTUALENV_ALWAYS_COPY=1 python3 -m pipenv install invoke
 # shellcheck disable=SC2086
-env PIPENV_HIDE_EMOJIS=1 python -m pipenv run \
+env PIPENV_HIDE_EMOJIS=1 python3 -m pipenv run \
   invoke build saltmaster-reclass \
     --require 'salt salt-formulas opnfv reclass tini-saltmaster' \
     --dist=ubuntu \
@@ -87,7 +86,7 @@ env PIPENV_HIDE_EMOJIS=1 python -m pipenv run \
         CACHE_INVALIDATE=\"${CACHE_INVALIDATE}\"" \
     ${DOCKER_PUSH}
 
-env PIPENV_HIDE_EMOJIS=1 python -m pipenv run \
+env PIPENV_HIDE_EMOJIS=1 python3 -m pipenv run \
   invoke build saltminion-maas \
     --require 'maas' \
     --dist=ubuntu \

diff --git a/mcp/patches/docker/0003-OPNFV-package-installation-Ubuntu-user.patch b/mcp/patches/docker/0003-OPNFV-package-installation-Ubuntu-user.patch
index 34db789..75da216 100644 (file)
--- a/mcp/patches/docker/0003-OPNFV-package-installation-Ubuntu-user.patch
+++ b/mcp/patches/docker/0003-OPNFV-package-installation-Ubuntu-user.patch
@@ -18,11 +18,15 @@ Subject: [PATCH] OPNFV package installation, Ubuntu user
   have to switch to 'root' login;
 * Preinstall `salt_minion_dependency_packages` and
   `salt_minion_reclass_dependencies`;
+* Pin dockermake to v0.8 to allow using python3.5 for virtualenv,
+  since python 3.6 is not easily available for Ubuntu Xenial
+  jump/build hosts.
 
 Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
 ---
  DockerMake.yml | 38 +++++++++++++++++++++++++++++++++++++-
- 1 file changed, 37 insertions(+), 1 deletion(-)
+ Pipfile        |  4 ++--
+ 2 files changed, 39 insertions(+), 3 deletions(-)
 
 diff --git a/DockerMake.yml b/DockerMake.yml
 index 2c75586..9ab7195 100644
@@ -87,3 +91,18 @@ index 2c75586..9ab7195 100644
  wheel:
    requires:
      - base
+diff --git a/Pipfile b/Pipfile
+index d3e8d66..340e125 100644
+--- a/Pipfile
++++ b/Pipfile
+@@ -7,8 +7,8 @@ name = "pypi"
+
+ [packages]
+ pygithub = "*"
+-docker-make = {git = "https://github.com/avirshup/DockerMake"}
+-dockermake = {git = "https://github.com/avirshup/DockerMake"}
++docker-make = {git = "https://github.com/avirshup/DockerMake", ref = "aeac230fd5ab468d806bf42b120aa97f91eb40a2"}
++dockermake = {git = "https://github.com/avirshup/DockerMake", ref = "aeac230fd5ab468d806bf42b120aa97f91eb40a2"}
+
+ [requires]
+ python_version = "3.6"

diff --git a/mcp/scripts/lib_template.sh b/mcp/scripts/lib_template.sh
index bec8e15..822d141 100644 (file)
--- a/mcp/scripts/lib_template.sh
+++ b/mcp/scripts/lib_template.sh
@@ -51,14 +51,14 @@ function do_templates_scenario {
     awk '/^(SALT|MCP|MAAS).*=/ { gsub(/=/,": "); print }' >> "${LOCAL_PDF}"
   j2args=$(find "${scenario_dir}" -name '*.j2' -exec echo -j {} \;)
   # shellcheck disable=SC2086
-  if ! "${PHAROS_GEN_CFG}" -y "${LOCAL_PDF}" ${j2args} -b -v \
+  if ! python3 "${PHAROS_GEN_CFG}" -y "${LOCAL_PDF}" ${j2args} -b -v \
     -i "$(dirname "$(readlink -f "${PHAROS_IA}")")"; then
     notify_e '[ERROR] Could not convert j2 scenario definitions!'
   fi
   for _yaml in "${extra_yaml[@]}"; do
     awk '/^---$/{f=1;next;}f' "${_yaml}" >> "${LOCAL_PDF}"
   done
-  if ! "${PHAROS_GEN_CFG}" -y "${LOCAL_PDF}" \
+  if ! python3 "${PHAROS_GEN_CFG}" -y "${LOCAL_PDF}" \
     -i "$(dirname "$(readlink -f "${PHAROS_IA}")")" \
     -j "${PHAROS_IA}" -v > "${image_dir}/pod_config.yml"; then
     notify_e "[ERROR] Could not convert PDF+IDF to reclass model input!"
@@ -84,7 +84,7 @@ function do_templates_cluster {
            "$(readlink -f docker-compose)" $(readlink -f ./*j2) \
            -name '*.j2' -exec echo -j {} \;)
   # shellcheck disable=SC2086
-  if ! "${PHAROS_GEN_CFG}" -y "${LOCAL_PDF}" ${j2args} -b -v \
+  if ! python3 "${PHAROS_GEN_CFG}" -y "${LOCAL_PDF}" ${j2args} -b -v \
     -i "$(dirname "$(readlink -f "${PHAROS_IA}")")"; then
     notify_e '[ERROR] Could not convert PDF to network definitions!'
   fi

diff --git a/mcp/scripts/requirements_deb.yaml b/mcp/scripts/requirements_deb.yaml
index 73374b6..58fc533 100644 (file)
--- a/mcp/scripts/requirements_deb.yaml
+++ b/mcp/scripts/requirements_deb.yaml
@@ -9,7 +9,10 @@
 build:
   # Common pkgs required for all builds, no matter the type, arch etc.
   common:
-    - python-pip
+    - curl
+    - git
+    - make
+    - python3-pip
 deploy:
   # Common pkgs required for all deploys, no matter the type, arch etc.
   common:
@@ -30,11 +33,10 @@ deploy:
     - uuid-runtime
     - virtinst
     # python is indirectly required for PDF parsing
-    - python
-    - python-ipaddress
-    - python-jinja2
-    - python-yaml
-    - python-jsonschema
+    - python3
+    - python3-jinja2
+    - python3-yaml
+    - python3-jsonschema
   # Optional, arch-specific requirements, matched by key name = $(uname -m)
   aarch64:
     # AArch64 VMs use AAVMF (guest UEFI)

diff --git a/mcp/scripts/requirements_rpm.yaml b/mcp/scripts/requirements_rpm.yaml
index ef9bfad..d21e49a 100644 (file)
--- a/mcp/scripts/requirements_rpm.yaml
+++ b/mcp/scripts/requirements_rpm.yaml
@@ -9,7 +9,9 @@
 build:
   # Common pkgs required for all builds, no matter the type, arch etc.
   common:
-    - python-pip
+    - git
+    - make
+    - python3-pip
 deploy:
   # Common pkgs required for all deploys, no matter the type, arch etc.
   common:
@@ -35,11 +37,10 @@ deploy:
     - virt-install
     - wget
     # For python is indirectly required for PDF parsing
-    - python
-    - python-ipaddress
-    - python-jinja2
-    - python-yaml
-    - python-jsonschema
+    - python3
+    - python3-jinja2
+    - python3-yaml
+    - python3-jsonschema
   # Optional, arch-specific requirements, matched by key name = $(uname -m)
   aarch64:
     # AArch64 VMs use AAVMF (guest UEFI)

diff --git a/mcp/scripts/xdf_data.sh.j2 b/mcp/scripts/xdf_data.sh.j2
index 79bc16c..6aca36f 100644 (file)
--- a/mcp/scripts/xdf_data.sh.j2
+++ b/mcp/scripts/xdf_data.sh.j2
@@ -39,7 +39,7 @@
         {%- if n not in V -%}{%- do V.update({n: {}}) -%}{%- endif -%}
         {%- set cpu_topo = 'cpu_topology' in V[n] and not conf.MCP_CMP_SS -%}
         {%- if 'numa' in V[n] and cpu_topo -%}
-          {%- for k, v in V[n].numa.iteritems() -%}
+          {%- for k, v in V[n].numa.items() -%}
             {%- set c = pack([k, v.memory, v.cpus]) -%}
             {%- do V[n].update({'s_numa': c if 's_numa' not in V[n] else pack([c, V[n].s_numa])}) -%}
           {%- endfor -%}