From 279195ddeb8e459a5931430f0841724ba6c06db1 Mon Sep 17 00:00:00 2001 From: Alexandru Avadanii Date: Sun, 26 Jan 2020 14:25:05 +0100 Subject: [PATCH] docker build, deploy: Switch tooling to python3 Python2.7 is deprecated and packages are starting to enforce py3 usage (e.g. dockermake recently started supporting only 3.6). Switch pipenv to python3, but allow pyhton3.5 by pinning dockermake to v0.8 since Ubuntu Xenial does not have python3.6 easily available. While at it, switch deploy tooling (PDF/IDF configuration parsing) from python2 to pyhton3 too and fix some jumphost package requirements. Change-Id: Id66d08d0f51a1bc35c1d78c1956df832a5536bde Signed-off-by: Alexandru Avadanii (cherry picked from commit fccf558f0c55c0b26448961a97d8edae3dd7db4d) --- ci/build.sh | 19 +++++++++---------- ...003-OPNFV-package-installation-Ubuntu-user.patch | 21 ++++++++++++++++++++- mcp/scripts/lib_template.sh | 6 +++--- mcp/scripts/requirements_deb.yaml | 14 ++++++++------ mcp/scripts/requirements_rpm.yaml | 13 +++++++------ mcp/scripts/xdf_data.sh.j2 | 2 +- 6 files changed, 48 insertions(+), 27 deletions(-) diff --git a/ci/build.sh b/ci/build.sh index 8fab2ca01..6119472ec 100755 --- a/ci/build.sh +++ b/ci/build.sh @@ -56,26 +56,25 @@ trap do_exit SIGINT SIGTERM EXIT # Set no restrictive umask so that Jenkins can remove any residuals umask 0000 -# Clone git submodules and apply our patches -make -C "${MCP_REPO_ROOT_PATH}/mcp/patches" deepclean patches-import - pushd "${DEPLOY_DIR}" > /dev/null # Install distro packages and pip-managed prerequisites -PYTHON_BIN_PATH="$(python -m site --user-base)/bin" -PATH="$PATH:$PYTHON_BIN_PATH" notify "[NOTE] Installing required build-time distro and pip pkgs" 2 jumpserver_pkg_install 'build' -python -m pip install --upgrade pipenv --user +PYTHON_BIN_PATH="$(python3 -m site --user-base)/bin" +PATH="$PATH:$PYTHON_BIN_PATH" +# Clone git submodules and apply our patches +make -C "${MCP_REPO_ROOT_PATH}/mcp/patches" deepclean patches-import +python3 -m pip install --upgrade pipenv --user docker_install popd > /dev/null pushd "${DOCKER_DIR}" > /dev/null -env PIPENV_HIDE_EMOJIS=1 VIRTUALENV_ALWAYS_COPY=1 python -m pipenv --two install -env PIPENV_HIDE_EMOJIS=1 VIRTUALENV_ALWAYS_COPY=1 python -m pipenv install invoke +env PIPENV_HIDE_EMOJIS=1 VIRTUALENV_ALWAYS_COPY=1 python3 -m pipenv --three install +env PIPENV_HIDE_EMOJIS=1 VIRTUALENV_ALWAYS_COPY=1 python3 -m pipenv install invoke # shellcheck disable=SC2086 -env PIPENV_HIDE_EMOJIS=1 python -m pipenv run \ +env PIPENV_HIDE_EMOJIS=1 python3 -m pipenv run \ invoke build saltmaster-reclass \ --require 'salt salt-formulas opnfv reclass tini-saltmaster' \ --dist=ubuntu \ @@ -87,7 +86,7 @@ env PIPENV_HIDE_EMOJIS=1 python -m pipenv run \ CACHE_INVALIDATE=\"${CACHE_INVALIDATE}\"" \ ${DOCKER_PUSH} -env PIPENV_HIDE_EMOJIS=1 python -m pipenv run \ +env PIPENV_HIDE_EMOJIS=1 python3 -m pipenv run \ invoke build saltminion-maas \ --require 'maas' \ --dist=ubuntu \ diff --git a/mcp/patches/docker/0003-OPNFV-package-installation-Ubuntu-user.patch b/mcp/patches/docker/0003-OPNFV-package-installation-Ubuntu-user.patch index 34db789a2..75da21633 100644 --- a/mcp/patches/docker/0003-OPNFV-package-installation-Ubuntu-user.patch +++ b/mcp/patches/docker/0003-OPNFV-package-installation-Ubuntu-user.patch @@ -18,11 +18,15 @@ Subject: [PATCH] OPNFV package installation, Ubuntu user have to switch to 'root' login; * Preinstall `salt_minion_dependency_packages` and `salt_minion_reclass_dependencies`; +* Pin dockermake to v0.8 to allow using python3.5 for virtualenv, + since python 3.6 is not easily available for Ubuntu Xenial + jump/build hosts. Signed-off-by: Alexandru Avadanii --- DockerMake.yml | 38 +++++++++++++++++++++++++++++++++++++- - 1 file changed, 37 insertions(+), 1 deletion(-) + Pipfile | 4 ++-- + 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/DockerMake.yml b/DockerMake.yml index 2c75586..9ab7195 100644 @@ -87,3 +91,18 @@ index 2c75586..9ab7195 100644 wheel: requires: - base +diff --git a/Pipfile b/Pipfile +index d3e8d66..340e125 100644 +--- a/Pipfile ++++ b/Pipfile +@@ -7,8 +7,8 @@ name = "pypi" + + [packages] + pygithub = "*" +-docker-make = {git = "https://github.com/avirshup/DockerMake"} +-dockermake = {git = "https://github.com/avirshup/DockerMake"} ++docker-make = {git = "https://github.com/avirshup/DockerMake", ref = "aeac230fd5ab468d806bf42b120aa97f91eb40a2"} ++dockermake = {git = "https://github.com/avirshup/DockerMake", ref = "aeac230fd5ab468d806bf42b120aa97f91eb40a2"} + + [requires] + python_version = "3.6" diff --git a/mcp/scripts/lib_template.sh b/mcp/scripts/lib_template.sh index bec8e159e..822d14116 100644 --- a/mcp/scripts/lib_template.sh +++ b/mcp/scripts/lib_template.sh @@ -51,14 +51,14 @@ function do_templates_scenario { awk '/^(SALT|MCP|MAAS).*=/ { gsub(/=/,": "); print }' >> "${LOCAL_PDF}" j2args=$(find "${scenario_dir}" -name '*.j2' -exec echo -j {} \;) # shellcheck disable=SC2086 - if ! "${PHAROS_GEN_CFG}" -y "${LOCAL_PDF}" ${j2args} -b -v \ + if ! python3 "${PHAROS_GEN_CFG}" -y "${LOCAL_PDF}" ${j2args} -b -v \ -i "$(dirname "$(readlink -f "${PHAROS_IA}")")"; then notify_e '[ERROR] Could not convert j2 scenario definitions!' fi for _yaml in "${extra_yaml[@]}"; do awk '/^---$/{f=1;next;}f' "${_yaml}" >> "${LOCAL_PDF}" done - if ! "${PHAROS_GEN_CFG}" -y "${LOCAL_PDF}" \ + if ! python3 "${PHAROS_GEN_CFG}" -y "${LOCAL_PDF}" \ -i "$(dirname "$(readlink -f "${PHAROS_IA}")")" \ -j "${PHAROS_IA}" -v > "${image_dir}/pod_config.yml"; then notify_e "[ERROR] Could not convert PDF+IDF to reclass model input!" @@ -84,7 +84,7 @@ function do_templates_cluster { "$(readlink -f docker-compose)" $(readlink -f ./*j2) \ -name '*.j2' -exec echo -j {} \;) # shellcheck disable=SC2086 - if ! "${PHAROS_GEN_CFG}" -y "${LOCAL_PDF}" ${j2args} -b -v \ + if ! python3 "${PHAROS_GEN_CFG}" -y "${LOCAL_PDF}" ${j2args} -b -v \ -i "$(dirname "$(readlink -f "${PHAROS_IA}")")"; then notify_e '[ERROR] Could not convert PDF to network definitions!' fi diff --git a/mcp/scripts/requirements_deb.yaml b/mcp/scripts/requirements_deb.yaml index 73374b6ef..58fc533b0 100644 --- a/mcp/scripts/requirements_deb.yaml +++ b/mcp/scripts/requirements_deb.yaml @@ -9,7 +9,10 @@ build: # Common pkgs required for all builds, no matter the type, arch etc. common: - - python-pip + - curl + - git + - make + - python3-pip deploy: # Common pkgs required for all deploys, no matter the type, arch etc. common: @@ -30,11 +33,10 @@ deploy: - uuid-runtime - virtinst # python is indirectly required for PDF parsing - - python - - python-ipaddress - - python-jinja2 - - python-yaml - - python-jsonschema + - python3 + - python3-jinja2 + - python3-yaml + - python3-jsonschema # Optional, arch-specific requirements, matched by key name = $(uname -m) aarch64: # AArch64 VMs use AAVMF (guest UEFI) diff --git a/mcp/scripts/requirements_rpm.yaml b/mcp/scripts/requirements_rpm.yaml index ef9bfad9c..d21e49a9d 100644 --- a/mcp/scripts/requirements_rpm.yaml +++ b/mcp/scripts/requirements_rpm.yaml @@ -9,7 +9,9 @@ build: # Common pkgs required for all builds, no matter the type, arch etc. common: - - python-pip + - git + - make + - python3-pip deploy: # Common pkgs required for all deploys, no matter the type, arch etc. common: @@ -35,11 +37,10 @@ deploy: - virt-install - wget # For python is indirectly required for PDF parsing - - python - - python-ipaddress - - python-jinja2 - - python-yaml - - python-jsonschema + - python3 + - python3-jinja2 + - python3-yaml + - python3-jsonschema # Optional, arch-specific requirements, matched by key name = $(uname -m) aarch64: # AArch64 VMs use AAVMF (guest UEFI) diff --git a/mcp/scripts/xdf_data.sh.j2 b/mcp/scripts/xdf_data.sh.j2 index 79bc16c81..6aca36f32 100644 --- a/mcp/scripts/xdf_data.sh.j2 +++ b/mcp/scripts/xdf_data.sh.j2 @@ -39,7 +39,7 @@ {%- if n not in V -%}{%- do V.update({n: {}}) -%}{%- endif -%} {%- set cpu_topo = 'cpu_topology' in V[n] and not conf.MCP_CMP_SS -%} {%- if 'numa' in V[n] and cpu_topo -%} - {%- for k, v in V[n].numa.iteritems() -%} + {%- for k, v in V[n].numa.items() -%} {%- set c = pack([k, v.memory, v.cpus]) -%} {%- do V[n].update({'s_numa': c if 's_numa' not in V[n] else pack([c, V[n].s_numa])}) -%} {%- endfor -%} -- 2.16.6