From: Markos Chandras <mchandras@suse.de>
Date: Fri, 23 Mar 2018 10:01:47 +0000 (+0000)
Subject: xci: install: Move all pip installations to virtualenv
X-Git-Tag: 6.0.0~74^2
X-Git-Url: https://gerrit.opnfv.org/gerrit/gitweb?a=commitdiff_plain;h=434466fafd40485eefba1c59af105ecf793a3c25;p=releng-xci.git
xci: install: Move all pip installations to virtualenv
Now that all pip installations are managed in the same way we can
move them inside a global virtual environment to further isolate the
XCI artifacts from the rest of the system. Moreover, we further simplify
the initial package installation to install everything at once instead
of calling the package manager for every single package that we need.
Change-Id: I6a170d2439fae8b0653f3141e0e8bb8ead67657e
Signed-off-by: Markos Chandras <mchandras@suse.de>
---
diff --git a/xci/files/install-lib.sh b/xci/files/install-lib.sh
index 2679233d..5c2dd288 100644
--- a/xci/files/install-lib.sh
+++ b/xci/files/install-lib.sh
@@ -17,6 +17,7 @@ function install_ansible() {
# Use the upper-constraints file from the pinned requirements repository.
local requirements_sha=$(awk '/requirements_git_install_branch:/ {print $2}' ${XCI_PATH}/xci/installer/osa/files/openstack_services.yml)
local uc="https://raw.githubusercontent.com/openstack/requirements/${requirements_sha}/upper-constraints.txt"
+ local install_map
declare -A PKG_MAP
@@ -49,6 +50,7 @@ function install_ansible() {
[lsb-release]=lsb-release
[make]=make
[net-tools]=net-tools
+ [pip]=python-pip
[python]=python
[python-devel]=python-devel
[venv]=python-virtualenv
@@ -75,12 +77,13 @@ function install_ansible() {
[lsb-release]=lsb-release
[make]=make
[net-tools]=net-tools
+ [pip]=python-pip
[python]=python-minimal
[python-devel]=libpython-dev
[venv]=python-virtualenv
[wget]=wget
)
- EXTRA_PKG_DEPS=()
+ EXTRA_PKG_DEPS=( apt-utils )
sudo apt-get update
;;
@@ -96,6 +99,7 @@ function install_ansible() {
[lsb-release]=redhat-lsb
[make]=make
[net-tools]=net-tools
+ [pip]=python2-pip
[python]=python
[python-devel]=python-devel
[venv]=python-virtualenv
@@ -108,51 +112,15 @@ function install_ansible() {
*) echo "ERROR: Supported package manager not found. Supported: apt, dnf, yum, zypper"; exit 1;;
esac
- if ! $(python --version &>/dev/null); then
- ${INSTALLER_CMD} ${PKG_MAP[python]}
- fi
- if ! $(gcc -v &>/dev/null); then
- ${INSTALLER_CMD} ${PKG_MAP[gcc]}
- fi
- if ! $(wget --version &>/dev/null); then
- ${INSTALLER_CMD} ${PKG_MAP[wget]}
- fi
-
- if ! $(python -m virtualenv --version &>/dev/null); then
- ${INSTALLER_CMD} ${PKG_MAP[venv]}
- fi
-
- for pkg in ${CHECK_CMD_PKGS[@]}; do
- if ! $(${CHECK_CMD} ${PKG_MAP[$pkg]} &>/dev/null); then
- ${INSTALLER_CMD} ${PKG_MAP[$pkg]}
- fi
+ # Build instllation map
+ for pkgmap in ${CHECK_CMD_PKGS[@]}; do
+ install_map+=(${PKG_MAP[$pkgmap]} )
done
- if [ -n "${EXTRA_PKG_DEPS-}" ]; then
- for pkg in ${EXTRA_PKG_DEPS}; do
- if ! $(${CHECK_CMD} ${pkg} &>/dev/null); then
- ${INSTALLER_CMD} ${pkg}
- fi
- done
- fi
+ install_map+=(${EXTRA_PKG_DEPS[@]} )
+
+ ${INSTALLER_CMD} ${install_map[@]}
- # If we're using a venv, we need to work around sudo not
- # keeping the path even with -E.
- PYTHON=$(which python)
-
- # To install python packages, we need pip.
- #
- # We can't use the apt packaged version of pip since
- # older versions of pip are incompatible with
- # requests, one of our indirect dependencies (bug 1459947).
- #
- # Note(cinerama): We use pip to install an updated pip plus our
- # other python requirements. pip breakages can seriously impact us,
- # so we've chosen to install/upgrade pip here rather than in
- # requirements (which are synced automatically from the global ones)
- # so we can quickly and easily adjust version parameters.
- # See bug 1536627.
- #
# Note(cinerama): If pip is linked to pip3, the rest of the install
# won't work. Remove the alternatives. This is due to ansible's
# python 2.x requirement.
@@ -160,16 +128,16 @@ function install_ansible() {
sudo -H update-alternatives --remove pip $(readlink -f /etc/alternatives/pip)
fi
- if ! which pip; then
- wget -O /tmp/get-pip.py https://bootstrap.pypa.io/get-pip.py
- sudo -H -E ${PYTHON} /tmp/get-pip.py
- fi
+ # We need to prepare our virtualenv now
+ virtualenv --quiet --no-site-packages ${XCI_VENV}
+ set +u
+ source ${XCI_VENV}/bin/activate
+ set -u
- PIP=$(which pip)
- echo "Using pip: $(${PIP} --version)"
- ${PIP} -q install --user --upgrade -c $uc ara virtualenv pip setuptools ansible==$XCI_ANSIBLE_PIP_VERSION
+ # We are inside the virtualenv now so we should be good to use pip and python from it.
+ pip -q install --upgrade -c $uc ara virtualenv pip setuptools ansible==$XCI_ANSIBLE_PIP_VERSION
- ara_location=$(${PYTHON} -c "import os,ara; print(os.path.dirname(ara.__file__))")
+ ara_location=$(python -c "import os,ara; print(os.path.dirname(ara.__file__))")
export ANSIBLE_CALLBACK_PLUGINS="/etc/ansible/roles/plugins/callback:${ara_location}/plugins/callbacks"
}
diff --git a/xci/files/xci-destroy-env.sh b/xci/files/xci-destroy-env.sh
index 8b811d0f..2e183bd7 100755
--- a/xci/files/xci-destroy-env.sh
+++ b/xci/files/xci-destroy-env.sh
@@ -23,7 +23,7 @@ rm -rf ${HOME}/.config/openstack
rm -rf ${HOME}/.ansible
# bifrost installs everything on venv so we need to look there if virtualbmc is not installed on the host.
-if which vbmc &>/dev/null || { [[ -e /opt/stack/bifrost/bin/activate ]] && source /opt/stack/bifrost/bin/activate; }; then
+if which vbmc &>/dev/null || { [[ -e ${XCI_VENV}/bifrost/bin/activate ]] && source ${XCI_VENV}/bifrost/bin/activate; }; then
# Delete all libvirt VMs and hosts from vbmc (look for a port number)
for vm in $(vbmc list | awk '/[0-9]/{{ print $2 }}'); do
if which virsh &>/dev/null; then
@@ -81,4 +81,6 @@ service ironic-api restart || true
service ironic-conductor start || true
service ironic-inspector restart || true
+rm -rf ${XCI_VENV}
+
# vim: set ts=4 sw=4 expandtab:
diff --git a/xci/infra/bifrost/scripts/bifrost-provision.sh b/xci/infra/bifrost/scripts/bifrost-provision.sh
index 939acb58..eedf350f 100755
--- a/xci/infra/bifrost/scripts/bifrost-provision.sh
+++ b/xci/infra/bifrost/scripts/bifrost-provision.sh
@@ -13,7 +13,8 @@ set -o pipefail
export PYTHONUNBUFFERED=1
SCRIPT_HOME="$(cd "$(dirname "$0")" && pwd)"
BIFROST_HOME=$SCRIPT_HOME/..
-ENABLE_VENV="false"
+ENABLE_VENV="true"
+export VENV=${XCI_VENV}/bifrost
PROVISION_WAIT_TIMEOUT=${PROVISION_WAIT_TIMEOUT:-3600}
# This is normally exported by XCI env but we should initialize it here
# in case we run this script on its own for debug purposes
@@ -93,7 +94,7 @@ if [[ -e ${XCI_PATH}/deployment_image.qcow2 ]]; then
fi
# Install missing dependencies
-$(which pip) install -q --user --upgrade -r "$(dirname $0)/../requirements.txt"
+pip install -q --upgrade -r "$(dirname $0)/../requirements.txt"
# Change working directory
cd $BIFROST_HOME/playbooks
diff --git a/xci/xci-deploy.sh b/xci/xci-deploy.sh
index fa9f3351..618377ec 100755
--- a/xci/xci-deploy.sh
+++ b/xci/xci-deploy.sh
@@ -59,6 +59,8 @@ fi
#-------------------------------------------------------------------------------
# find where are we
export XCI_PATH="$(git rev-parse --show-toplevel)"
+# Declare our virtualenv
+export XCI_VENV=${XCI_PATH}/venv/
# source user vars
source $XCI_PATH/xci/config/user-vars
# source pinned versions