patches/opnfv-fuel/upstream-backports/0005-CI-deploy-cache-Store-and-reuse-deploy-artifacts.patch

   1 From: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
   2 Date: Thu, 24 Nov 2016 23:02:04 +0100
   3 Subject: [PATCH] CI: deploy-cache: Store and reuse deploy artifacts
   4
   5 Add support for caching deploy artifacts, like bootstraps and
   6 target images, which take a lot of time at each deploy to be built,
   7 considering it requires a cross-debootstrap via qemu-user-static and
   8 binfmt.
   9
  10 For OPNFV CI, the cache will piggy back on the <iso_mount> mechanism,
  11 and be located at:
  12 /iso_mount/opnfv_ci/<branch>/deploy-cache
  13
  14 TODO: Use dea interface adapter in target images fingerprinting.
  15 TODO: remote fingerprinting
  16 TODO: differentiate between bootstraps and targetimages, so we don't
  17 end up trying to use one cache artifact type as the other.
  18
  19 JIRA: ARMBAND-172
  20
  21 Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
  22 ---
  23  ...p_admin_node.sh-deploy_cache-install-hook.patch |  69 +++++
  24  ci/deploy.sh                                       |  14 +-
  25  deploy/cloud/deploy.py                             |  11 +
  26  deploy/deploy.py                                   |  25 +-
  27  deploy/deploy_cache.py                             | 319 +++++++++++++++++++++
  28  deploy/deploy_env.py                               |  13 +-
  29  deploy/install_fuel_master.py                      |   9 +-
  30  7 files changed, 451 insertions(+), 9 deletions(-)
  31  create mode 100644 build/f_repos/patch/fuel-main/0006-bootstrap_admin_node.sh-deploy_cache-install-hook.patch
  32  create mode 100644 deploy/deploy_cache.py
  33
  34 diff --git a/build/f_repos/patch/fuel-main/0006-bootstrap_admin_node.sh-deploy_cache-install-hook.patch b/build/f_repos/patch/fuel-main/0006-bootstrap_admin_node.sh-deploy_cache-install-hook.patch
  35 new file mode 100644
  36 index 0000000..d5b7646
  37 --- /dev/null
  38 +++ b/build/f_repos/patch/fuel-main/0006-bootstrap_admin_node.sh-deploy_cache-install-hook.patch
  39 @@ -0,0 +1,69 @@
  40 +From: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
  41 +Date: Mon, 28 Nov 2016 14:27:48 +0100
  42 +Subject: [PATCH] bootstrap_admin_node.sh: deploy_cache install hook
  43 +
  44 +Tooling on the automatic deploy side was updated to support deploy
  45 +caching of artifacts like bootstrap (and id_rsa keypair), target
  46 +images etc.
  47 +
  48 +Add installation hook that calls `fuel-bootstrap import` instead of
  49 +`build` when a bootstrap tar is available in the agreed location,
  50 +/var/lib/opnfv/cache/bootstraps/.
  51 +
  52 +JIRA: ARMBAND-172
  53 +
  54 +Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
  55 +---
  56 + iso/bootstrap_admin_node.sh | 20 +++++++++++++++++++-
  57 + 1 file changed, 19 insertions(+), 1 deletion(-)
  58 +
  59 +diff --git a/iso/bootstrap_admin_node.sh b/iso/bootstrap_admin_node.sh
  60 +index abc5ffb..15e6261 100755
  61 +--- a/iso/bootstrap_admin_node.sh
  62 ++++ b/iso/bootstrap_admin_node.sh
  63 +@@ -61,6 +61,8 @@ wget \
  64 +
  65 + ASTUTE_YAML='/etc/fuel/astute.yaml'
  66 + BOOTSTRAP_NODE_CONFIG="/etc/fuel/bootstrap_admin_node.conf"
  67 ++OPNFV_CACHE_PATH="/var/lib/opnfv/cache/bootstraps"
  68 ++OPNFV_CACHE_TAR="opnfv-bootstraps-cache.tar"
  69 + bs_build_log='/var/log/fuel-bootstrap-image-build.log'
  70 + bs_status=0
  71 + # Backup network configs to this folder. Folder will be created only if
  72 +@@ -94,6 +96,7 @@ image becomes available, reboot nodes that failed to be discovered."
  73 + bs_done_message="Default bootstrap image building done. Now you can boot new \
  74 + nodes over PXE, they will be discovered and become available for installing \
  75 + OpenStack on them"
  76 ++bs_cache_message="OPNFV deploy cache: bootstrap image injected."
  77 + # Update issues messages
  78 + update_warn_message="There is an issue connecting to update repository of \
  79 + your distributions of OpenStack. \
  80 +@@ -500,12 +503,27 @@ set_ui_bootstrap_error () {
  81 +       EOF
  82 + }
  83 +
  84 ++function inject_cached_ubuntu_bootstrap () {
  85 ++        if [ -f "${OPNFV_CACHE_PATH}/${OPNFV_CACHE_TAR}" -a \
  86 ++             -f "${OPNFV_CACHE_PATH}/id_rsa.pub" -a \
  87 ++             -f "${OPNFV_CACHE_PATH}/id_rsa" ]; then
  88 ++          if cp "${OPNFV_CACHE_PATH}/id_rsa{,.pub}" "~/.ssh/" && \
  89 ++                fuel-bootstrap -v --debug import --activate \
  90 ++                "${OPNFV_CACHE_PATH}/${OPNFV_CACHE_TAR}" >>"$bs_build_log" 2>&1; then
  91 ++            fuel notify --topic "done" --send "${bs_cache_message}"
  92 ++            return 0
  93 ++          fi
  94 ++        fi
  95 ++        return 1
  96 ++}
  97 ++
  98 + # Actually build the bootstrap image
  99 + build_ubuntu_bootstrap () {
 100 +         local ret=1
 101 +         echo ${bs_progress_message} >&2
 102 +         set_ui_bootstrap_error "${bs_progress_message}" >&2
 103 +-        if fuel-bootstrap -v --debug build --target_arch arm64 --activate >>"$bs_build_log" 2>&1; then
 104 ++        if inject_cached_ubuntu_bootstrap || fuel-bootstrap -v --debug \
 105 ++          build --activate --target_arch arm64 >>"$bs_build_log" 2>&1; then
 106 +           ret=0
 107 +           fuel notify --topic "done" --send "${bs_done_message}"
 108 +         else
 109 diff --git a/ci/deploy.sh b/ci/deploy.sh
 110 index 081806c..4b1ae0e 100755
 111 --- a/ci/deploy.sh
 112 +++ b/ci/deploy.sh
 113 @@ -29,7 +29,7 @@ cat << EOF
 114  xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
 115  `basename $0`: Deploys the Fuel@OPNFV stack
 116
 117 -usage: `basename $0` -b base-uri [-B PXE Bridge] [-f] [-F] [-H] -l lab-name -p pod-name -s deploy-scenario [-S image-dir] [-T timeout] -i iso
 118 +usage: `basename $0` -b base-uri [-B PXE Bridge] [-f] [-F] [-H] -l lab-name -p pod-name -s deploy-scenario [-S image-dir] [-C deploy-cache-dir] [-T timeout] -i iso
 119         -s deployment-scenario [-S optional Deploy-scenario path URI]
 120         [-R optional local relen repo (containing deployment Scenarios]
 121
 122 @@ -47,6 +47,7 @@ OPTIONS:
 123    -p  Pod-name
 124    -s  Deploy-scenario short-name/base-file-name
 125    -S  Storage dir for VM images
 126 +  -C  Deploy cache dir for storing image artifacts
 127    -T  Timeout, in minutes, for the deploy.
 128    -i  iso url
 129
 130 @@ -79,6 +80,7 @@ Input parameters to the build script is:
 131     or a deployment short-name as defined by scenario.yaml in the deployment
 132     scenario path.
 133  -S Storage dir for VM images, default is fuel/deploy/images
 134 +-C Deploy cache dir for bootstrap and target image artifacts, optional
 135  -T Timeout, in minutes, for the deploy. It defaults to using the DEPLOY_TIMEOUT
 136     environment variable when defined, or to the default in deploy.py otherwise
 137  -i .iso image to be deployed (needs to be provided in a URI
 138 @@ -116,6 +118,7 @@ FUEL_CREATION_ONLY=''
 139  NO_DEPLOY_ENVIRONMENT=''
 140  STORAGE_DIR=''
 141  DRY_RUN=0
 142 +DEPLOY_CACHE_DIR=''
 143  if ! [ -z $DEPLOY_TIMEOUT ]; then
 144      DEPLOY_TIMEOUT="-dt $DEPLOY_TIMEOUT"
 145  else
 146 @@ -128,7 +131,7 @@ fi
 147  ############################################################################
 148  # BEGIN of main
 149  #
 150 -while getopts "b:B:dfFHl:L:p:s:S:T:i:he" OPTION
 151 +while getopts "b:B:dfFHl:L:p:s:S:C:T:i:he" OPTION
 152  do
 153      case $OPTION in
 154          b)
 155 @@ -179,6 +182,9 @@ do
 156                  STORAGE_DIR="-s ${OPTARG}"
 157              fi
 158              ;;
 159 +        C)
 160 +            DEPLOY_CACHE_DIR="-dc ${OPTARG}"
 161 +            ;;
 162          T)
 163              DEPLOY_TIMEOUT="-dt ${OPTARG}"
 164              ;;
 165 @@ -243,8 +249,8 @@ if [ $DRY_RUN -eq 0 ]; then
 166          ISO=${SCRIPT_PATH}/ISO/image.iso
 167      fi
 168      # Start deployment
 169 -    echo "python deploy.py $DEPLOY_LOG $STORAGE_DIR $PXE_BRIDGE $USE_EXISTING_FUEL $FUEL_CREATION_ONLY $NO_HEALTH_CHECK $NO_DEPLOY_ENVIRONMENT -dea ${SCRIPT_PATH}/config/dea.yaml -dha ${SCRIPT_PATH}/config/dha.yaml -iso $ISO $DEPLOY_TIMEOUT"
 170 -    python deploy.py $DEPLOY_LOG $STORAGE_DIR $PXE_BRIDGE $USE_EXISTING_FUEL $FUEL_CREATION_ONLY $NO_HEALTH_CHECK $NO_DEPLOY_ENVIRONMENT -dea ${SCRIPT_PATH}/config/dea.yaml -dha ${SCRIPT_PATH}/config/dha.yaml -iso $ISO $DEPLOY_TIMEOUT
 171 +    echo "python deploy.py $DEPLOY_LOG $STORAGE_DIR $PXE_BRIDGE $USE_EXISTING_FUEL $FUEL_CREATION_ONLY $NO_HEALTH_CHECK $NO_DEPLOY_ENVIRONMENT -dea ${SCRIPT_PATH}/config/dea.yaml -dha ${SCRIPT_PATH}/config/dha.yaml -iso $ISO $DEPLOY_TIMEOUT $DEPLOY_CACHE_DIR"
 172 +    python deploy.py $DEPLOY_LOG $STORAGE_DIR $PXE_BRIDGE $USE_EXISTING_FUEL $FUEL_CREATION_ONLY $NO_HEALTH_CHECK $NO_DEPLOY_ENVIRONMENT -dea ${SCRIPT_PATH}/config/dea.yaml -dha ${SCRIPT_PATH}/config/dha.yaml -iso $ISO $DEPLOY_TIMEOUT $DEPLOY_CACHE_DIR
 173  fi
 174  popd > /dev/null
 175
 176 diff --git a/deploy/cloud/deploy.py b/deploy/cloud/deploy.py
 177 index e00934b..b39e5fc 100644
 178 --- a/deploy/cloud/deploy.py
 179 +++ b/deploy/cloud/deploy.py
 180 @@ -14,6 +14,7 @@ import io
 181  from dea import DeploymentEnvironmentAdapter
 182  from configure_environment import ConfigureEnvironment
 183  from deployment import Deployment
 184 +from deploy_cache import DeployCache
 185
 186  from common import (
 187      R,
 188 @@ -61,6 +62,12 @@ class Deploy(object):
 189          config_env.configure_environment()
 190          self.env_id = config_env.env_id
 191
 192 +    def deploy_cache_install_targetimages(self):
 193 +        DeployCache.install_targetimages_for_env(self.env_id)
 194 +
 195 +    def deploy_cache_extract_targetimages(self):
 196 +        DeployCache.extract_targetimages_from_env(self.env_id)
 197 +
 198      def deploy_cloud(self):
 199          dep = Deployment(self.dea, YAML_CONF_DIR, self.env_id,
 200                           self.node_roles_dict, self.no_health_check,
 201 @@ -76,8 +83,12 @@ class Deploy(object):
 202
 203          self.configure_environment()
 204
 205 +        self.deploy_cache_install_targetimages()
 206 +
 207          self.deploy_cloud()
 208
 209 +        self.deploy_cache_extract_targetimages()
 210 +
 211
 212  def parse_arguments():
 213      parser = ArgParser(prog='python %s' % __file__)
 214 diff --git a/deploy/deploy.py b/deploy/deploy.py
 215 index 08702d2..1a55361 100755
 216 --- a/deploy/deploy.py
 217 +++ b/deploy/deploy.py
 218 @@ -23,6 +23,7 @@ from dea import DeploymentEnvironmentAdapter
 219  from dha import DeploymentHardwareAdapter
 220  from install_fuel_master import InstallFuelMaster
 221  from deploy_env import CloudDeploy
 222 +from deploy_cache import DeployCache
 223  from execution_environment import ExecutionEnvironment
 224
 225  from common import (
 226 @@ -62,7 +63,8 @@ class AutoDeploy(object):
 227      def __init__(self, no_fuel, fuel_only, no_health_check, cleanup_only,
 228                   cleanup, storage_dir, pxe_bridge, iso_file, dea_file,
 229                   dha_file, fuel_plugins_dir, fuel_plugins_conf_dir,
 230 -                 no_plugins, deploy_timeout, no_deploy_environment, deploy_log):
 231 +                 no_plugins, deploy_cache_dir, deploy_timeout,
 232 +                 no_deploy_environment, deploy_log):
 233          self.no_fuel = no_fuel
 234          self.fuel_only = fuel_only
 235          self.no_health_check = no_health_check
 236 @@ -76,6 +78,7 @@ class AutoDeploy(object):
 237          self.fuel_plugins_dir = fuel_plugins_dir
 238          self.fuel_plugins_conf_dir = fuel_plugins_conf_dir
 239          self.no_plugins = no_plugins
 240 +        self.deploy_cache_dir = deploy_cache_dir
 241          self.deploy_timeout = deploy_timeout
 242          self.no_deploy_environment = no_deploy_environment
 243          self.deploy_log = deploy_log
 244 @@ -117,7 +120,7 @@ class AutoDeploy(object):
 245                                    self.fuel_username, self.fuel_password,
 246                                    self.dea_file, self.fuel_plugins_conf_dir,
 247                                    WORK_DIR, self.no_health_check,
 248 -                                  self.deploy_timeout,
 249 +                                  self.deploy_cache_dir, self.deploy_timeout,
 250                                    self.no_deploy_environment, self.deploy_log)
 251              with old_dep.ssh:
 252                  old_dep.check_previous_installation()
 253 @@ -129,6 +132,7 @@ class AutoDeploy(object):
 254                                   self.fuel_conf['ip'], self.fuel_username,
 255                                   self.fuel_password, self.fuel_node_id,
 256                                   self.iso_file, WORK_DIR,
 257 +                                 self.deploy_cache_dir,
 258                                   self.fuel_plugins_dir, self.no_plugins)
 259          fuel.install()
 260
 261 @@ -137,6 +141,7 @@ class AutoDeploy(object):
 262          tmp_new_dir = '%s/newiso' % self.tmp_dir
 263          try:
 264              self.copy(tmp_orig_dir, tmp_new_dir)
 265 +            self.deploy_cache_fingerprints(tmp_new_dir)
 266              self.patch(tmp_new_dir, new_iso)
 267          except Exception as e:
 268              exec_cmd('fusermount -u %s' % tmp_orig_dir, False)
 269 @@ -157,6 +162,12 @@ class AutoDeploy(object):
 270          delete(tmp_orig_dir)
 271          exec_cmd('chmod -R 755 %s' % tmp_new_dir)
 272
 273 +    def deploy_cache_fingerprints(self, tmp_new_dir):
 274 +        if self.deploy_cache_dir:
 275 +            log('Deploy cache: Collecting fingerprints...')
 276 +            deploy_cache = DeployCache(self.deploy_cache_dir)
 277 +            deploy_cache.do_fingerprints(tmp_new_dir, self.dea_file)
 278 +
 279      def patch(self, tmp_new_dir, new_iso):
 280          log('Patching...')
 281          patch_dir = '%s/%s' % (CWD, PATCH_DIR)
 282 @@ -219,7 +230,8 @@ class AutoDeploy(object):
 283          dep = CloudDeploy(self.dea, self.dha, self.fuel_conf['ip'],
 284                            self.fuel_username, self.fuel_password,
 285                            self.dea_file, self.fuel_plugins_conf_dir,
 286 -                          WORK_DIR, self.no_health_check, self.deploy_timeout,
 287 +                          WORK_DIR, self.no_health_check,
 288 +                          self.deploy_cache_dir, self.deploy_timeout,
 289                            self.no_deploy_environment, self.deploy_log)
 290          return dep.deploy()
 291
 292 @@ -344,6 +356,8 @@ def parse_arguments():
 293                          help='Fuel Plugins Configuration directory')
 294      parser.add_argument('-np', dest='no_plugins', action='store_true',
 295                          default=False, help='Do not install Fuel Plugins')
 296 +    parser.add_argument('-dc', dest='deploy_cache_dir', action='store',
 297 +                        help='Deploy Cache Directory')
 298      parser.add_argument('-dt', dest='deploy_timeout', action='store',
 299                          default=240, help='Deployment timeout (in minutes) '
 300                          '[default: 240]')
 301 @@ -377,6 +391,10 @@ def parse_arguments():
 302          for bridge in args.pxe_bridge:
 303              check_bridge(bridge, args.dha_file)
 304
 305 +    if args.deploy_cache_dir:
 306 +        log('Using deploy cache directory: %s' % args.deploy_cache_dir)
 307 +        create_dir_if_not_exists(args.deploy_cache_dir)
 308 +
 309
 310      kwargs = {'no_fuel': args.no_fuel, 'fuel_only': args.fuel_only,
 311                'no_health_check': args.no_health_check,
 312 @@ -387,6 +405,7 @@ def parse_arguments():
 313                'fuel_plugins_dir': args.fuel_plugins_dir,
 314                'fuel_plugins_conf_dir': args.fuel_plugins_conf_dir,
 315                'no_plugins': args.no_plugins,
 316 +              'deploy_cache_dir': args.deploy_cache_dir,
 317                'deploy_timeout': args.deploy_timeout,
 318                'no_deploy_environment': args.no_deploy_environment,
 319                'deploy_log': args.deploy_log}
 320 diff --git a/deploy/deploy_cache.py b/deploy/deploy_cache.py
 321 new file mode 100644
 322 index 0000000..d7ec1c7
 323 --- /dev/null
 324 +++ b/deploy/deploy_cache.py
 325 @@ -0,0 +1,319 @@
 326 +###############################################################################
 327 +# Copyright (c) 2016 Enea AB and others.
 328 +# Alexandru.Avadanii@enea.com
 329 +# All rights reserved. This program and the accompanying materials
 330 +# are made available under the terms of the Apache License, Version 2.0
 331 +# which accompanies this distribution, and is available at
 332 +# http://www.apache.org/licenses/LICENSE-2.0
 333 +###############################################################################
 334 +
 335 +import glob
 336 +import hashlib
 337 +import io
 338 +import json
 339 +import os
 340 +import shutil
 341 +import yaml
 342 +
 343 +from common import (
 344 +    exec_cmd,
 345 +    log,
 346 +)
 347 +
 348 +###############################################################################
 349 +# Deploy Cache Flow Overview
 350 +###############################################################################
 351 +# 1. do_fingerprints
 352 +#    Can be called as soon as a Fuel Master ISO chroot is available.
 353 +#    This will gather all required information for uniquely identifying the
 354 +#    objects in cache (bootstraps, targetimages).
 355 +# 2. inject_cache
 356 +#    Can be called as soon as we have a steady SSH connection to the Fuel
 357 +#    Master node. It will inject cached artifacts over SSH, for later install.
 358 +# 3. (external, async) install cached bootstrap instead of building a new one
 359 +#    /sbin/bootstrap_admin_node.sh will check for cached bootstrap images
 360 +#    (with id_rsa, id_rsa.pub attached) and will install those via
 361 +#    $ fuel-bootstrap import opfnv-bootstraps-cache.tar
 362 +# 4. install_targetimages_for_env
 363 +#    Should be called before cloud deploy is started, to install env-generic
 364 +#    'env_X_...' cached images for the current environment ID.
 365 +#    Static method, to be used on the remote Fuel Master node; does not require
 366 +#    access to the deploy cache, it only moves around some local files.
 367 +# 5. extract_targetimages_from_env
 368 +#    Should be called at env deploy finish, to prepare artifacts for caching.
 369 +#    Static method, same observations as above apply.
 370 +# 6. collect_artifacts
 371 +#    Call last, to collect all artifacts.
 372 +###############################################################################
 373 +
 374 +###############################################################################
 375 +# Deploy cache artifacts:
 376 +# - id_rsa
 377 +# - bootstrap image (Ubuntu)
 378 +# - environment target image (Ubuntu)
 379 +###############################################################################
 380 +# Cache fingerprint covers:
 381 +# - bootstrap:
 382 +#   - local mirror contents
 383 +#   - package list (and everything else in fuel_bootstrap_cli.yaml)
 384 +# - target image:
 385 +#   - local mirror contents
 386 +#   - package list (determined from DEA)
 387 +###############################################################################
 388 +# WARN: Cache fingerprint does NOT yet cover:
 389 +# - image_data (always assume the default /boot, /);
 390 +# - output_dir (always assume the default /var/www/nailgun/targetimages;
 391 +# - codename (always assume the default, currently 'trusty');
 392 +# - extra_dirs: /usr/share/fuel_bootstrap_cli/files/trusty
 393 +# - root_ssh_authorized_file, inluding the contents of /root/.ssh/id_rsa.pub
 394 +# - Auxiliary repo  .../mitaka-9.0/ubuntu/auxiliary
 395 +# If the above change without triggering a cache miss, try clearing the cache.
 396 +###############################################################################
 397 +# WARN: Bootstrap caching implies RSA keypair to be reused!
 398 +###############################################################################
 399 +
 400 +# Local mirrros will be used on Fuel Master for both bootstrap and target image
 401 +# build, from `http://127.0.0.1:8080/...` or `http://10.20.0.2:8080/...`:
 402 +# - MOS        .../mitaka-9.0/ubuntu/x86_64
 403 +# - Ubuntu     .../mirrors/ubuntu/
 404 +# All these reside on Fuel Master at local path:
 405 +NAILGUN_PATH = '/var/www/nailgun/'
 406 +
 407 +# Artifact names (corresponding to nailgun subdirs)
 408 +MIRRORS = 'mirrors'
 409 +BOOTSTRAPS = 'bootstraps'
 410 +TARGETIMAGES = 'targetimages'
 411 +
 412 +# Info for collecting RSA keypair
 413 +RSA_KEYPAIR_PATH = '/root/.ssh'
 414 +RSA_KEYPAIR_FILES = ['id_rsa', 'id_rsa.pub']
 415 +
 416 +# Relative path for collecting the active bootstrap image(s) after env deploy
 417 +NAILGUN_ACT_BOOTSTRAP_SUBDIR = '%s/active_bootstrap' % BOOTSTRAPS
 418 +
 419 +# Relative path for collecting target image(s) for deployed enviroment
 420 +NAILGUN_TIMAGES_SUBDIR = TARGETIMAGES
 421 +
 422 +# OPNFV Fuel bootstrap settings file that will be injected at deploy
 423 +ISO_BOOTSTRAP_CLI_YAML = '/opnfv/fuel_bootstrap_cli.yaml'
 424 +
 425 +# OPNFV Deploy Cache path on Fuel Master, where artifacts will be injected
 426 +REMOTE_CACHE_PATH = '/var/lib/opnfv/cache'
 427 +
 428 +# OPNFV Bootstrap Cache tar archive name, to be used by bootstrap_admin_node.sh
 429 +BOOTSTRAP_ARCHIVE = 'opnfv-bootstraps-cache.tar'
 430 +
 431 +# Env-ID indep prefix
 432 +ENVX = 'env_X_'
 433 +
 434 +class DeployCache(object):
 435 +    """OPNFV Deploy Cache - managed storage for cacheable artifacts"""
 436 +
 437 +    def __init__(self, cache_dir,
 438 +                 fingerprints_yaml='deploy_cache_fingerprints.yaml'):
 439 +        self.cache_dir = cache_dir
 440 +        self.fingerprints_yaml = fingerprints_yaml
 441 +        self.fingerprints = {BOOTSTRAPS: None,
 442 +                             MIRRORS: None,
 443 +                             TARGETIMAGES: None}
 444 +
 445 +    def __load_fingerprints(self):
 446 +        """Load deploy cache yaml config holding fingerprints"""
 447 +        if os.path.isfile(self.fingerprints_yaml):
 448 +            cache_fingerprints = open(self.fingerprints_yaml).read()
 449 +            self.fingerprints = yaml.load(cache_fingerprints)
 450 +
 451 +    def __save_fingerprints(self):
 452 +        """Update deploy cache yaml config holding fingerprints"""
 453 +        with open(self.fingerprints_yaml, 'w') as outfile:
 454 +            outfile.write(yaml.safe_dump(self.fingerprints,
 455 +                          default_flow_style=False))
 456 +
 457 +    def __fingerprint_mirrors(self, chroot_path):
 458 +        """Collect repo mirror fingerprints"""
 459 +        md5sums = list()
 460 +        # Scan all ISO for deb repo metadata and collect MD5 from Release files
 461 +        for root, _, files in os.walk(chroot_path):
 462 +            for relf in files:
 463 +                if relf == 'Release' and 'binary' not in root:
 464 +                    collect_sums = False
 465 +                    filepath = os.path.join(root, relf)
 466 +                    with open(filepath, "r") as release_file:
 467 +                        for line in release_file:
 468 +                            if collect_sums:
 469 +                                if line.startswith(' '):
 470 +                                    md5sums += [line[1:33]]
 471 +                                else:
 472 +                                    break
 473 +                            elif line.startswith('MD5Sum:'):
 474 +                                collect_sums = True
 475 +        sorted_md5sums = json.dumps(md5sums, sort_keys=True)
 476 +        self.fingerprints[MIRRORS] = hashlib.sha1(sorted_md5sums).hexdigest()
 477 +
 478 +    def __fingerprint_bootstrap(self, chroot_path):
 479 +        """Collect bootstrap image metadata fingerprints"""
 480 +        # FIXME(armband): include 'extra_dirs' contents
 481 +        cli_yaml_path = os.path.join(chroot_path, ISO_BOOTSTRAP_CLI_YAML[1:])
 482 +        bootstrap_cli_yaml = open(cli_yaml_path).read()
 483 +        bootstrap_data = yaml.load(bootstrap_cli_yaml)
 484 +        sorted_data = json.dumps(bootstrap_data, sort_keys=True)
 485 +        self.fingerprints[BOOTSTRAPS] = hashlib.sha1(sorted_data).hexdigest()
 486 +
 487 +    def __fingerprint_target(self, dea_file):
 488 +        """Collect target image metadata fingerprints"""
 489 +        # FIXME(armband): include 'image_data', 'codename', 'output'
 490 +        with io.open(dea_file) as stream:
 491 +            dea = yaml.load(stream)
 492 +            editable = dea['settings']['editable']
 493 +            target_data = {'packages': editable['provision']['packages'],
 494 +                           'repos': editable['repo_setup']['repos']}
 495 +            s_data = json.dumps(target_data, sort_keys=True)
 496 +            self.fingerprints[TARGETIMAGES] = hashlib.sha1(s_data).hexdigest()
 497 +
 498 +    def do_fingerprints(self, chroot_path, dea_file):
 499 +        """Collect SHA1 fingerprints based on chroot contents, DEA settings"""
 500 +        try:
 501 +            self.__load_fingerprints()
 502 +            self.__fingerprint_mirrors(chroot_path)
 503 +            self.__fingerprint_bootstrap(chroot_path)
 504 +            self.__fingerprint_target(dea_file)
 505 +            self.__save_fingerprints()
 506 +        except Exception as ex:
 507 +            log('Failed to get cache fingerprint: %s' % str(ex))
 508 +
 509 +    def __lookup_cache(self, sha):
 510 +        """Search for object in cache based on SHA fingerprint"""
 511 +        cache_sha_dir = os.path.join(self.cache_dir, sha)
 512 +        if not os.path.isdir(cache_sha_dir) or not os.listdir(cache_sha_dir):
 513 +            return None
 514 +        return cache_sha_dir
 515 +
 516 +    def __inject_cache_dir(self, ssh, sha, artifact):
 517 +        """Stage cached object (dir) in Fuel Master OPNFV local cache"""
 518 +        local_path = self.__lookup_cache(sha)
 519 +        if local_path:
 520 +            remote_path = os.path.join(REMOTE_CACHE_PATH, artifact)
 521 +            with ssh:
 522 +                ssh.exec_cmd('mkdir -p %s' % remote_path)
 523 +                for cachedfile in glob.glob('%s/*' % local_path):
 524 +                    ssh.scp_put(cachedfile, remote_path)
 525 +        return local_path
 526 +
 527 +    def __mix_fingerprints(self, f1, f2):
 528 +        """Compute composite fingerprint"""
 529 +        if self.fingerprints[f1] is None or self.fingerprints[f2] is None:
 530 +            return None
 531 +        return hashlib.sha1('%s%s' %
 532 +            (self.fingerprints[f1], self.fingerprints[f2])).hexdigest()
 533 +
 534 +    def inject_cache(self, ssh):
 535 +        """Lookup artifacts in cache and inject them over SSH/SCP into Fuel"""
 536 +        try:
 537 +            self.__load_fingerprints()
 538 +            for artifact in [BOOTSTRAPS, TARGETIMAGES]:
 539 +                sha = self.__mix_fingerprints(MIRRORS, artifact)
 540 +                if sha is None:
 541 +                    log('Missing fingerprint for: %s' % artifact)
 542 +                    continue
 543 +                if not self.__inject_cache_dir(ssh, sha, artifact):
 544 +                    log('SHA1 not in cache: %s (%s)' % (str(sha), artifact))
 545 +                else:
 546 +                    log('SHA1 injected: %s (%s)' % (str(sha), artifact))
 547 +        except Exception as ex:
 548 +            log('Failed to inject cached artifacts into Fuel: %s' % str(ex))
 549 +
 550 +    def __extract_bootstraps(self, ssh, cache_sha_dir):
 551 +        """Collect bootstrap artifacts from Fuel over SSH/SCP"""
 552 +        remote_tar = os.path.join(REMOTE_CACHE_PATH, BOOTSTRAP_ARCHIVE)
 553 +        local_tar = os.path.join(cache_sha_dir, BOOTSTRAP_ARCHIVE)
 554 +        with ssh:
 555 +            for k in RSA_KEYPAIR_FILES:
 556 +                ssh.scp_get(os.path.join(RSA_KEYPAIR_PATH, k),
 557 +                    local=os.path.join(cache_sha_dir, k))
 558 +            ssh.exec_cmd('tar cf %s %s/*', remote_tar,
 559 +                os.path.join(NAILGUN_PATH, NAILGUN_ACT_BOOTSTRAP_SUBDIR))
 560 +            ssh.scp_get(remote_tar, local=local_tar)
 561 +            ssh.exec_cmd('rm -f %s', remote_tar)
 562 +
 563 +    def __extract_targetimages(self, ssh, cache_sha_dir):
 564 +        """Collect target image artifacts from Fuel over SSH/SCP"""
 565 +        cti_path = os.path.join(REMOTE_CACHE_PATH, TARGETIMAGES)
 566 +        with ssh:
 567 +            ssh.scp_get('%s/%s*' % (cti_path, ENVX), local=cache_sha_dir)
 568 +
 569 +    def collect_artifacts(self, ssh):
 570 +        """Collect artifacts from Fuel over SSH/SCP and add them to cache"""
 571 +        try:
 572 +            self.__load_fingerprints()
 573 +            for artifact, func in {
 574 +                    BOOTSTRAPS: self.__extract_bootstraps,
 575 +                    TARGETIMAGES: self.__extract_targetimages
 576 +                }.iteritems():
 577 +                sha = self.__mix_fingerprints(MIRRORS, artifact)
 578 +                if sha is None:
 579 +                    log('WARN: Skip caching, NO fingerprint: %s' % artifact)
 580 +                    continue
 581 +                local_path = self.__lookup_cache(sha)
 582 +                if local_path:
 583 +                    log('SHA1 already in cache: %s (%s)' % (str(sha), artifact))
 584 +                else:
 585 +                    log('New cache SHA1: %s (%s)' % (str(sha), artifact))
 586 +                    cache_sha_dir = os.path.join(self.cache_dir, sha)
 587 +                    exec_cmd('mkdir -p %s' % cache_sha_dir)
 588 +                    func(ssh, cache_sha_dir)
 589 +        except Exception as ex:
 590 +            log('Failed to extract artifacts from Fuel: %s' % str(ex))
 591 +
 592 +    @staticmethod
 593 +    def extract_targetimages_from_env(env_id):
 594 +        """Prepare targetimages from env ID for storage in deploy cache
 595 +
 596 +        NOTE: This method should be executed locally ON the Fuel Master node.
 597 +        WARN: This method overwrites targetimages cache on Fuel Master node.
 598 +        """
 599 +        env_n = 'env_%s_' % str(env_id)
 600 +        cti_path = os.path.join(REMOTE_CACHE_PATH, TARGETIMAGES)
 601 +        ti_path = os.path.join(NAILGUN_PATH, NAILGUN_TIMAGES_SUBDIR)
 602 +        try:
 603 +            exec_cmd('rm -rf %s && mkdir -p %s' % (cti_path, cti_path))
 604 +            for root, _, files in os.walk(ti_path):
 605 +                for tif in files:
 606 +                    if tif.startswith(env_n):
 607 +                        src = os.path.join(root, tif)
 608 +                        dest = os.path.join(cti_path, tif.replace(env_n, ENVX))
 609 +                        if tif.endswith('.yaml'):
 610 +                            shutil.copy(src, dest)
 611 +                            exec_cmd('sed -i "s|%s|%s|g" %s' %
 612 +                                     (env_n, ENVX, dest))
 613 +                        else:
 614 +                            os.link(src, dest)
 615 +        except Exception as ex:
 616 +            log('Failed to extract targetimages artifacts from env %s: %s' %
 617 +                (str(env_id), str(ex)))
 618 +
 619 +    @staticmethod
 620 +    def install_targetimages_for_env(env_id):
 621 +        """Install targetimages artifacts for a specific env ID
 622 +
 623 +        NOTE: This method should be executed locally ON the Fuel Master node.
 624 +        """
 625 +        env_n = 'env_%s_' % str(env_id)
 626 +        cti_path = os.path.join(REMOTE_CACHE_PATH, TARGETIMAGES)
 627 +        ti_path = os.path.join(NAILGUN_PATH, NAILGUN_TIMAGES_SUBDIR)
 628 +        if not os.path.isdir(cti_path):
 629 +            log('%s cache dir not found: %s' % (TARGETIMAGES, cti_path))
 630 +        else:
 631 +            try:
 632 +                for root, _, files in os.walk(cti_path):
 633 +                    for tif in files:
 634 +                        src = os.path.join(root, tif)
 635 +                        dest = os.path.join(ti_path, tif.replace(ENVX, env_n))
 636 +                        if tif.endswith('.yaml'):
 637 +                            shutil.copy(src, dest)
 638 +                            exec_cmd('sed -i "s|%s|%s|g" %s' %
 639 +                                     (ENVX, env_n, dest))
 640 +                        else:
 641 +                            os.link(src, dest)
 642 +            except Exception as ex:
 643 +                log('Failed to install targetimages for env %s: %s' %
 644 +                    (str(env_id), str(ex)))
 645 diff --git a/deploy/deploy_env.py b/deploy/deploy_env.py
 646 index 1d2dfeb..2375f51 100644
 647 --- a/deploy/deploy_env.py
 648 +++ b/deploy/deploy_env.py
 649 @@ -15,6 +15,7 @@ import glob
 650  import time
 651  import shutil
 652
 653 +from deploy_cache import DeployCache
 654  from ssh_client import SSHClient
 655
 656  from common import (
 657 @@ -36,7 +37,8 @@ class CloudDeploy(object):
 658
 659      def __init__(self, dea, dha, fuel_ip, fuel_username, fuel_password,
 660                   dea_file, fuel_plugins_conf_dir, work_dir, no_health_check,
 661 -                 deploy_timeout, no_deploy_environment, deploy_log):
 662 +                 deploy_cache_dir, deploy_timeout,
 663 +                 no_deploy_environment, deploy_log):
 664          self.dea = dea
 665          self.dha = dha
 666          self.fuel_ip = fuel_ip
 667 @@ -50,6 +52,8 @@ class CloudDeploy(object):
 668          self.fuel_plugins_conf_dir = fuel_plugins_conf_dir
 669          self.work_dir = work_dir
 670          self.no_health_check = no_health_check
 671 +        self.deploy_cache = ( DeployCache(deploy_cache_dir)
 672 +                              if deploy_cache_dir else None )
 673          self.deploy_timeout = deploy_timeout
 674          self.no_deploy_environment = no_deploy_environment
 675          self.deploy_log = deploy_log
 676 @@ -83,9 +87,14 @@ class CloudDeploy(object):
 677                  self.work_dir, os.path.basename(self.dea_file)))
 678              s.scp_put('%s/common.py' % self.file_dir, self.work_dir)
 679              s.scp_put('%s/dea.py' % self.file_dir, self.work_dir)
 680 +            s.scp_put('%s/deploy_cache.py' % self.file_dir, self.work_dir)
 681              for f in glob.glob('%s/cloud/*' % self.file_dir):
 682                  s.scp_put(f, self.work_dir)
 683
 684 +    def deploy_cache_collect_artifacts(self):
 685 +        if self.deploy_cache:
 686 +            self.deploy_cache.collect_artifacts(self.ssh)
 687 +
 688      def power_off_nodes(self):
 689          for node_id in self.node_ids:
 690              self.dha.node_power_off(node_id)
 691 @@ -284,4 +293,6 @@ class CloudDeploy(object):
 692
 693          self.get_put_deploy_log()
 694
 695 +        self.deploy_cache_collect_artifacts()
 696 +
 697          return rc
 698 diff --git a/deploy/install_fuel_master.py b/deploy/install_fuel_master.py
 699 index ccc18d3..2615818 100644
 700 --- a/deploy/install_fuel_master.py
 701 +++ b/deploy/install_fuel_master.py
 702 @@ -10,6 +10,7 @@
 703  import time
 704  import os
 705  import glob
 706 +from deploy_cache import DeployCache
 707  from ssh_client import SSHClient
 708  from dha_adapters.libvirt_adapter import LibvirtAdapter
 709
 710 @@ -33,7 +34,7 @@ class InstallFuelMaster(object):
 711
 712      def __init__(self, dea_file, dha_file, fuel_ip, fuel_username,
 713                   fuel_password, fuel_node_id, iso_file, work_dir,
 714 -                 fuel_plugins_dir, no_plugins):
 715 +                 deploy_cache_dir, fuel_plugins_dir, no_plugins):
 716          self.dea_file = dea_file
 717          self.dha = LibvirtAdapter(dha_file)
 718          self.fuel_ip = fuel_ip
 719 @@ -43,6 +44,8 @@ class InstallFuelMaster(object):
 720          self.iso_file = iso_file
 721          self.iso_dir = os.path.dirname(self.iso_file)
 722          self.work_dir = work_dir
 723 +        self.deploy_cache = ( DeployCache(deploy_cache_dir)
 724 +                              if deploy_cache_dir else None )
 725          self.fuel_plugins_dir = fuel_plugins_dir
 726          self.no_plugins = no_plugins
 727          self.file_dir = os.path.dirname(os.path.realpath(__file__))
 728 @@ -84,6 +87,10 @@ class InstallFuelMaster(object):
 729          log('Wait until Fuel menu is up')
 730          fuel_menu_pid = self.wait_until_fuel_menu_up()
 731
 732 +        if self.deploy_cache:
 733 +            log('Deploy cache: Injecting bootstraps and targetimages')
 734 +            self.deploy_cache.inject_cache(self.ssh)
 735 +
 736          log('Inject our own astute.yaml and fuel_bootstrap_cli.yaml settings')
 737          self.inject_own_astute_and_bootstrap_yaml()
 738