patches/opnfv-fuel/upstream-backports/0005-CI-deploy-cache-Store-and-reuse-deploy-artifacts.patch

   1 From: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
   2 Date: Thu, 24 Nov 2016 23:02:04 +0100
   3 Subject: [PATCH] CI: deploy-cache: Store and reuse deploy artifacts
   4
   5 Add support for caching deploy artifacts, like bootstraps and
   6 target images, which take a lot of time at each deploy to be built,
   7 considering it requires a cross-debootstrap via qemu-user-static and
   8 binfmt.
   9
  10 For OPNFV CI, the cache will piggy back on the <iso_mount> mechanism,
  11 and be located at:
  12 /iso_mount/opnfv_ci/<branch>/deploy-cache
  13
  14 TODO: Use dea interface adapter in target images fingerprinting.
  15 TODO: remote fingerprinting
  16 TODO: differentiate between bootstraps and targetimages, so we don't
  17 end up trying to use one cache artifact type as the other.
  18 TODO: implement sanity checks for bootstrap and target images;
  19 TODO: switch `exec_cmd('mkdir ...')` to `create_dir_if_not_exists`;
  20
  21 JIRA: ARMBAND-172
  22
  23 Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
  24 ---
  25  ...p_admin_node.sh-deploy_cache-install-hook.patch |  73 +++++
  26  ci/deploy.sh                                       |  14 +-
  27  deploy/cloud/deployment.py                         |  12 +
  28  deploy/deploy.py                                   |  25 +-
  29  deploy/deploy_cache.py                             | 312 +++++++++++++++++++++
  30  deploy/deploy_env.py                               |  13 +-
  31  deploy/install_fuel_master.py                      |   9 +-
  32  7 files changed, 449 insertions(+), 9 deletions(-)
  33  create mode 100644 build/f_repos/patch/fuel-main/0006-bootstrap_admin_node.sh-deploy_cache-install-hook.patch
  34  create mode 100644 deploy/deploy_cache.py
  35
  36 diff --git a/build/f_repos/patch/fuel-main/0006-bootstrap_admin_node.sh-deploy_cache-install-hook.patch b/build/f_repos/patch/fuel-main/0006-bootstrap_admin_node.sh-deploy_cache-install-hook.patch
  37 new file mode 100644
  38 index 0000000..71eae61
  39 --- /dev/null
  40 +++ b/build/f_repos/patch/fuel-main/0006-bootstrap_admin_node.sh-deploy_cache-install-hook.patch
  41 @@ -0,0 +1,73 @@
  42 +From: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
  43 +Date: Mon, 28 Nov 2016 14:27:48 +0100
  44 +Subject: [PATCH] bootstrap_admin_node.sh: deploy_cache install hook
  45 +
  46 +Tooling on the automatic deploy side was updated to support deploy
  47 +caching of artifacts like bootstrap (and id_rsa keypair), target
  48 +images etc.
  49 +
  50 +Add installation hook that calls `fuel-bootstrap import` instead of
  51 +`build` when a bootstrap tar is available in the agreed location,
  52 +/var/lib/opnfv/cache/bootstraps/.
  53 +
  54 +JIRA: ARMBAND-172
  55 +
  56 +Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
  57 +---
  58 + iso/bootstrap_admin_node.sh | 20 +++++++++++++++++++-
  59 + 1 file changed, 19 insertions(+), 1 deletion(-)
  60 +
  61 +diff --git a/iso/bootstrap_admin_node.sh b/iso/bootstrap_admin_node.sh
  62 +index abc5ffb..15e6261 100755
  63 +--- a/iso/bootstrap_admin_node.sh
  64 ++++ b/iso/bootstrap_admin_node.sh
  65 +@@ -61,6 +61,8 @@ wget \
  66 + ASTUTE_YAML='/etc/fuel/astute.yaml'
  67 + BOOTSTRAP_NODE_CONFIG="/etc/fuel/bootstrap_admin_node.conf"
  68 + CUSTOM_REPOS="/root/default_deb_repos.yaml"
  69 ++OPNFV_CACHE_PATH="/var/cache/opnfv/bootstraps"
  70 ++OPNFV_CACHE_TAR="opnfv-bootstraps-cache.tar"
  71 + bs_build_log='/var/log/fuel-bootstrap-image-build.log'
  72 + bs_status=0
  73 + # Backup network configs to this folder. Folder will be created only if
  74 +@@ -94,6 +96,7 @@ image becomes available, reboot nodes that failed to be discovered."
  75 + bs_done_message="Default bootstrap image building done. Now you can boot new \
  76 + nodes over PXE, they will be discovered and become available for installing \
  77 + OpenStack on them"
  78 ++bs_cache_message="OPNFV deploy cache: bootstrap image injected."
  79 + # Update issues messages
  80 + update_warn_message="There is an issue connecting to update repository of \
  81 + your distributions of OpenStack. \
  82 +@@ -500,12 +503,31 @@ set_ui_bootstrap_error () {
  83 +       EOF
  84 + }
  85 +
  86 ++function inject_cached_ubuntu_bootstrap () {
  87 ++        if [ -f "${OPNFV_CACHE_PATH}/${OPNFV_CACHE_TAR}" -a \
  88 ++             -f "${OPNFV_CACHE_PATH}/id_rsa.pub" -a \
  89 ++             -f "${OPNFV_CACHE_PATH}/id_rsa" ]; then
  90 ++          if cp "${OPNFV_CACHE_PATH}/id_rsa"* "/root/.ssh/" && \
  91 ++             cp "/root/.ssh/id_rsa.pub" "/root/.ssh/authorized_keys" && \
  92 ++             cp "/root/.ssh/id_rsa.pub" "/etc/cobbler/authorized_keys" && \
  93 ++                sed -i -e "s|\"ssh-rsa .*\"|\"$(cat /root/.ssh/id_rsa.pub)\"|g" \
  94 ++                /etc/nailgun/settings.yaml && \
  95 ++                fuel-bootstrap -v --debug import --activate \
  96 ++                "${OPNFV_CACHE_PATH}/${OPNFV_CACHE_TAR}" >>"$bs_build_log" 2>&1; then
  97 ++                    fuel notify --topic "done" --send "${bs_cache_message}"
  98 ++                    return 0
  99 ++          fi
 100 ++        fi
 101 ++        return 1
 102 ++}
 103 ++
 104 + # Actually build the bootstrap image
 105 + build_ubuntu_bootstrap () {
 106 +         local ret=1
 107 +         echo ${bs_progress_message} >&2
 108 +         set_ui_bootstrap_error "${bs_progress_message}" >&2
 109 +-        if fuel-bootstrap -v --debug build --target_arch arm64 --activate >>"$bs_build_log" 2>&1; then
 110 ++        if inject_cached_ubuntu_bootstrap || fuel-bootstrap -v --debug \
 111 ++          build --activate --target_arch arm64 >>"$bs_build_log" 2>&1; then
 112 +           ret=0
 113 +           fuel notify --topic "done" --send "${bs_done_message}"
 114 +         else
 115 diff --git a/ci/deploy.sh b/ci/deploy.sh
 116 index 081806c..4b1ae0e 100755
 117 --- a/ci/deploy.sh
 118 +++ b/ci/deploy.sh
 119 @@ -29,7 +29,7 @@ cat << EOF
 120  xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
 121  `basename $0`: Deploys the Fuel@OPNFV stack
 122
 123 -usage: `basename $0` -b base-uri [-B PXE Bridge] [-f] [-F] [-H] -l lab-name -p pod-name -s deploy-scenario [-S image-dir] [-T timeout] -i iso
 124 +usage: `basename $0` -b base-uri [-B PXE Bridge] [-f] [-F] [-H] -l lab-name -p pod-name -s deploy-scenario [-S image-dir] [-C deploy-cache-dir] [-T timeout] -i iso
 125         -s deployment-scenario [-S optional Deploy-scenario path URI]
 126         [-R optional local relen repo (containing deployment Scenarios]
 127
 128 @@ -47,6 +47,7 @@ OPTIONS:
 129    -p  Pod-name
 130    -s  Deploy-scenario short-name/base-file-name
 131    -S  Storage dir for VM images
 132 +  -C  Deploy cache dir for storing image artifacts
 133    -T  Timeout, in minutes, for the deploy.
 134    -i  iso url
 135
 136 @@ -79,6 +80,7 @@ Input parameters to the build script is:
 137     or a deployment short-name as defined by scenario.yaml in the deployment
 138     scenario path.
 139  -S Storage dir for VM images, default is fuel/deploy/images
 140 +-C Deploy cache dir for bootstrap and target image artifacts, optional
 141  -T Timeout, in minutes, for the deploy. It defaults to using the DEPLOY_TIMEOUT
 142     environment variable when defined, or to the default in deploy.py otherwise
 143  -i .iso image to be deployed (needs to be provided in a URI
 144 @@ -116,6 +118,7 @@ FUEL_CREATION_ONLY=''
 145  NO_DEPLOY_ENVIRONMENT=''
 146  STORAGE_DIR=''
 147  DRY_RUN=0
 148 +DEPLOY_CACHE_DIR=''
 149  if ! [ -z $DEPLOY_TIMEOUT ]; then
 150      DEPLOY_TIMEOUT="-dt $DEPLOY_TIMEOUT"
 151  else
 152 @@ -128,7 +131,7 @@ fi
 153  ############################################################################
 154  # BEGIN of main
 155  #
 156 -while getopts "b:B:dfFHl:L:p:s:S:T:i:he" OPTION
 157 +while getopts "b:B:dfFHl:L:p:s:S:C:T:i:he" OPTION
 158  do
 159      case $OPTION in
 160          b)
 161 @@ -179,6 +182,9 @@ do
 162                  STORAGE_DIR="-s ${OPTARG}"
 163              fi
 164              ;;
 165 +        C)
 166 +            DEPLOY_CACHE_DIR="-dc ${OPTARG}"
 167 +            ;;
 168          T)
 169              DEPLOY_TIMEOUT="-dt ${OPTARG}"
 170              ;;
 171 @@ -243,8 +249,8 @@ if [ $DRY_RUN -eq 0 ]; then
 172          ISO=${SCRIPT_PATH}/ISO/image.iso
 173      fi
 174      # Start deployment
 175 -    echo "python deploy.py $DEPLOY_LOG $STORAGE_DIR $PXE_BRIDGE $USE_EXISTING_FUEL $FUEL_CREATION_ONLY $NO_HEALTH_CHECK $NO_DEPLOY_ENVIRONMENT -dea ${SCRIPT_PATH}/config/dea.yaml -dha ${SCRIPT_PATH}/config/dha.yaml -iso $ISO $DEPLOY_TIMEOUT"
 176 -    python deploy.py $DEPLOY_LOG $STORAGE_DIR $PXE_BRIDGE $USE_EXISTING_FUEL $FUEL_CREATION_ONLY $NO_HEALTH_CHECK $NO_DEPLOY_ENVIRONMENT -dea ${SCRIPT_PATH}/config/dea.yaml -dha ${SCRIPT_PATH}/config/dha.yaml -iso $ISO $DEPLOY_TIMEOUT
 177 +    echo "python deploy.py $DEPLOY_LOG $STORAGE_DIR $PXE_BRIDGE $USE_EXISTING_FUEL $FUEL_CREATION_ONLY $NO_HEALTH_CHECK $NO_DEPLOY_ENVIRONMENT -dea ${SCRIPT_PATH}/config/dea.yaml -dha ${SCRIPT_PATH}/config/dha.yaml -iso $ISO $DEPLOY_TIMEOUT $DEPLOY_CACHE_DIR"
 178 +    python deploy.py $DEPLOY_LOG $STORAGE_DIR $PXE_BRIDGE $USE_EXISTING_FUEL $FUEL_CREATION_ONLY $NO_HEALTH_CHECK $NO_DEPLOY_ENVIRONMENT -dea ${SCRIPT_PATH}/config/dea.yaml -dha ${SCRIPT_PATH}/config/dha.yaml -iso $ISO $DEPLOY_TIMEOUT $DEPLOY_CACHE_DIR
 179  fi
 180  popd > /dev/null
 181
 182 diff --git a/deploy/cloud/deployment.py b/deploy/cloud/deployment.py
 183 index 28bcfdf..b0bfdcc 100644
 184 --- a/deploy/cloud/deployment.py
 185 +++ b/deploy/cloud/deployment.py
 186 @@ -19,6 +19,8 @@ from common import (
 187      log,
 188  )
 189
 190 +from deploy_cache import DeployCache
 191 +
 192  SEARCH_TEXT = '(err)'
 193  LOG_FILE = '/var/log/puppet.log'
 194  GREP_LINES_OF_LEADING_CONTEXT = 100
 195 @@ -47,6 +49,14 @@ class Deployment(object):
 196          self.pattern = re.compile(
 197              '\d\d\d\d-\d\d-\d\d\s\d\d:\d\d:\d\d')
 198
 199 +    def deploy_cache_install_targetimages(self):
 200 +        log('Using target images from deploy cache')
 201 +        DeployCache.install_targetimages_for_env(self.env_id)
 202 +
 203 +    def deploy_cache_extract_targetimages(self):
 204 +        log('Collecting Fuel target image files for deploy cache')
 205 +        DeployCache.extract_targetimages_from_env(self.env_id)
 206 +
 207      def collect_error_logs(self):
 208          for node_id, roles_blade in self.node_id_roles_dict.iteritems():
 209              log_list = []
 210 @@ -108,6 +118,7 @@ class Deployment(object):
 211          start = time.time()
 212
 213          log('Starting deployment of environment %s' % self.env_id)
 214 +        self.deploy_cache_install_targetimages()
 215          deploy_id = None
 216          ready = False
 217          timeout = False
 218 @@ -140,6 +151,7 @@ class Deployment(object):
 219              err('Deployment timed out, environment %s is not operational, '
 220                  'snapshot will not be performed'
 221                  % self.env_id)
 222 +        self.deploy_cache_extract_targetimages()
 223          if ready:
 224              log('Environment %s successfully deployed'
 225                  % self.env_id)
 226 diff --git a/deploy/deploy.py b/deploy/deploy.py
 227 index 7648baf..ee3cb7a 100755
 228 --- a/deploy/deploy.py
 229 +++ b/deploy/deploy.py
 230 @@ -22,6 +22,7 @@ from dea import DeploymentEnvironmentAdapter
 231  from dha import DeploymentHardwareAdapter
 232  from install_fuel_master import InstallFuelMaster
 233  from deploy_env import CloudDeploy
 234 +from deploy_cache import DeployCache
 235  from execution_environment import ExecutionEnvironment
 236
 237  from common import (
 238 @@ -61,7 +62,8 @@ class AutoDeploy(object):
 239      def __init__(self, no_fuel, fuel_only, no_health_check, cleanup_only,
 240                   cleanup, storage_dir, pxe_bridge, iso_file, dea_file,
 241                   dha_file, fuel_plugins_dir, fuel_plugins_conf_dir,
 242 -                 no_plugins, deploy_timeout, no_deploy_environment, deploy_log):
 243 +                 no_plugins, deploy_cache_dir, deploy_timeout,
 244 +                 no_deploy_environment, deploy_log):
 245          self.no_fuel = no_fuel
 246          self.fuel_only = fuel_only
 247          self.no_health_check = no_health_check
 248 @@ -75,6 +77,7 @@ class AutoDeploy(object):
 249          self.fuel_plugins_dir = fuel_plugins_dir
 250          self.fuel_plugins_conf_dir = fuel_plugins_conf_dir
 251          self.no_plugins = no_plugins
 252 +        self.deploy_cache_dir = deploy_cache_dir
 253          self.deploy_timeout = deploy_timeout
 254          self.no_deploy_environment = no_deploy_environment
 255          self.deploy_log = deploy_log
 256 @@ -116,7 +119,7 @@ class AutoDeploy(object):
 257                                    self.fuel_username, self.fuel_password,
 258                                    self.dea_file, self.fuel_plugins_conf_dir,
 259                                    WORK_DIR, self.no_health_check,
 260 -                                  self.deploy_timeout,
 261 +                                  self.deploy_cache_dir, self.deploy_timeout,
 262                                    self.no_deploy_environment, self.deploy_log)
 263              with old_dep.ssh:
 264                  old_dep.check_previous_installation()
 265 @@ -128,6 +131,7 @@ class AutoDeploy(object):
 266                                   self.fuel_conf['ip'], self.fuel_username,
 267                                   self.fuel_password, self.fuel_node_id,
 268                                   self.iso_file, WORK_DIR,
 269 +                                 self.deploy_cache_dir,
 270                                   self.fuel_plugins_dir, self.no_plugins)
 271          fuel.install()
 272
 273 @@ -136,6 +140,7 @@ class AutoDeploy(object):
 274          tmp_new_dir = '%s/newiso' % self.tmp_dir
 275          try:
 276              self.copy(tmp_orig_dir, tmp_new_dir)
 277 +            self.deploy_cache_fingerprints(tmp_new_dir)
 278              self.patch(tmp_new_dir, new_iso)
 279          except Exception as e:
 280              exec_cmd('fusermount -u %s' % tmp_orig_dir, False)
 281 @@ -156,6 +161,12 @@ class AutoDeploy(object):
 282          delete(tmp_orig_dir)
 283          exec_cmd('chmod -R 755 %s' % tmp_new_dir)
 284
 285 +    def deploy_cache_fingerprints(self, tmp_new_dir):
 286 +        if self.deploy_cache_dir:
 287 +            log('Deploy cache: Collecting fingerprints...')
 288 +            deploy_cache = DeployCache(self.deploy_cache_dir)
 289 +            deploy_cache.do_fingerprints(tmp_new_dir, self.dea_file)
 290 +
 291      def patch(self, tmp_new_dir, new_iso):
 292          log('Patching...')
 293          patch_dir = '%s/%s' % (CWD, PATCH_DIR)
 294 @@ -218,7 +229,8 @@ class AutoDeploy(object):
 295          dep = CloudDeploy(self.dea, self.dha, self.fuel_conf['ip'],
 296                            self.fuel_username, self.fuel_password,
 297                            self.dea_file, self.fuel_plugins_conf_dir,
 298 -                          WORK_DIR, self.no_health_check, self.deploy_timeout,
 299 +                          WORK_DIR, self.no_health_check,
 300 +                          self.deploy_cache_dir, self.deploy_timeout,
 301                            self.no_deploy_environment, self.deploy_log)
 302          return dep.deploy()
 303
 304 @@ -343,6 +355,8 @@ def parse_arguments():
 305                          help='Fuel Plugins Configuration directory')
 306      parser.add_argument('-np', dest='no_plugins', action='store_true',
 307                          default=False, help='Do not install Fuel Plugins')
 308 +    parser.add_argument('-dc', dest='deploy_cache_dir', action='store',
 309 +                        help='Deploy Cache Directory')
 310      parser.add_argument('-dt', dest='deploy_timeout', action='store',
 311                          default=240, help='Deployment timeout (in minutes) '
 312                          '[default: 240]')
 313 @@ -376,6 +390,10 @@ def parse_arguments():
 314          for bridge in args.pxe_bridge:
 315              check_bridge(bridge, args.dha_file)
 316
 317 +    if args.deploy_cache_dir:
 318 +        log('Using deploy cache directory: %s' % args.deploy_cache_dir)
 319 +        create_dir_if_not_exists(args.deploy_cache_dir)
 320 +
 321
 322      kwargs = {'no_fuel': args.no_fuel, 'fuel_only': args.fuel_only,
 323                'no_health_check': args.no_health_check,
 324 @@ -386,6 +404,7 @@ def parse_arguments():
 325                'fuel_plugins_dir': args.fuel_plugins_dir,
 326                'fuel_plugins_conf_dir': args.fuel_plugins_conf_dir,
 327                'no_plugins': args.no_plugins,
 328 +              'deploy_cache_dir': args.deploy_cache_dir,
 329                'deploy_timeout': args.deploy_timeout,
 330                'no_deploy_environment': args.no_deploy_environment,
 331                'deploy_log': args.deploy_log}
 332 diff --git a/deploy/deploy_cache.py b/deploy/deploy_cache.py
 333 new file mode 100644
 334 index 0000000..7df43c6
 335 --- /dev/null
 336 +++ b/deploy/deploy_cache.py
 337 @@ -0,0 +1,312 @@
 338 +###############################################################################
 339 +# Copyright (c) 2016 Enea AB and others.
 340 +# Alexandru.Avadanii@enea.com
 341 +# All rights reserved. This program and the accompanying materials
 342 +# are made available under the terms of the Apache License, Version 2.0
 343 +# which accompanies this distribution, and is available at
 344 +# http://www.apache.org/licenses/LICENSE-2.0
 345 +###############################################################################
 346 +
 347 +import glob
 348 +import hashlib
 349 +import io
 350 +import json
 351 +import os
 352 +import shutil
 353 +import yaml
 354 +
 355 +from common import (
 356 +    exec_cmd,
 357 +    log,
 358 +)
 359 +
 360 +###############################################################################
 361 +# Deploy Cache Flow Overview
 362 +###############################################################################
 363 +# 1. do_fingerprints
 364 +#    Can be called as soon as a Fuel Master ISO chroot is available.
 365 +#    This will gather all required information for uniquely identifying the
 366 +#    objects in cache (bootstraps, targetimages).
 367 +# 2. inject_cache
 368 +#    Can be called as soon as we have a steady SSH connection to the Fuel
 369 +#    Master node. It will inject cached artifacts over SSH, for later install.
 370 +# 3. (external, async) install cached bootstrap instead of building a new one
 371 +#    /sbin/bootstrap_admin_node.sh will check for cached bootstrap images
 372 +#    (with id_rsa, id_rsa.pub attached) and will install those via
 373 +#    $ fuel-bootstrap import opfnv-bootstraps-cache.tar
 374 +# 4. install_targetimages_for_env
 375 +#    Should be called before cloud deploy is started, to install env-generic
 376 +#    'env_X_...' cached images for the current environment ID.
 377 +#    Static method, to be used on the remote Fuel Master node; does not require
 378 +#    access to the deploy cache, it only moves around some local files.
 379 +# 5. extract_targetimages_from_env
 380 +#    Should be called at env deploy finish, to prepare artifacts for caching.
 381 +#    Static method, same observations as above apply.
 382 +# 6. collect_artifacts
 383 +#    Call last, to collect all artifacts.
 384 +###############################################################################
 385 +
 386 +###############################################################################
 387 +# Deploy cache artifacts:
 388 +# - id_rsa
 389 +# - bootstrap image (Ubuntu)
 390 +# - environment target image (Ubuntu)
 391 +###############################################################################
 392 +# Cache fingerprint covers:
 393 +# - bootstrap:
 394 +#   - local mirror contents
 395 +#   - package list (and everything else in fuel_bootstrap_cli.yaml)
 396 +# - target image:
 397 +#   - local mirror contents
 398 +#   - package list (determined from DEA)
 399 +###############################################################################
 400 +# WARN: Cache fingerprint does NOT yet cover:
 401 +# - image_data (always assume the default /boot, /);
 402 +# - output_dir (always assume the default /var/www/nailgun/targetimages;
 403 +# - codename (always assume the default, currently 'trusty');
 404 +# - extra_dirs: /usr/share/fuel_bootstrap_cli/files/trusty
 405 +# - root_ssh_authorized_file, inluding the contents of /root/.ssh/id_rsa.pub
 406 +# - Auxiliary repo  .../mitaka-9.0/ubuntu/auxiliary
 407 +# If the above change without triggering a cache miss, try clearing the cache.
 408 +###############################################################################
 409 +# WARN: Bootstrap caching implies RSA keypair to be reused!
 410 +###############################################################################
 411 +
 412 +# Local mirrros will be used on Fuel Master for both bootstrap and target image
 413 +# build, from `http://127.0.0.1:8080/...` or `http://10.20.0.2:8080/...`:
 414 +# - MOS        .../mitaka-9.0/ubuntu/x86_64
 415 +# - Ubuntu     .../mirrors/ubuntu/
 416 +# All these reside on Fuel Master at local path:
 417 +NAILGUN_PATH = '/var/www/nailgun/'
 418 +
 419 +# Artifact names (corresponding to nailgun subdirs)
 420 +MIRRORS = 'mirrors'
 421 +BOOTSTRAPS = 'bootstraps'
 422 +TARGETIMAGES = 'targetimages'
 423 +
 424 +# Info for collecting RSA keypair
 425 +RSA_KEYPAIR_PATH = '/root/.ssh'
 426 +RSA_KEYPAIR_FILES = ['id_rsa', 'id_rsa.pub']
 427 +
 428 +# Relative path for collecting the active bootstrap image(s) after env deploy
 429 +NAILGUN_ACT_BOOTSTRAP_SUBDIR = '%s/active_bootstrap' % BOOTSTRAPS
 430 +
 431 +# Relative path for collecting target image(s) for deployed enviroment
 432 +NAILGUN_TIMAGES_SUBDIR = TARGETIMAGES
 433 +
 434 +# OPNFV Fuel bootstrap settings file that will be injected at deploy
 435 +ISO_BOOTSTRAP_CLI_YAML = '/opnfv/fuel_bootstrap_cli.yaml'
 436 +
 437 +# OPNFV Deploy Cache path on Fuel Master, where artifacts will be injected
 438 +REMOTE_CACHE_PATH = '/var/cache/opnfv'
 439 +
 440 +# OPNFV Bootstrap Cache tar archive name, to be used by bootstrap_admin_node.sh
 441 +BOOTSTRAP_ARCHIVE = 'opnfv-bootstraps-cache.tar'
 442 +
 443 +# Env-ID indep prefix
 444 +ENVX = 'env_X_'
 445 +
 446 +class DeployCache(object):
 447 +    """OPNFV Deploy Cache - managed storage for cacheable artifacts"""
 448 +
 449 +    def __init__(self, cache_dir,
 450 +                 fingerprints_yaml='deploy_cache_fingerprints.yaml'):
 451 +        self.cache_dir = cache_dir
 452 +        self.fingerprints_yaml = fingerprints_yaml
 453 +        self.fingerprints = {BOOTSTRAPS: None,
 454 +                             MIRRORS: None,
 455 +                             TARGETIMAGES: None}
 456 +
 457 +    def __load_fingerprints(self):
 458 +        """Load deploy cache yaml config holding fingerprints"""
 459 +        if os.path.isfile(self.fingerprints_yaml):
 460 +            cache_fingerprints = open(self.fingerprints_yaml).read()
 461 +            self.fingerprints = yaml.load(cache_fingerprints)
 462 +
 463 +    def __save_fingerprints(self):
 464 +        """Update deploy cache yaml config holding fingerprints"""
 465 +        with open(self.fingerprints_yaml, 'w') as outfile:
 466 +            outfile.write(yaml.safe_dump(self.fingerprints,
 467 +                          default_flow_style=False))
 468 +
 469 +    def __fingerprint_mirrors(self, chroot_path):
 470 +        """Collect repo mirror fingerprints"""
 471 +        deb_packages = list()
 472 +        # Scan ISO for deb files (MOS mirror + Ubuntu mirror, no plugins)
 473 +        for repo_dir in ['ubuntu', 'opnfv/nailgun/mirrors/ubuntu']:
 474 +            for _, _, files in os.walk(os.path.join(chroot_path, repo_dir)):
 475 +                for fdeb in files:
 476 +                    if fdeb.endswith(".deb"):
 477 +                        deb_packages.append(fdeb)
 478 +        sorted_debs = json.dumps(deb_packages, sort_keys=True)
 479 +        self.fingerprints[MIRRORS] = hashlib.sha1(sorted_debs).hexdigest()
 480 +
 481 +    def __fingerprint_bootstrap(self, chroot_path):
 482 +        """Collect bootstrap image metadata fingerprints"""
 483 +        # FIXME(armband): include 'extra_dirs' contents
 484 +        cli_yaml_path = os.path.join(chroot_path, ISO_BOOTSTRAP_CLI_YAML[1:])
 485 +        bootstrap_cli_yaml = open(cli_yaml_path).read()
 486 +        bootstrap_data = yaml.load(bootstrap_cli_yaml)
 487 +        sorted_data = json.dumps(bootstrap_data, sort_keys=True)
 488 +        self.fingerprints[BOOTSTRAPS] = hashlib.sha1(sorted_data).hexdigest()
 489 +
 490 +    def __fingerprint_target(self, dea_file):
 491 +        """Collect target image metadata fingerprints"""
 492 +        # FIXME(armband): include 'image_data', 'codename', 'output'
 493 +        with io.open(dea_file) as stream:
 494 +            dea = yaml.load(stream)
 495 +            editable = dea['settings']['editable']
 496 +            target_data = {'packages': editable['provision']['packages'],
 497 +                           'repos': editable['repo_setup']['repos']}
 498 +            s_data = json.dumps(target_data, sort_keys=True)
 499 +            self.fingerprints[TARGETIMAGES] = hashlib.sha1(s_data).hexdigest()
 500 +
 501 +    def do_fingerprints(self, chroot_path, dea_file):
 502 +        """Collect SHA1 fingerprints based on chroot contents, DEA settings"""
 503 +        try:
 504 +            self.__load_fingerprints()
 505 +            self.__fingerprint_mirrors(chroot_path)
 506 +            self.__fingerprint_bootstrap(chroot_path)
 507 +            self.__fingerprint_target(dea_file)
 508 +            self.__save_fingerprints()
 509 +        except Exception as ex:
 510 +            log('Failed to get cache fingerprint: %s' % str(ex))
 511 +
 512 +    def __lookup_cache(self, sha):
 513 +        """Search for object in cache based on SHA fingerprint"""
 514 +        cache_sha_dir = os.path.join(self.cache_dir, sha)
 515 +        if not os.path.isdir(cache_sha_dir) or not os.listdir(cache_sha_dir):
 516 +            return None
 517 +        return cache_sha_dir
 518 +
 519 +    def __inject_cache_dir(self, ssh, sha, artifact):
 520 +        """Stage cached object (dir) in Fuel Master OPNFV local cache"""
 521 +        local_path = self.__lookup_cache(sha)
 522 +        if local_path:
 523 +            remote_path = os.path.join(REMOTE_CACHE_PATH, artifact)
 524 +            with ssh:
 525 +                ssh.exec_cmd('mkdir -p %s' % remote_path)
 526 +                for cachedfile in glob.glob('%s/*' % local_path):
 527 +                    ssh.scp_put(cachedfile, remote_path)
 528 +        return local_path
 529 +
 530 +    def __mix_fingerprints(self, f1, f2):
 531 +        """Compute composite fingerprint"""
 532 +        if self.fingerprints[f1] is None or self.fingerprints[f2] is None:
 533 +            return None
 534 +        return hashlib.sha1('%s%s' %
 535 +            (self.fingerprints[f1], self.fingerprints[f2])).hexdigest()
 536 +
 537 +    def inject_cache(self, ssh):
 538 +        """Lookup artifacts in cache and inject them over SSH/SCP into Fuel"""
 539 +        try:
 540 +            self.__load_fingerprints()
 541 +            for artifact in [BOOTSTRAPS, TARGETIMAGES]:
 542 +                sha = self.__mix_fingerprints(MIRRORS, artifact)
 543 +                if sha is None:
 544 +                    log('Missing fingerprint for: %s' % artifact)
 545 +                    continue
 546 +                if not self.__inject_cache_dir(ssh, sha, artifact):
 547 +                    log('SHA1 not in cache: %s (%s)' % (str(sha), artifact))
 548 +                else:
 549 +                    log('SHA1 injected: %s (%s)' % (str(sha), artifact))
 550 +        except Exception as ex:
 551 +            log('Failed to inject cached artifacts into Fuel: %s' % str(ex))
 552 +
 553 +    def __extract_bootstraps(self, ssh, cache_sha_dir):
 554 +        """Collect bootstrap artifacts from Fuel over SSH/SCP"""
 555 +        remote_tar = os.path.join(REMOTE_CACHE_PATH, BOOTSTRAP_ARCHIVE)
 556 +        local_tar = os.path.join(cache_sha_dir, BOOTSTRAP_ARCHIVE)
 557 +        with ssh:
 558 +            for k in RSA_KEYPAIR_FILES:
 559 +                ssh.scp_get(os.path.join(RSA_KEYPAIR_PATH, k),
 560 +                    local=os.path.join(cache_sha_dir, k))
 561 +            ssh.exec_cmd('mkdir -p %s && cd %s && tar cf %s *' %
 562 +                (REMOTE_CACHE_PATH,
 563 +                os.path.join(NAILGUN_PATH, NAILGUN_ACT_BOOTSTRAP_SUBDIR),
 564 +                remote_tar))
 565 +            ssh.scp_get(remote_tar, local=local_tar)
 566 +            ssh.exec_cmd('rm -f %s' % remote_tar)
 567 +
 568 +    def __extract_targetimages(self, ssh, cache_sha_dir):
 569 +        """Collect target image artifacts from Fuel over SSH/SCP"""
 570 +        cti_path = os.path.join(REMOTE_CACHE_PATH, TARGETIMAGES)
 571 +        with ssh:
 572 +            ssh.scp_get('%s/%s*' % (cti_path, ENVX), local=cache_sha_dir)
 573 +
 574 +    def collect_artifacts(self, ssh):
 575 +        """Collect artifacts from Fuel over SSH/SCP and add them to cache"""
 576 +        try:
 577 +            self.__load_fingerprints()
 578 +            for artifact, func in {
 579 +                    BOOTSTRAPS: self.__extract_bootstraps,
 580 +                    TARGETIMAGES: self.__extract_targetimages
 581 +                }.iteritems():
 582 +                sha = self.__mix_fingerprints(MIRRORS, artifact)
 583 +                if sha is None:
 584 +                    log('WARN: Skip caching, NO fingerprint: %s' % artifact)
 585 +                    continue
 586 +                local_path = self.__lookup_cache(sha)
 587 +                if local_path:
 588 +                    log('SHA1 already in cache: %s (%s)' % (str(sha), artifact))
 589 +                else:
 590 +                    log('New cache SHA1: %s (%s)' % (str(sha), artifact))
 591 +                    cache_sha_dir = os.path.join(self.cache_dir, sha)
 592 +                    exec_cmd('mkdir -p %s' % cache_sha_dir)
 593 +                    func(ssh, cache_sha_dir)
 594 +        except Exception as ex:
 595 +            log('Failed to extract artifacts from Fuel: %s' % str(ex))
 596 +
 597 +    @staticmethod
 598 +    def extract_targetimages_from_env(env_id):
 599 +        """Prepare targetimages from env ID for storage in deploy cache
 600 +
 601 +        NOTE: This method should be executed locally ON the Fuel Master node.
 602 +        WARN: This method overwrites targetimages cache on Fuel Master node.
 603 +        """
 604 +        env_n = 'env_%s_' % str(env_id)
 605 +        cti_path = os.path.join(REMOTE_CACHE_PATH, TARGETIMAGES)
 606 +        ti_path = os.path.join(NAILGUN_PATH, NAILGUN_TIMAGES_SUBDIR)
 607 +        try:
 608 +            exec_cmd('rm -rf %s && mkdir -p %s' % (cti_path, cti_path))
 609 +            for root, _, files in os.walk(ti_path):
 610 +                for tif in files:
 611 +                    if tif.startswith(env_n):
 612 +                        src = os.path.join(root, tif)
 613 +                        dest = os.path.join(cti_path, tif.replace(env_n, ENVX))
 614 +                        if tif.endswith('.yaml'):
 615 +                            shutil.copy(src, dest)
 616 +                            exec_cmd('sed -i "s|%s|%s|g" %s' %
 617 +                                     (env_n, ENVX, dest))
 618 +                        else:
 619 +                            os.link(src, dest)
 620 +        except Exception as ex:
 621 +            log('Failed to extract targetimages artifacts from env %s: %s' %
 622 +                (str(env_id), str(ex)))
 623 +
 624 +    @staticmethod
 625 +    def install_targetimages_for_env(env_id):
 626 +        """Install targetimages artifacts for a specific env ID
 627 +
 628 +        NOTE: This method should be executed locally ON the Fuel Master node.
 629 +        """
 630 +        env_n = 'env_%s_' % str(env_id)
 631 +        cti_path = os.path.join(REMOTE_CACHE_PATH, TARGETIMAGES)
 632 +        ti_path = os.path.join(NAILGUN_PATH, NAILGUN_TIMAGES_SUBDIR)
 633 +        if not os.path.isdir(cti_path):
 634 +            log('%s cache dir not found: %s' % (TARGETIMAGES, cti_path))
 635 +        else:
 636 +            try:
 637 +                for root, _, files in os.walk(cti_path):
 638 +                    for tif in files:
 639 +                        src = os.path.join(root, tif)
 640 +                        dest = os.path.join(ti_path, tif.replace(ENVX, env_n))
 641 +                        if tif.endswith('.yaml'):
 642 +                            shutil.copy(src, dest)
 643 +                            exec_cmd('sed -i "s|%s|%s|g" %s' %
 644 +                                     (ENVX, env_n, dest))
 645 +                        else:
 646 +                            os.link(src, dest)
 647 +            except Exception as ex:
 648 +                log('Failed to install targetimages for env %s: %s' %
 649 +                    (str(env_id), str(ex)))
 650 diff --git a/deploy/deploy_env.py b/deploy/deploy_env.py
 651 index aa8c4cb..e9c50bb 100644
 652 --- a/deploy/deploy_env.py
 653 +++ b/deploy/deploy_env.py
 654 @@ -15,6 +15,7 @@ import glob
 655  import time
 656  import shutil
 657
 658 +from deploy_cache import DeployCache
 659  from ssh_client import SSHClient
 660
 661  from common import (
 662 @@ -35,7 +36,8 @@ class CloudDeploy(object):
 663
 664      def __init__(self, dea, dha, fuel_ip, fuel_username, fuel_password,
 665                   dea_file, fuel_plugins_conf_dir, work_dir, no_health_check,
 666 -                 deploy_timeout, no_deploy_environment, deploy_log):
 667 +                 deploy_cache_dir, deploy_timeout,
 668 +                 no_deploy_environment, deploy_log):
 669          self.dea = dea
 670          self.dha = dha
 671          self.fuel_ip = fuel_ip
 672 @@ -49,6 +51,8 @@ class CloudDeploy(object):
 673          self.fuel_plugins_conf_dir = fuel_plugins_conf_dir
 674          self.work_dir = work_dir
 675          self.no_health_check = no_health_check
 676 +        self.deploy_cache = ( DeployCache(deploy_cache_dir)
 677 +                              if deploy_cache_dir else None )
 678          self.deploy_timeout = deploy_timeout
 679          self.no_deploy_environment = no_deploy_environment
 680          self.deploy_log = deploy_log
 681 @@ -82,9 +86,14 @@ class CloudDeploy(object):
 682                  self.work_dir, os.path.basename(self.dea_file)))
 683              s.scp_put('%s/common.py' % self.file_dir, self.work_dir)
 684              s.scp_put('%s/dea.py' % self.file_dir, self.work_dir)
 685 +            s.scp_put('%s/deploy_cache.py' % self.file_dir, self.work_dir)
 686              for f in glob.glob('%s/cloud/*' % self.file_dir):
 687                  s.scp_put(f, self.work_dir)
 688
 689 +    def deploy_cache_collect_artifacts(self):
 690 +        if self.deploy_cache:
 691 +            self.deploy_cache.collect_artifacts(self.ssh)
 692 +
 693      def power_off_nodes(self):
 694          for node_id in self.node_ids:
 695              self.dha.node_power_off(node_id)
 696 @@ -283,4 +292,6 @@ class CloudDeploy(object):
 697
 698          self.get_put_deploy_log()
 699
 700 +        self.deploy_cache_collect_artifacts()
 701 +
 702          return rc
 703 diff --git a/deploy/install_fuel_master.py b/deploy/install_fuel_master.py
 704 index b731c6b..83d31fb 100644
 705 --- a/deploy/install_fuel_master.py
 706 +++ b/deploy/install_fuel_master.py
 707 @@ -10,6 +10,7 @@
 708  import time
 709  import os
 710  import glob
 711 +from deploy_cache import DeployCache
 712  from ssh_client import SSHClient
 713  from dha_adapters.libvirt_adapter import LibvirtAdapter
 714
 715 @@ -32,7 +33,7 @@ class InstallFuelMaster(object):
 716
 717      def __init__(self, dea_file, dha_file, fuel_ip, fuel_username,
 718                   fuel_password, fuel_node_id, iso_file, work_dir,
 719 -                 fuel_plugins_dir, no_plugins):
 720 +                 deploy_cache_dir, fuel_plugins_dir, no_plugins):
 721          self.dea_file = dea_file
 722          self.dha = LibvirtAdapter(dha_file)
 723          self.fuel_ip = fuel_ip
 724 @@ -42,6 +43,8 @@ class InstallFuelMaster(object):
 725          self.iso_file = iso_file
 726          self.iso_dir = os.path.dirname(self.iso_file)
 727          self.work_dir = work_dir
 728 +        self.deploy_cache = ( DeployCache(deploy_cache_dir)
 729 +                              if deploy_cache_dir else None )
 730          self.fuel_plugins_dir = fuel_plugins_dir
 731          self.no_plugins = no_plugins
 732          self.file_dir = os.path.dirname(os.path.realpath(__file__))
 733 @@ -83,6 +86,10 @@ class InstallFuelMaster(object):
 734          log('Wait until Fuel menu is up')
 735          fuel_menu_pid = self.wait_until_fuel_menu_up()
 736
 737 +        if self.deploy_cache:
 738 +            log('Deploy cache: Injecting bootstraps and targetimages')
 739 +            self.deploy_cache.inject_cache(self.ssh)
 740 +
 741          log('Inject our own astute.yaml and fuel_bootstrap_cli.yaml settings')
 742          self.inject_own_astute_and_bootstrap_yaml()
 743