patches/opnfv-fuel/upstream-backports/0005-CI-deploy-cache-Store-and-reuse-deploy-artifacts.patch

   1 From: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
   2 Date: Thu, 24 Nov 2016 23:02:04 +0100
   3 Subject: [PATCH] CI: deploy-cache: Store and reuse deploy artifacts
   4
   5 Add support for caching deploy artifacts, like bootstraps and
   6 target images, which take a lot of time at each deploy to be built,
   7 considering it requires a cross-debootstrap via qemu-user-static and
   8 binfmt.
   9
  10 For OPNFV CI, the cache will piggy back on the <iso_mount> mechanism,
  11 and be located at:
  12 /iso_mount/opnfv_ci/<branch>/deploy-cache
  13
  14 TODO: Use dea interface adapter in target images fingerprinting.
  15 TODO: remote fingerprinting
  16 TODO: differentiate between bootstraps and targetimages, so we don't
  17 end up trying to use one cache artifact type as the other.
  18 TODO: implement sanity checks for bootstrap and target images;
  19 TODO: switch `exec_cmd('mkdir ...')` to `create_dir_if_not_exists`;
  20
  21 JIRA: ARMBAND-172
  22
  23 Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
  24 ---
  25  ...p_admin_node.sh-deploy_cache-install-hook.patch |  69 +++++
  26  ci/deploy.sh                                       |  14 +-
  27  deploy/cloud/deployment.py                         |  12 +
  28  deploy/deploy.py                                   |  25 +-
  29  deploy/deploy_cache.py                             | 321 +++++++++++++++++++++
  30  deploy/deploy_env.py                               |  13 +-
  31  deploy/install_fuel_master.py                      |   9 +-
  32  7 files changed, 454 insertions(+), 9 deletions(-)
  33  create mode 100644 build/f_repos/patch/fuel-main/0006-bootstrap_admin_node.sh-deploy_cache-install-hook.patch
  34  create mode 100644 deploy/deploy_cache.py
  35
  36 diff --git a/build/f_repos/patch/fuel-main/0006-bootstrap_admin_node.sh-deploy_cache-install-hook.patch b/build/f_repos/patch/fuel-main/0006-bootstrap_admin_node.sh-deploy_cache-install-hook.patch
  37 new file mode 100644
  38 index 0000000..80cd0f4
  39 --- /dev/null
  40 +++ b/build/f_repos/patch/fuel-main/0006-bootstrap_admin_node.sh-deploy_cache-install-hook.patch
  41 @@ -0,0 +1,69 @@
  42 +From: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
  43 +Date: Mon, 28 Nov 2016 14:27:48 +0100
  44 +Subject: [PATCH] bootstrap_admin_node.sh: deploy_cache install hook
  45 +
  46 +Tooling on the automatic deploy side was updated to support deploy
  47 +caching of artifacts like bootstrap (and id_rsa keypair), target
  48 +images etc.
  49 +
  50 +Add installation hook that calls `fuel-bootstrap import` instead of
  51 +`build` when a bootstrap tar is available in the agreed location,
  52 +/var/lib/opnfv/cache/bootstraps/.
  53 +
  54 +JIRA: ARMBAND-172
  55 +
  56 +Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
  57 +---
  58 + iso/bootstrap_admin_node.sh | 20 +++++++++++++++++++-
  59 + 1 file changed, 19 insertions(+), 1 deletion(-)
  60 +
  61 +diff --git a/iso/bootstrap_admin_node.sh b/iso/bootstrap_admin_node.sh
  62 +index abc5ffb..15e6261 100755
  63 +--- a/iso/bootstrap_admin_node.sh
  64 ++++ b/iso/bootstrap_admin_node.sh
  65 +@@ -61,6 +61,8 @@ wget \
  66 +
  67 + ASTUTE_YAML='/etc/fuel/astute.yaml'
  68 + BOOTSTRAP_NODE_CONFIG="/etc/fuel/bootstrap_admin_node.conf"
  69 ++OPNFV_CACHE_PATH="/var/cache/opnfv/bootstraps"
  70 ++OPNFV_CACHE_TAR="opnfv-bootstraps-cache.tar"
  71 + bs_build_log='/var/log/fuel-bootstrap-image-build.log'
  72 + bs_status=0
  73 + # Backup network configs to this folder. Folder will be created only if
  74 +@@ -94,6 +96,7 @@ image becomes available, reboot nodes that failed to be discovered."
  75 + bs_done_message="Default bootstrap image building done. Now you can boot new \
  76 + nodes over PXE, they will be discovered and become available for installing \
  77 + OpenStack on them"
  78 ++bs_cache_message="OPNFV deploy cache: bootstrap image injected."
  79 + # Update issues messages
  80 + update_warn_message="There is an issue connecting to update repository of \
  81 + your distributions of OpenStack. \
  82 +@@ -500,12 +503,27 @@ set_ui_bootstrap_error () {
  83 +       EOF
  84 + }
  85 +
  86 ++function inject_cached_ubuntu_bootstrap () {
  87 ++        if [ -f "${OPNFV_CACHE_PATH}/${OPNFV_CACHE_TAR}" -a \
  88 ++             -f "${OPNFV_CACHE_PATH}/id_rsa.pub" -a \
  89 ++             -f "${OPNFV_CACHE_PATH}/id_rsa" ]; then
  90 ++          if cp "${OPNFV_CACHE_PATH}/id_rsa"* "~/.ssh/" && \
  91 ++                fuel-bootstrap -v --debug import --activate \
  92 ++                "${OPNFV_CACHE_PATH}/${OPNFV_CACHE_TAR}" >>"$bs_build_log" 2>&1; then
  93 ++            fuel notify --topic "done" --send "${bs_cache_message}"
  94 ++            return 0
  95 ++          fi
  96 ++        fi
  97 ++        return 1
  98 ++}
  99 ++
 100 + # Actually build the bootstrap image
 101 + build_ubuntu_bootstrap () {
 102 +         local ret=1
 103 +         echo ${bs_progress_message} >&2
 104 +         set_ui_bootstrap_error "${bs_progress_message}" >&2
 105 +-        if fuel-bootstrap -v --debug build --target_arch arm64 --activate >>"$bs_build_log" 2>&1; then
 106 ++        if inject_cached_ubuntu_bootstrap || fuel-bootstrap -v --debug \
 107 ++          build --activate --target_arch arm64 >>"$bs_build_log" 2>&1; then
 108 +           ret=0
 109 +           fuel notify --topic "done" --send "${bs_done_message}"
 110 +         else
 111 diff --git a/ci/deploy.sh b/ci/deploy.sh
 112 index 081806c..4b1ae0e 100755
 113 --- a/ci/deploy.sh
 114 +++ b/ci/deploy.sh
 115 @@ -29,7 +29,7 @@ cat << EOF
 116  xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
 117  `basename $0`: Deploys the Fuel@OPNFV stack
 118
 119 -usage: `basename $0` -b base-uri [-B PXE Bridge] [-f] [-F] [-H] -l lab-name -p pod-name -s deploy-scenario [-S image-dir] [-T timeout] -i iso
 120 +usage: `basename $0` -b base-uri [-B PXE Bridge] [-f] [-F] [-H] -l lab-name -p pod-name -s deploy-scenario [-S image-dir] [-C deploy-cache-dir] [-T timeout] -i iso
 121         -s deployment-scenario [-S optional Deploy-scenario path URI]
 122         [-R optional local relen repo (containing deployment Scenarios]
 123
 124 @@ -47,6 +47,7 @@ OPTIONS:
 125    -p  Pod-name
 126    -s  Deploy-scenario short-name/base-file-name
 127    -S  Storage dir for VM images
 128 +  -C  Deploy cache dir for storing image artifacts
 129    -T  Timeout, in minutes, for the deploy.
 130    -i  iso url
 131
 132 @@ -79,6 +80,7 @@ Input parameters to the build script is:
 133     or a deployment short-name as defined by scenario.yaml in the deployment
 134     scenario path.
 135  -S Storage dir for VM images, default is fuel/deploy/images
 136 +-C Deploy cache dir for bootstrap and target image artifacts, optional
 137  -T Timeout, in minutes, for the deploy. It defaults to using the DEPLOY_TIMEOUT
 138     environment variable when defined, or to the default in deploy.py otherwise
 139  -i .iso image to be deployed (needs to be provided in a URI
 140 @@ -116,6 +118,7 @@ FUEL_CREATION_ONLY=''
 141  NO_DEPLOY_ENVIRONMENT=''
 142  STORAGE_DIR=''
 143  DRY_RUN=0
 144 +DEPLOY_CACHE_DIR=''
 145  if ! [ -z $DEPLOY_TIMEOUT ]; then
 146      DEPLOY_TIMEOUT="-dt $DEPLOY_TIMEOUT"
 147  else
 148 @@ -128,7 +131,7 @@ fi
 149  ############################################################################
 150  # BEGIN of main
 151  #
 152 -while getopts "b:B:dfFHl:L:p:s:S:T:i:he" OPTION
 153 +while getopts "b:B:dfFHl:L:p:s:S:C:T:i:he" OPTION
 154  do
 155      case $OPTION in
 156          b)
 157 @@ -179,6 +182,9 @@ do
 158                  STORAGE_DIR="-s ${OPTARG}"
 159              fi
 160              ;;
 161 +        C)
 162 +            DEPLOY_CACHE_DIR="-dc ${OPTARG}"
 163 +            ;;
 164          T)
 165              DEPLOY_TIMEOUT="-dt ${OPTARG}"
 166              ;;
 167 @@ -243,8 +249,8 @@ if [ $DRY_RUN -eq 0 ]; then
 168          ISO=${SCRIPT_PATH}/ISO/image.iso
 169      fi
 170      # Start deployment
 171 -    echo "python deploy.py $DEPLOY_LOG $STORAGE_DIR $PXE_BRIDGE $USE_EXISTING_FUEL $FUEL_CREATION_ONLY $NO_HEALTH_CHECK $NO_DEPLOY_ENVIRONMENT -dea ${SCRIPT_PATH}/config/dea.yaml -dha ${SCRIPT_PATH}/config/dha.yaml -iso $ISO $DEPLOY_TIMEOUT"
 172 -    python deploy.py $DEPLOY_LOG $STORAGE_DIR $PXE_BRIDGE $USE_EXISTING_FUEL $FUEL_CREATION_ONLY $NO_HEALTH_CHECK $NO_DEPLOY_ENVIRONMENT -dea ${SCRIPT_PATH}/config/dea.yaml -dha ${SCRIPT_PATH}/config/dha.yaml -iso $ISO $DEPLOY_TIMEOUT
 173 +    echo "python deploy.py $DEPLOY_LOG $STORAGE_DIR $PXE_BRIDGE $USE_EXISTING_FUEL $FUEL_CREATION_ONLY $NO_HEALTH_CHECK $NO_DEPLOY_ENVIRONMENT -dea ${SCRIPT_PATH}/config/dea.yaml -dha ${SCRIPT_PATH}/config/dha.yaml -iso $ISO $DEPLOY_TIMEOUT $DEPLOY_CACHE_DIR"
 174 +    python deploy.py $DEPLOY_LOG $STORAGE_DIR $PXE_BRIDGE $USE_EXISTING_FUEL $FUEL_CREATION_ONLY $NO_HEALTH_CHECK $NO_DEPLOY_ENVIRONMENT -dea ${SCRIPT_PATH}/config/dea.yaml -dha ${SCRIPT_PATH}/config/dha.yaml -iso $ISO $DEPLOY_TIMEOUT $DEPLOY_CACHE_DIR
 175  fi
 176  popd > /dev/null
 177
 178 diff --git a/deploy/cloud/deployment.py b/deploy/cloud/deployment.py
 179 index 5dd0263..3db4c0d 100644
 180 --- a/deploy/cloud/deployment.py
 181 +++ b/deploy/cloud/deployment.py
 182 @@ -24,6 +24,8 @@ from common import (
 183      delete,
 184  )
 185
 186 +from deploy_cache import DeployCache
 187 +
 188  SEARCH_TEXT = '(err)'
 189  LOG_FILE = '/var/log/puppet.log'
 190  GREP_LINES_OF_LEADING_CONTEXT = 100
 191 @@ -52,6 +54,14 @@ class Deployment(object):
 192          self.pattern = re.compile(
 193              '\d\d\d\d-\d\d-\d\d\s\d\d:\d\d:\d\d')
 194
 195 +    def deploy_cache_install_targetimages(self):
 196 +        log('Using target images from deploy cache')
 197 +        DeployCache.install_targetimages_for_env(self.env_id)
 198 +
 199 +    def deploy_cache_extract_targetimages(self):
 200 +        log('Collecting Fuel target image files for deploy cache')
 201 +        DeployCache.extract_targetimages_from_env(self.env_id)
 202 +
 203      def collect_error_logs(self):
 204          for node_id, roles_blade in self.node_id_roles_dict.iteritems():
 205              log_list = []
 206 @@ -113,6 +123,7 @@ class Deployment(object):
 207          start = time.time()
 208
 209          log('Starting deployment of environment %s' % self.env_id)
 210 +        self.deploy_cache_install_targetimages()
 211          deploy_id = None
 212          ready = False
 213          timeout = False
 214 @@ -145,6 +156,7 @@ class Deployment(object):
 215              err('Deployment timed out, environment %s is not operational, '
 216                  'snapshot will not be performed'
 217                  % self.env_id)
 218 +        self.deploy_cache_extract_targetimages()
 219          if ready:
 220              log('Environment %s successfully deployed'
 221                  % self.env_id)
 222 diff --git a/deploy/deploy.py b/deploy/deploy.py
 223 index 08702d2..1a55361 100755
 224 --- a/deploy/deploy.py
 225 +++ b/deploy/deploy.py
 226 @@ -23,6 +23,7 @@ from dea import DeploymentEnvironmentAdapter
 227  from dha import DeploymentHardwareAdapter
 228  from install_fuel_master import InstallFuelMaster
 229  from deploy_env import CloudDeploy
 230 +from deploy_cache import DeployCache
 231  from execution_environment import ExecutionEnvironment
 232
 233  from common import (
 234 @@ -62,7 +63,8 @@ class AutoDeploy(object):
 235      def __init__(self, no_fuel, fuel_only, no_health_check, cleanup_only,
 236                   cleanup, storage_dir, pxe_bridge, iso_file, dea_file,
 237                   dha_file, fuel_plugins_dir, fuel_plugins_conf_dir,
 238 -                 no_plugins, deploy_timeout, no_deploy_environment, deploy_log):
 239 +                 no_plugins, deploy_cache_dir, deploy_timeout,
 240 +                 no_deploy_environment, deploy_log):
 241          self.no_fuel = no_fuel
 242          self.fuel_only = fuel_only
 243          self.no_health_check = no_health_check
 244 @@ -76,6 +78,7 @@ class AutoDeploy(object):
 245          self.fuel_plugins_dir = fuel_plugins_dir
 246          self.fuel_plugins_conf_dir = fuel_plugins_conf_dir
 247          self.no_plugins = no_plugins
 248 +        self.deploy_cache_dir = deploy_cache_dir
 249          self.deploy_timeout = deploy_timeout
 250          self.no_deploy_environment = no_deploy_environment
 251          self.deploy_log = deploy_log
 252 @@ -117,7 +120,7 @@ class AutoDeploy(object):
 253                                    self.fuel_username, self.fuel_password,
 254                                    self.dea_file, self.fuel_plugins_conf_dir,
 255                                    WORK_DIR, self.no_health_check,
 256 -                                  self.deploy_timeout,
 257 +                                  self.deploy_cache_dir, self.deploy_timeout,
 258                                    self.no_deploy_environment, self.deploy_log)
 259              with old_dep.ssh:
 260                  old_dep.check_previous_installation()
 261 @@ -129,6 +132,7 @@ class AutoDeploy(object):
 262                                   self.fuel_conf['ip'], self.fuel_username,
 263                                   self.fuel_password, self.fuel_node_id,
 264                                   self.iso_file, WORK_DIR,
 265 +                                 self.deploy_cache_dir,
 266                                   self.fuel_plugins_dir, self.no_plugins)
 267          fuel.install()
 268
 269 @@ -137,6 +141,7 @@ class AutoDeploy(object):
 270          tmp_new_dir = '%s/newiso' % self.tmp_dir
 271          try:
 272              self.copy(tmp_orig_dir, tmp_new_dir)
 273 +            self.deploy_cache_fingerprints(tmp_new_dir)
 274              self.patch(tmp_new_dir, new_iso)
 275          except Exception as e:
 276              exec_cmd('fusermount -u %s' % tmp_orig_dir, False)
 277 @@ -157,6 +162,12 @@ class AutoDeploy(object):
 278          delete(tmp_orig_dir)
 279          exec_cmd('chmod -R 755 %s' % tmp_new_dir)
 280
 281 +    def deploy_cache_fingerprints(self, tmp_new_dir):
 282 +        if self.deploy_cache_dir:
 283 +            log('Deploy cache: Collecting fingerprints...')
 284 +            deploy_cache = DeployCache(self.deploy_cache_dir)
 285 +            deploy_cache.do_fingerprints(tmp_new_dir, self.dea_file)
 286 +
 287      def patch(self, tmp_new_dir, new_iso):
 288          log('Patching...')
 289          patch_dir = '%s/%s' % (CWD, PATCH_DIR)
 290 @@ -219,7 +230,8 @@ class AutoDeploy(object):
 291          dep = CloudDeploy(self.dea, self.dha, self.fuel_conf['ip'],
 292                            self.fuel_username, self.fuel_password,
 293                            self.dea_file, self.fuel_plugins_conf_dir,
 294 -                          WORK_DIR, self.no_health_check, self.deploy_timeout,
 295 +                          WORK_DIR, self.no_health_check,
 296 +                          self.deploy_cache_dir, self.deploy_timeout,
 297                            self.no_deploy_environment, self.deploy_log)
 298          return dep.deploy()
 299
 300 @@ -344,6 +356,8 @@ def parse_arguments():
 301                          help='Fuel Plugins Configuration directory')
 302      parser.add_argument('-np', dest='no_plugins', action='store_true',
 303                          default=False, help='Do not install Fuel Plugins')
 304 +    parser.add_argument('-dc', dest='deploy_cache_dir', action='store',
 305 +                        help='Deploy Cache Directory')
 306      parser.add_argument('-dt', dest='deploy_timeout', action='store',
 307                          default=240, help='Deployment timeout (in minutes) '
 308                          '[default: 240]')
 309 @@ -377,6 +391,10 @@ def parse_arguments():
 310          for bridge in args.pxe_bridge:
 311              check_bridge(bridge, args.dha_file)
 312
 313 +    if args.deploy_cache_dir:
 314 +        log('Using deploy cache directory: %s' % args.deploy_cache_dir)
 315 +        create_dir_if_not_exists(args.deploy_cache_dir)
 316 +
 317
 318      kwargs = {'no_fuel': args.no_fuel, 'fuel_only': args.fuel_only,
 319                'no_health_check': args.no_health_check,
 320 @@ -387,6 +405,7 @@ def parse_arguments():
 321                'fuel_plugins_dir': args.fuel_plugins_dir,
 322                'fuel_plugins_conf_dir': args.fuel_plugins_conf_dir,
 323                'no_plugins': args.no_plugins,
 324 +              'deploy_cache_dir': args.deploy_cache_dir,
 325                'deploy_timeout': args.deploy_timeout,
 326                'no_deploy_environment': args.no_deploy_environment,
 327                'deploy_log': args.deploy_log}
 328 diff --git a/deploy/deploy_cache.py b/deploy/deploy_cache.py
 329 new file mode 100644
 330 index 0000000..76fb1b9
 331 --- /dev/null
 332 +++ b/deploy/deploy_cache.py
 333 @@ -0,0 +1,321 @@
 334 +###############################################################################
 335 +# Copyright (c) 2016 Enea AB and others.
 336 +# Alexandru.Avadanii@enea.com
 337 +# All rights reserved. This program and the accompanying materials
 338 +# are made available under the terms of the Apache License, Version 2.0
 339 +# which accompanies this distribution, and is available at
 340 +# http://www.apache.org/licenses/LICENSE-2.0
 341 +###############################################################################
 342 +
 343 +import glob
 344 +import hashlib
 345 +import io
 346 +import json
 347 +import os
 348 +import shutil
 349 +import yaml
 350 +
 351 +from common import (
 352 +    exec_cmd,
 353 +    log,
 354 +)
 355 +
 356 +###############################################################################
 357 +# Deploy Cache Flow Overview
 358 +###############################################################################
 359 +# 1. do_fingerprints
 360 +#    Can be called as soon as a Fuel Master ISO chroot is available.
 361 +#    This will gather all required information for uniquely identifying the
 362 +#    objects in cache (bootstraps, targetimages).
 363 +# 2. inject_cache
 364 +#    Can be called as soon as we have a steady SSH connection to the Fuel
 365 +#    Master node. It will inject cached artifacts over SSH, for later install.
 366 +# 3. (external, async) install cached bootstrap instead of building a new one
 367 +#    /sbin/bootstrap_admin_node.sh will check for cached bootstrap images
 368 +#    (with id_rsa, id_rsa.pub attached) and will install those via
 369 +#    $ fuel-bootstrap import opfnv-bootstraps-cache.tar
 370 +# 4. install_targetimages_for_env
 371 +#    Should be called before cloud deploy is started, to install env-generic
 372 +#    'env_X_...' cached images for the current environment ID.
 373 +#    Static method, to be used on the remote Fuel Master node; does not require
 374 +#    access to the deploy cache, it only moves around some local files.
 375 +# 5. extract_targetimages_from_env
 376 +#    Should be called at env deploy finish, to prepare artifacts for caching.
 377 +#    Static method, same observations as above apply.
 378 +# 6. collect_artifacts
 379 +#    Call last, to collect all artifacts.
 380 +###############################################################################
 381 +
 382 +###############################################################################
 383 +# Deploy cache artifacts:
 384 +# - id_rsa
 385 +# - bootstrap image (Ubuntu)
 386 +# - environment target image (Ubuntu)
 387 +###############################################################################
 388 +# Cache fingerprint covers:
 389 +# - bootstrap:
 390 +#   - local mirror contents
 391 +#   - package list (and everything else in fuel_bootstrap_cli.yaml)
 392 +# - target image:
 393 +#   - local mirror contents
 394 +#   - package list (determined from DEA)
 395 +###############################################################################
 396 +# WARN: Cache fingerprint does NOT yet cover:
 397 +# - image_data (always assume the default /boot, /);
 398 +# - output_dir (always assume the default /var/www/nailgun/targetimages;
 399 +# - codename (always assume the default, currently 'trusty');
 400 +# - extra_dirs: /usr/share/fuel_bootstrap_cli/files/trusty
 401 +# - root_ssh_authorized_file, inluding the contents of /root/.ssh/id_rsa.pub
 402 +# - Auxiliary repo  .../mitaka-9.0/ubuntu/auxiliary
 403 +# If the above change without triggering a cache miss, try clearing the cache.
 404 +###############################################################################
 405 +# WARN: Bootstrap caching implies RSA keypair to be reused!
 406 +###############################################################################
 407 +
 408 +# Local mirrros will be used on Fuel Master for both bootstrap and target image
 409 +# build, from `http://127.0.0.1:8080/...` or `http://10.20.0.2:8080/...`:
 410 +# - MOS        .../mitaka-9.0/ubuntu/x86_64
 411 +# - Ubuntu     .../mirrors/ubuntu/
 412 +# All these reside on Fuel Master at local path:
 413 +NAILGUN_PATH = '/var/www/nailgun/'
 414 +
 415 +# Artifact names (corresponding to nailgun subdirs)
 416 +MIRRORS = 'mirrors'
 417 +BOOTSTRAPS = 'bootstraps'
 418 +TARGETIMAGES = 'targetimages'
 419 +
 420 +# Info for collecting RSA keypair
 421 +RSA_KEYPAIR_PATH = '/root/.ssh'
 422 +RSA_KEYPAIR_FILES = ['id_rsa', 'id_rsa.pub']
 423 +
 424 +# Relative path for collecting the active bootstrap image(s) after env deploy
 425 +NAILGUN_ACT_BOOTSTRAP_SUBDIR = '%s/active_bootstrap' % BOOTSTRAPS
 426 +
 427 +# Relative path for collecting target image(s) for deployed enviroment
 428 +NAILGUN_TIMAGES_SUBDIR = TARGETIMAGES
 429 +
 430 +# OPNFV Fuel bootstrap settings file that will be injected at deploy
 431 +ISO_BOOTSTRAP_CLI_YAML = '/opnfv/fuel_bootstrap_cli.yaml'
 432 +
 433 +# OPNFV Deploy Cache path on Fuel Master, where artifacts will be injected
 434 +REMOTE_CACHE_PATH = '/var/cache/opnfv'
 435 +
 436 +# OPNFV Bootstrap Cache tar archive name, to be used by bootstrap_admin_node.sh
 437 +BOOTSTRAP_ARCHIVE = 'opnfv-bootstraps-cache.tar'
 438 +
 439 +# Env-ID indep prefix
 440 +ENVX = 'env_X_'
 441 +
 442 +class DeployCache(object):
 443 +    """OPNFV Deploy Cache - managed storage for cacheable artifacts"""
 444 +
 445 +    def __init__(self, cache_dir,
 446 +                 fingerprints_yaml='deploy_cache_fingerprints.yaml'):
 447 +        self.cache_dir = cache_dir
 448 +        self.fingerprints_yaml = fingerprints_yaml
 449 +        self.fingerprints = {BOOTSTRAPS: None,
 450 +                             MIRRORS: None,
 451 +                             TARGETIMAGES: None}
 452 +
 453 +    def __load_fingerprints(self):
 454 +        """Load deploy cache yaml config holding fingerprints"""
 455 +        if os.path.isfile(self.fingerprints_yaml):
 456 +            cache_fingerprints = open(self.fingerprints_yaml).read()
 457 +            self.fingerprints = yaml.load(cache_fingerprints)
 458 +
 459 +    def __save_fingerprints(self):
 460 +        """Update deploy cache yaml config holding fingerprints"""
 461 +        with open(self.fingerprints_yaml, 'w') as outfile:
 462 +            outfile.write(yaml.safe_dump(self.fingerprints,
 463 +                          default_flow_style=False))
 464 +
 465 +    def __fingerprint_mirrors(self, chroot_path):
 466 +        """Collect repo mirror fingerprints"""
 467 +        md5sums = list()
 468 +        # Scan all ISO for deb repo metadata and collect MD5 from Release files
 469 +        for root, _, files in os.walk(chroot_path):
 470 +            for relf in files:
 471 +                if relf == 'Release' and 'binary' not in root:
 472 +                    collect_sums = False
 473 +                    filepath = os.path.join(root, relf)
 474 +                    with open(filepath, "r") as release_file:
 475 +                        for line in release_file:
 476 +                            if collect_sums:
 477 +                                if line.startswith(' '):
 478 +                                    md5sums += [line[1:33]]
 479 +                                else:
 480 +                                    break
 481 +                            elif line.startswith('MD5Sum:'):
 482 +                                collect_sums = True
 483 +        sorted_md5sums = json.dumps(md5sums, sort_keys=True)
 484 +        self.fingerprints[MIRRORS] = hashlib.sha1(sorted_md5sums).hexdigest()
 485 +
 486 +    def __fingerprint_bootstrap(self, chroot_path):
 487 +        """Collect bootstrap image metadata fingerprints"""
 488 +        # FIXME(armband): include 'extra_dirs' contents
 489 +        cli_yaml_path = os.path.join(chroot_path, ISO_BOOTSTRAP_CLI_YAML[1:])
 490 +        bootstrap_cli_yaml = open(cli_yaml_path).read()
 491 +        bootstrap_data = yaml.load(bootstrap_cli_yaml)
 492 +        sorted_data = json.dumps(bootstrap_data, sort_keys=True)
 493 +        self.fingerprints[BOOTSTRAPS] = hashlib.sha1(sorted_data).hexdigest()
 494 +
 495 +    def __fingerprint_target(self, dea_file):
 496 +        """Collect target image metadata fingerprints"""
 497 +        # FIXME(armband): include 'image_data', 'codename', 'output'
 498 +        with io.open(dea_file) as stream:
 499 +            dea = yaml.load(stream)
 500 +            editable = dea['settings']['editable']
 501 +            target_data = {'packages': editable['provision']['packages'],
 502 +                           'repos': editable['repo_setup']['repos']}
 503 +            s_data = json.dumps(target_data, sort_keys=True)
 504 +            self.fingerprints[TARGETIMAGES] = hashlib.sha1(s_data).hexdigest()
 505 +
 506 +    def do_fingerprints(self, chroot_path, dea_file):
 507 +        """Collect SHA1 fingerprints based on chroot contents, DEA settings"""
 508 +        try:
 509 +            self.__load_fingerprints()
 510 +            self.__fingerprint_mirrors(chroot_path)
 511 +            self.__fingerprint_bootstrap(chroot_path)
 512 +            self.__fingerprint_target(dea_file)
 513 +            self.__save_fingerprints()
 514 +        except Exception as ex:
 515 +            log('Failed to get cache fingerprint: %s' % str(ex))
 516 +
 517 +    def __lookup_cache(self, sha):
 518 +        """Search for object in cache based on SHA fingerprint"""
 519 +        cache_sha_dir = os.path.join(self.cache_dir, sha)
 520 +        if not os.path.isdir(cache_sha_dir) or not os.listdir(cache_sha_dir):
 521 +            return None
 522 +        return cache_sha_dir
 523 +
 524 +    def __inject_cache_dir(self, ssh, sha, artifact):
 525 +        """Stage cached object (dir) in Fuel Master OPNFV local cache"""
 526 +        local_path = self.__lookup_cache(sha)
 527 +        if local_path:
 528 +            remote_path = os.path.join(REMOTE_CACHE_PATH, artifact)
 529 +            with ssh:
 530 +                ssh.exec_cmd('mkdir -p %s' % remote_path)
 531 +                for cachedfile in glob.glob('%s/*' % local_path):
 532 +                    ssh.scp_put(cachedfile, remote_path)
 533 +        return local_path
 534 +
 535 +    def __mix_fingerprints(self, f1, f2):
 536 +        """Compute composite fingerprint"""
 537 +        if self.fingerprints[f1] is None or self.fingerprints[f2] is None:
 538 +            return None
 539 +        return hashlib.sha1('%s%s' %
 540 +            (self.fingerprints[f1], self.fingerprints[f2])).hexdigest()
 541 +
 542 +    def inject_cache(self, ssh):
 543 +        """Lookup artifacts in cache and inject them over SSH/SCP into Fuel"""
 544 +        try:
 545 +            self.__load_fingerprints()
 546 +            for artifact in [BOOTSTRAPS, TARGETIMAGES]:
 547 +                sha = self.__mix_fingerprints(MIRRORS, artifact)
 548 +                if sha is None:
 549 +                    log('Missing fingerprint for: %s' % artifact)
 550 +                    continue
 551 +                if not self.__inject_cache_dir(ssh, sha, artifact):
 552 +                    log('SHA1 not in cache: %s (%s)' % (str(sha), artifact))
 553 +                else:
 554 +                    log('SHA1 injected: %s (%s)' % (str(sha), artifact))
 555 +        except Exception as ex:
 556 +            log('Failed to inject cached artifacts into Fuel: %s' % str(ex))
 557 +
 558 +    def __extract_bootstraps(self, ssh, cache_sha_dir):
 559 +        """Collect bootstrap artifacts from Fuel over SSH/SCP"""
 560 +        remote_tar = os.path.join(REMOTE_CACHE_PATH, BOOTSTRAP_ARCHIVE)
 561 +        local_tar = os.path.join(cache_sha_dir, BOOTSTRAP_ARCHIVE)
 562 +        with ssh:
 563 +            for k in RSA_KEYPAIR_FILES:
 564 +                ssh.scp_get(os.path.join(RSA_KEYPAIR_PATH, k),
 565 +                    local=os.path.join(cache_sha_dir, k))
 566 +            ssh.exec_cmd('mkdir -p %s && cd %s && tar cf %s *' %
 567 +                (REMOTE_CACHE_PATH,
 568 +                os.path.join(NAILGUN_PATH, NAILGUN_ACT_BOOTSTRAP_SUBDIR),
 569 +                remote_tar))
 570 +            ssh.scp_get(remote_tar, local=local_tar)
 571 +            ssh.exec_cmd('rm -f %s' % remote_tar)
 572 +
 573 +    def __extract_targetimages(self, ssh, cache_sha_dir):
 574 +        """Collect target image artifacts from Fuel over SSH/SCP"""
 575 +        cti_path = os.path.join(REMOTE_CACHE_PATH, TARGETIMAGES)
 576 +        with ssh:
 577 +            ssh.scp_get('%s/%s*' % (cti_path, ENVX), local=cache_sha_dir)
 578 +
 579 +    def collect_artifacts(self, ssh):
 580 +        """Collect artifacts from Fuel over SSH/SCP and add them to cache"""
 581 +        try:
 582 +            self.__load_fingerprints()
 583 +            for artifact, func in {
 584 +                    BOOTSTRAPS: self.__extract_bootstraps,
 585 +                    TARGETIMAGES: self.__extract_targetimages
 586 +                }.iteritems():
 587 +                sha = self.__mix_fingerprints(MIRRORS, artifact)
 588 +                if sha is None:
 589 +                    log('WARN: Skip caching, NO fingerprint: %s' % artifact)
 590 +                    continue
 591 +                local_path = self.__lookup_cache(sha)
 592 +                if local_path:
 593 +                    log('SHA1 already in cache: %s (%s)' % (str(sha), artifact))
 594 +                else:
 595 +                    log('New cache SHA1: %s (%s)' % (str(sha), artifact))
 596 +                    cache_sha_dir = os.path.join(self.cache_dir, sha)
 597 +                    exec_cmd('mkdir -p %s' % cache_sha_dir)
 598 +                    func(ssh, cache_sha_dir)
 599 +        except Exception as ex:
 600 +            log('Failed to extract artifacts from Fuel: %s' % str(ex))
 601 +
 602 +    @staticmethod
 603 +    def extract_targetimages_from_env(env_id):
 604 +        """Prepare targetimages from env ID for storage in deploy cache
 605 +
 606 +        NOTE: This method should be executed locally ON the Fuel Master node.
 607 +        WARN: This method overwrites targetimages cache on Fuel Master node.
 608 +        """
 609 +        env_n = 'env_%s_' % str(env_id)
 610 +        cti_path = os.path.join(REMOTE_CACHE_PATH, TARGETIMAGES)
 611 +        ti_path = os.path.join(NAILGUN_PATH, NAILGUN_TIMAGES_SUBDIR)
 612 +        try:
 613 +            exec_cmd('rm -rf %s && mkdir -p %s' % (cti_path, cti_path))
 614 +            for root, _, files in os.walk(ti_path):
 615 +                for tif in files:
 616 +                    if tif.startswith(env_n):
 617 +                        src = os.path.join(root, tif)
 618 +                        dest = os.path.join(cti_path, tif.replace(env_n, ENVX))
 619 +                        if tif.endswith('.yaml'):
 620 +                            shutil.copy(src, dest)
 621 +                            exec_cmd('sed -i "s|%s|%s|g" %s' %
 622 +                                     (env_n, ENVX, dest))
 623 +                        else:
 624 +                            os.link(src, dest)
 625 +        except Exception as ex:
 626 +            log('Failed to extract targetimages artifacts from env %s: %s' %
 627 +                (str(env_id), str(ex)))
 628 +
 629 +    @staticmethod
 630 +    def install_targetimages_for_env(env_id):
 631 +        """Install targetimages artifacts for a specific env ID
 632 +
 633 +        NOTE: This method should be executed locally ON the Fuel Master node.
 634 +        """
 635 +        env_n = 'env_%s_' % str(env_id)
 636 +        cti_path = os.path.join(REMOTE_CACHE_PATH, TARGETIMAGES)
 637 +        ti_path = os.path.join(NAILGUN_PATH, NAILGUN_TIMAGES_SUBDIR)
 638 +        if not os.path.isdir(cti_path):
 639 +            log('%s cache dir not found: %s' % (TARGETIMAGES, cti_path))
 640 +        else:
 641 +            try:
 642 +                for root, _, files in os.walk(cti_path):
 643 +                    for tif in files:
 644 +                        src = os.path.join(root, tif)
 645 +                        dest = os.path.join(ti_path, tif.replace(ENVX, env_n))
 646 +                        if tif.endswith('.yaml'):
 647 +                            shutil.copy(src, dest)
 648 +                            exec_cmd('sed -i "s|%s|%s|g" %s' %
 649 +                                     (ENVX, env_n, dest))
 650 +                        else:
 651 +                            os.link(src, dest)
 652 +            except Exception as ex:
 653 +                log('Failed to install targetimages for env %s: %s' %
 654 +                    (str(env_id), str(ex)))
 655 diff --git a/deploy/deploy_env.py b/deploy/deploy_env.py
 656 index 1d2dfeb..2375f51 100644
 657 --- a/deploy/deploy_env.py
 658 +++ b/deploy/deploy_env.py
 659 @@ -15,6 +15,7 @@ import glob
 660  import time
 661  import shutil
 662
 663 +from deploy_cache import DeployCache
 664  from ssh_client import SSHClient
 665
 666  from common import (
 667 @@ -36,7 +37,8 @@ class CloudDeploy(object):
 668
 669      def __init__(self, dea, dha, fuel_ip, fuel_username, fuel_password,
 670                   dea_file, fuel_plugins_conf_dir, work_dir, no_health_check,
 671 -                 deploy_timeout, no_deploy_environment, deploy_log):
 672 +                 deploy_cache_dir, deploy_timeout,
 673 +                 no_deploy_environment, deploy_log):
 674          self.dea = dea
 675          self.dha = dha
 676          self.fuel_ip = fuel_ip
 677 @@ -50,6 +52,8 @@ class CloudDeploy(object):
 678          self.fuel_plugins_conf_dir = fuel_plugins_conf_dir
 679          self.work_dir = work_dir
 680          self.no_health_check = no_health_check
 681 +        self.deploy_cache = ( DeployCache(deploy_cache_dir)
 682 +                              if deploy_cache_dir else None )
 683          self.deploy_timeout = deploy_timeout
 684          self.no_deploy_environment = no_deploy_environment
 685          self.deploy_log = deploy_log
 686 @@ -83,9 +87,14 @@ class CloudDeploy(object):
 687                  self.work_dir, os.path.basename(self.dea_file)))
 688              s.scp_put('%s/common.py' % self.file_dir, self.work_dir)
 689              s.scp_put('%s/dea.py' % self.file_dir, self.work_dir)
 690 +            s.scp_put('%s/deploy_cache.py' % self.file_dir, self.work_dir)
 691              for f in glob.glob('%s/cloud/*' % self.file_dir):
 692                  s.scp_put(f, self.work_dir)
 693
 694 +    def deploy_cache_collect_artifacts(self):
 695 +        if self.deploy_cache:
 696 +            self.deploy_cache.collect_artifacts(self.ssh)
 697 +
 698      def power_off_nodes(self):
 699          for node_id in self.node_ids:
 700              self.dha.node_power_off(node_id)
 701 @@ -284,4 +293,6 @@ class CloudDeploy(object):
 702
 703          self.get_put_deploy_log()
 704
 705 +        self.deploy_cache_collect_artifacts()
 706 +
 707          return rc
 708 diff --git a/deploy/install_fuel_master.py b/deploy/install_fuel_master.py
 709 index ccc18d3..2615818 100644
 710 --- a/deploy/install_fuel_master.py
 711 +++ b/deploy/install_fuel_master.py
 712 @@ -10,6 +10,7 @@
 713  import time
 714  import os
 715  import glob
 716 +from deploy_cache import DeployCache
 717  from ssh_client import SSHClient
 718  from dha_adapters.libvirt_adapter import LibvirtAdapter
 719
 720 @@ -33,7 +34,7 @@ class InstallFuelMaster(object):
 721
 722      def __init__(self, dea_file, dha_file, fuel_ip, fuel_username,
 723                   fuel_password, fuel_node_id, iso_file, work_dir,
 724 -                 fuel_plugins_dir, no_plugins):
 725 +                 deploy_cache_dir, fuel_plugins_dir, no_plugins):
 726          self.dea_file = dea_file
 727          self.dha = LibvirtAdapter(dha_file)
 728          self.fuel_ip = fuel_ip
 729 @@ -43,6 +44,8 @@ class InstallFuelMaster(object):
 730          self.iso_file = iso_file
 731          self.iso_dir = os.path.dirname(self.iso_file)
 732          self.work_dir = work_dir
 733 +        self.deploy_cache = ( DeployCache(deploy_cache_dir)
 734 +                              if deploy_cache_dir else None )
 735          self.fuel_plugins_dir = fuel_plugins_dir
 736          self.no_plugins = no_plugins
 737          self.file_dir = os.path.dirname(os.path.realpath(__file__))
 738 @@ -84,6 +87,10 @@ class InstallFuelMaster(object):
 739          log('Wait until Fuel menu is up')
 740          fuel_menu_pid = self.wait_until_fuel_menu_up()
 741
 742 +        if self.deploy_cache:
 743 +            log('Deploy cache: Injecting bootstraps and targetimages')
 744 +            self.deploy_cache.inject_cache(self.ssh)
 745 +
 746          log('Inject our own astute.yaml and fuel_bootstrap_cli.yaml settings')
 747          self.inject_own_astute_and_bootstrap_yaml()
 748