1 From: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
2 Date: Thu, 24 Nov 2016 23:02:04 +0100
3 Subject: [PATCH] CI: deploy-cache: Store and reuse deploy artifacts
5 Add support for caching deploy artifacts, like bootstraps and
6 target images, which take a lot of time at each deploy to be built,
7 considering it requires a cross-debootstrap via qemu-user-static and
10 For OPNFV CI, the cache will piggy back on the <iso_mount> mechanism,
12 /iso_mount/opnfv_ci/<branch>/deploy-cache
14 TODO: Use dea interface adapter in target images fingerprinting.
15 TODO: remote fingerprinting
16 TODO: differentiate between bootstraps and targetimages, so we don't
17 end up trying to use one cache artifact type as the other.
18 TODO: implement sanity checks for bootstrap and target images;
19 TODO: switch `exec_cmd('mkdir ...')` to `create_dir_if_not_exists`;
23 Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
25 ...p_admin_node.sh-deploy_cache-install-hook.patch | 73 +++++
27 deploy/cloud/deployment.py | 12 +
28 deploy/deploy.py | 25 +-
29 deploy/deploy_cache.py | 312 +++++++++++++++++++++
30 deploy/deploy_env.py | 13 +-
31 deploy/install_fuel_master.py | 9 +-
32 7 files changed, 449 insertions(+), 9 deletions(-)
33 create mode 100644 build/f_repos/patch/fuel-main/0006-bootstrap_admin_node.sh-deploy_cache-install-hook.patch
34 create mode 100644 deploy/deploy_cache.py
36 diff --git a/build/f_repos/patch/fuel-main/0006-bootstrap_admin_node.sh-deploy_cache-install-hook.patch b/build/f_repos/patch/fuel-main/0006-bootstrap_admin_node.sh-deploy_cache-install-hook.patch
38 index 0000000..71eae61
40 +++ b/build/f_repos/patch/fuel-main/0006-bootstrap_admin_node.sh-deploy_cache-install-hook.patch
42 +From: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
43 +Date: Mon, 28 Nov 2016 14:27:48 +0100
44 +Subject: [PATCH] bootstrap_admin_node.sh: deploy_cache install hook
46 +Tooling on the automatic deploy side was updated to support deploy
47 +caching of artifacts like bootstrap (and id_rsa keypair), target
50 +Add installation hook that calls `fuel-bootstrap import` instead of
51 +`build` when a bootstrap tar is available in the agreed location,
52 +/var/lib/opnfv/cache/bootstraps/.
56 +Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
58 + iso/bootstrap_admin_node.sh | 20 +++++++++++++++++++-
59 + 1 file changed, 19 insertions(+), 1 deletion(-)
61 +diff --git a/iso/bootstrap_admin_node.sh b/iso/bootstrap_admin_node.sh
62 +index abc5ffb..15e6261 100755
63 +--- a/iso/bootstrap_admin_node.sh
64 ++++ b/iso/bootstrap_admin_node.sh
65 +@@ -61,6 +61,8 @@ wget \
66 + ASTUTE_YAML='/etc/fuel/astute.yaml'
67 + BOOTSTRAP_NODE_CONFIG="/etc/fuel/bootstrap_admin_node.conf"
68 + CUSTOM_REPOS="/root/default_deb_repos.yaml"
69 ++OPNFV_CACHE_PATH="/var/cache/opnfv/bootstraps"
70 ++OPNFV_CACHE_TAR="opnfv-bootstraps-cache.tar"
71 + bs_build_log='/var/log/fuel-bootstrap-image-build.log'
73 + # Backup network configs to this folder. Folder will be created only if
74 +@@ -94,6 +96,7 @@ image becomes available, reboot nodes that failed to be discovered."
75 + bs_done_message="Default bootstrap image building done. Now you can boot new \
76 + nodes over PXE, they will be discovered and become available for installing \
78 ++bs_cache_message="OPNFV deploy cache: bootstrap image injected."
79 + # Update issues messages
80 + update_warn_message="There is an issue connecting to update repository of \
81 + your distributions of OpenStack. \
82 +@@ -500,12 +503,31 @@ set_ui_bootstrap_error () {
86 ++function inject_cached_ubuntu_bootstrap () {
87 ++ if [ -f "${OPNFV_CACHE_PATH}/${OPNFV_CACHE_TAR}" -a \
88 ++ -f "${OPNFV_CACHE_PATH}/id_rsa.pub" -a \
89 ++ -f "${OPNFV_CACHE_PATH}/id_rsa" ]; then
90 ++ if cp "${OPNFV_CACHE_PATH}/id_rsa"* "/root/.ssh/" && \
91 ++ cp "/root/.ssh/id_rsa.pub" "/root/.ssh/authorized_keys" && \
92 ++ cp "/root/.ssh/id_rsa.pub" "/etc/cobbler/authorized_keys" && \
93 ++ sed -i -e "s|\"ssh-rsa .*\"|\"$(cat /root/.ssh/id_rsa.pub)\"|g" \
94 ++ /etc/nailgun/settings.yaml && \
95 ++ fuel-bootstrap -v --debug import --activate \
96 ++ "${OPNFV_CACHE_PATH}/${OPNFV_CACHE_TAR}" >>"$bs_build_log" 2>&1; then
97 ++ fuel notify --topic "done" --send "${bs_cache_message}"
104 + # Actually build the bootstrap image
105 + build_ubuntu_bootstrap () {
107 + echo ${bs_progress_message} >&2
108 + set_ui_bootstrap_error "${bs_progress_message}" >&2
109 +- if fuel-bootstrap -v --debug build --target_arch arm64 --activate >>"$bs_build_log" 2>&1; then
110 ++ if inject_cached_ubuntu_bootstrap || fuel-bootstrap -v --debug \
111 ++ build --activate --target_arch arm64 >>"$bs_build_log" 2>&1; then
113 + fuel notify --topic "done" --send "${bs_done_message}"
115 diff --git a/ci/deploy.sh b/ci/deploy.sh
116 index 081806c..4b1ae0e 100755
119 @@ -29,7 +29,7 @@ cat << EOF
120 xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
121 `basename $0`: Deploys the Fuel@OPNFV stack
123 -usage: `basename $0` -b base-uri [-B PXE Bridge] [-f] [-F] [-H] -l lab-name -p pod-name -s deploy-scenario [-S image-dir] [-T timeout] -i iso
124 +usage: `basename $0` -b base-uri [-B PXE Bridge] [-f] [-F] [-H] -l lab-name -p pod-name -s deploy-scenario [-S image-dir] [-C deploy-cache-dir] [-T timeout] -i iso
125 -s deployment-scenario [-S optional Deploy-scenario path URI]
126 [-R optional local relen repo (containing deployment Scenarios]
128 @@ -47,6 +47,7 @@ OPTIONS:
130 -s Deploy-scenario short-name/base-file-name
131 -S Storage dir for VM images
132 + -C Deploy cache dir for storing image artifacts
133 -T Timeout, in minutes, for the deploy.
136 @@ -79,6 +80,7 @@ Input parameters to the build script is:
137 or a deployment short-name as defined by scenario.yaml in the deployment
139 -S Storage dir for VM images, default is fuel/deploy/images
140 +-C Deploy cache dir for bootstrap and target image artifacts, optional
141 -T Timeout, in minutes, for the deploy. It defaults to using the DEPLOY_TIMEOUT
142 environment variable when defined, or to the default in deploy.py otherwise
143 -i .iso image to be deployed (needs to be provided in a URI
144 @@ -116,6 +118,7 @@ FUEL_CREATION_ONLY=''
145 NO_DEPLOY_ENVIRONMENT=''
149 if ! [ -z $DEPLOY_TIMEOUT ]; then
150 DEPLOY_TIMEOUT="-dt $DEPLOY_TIMEOUT"
152 @@ -128,7 +131,7 @@ fi
153 ############################################################################
156 -while getopts "b:B:dfFHl:L:p:s:S:T:i:he" OPTION
157 +while getopts "b:B:dfFHl:L:p:s:S:C:T:i:he" OPTION
161 @@ -179,6 +182,9 @@ do
162 STORAGE_DIR="-s ${OPTARG}"
166 + DEPLOY_CACHE_DIR="-dc ${OPTARG}"
169 DEPLOY_TIMEOUT="-dt ${OPTARG}"
171 @@ -243,8 +249,8 @@ if [ $DRY_RUN -eq 0 ]; then
172 ISO=${SCRIPT_PATH}/ISO/image.iso
175 - echo "python deploy.py $DEPLOY_LOG $STORAGE_DIR $PXE_BRIDGE $USE_EXISTING_FUEL $FUEL_CREATION_ONLY $NO_HEALTH_CHECK $NO_DEPLOY_ENVIRONMENT -dea ${SCRIPT_PATH}/config/dea.yaml -dha ${SCRIPT_PATH}/config/dha.yaml -iso $ISO $DEPLOY_TIMEOUT"
176 - python deploy.py $DEPLOY_LOG $STORAGE_DIR $PXE_BRIDGE $USE_EXISTING_FUEL $FUEL_CREATION_ONLY $NO_HEALTH_CHECK $NO_DEPLOY_ENVIRONMENT -dea ${SCRIPT_PATH}/config/dea.yaml -dha ${SCRIPT_PATH}/config/dha.yaml -iso $ISO $DEPLOY_TIMEOUT
177 + echo "python deploy.py $DEPLOY_LOG $STORAGE_DIR $PXE_BRIDGE $USE_EXISTING_FUEL $FUEL_CREATION_ONLY $NO_HEALTH_CHECK $NO_DEPLOY_ENVIRONMENT -dea ${SCRIPT_PATH}/config/dea.yaml -dha ${SCRIPT_PATH}/config/dha.yaml -iso $ISO $DEPLOY_TIMEOUT $DEPLOY_CACHE_DIR"
178 + python deploy.py $DEPLOY_LOG $STORAGE_DIR $PXE_BRIDGE $USE_EXISTING_FUEL $FUEL_CREATION_ONLY $NO_HEALTH_CHECK $NO_DEPLOY_ENVIRONMENT -dea ${SCRIPT_PATH}/config/dea.yaml -dha ${SCRIPT_PATH}/config/dha.yaml -iso $ISO $DEPLOY_TIMEOUT $DEPLOY_CACHE_DIR
182 diff --git a/deploy/cloud/deployment.py b/deploy/cloud/deployment.py
183 index 28bcfdf..b0bfdcc 100644
184 --- a/deploy/cloud/deployment.py
185 +++ b/deploy/cloud/deployment.py
186 @@ -19,6 +19,8 @@ from common import (
190 +from deploy_cache import DeployCache
192 SEARCH_TEXT = '(err)'
193 LOG_FILE = '/var/log/puppet.log'
194 GREP_LINES_OF_LEADING_CONTEXT = 100
195 @@ -47,6 +49,14 @@ class Deployment(object):
196 self.pattern = re.compile(
197 '\d\d\d\d-\d\d-\d\d\s\d\d:\d\d:\d\d')
199 + def deploy_cache_install_targetimages(self):
200 + log('Using target images from deploy cache')
201 + DeployCache.install_targetimages_for_env(self.env_id)
203 + def deploy_cache_extract_targetimages(self):
204 + log('Collecting Fuel target image files for deploy cache')
205 + DeployCache.extract_targetimages_from_env(self.env_id)
207 def collect_error_logs(self):
208 for node_id, roles_blade in self.node_id_roles_dict.iteritems():
210 @@ -108,6 +118,7 @@ class Deployment(object):
213 log('Starting deployment of environment %s' % self.env_id)
214 + self.deploy_cache_install_targetimages()
218 @@ -140,6 +151,7 @@ class Deployment(object):
219 err('Deployment timed out, environment %s is not operational, '
220 'snapshot will not be performed'
222 + self.deploy_cache_extract_targetimages()
224 log('Environment %s successfully deployed'
226 diff --git a/deploy/deploy.py b/deploy/deploy.py
227 index 7648baf..ee3cb7a 100755
228 --- a/deploy/deploy.py
229 +++ b/deploy/deploy.py
230 @@ -22,6 +22,7 @@ from dea import DeploymentEnvironmentAdapter
231 from dha import DeploymentHardwareAdapter
232 from install_fuel_master import InstallFuelMaster
233 from deploy_env import CloudDeploy
234 +from deploy_cache import DeployCache
235 from execution_environment import ExecutionEnvironment
238 @@ -61,7 +62,8 @@ class AutoDeploy(object):
239 def __init__(self, no_fuel, fuel_only, no_health_check, cleanup_only,
240 cleanup, storage_dir, pxe_bridge, iso_file, dea_file,
241 dha_file, fuel_plugins_dir, fuel_plugins_conf_dir,
242 - no_plugins, deploy_timeout, no_deploy_environment, deploy_log):
243 + no_plugins, deploy_cache_dir, deploy_timeout,
244 + no_deploy_environment, deploy_log):
245 self.no_fuel = no_fuel
246 self.fuel_only = fuel_only
247 self.no_health_check = no_health_check
248 @@ -75,6 +77,7 @@ class AutoDeploy(object):
249 self.fuel_plugins_dir = fuel_plugins_dir
250 self.fuel_plugins_conf_dir = fuel_plugins_conf_dir
251 self.no_plugins = no_plugins
252 + self.deploy_cache_dir = deploy_cache_dir
253 self.deploy_timeout = deploy_timeout
254 self.no_deploy_environment = no_deploy_environment
255 self.deploy_log = deploy_log
256 @@ -116,7 +119,7 @@ class AutoDeploy(object):
257 self.fuel_username, self.fuel_password,
258 self.dea_file, self.fuel_plugins_conf_dir,
259 WORK_DIR, self.no_health_check,
260 - self.deploy_timeout,
261 + self.deploy_cache_dir, self.deploy_timeout,
262 self.no_deploy_environment, self.deploy_log)
264 old_dep.check_previous_installation()
265 @@ -128,6 +131,7 @@ class AutoDeploy(object):
266 self.fuel_conf['ip'], self.fuel_username,
267 self.fuel_password, self.fuel_node_id,
268 self.iso_file, WORK_DIR,
269 + self.deploy_cache_dir,
270 self.fuel_plugins_dir, self.no_plugins)
273 @@ -136,6 +140,7 @@ class AutoDeploy(object):
274 tmp_new_dir = '%s/newiso' % self.tmp_dir
276 self.copy(tmp_orig_dir, tmp_new_dir)
277 + self.deploy_cache_fingerprints(tmp_new_dir)
278 self.patch(tmp_new_dir, new_iso)
279 except Exception as e:
280 exec_cmd('fusermount -u %s' % tmp_orig_dir, False)
281 @@ -156,6 +161,12 @@ class AutoDeploy(object):
283 exec_cmd('chmod -R 755 %s' % tmp_new_dir)
285 + def deploy_cache_fingerprints(self, tmp_new_dir):
286 + if self.deploy_cache_dir:
287 + log('Deploy cache: Collecting fingerprints...')
288 + deploy_cache = DeployCache(self.deploy_cache_dir)
289 + deploy_cache.do_fingerprints(tmp_new_dir, self.dea_file)
291 def patch(self, tmp_new_dir, new_iso):
293 patch_dir = '%s/%s' % (CWD, PATCH_DIR)
294 @@ -218,7 +229,8 @@ class AutoDeploy(object):
295 dep = CloudDeploy(self.dea, self.dha, self.fuel_conf['ip'],
296 self.fuel_username, self.fuel_password,
297 self.dea_file, self.fuel_plugins_conf_dir,
298 - WORK_DIR, self.no_health_check, self.deploy_timeout,
299 + WORK_DIR, self.no_health_check,
300 + self.deploy_cache_dir, self.deploy_timeout,
301 self.no_deploy_environment, self.deploy_log)
304 @@ -343,6 +355,8 @@ def parse_arguments():
305 help='Fuel Plugins Configuration directory')
306 parser.add_argument('-np', dest='no_plugins', action='store_true',
307 default=False, help='Do not install Fuel Plugins')
308 + parser.add_argument('-dc', dest='deploy_cache_dir', action='store',
309 + help='Deploy Cache Directory')
310 parser.add_argument('-dt', dest='deploy_timeout', action='store',
311 default=240, help='Deployment timeout (in minutes) '
313 @@ -376,6 +390,10 @@ def parse_arguments():
314 for bridge in args.pxe_bridge:
315 check_bridge(bridge, args.dha_file)
317 + if args.deploy_cache_dir:
318 + log('Using deploy cache directory: %s' % args.deploy_cache_dir)
319 + create_dir_if_not_exists(args.deploy_cache_dir)
322 kwargs = {'no_fuel': args.no_fuel, 'fuel_only': args.fuel_only,
323 'no_health_check': args.no_health_check,
324 @@ -386,6 +404,7 @@ def parse_arguments():
325 'fuel_plugins_dir': args.fuel_plugins_dir,
326 'fuel_plugins_conf_dir': args.fuel_plugins_conf_dir,
327 'no_plugins': args.no_plugins,
328 + 'deploy_cache_dir': args.deploy_cache_dir,
329 'deploy_timeout': args.deploy_timeout,
330 'no_deploy_environment': args.no_deploy_environment,
331 'deploy_log': args.deploy_log}
332 diff --git a/deploy/deploy_cache.py b/deploy/deploy_cache.py
334 index 0000000..7df43c6
336 +++ b/deploy/deploy_cache.py
338 +###############################################################################
339 +# Copyright (c) 2016 Enea AB and others.
340 +# Alexandru.Avadanii@enea.com
341 +# All rights reserved. This program and the accompanying materials
342 +# are made available under the terms of the Apache License, Version 2.0
343 +# which accompanies this distribution, and is available at
344 +# http://www.apache.org/licenses/LICENSE-2.0
345 +###############################################################################
355 +from common import (
360 +###############################################################################
361 +# Deploy Cache Flow Overview
362 +###############################################################################
363 +# 1. do_fingerprints
364 +# Can be called as soon as a Fuel Master ISO chroot is available.
365 +# This will gather all required information for uniquely identifying the
366 +# objects in cache (bootstraps, targetimages).
368 +# Can be called as soon as we have a steady SSH connection to the Fuel
369 +# Master node. It will inject cached artifacts over SSH, for later install.
370 +# 3. (external, async) install cached bootstrap instead of building a new one
371 +# /sbin/bootstrap_admin_node.sh will check for cached bootstrap images
372 +# (with id_rsa, id_rsa.pub attached) and will install those via
373 +# $ fuel-bootstrap import opfnv-bootstraps-cache.tar
374 +# 4. install_targetimages_for_env
375 +# Should be called before cloud deploy is started, to install env-generic
376 +# 'env_X_...' cached images for the current environment ID.
377 +# Static method, to be used on the remote Fuel Master node; does not require
378 +# access to the deploy cache, it only moves around some local files.
379 +# 5. extract_targetimages_from_env
380 +# Should be called at env deploy finish, to prepare artifacts for caching.
381 +# Static method, same observations as above apply.
382 +# 6. collect_artifacts
383 +# Call last, to collect all artifacts.
384 +###############################################################################
386 +###############################################################################
387 +# Deploy cache artifacts:
389 +# - bootstrap image (Ubuntu)
390 +# - environment target image (Ubuntu)
391 +###############################################################################
392 +# Cache fingerprint covers:
394 +# - local mirror contents
395 +# - package list (and everything else in fuel_bootstrap_cli.yaml)
397 +# - local mirror contents
398 +# - package list (determined from DEA)
399 +###############################################################################
400 +# WARN: Cache fingerprint does NOT yet cover:
401 +# - image_data (always assume the default /boot, /);
402 +# - output_dir (always assume the default /var/www/nailgun/targetimages;
403 +# - codename (always assume the default, currently 'trusty');
404 +# - extra_dirs: /usr/share/fuel_bootstrap_cli/files/trusty
405 +# - root_ssh_authorized_file, inluding the contents of /root/.ssh/id_rsa.pub
406 +# - Auxiliary repo .../mitaka-9.0/ubuntu/auxiliary
407 +# If the above change without triggering a cache miss, try clearing the cache.
408 +###############################################################################
409 +# WARN: Bootstrap caching implies RSA keypair to be reused!
410 +###############################################################################
412 +# Local mirrros will be used on Fuel Master for both bootstrap and target image
413 +# build, from `http://127.0.0.1:8080/...` or `http://10.20.0.2:8080/...`:
414 +# - MOS .../mitaka-9.0/ubuntu/x86_64
415 +# - Ubuntu .../mirrors/ubuntu/
416 +# All these reside on Fuel Master at local path:
417 +NAILGUN_PATH = '/var/www/nailgun/'
419 +# Artifact names (corresponding to nailgun subdirs)
421 +BOOTSTRAPS = 'bootstraps'
422 +TARGETIMAGES = 'targetimages'
424 +# Info for collecting RSA keypair
425 +RSA_KEYPAIR_PATH = '/root/.ssh'
426 +RSA_KEYPAIR_FILES = ['id_rsa', 'id_rsa.pub']
428 +# Relative path for collecting the active bootstrap image(s) after env deploy
429 +NAILGUN_ACT_BOOTSTRAP_SUBDIR = '%s/active_bootstrap' % BOOTSTRAPS
431 +# Relative path for collecting target image(s) for deployed enviroment
432 +NAILGUN_TIMAGES_SUBDIR = TARGETIMAGES
434 +# OPNFV Fuel bootstrap settings file that will be injected at deploy
435 +ISO_BOOTSTRAP_CLI_YAML = '/opnfv/fuel_bootstrap_cli.yaml'
437 +# OPNFV Deploy Cache path on Fuel Master, where artifacts will be injected
438 +REMOTE_CACHE_PATH = '/var/cache/opnfv'
440 +# OPNFV Bootstrap Cache tar archive name, to be used by bootstrap_admin_node.sh
441 +BOOTSTRAP_ARCHIVE = 'opnfv-bootstraps-cache.tar'
443 +# Env-ID indep prefix
446 +class DeployCache(object):
447 + """OPNFV Deploy Cache - managed storage for cacheable artifacts"""
449 + def __init__(self, cache_dir,
450 + fingerprints_yaml='deploy_cache_fingerprints.yaml'):
451 + self.cache_dir = cache_dir
452 + self.fingerprints_yaml = fingerprints_yaml
453 + self.fingerprints = {BOOTSTRAPS: None,
455 + TARGETIMAGES: None}
457 + def __load_fingerprints(self):
458 + """Load deploy cache yaml config holding fingerprints"""
459 + if os.path.isfile(self.fingerprints_yaml):
460 + cache_fingerprints = open(self.fingerprints_yaml).read()
461 + self.fingerprints = yaml.load(cache_fingerprints)
463 + def __save_fingerprints(self):
464 + """Update deploy cache yaml config holding fingerprints"""
465 + with open(self.fingerprints_yaml, 'w') as outfile:
466 + outfile.write(yaml.safe_dump(self.fingerprints,
467 + default_flow_style=False))
469 + def __fingerprint_mirrors(self, chroot_path):
470 + """Collect repo mirror fingerprints"""
471 + deb_packages = list()
472 + # Scan ISO for deb files (MOS mirror + Ubuntu mirror, no plugins)
473 + for repo_dir in ['ubuntu', 'opnfv/nailgun/mirrors/ubuntu']:
474 + for _, _, files in os.walk(os.path.join(chroot_path, repo_dir)):
476 + if fdeb.endswith(".deb"):
477 + deb_packages.append(fdeb)
478 + sorted_debs = json.dumps(deb_packages, sort_keys=True)
479 + self.fingerprints[MIRRORS] = hashlib.sha1(sorted_debs).hexdigest()
481 + def __fingerprint_bootstrap(self, chroot_path):
482 + """Collect bootstrap image metadata fingerprints"""
483 + # FIXME(armband): include 'extra_dirs' contents
484 + cli_yaml_path = os.path.join(chroot_path, ISO_BOOTSTRAP_CLI_YAML[1:])
485 + bootstrap_cli_yaml = open(cli_yaml_path).read()
486 + bootstrap_data = yaml.load(bootstrap_cli_yaml)
487 + sorted_data = json.dumps(bootstrap_data, sort_keys=True)
488 + self.fingerprints[BOOTSTRAPS] = hashlib.sha1(sorted_data).hexdigest()
490 + def __fingerprint_target(self, dea_file):
491 + """Collect target image metadata fingerprints"""
492 + # FIXME(armband): include 'image_data', 'codename', 'output'
493 + with io.open(dea_file) as stream:
494 + dea = yaml.load(stream)
495 + editable = dea['settings']['editable']
496 + target_data = {'packages': editable['provision']['packages'],
497 + 'repos': editable['repo_setup']['repos']}
498 + s_data = json.dumps(target_data, sort_keys=True)
499 + self.fingerprints[TARGETIMAGES] = hashlib.sha1(s_data).hexdigest()
501 + def do_fingerprints(self, chroot_path, dea_file):
502 + """Collect SHA1 fingerprints based on chroot contents, DEA settings"""
504 + self.__load_fingerprints()
505 + self.__fingerprint_mirrors(chroot_path)
506 + self.__fingerprint_bootstrap(chroot_path)
507 + self.__fingerprint_target(dea_file)
508 + self.__save_fingerprints()
509 + except Exception as ex:
510 + log('Failed to get cache fingerprint: %s' % str(ex))
512 + def __lookup_cache(self, sha):
513 + """Search for object in cache based on SHA fingerprint"""
514 + cache_sha_dir = os.path.join(self.cache_dir, sha)
515 + if not os.path.isdir(cache_sha_dir) or not os.listdir(cache_sha_dir):
517 + return cache_sha_dir
519 + def __inject_cache_dir(self, ssh, sha, artifact):
520 + """Stage cached object (dir) in Fuel Master OPNFV local cache"""
521 + local_path = self.__lookup_cache(sha)
523 + remote_path = os.path.join(REMOTE_CACHE_PATH, artifact)
525 + ssh.exec_cmd('mkdir -p %s' % remote_path)
526 + for cachedfile in glob.glob('%s/*' % local_path):
527 + ssh.scp_put(cachedfile, remote_path)
530 + def __mix_fingerprints(self, f1, f2):
531 + """Compute composite fingerprint"""
532 + if self.fingerprints[f1] is None or self.fingerprints[f2] is None:
534 + return hashlib.sha1('%s%s' %
535 + (self.fingerprints[f1], self.fingerprints[f2])).hexdigest()
537 + def inject_cache(self, ssh):
538 + """Lookup artifacts in cache and inject them over SSH/SCP into Fuel"""
540 + self.__load_fingerprints()
541 + for artifact in [BOOTSTRAPS, TARGETIMAGES]:
542 + sha = self.__mix_fingerprints(MIRRORS, artifact)
544 + log('Missing fingerprint for: %s' % artifact)
546 + if not self.__inject_cache_dir(ssh, sha, artifact):
547 + log('SHA1 not in cache: %s (%s)' % (str(sha), artifact))
549 + log('SHA1 injected: %s (%s)' % (str(sha), artifact))
550 + except Exception as ex:
551 + log('Failed to inject cached artifacts into Fuel: %s' % str(ex))
553 + def __extract_bootstraps(self, ssh, cache_sha_dir):
554 + """Collect bootstrap artifacts from Fuel over SSH/SCP"""
555 + remote_tar = os.path.join(REMOTE_CACHE_PATH, BOOTSTRAP_ARCHIVE)
556 + local_tar = os.path.join(cache_sha_dir, BOOTSTRAP_ARCHIVE)
558 + for k in RSA_KEYPAIR_FILES:
559 + ssh.scp_get(os.path.join(RSA_KEYPAIR_PATH, k),
560 + local=os.path.join(cache_sha_dir, k))
561 + ssh.exec_cmd('mkdir -p %s && cd %s && tar cf %s *' %
562 + (REMOTE_CACHE_PATH,
563 + os.path.join(NAILGUN_PATH, NAILGUN_ACT_BOOTSTRAP_SUBDIR),
565 + ssh.scp_get(remote_tar, local=local_tar)
566 + ssh.exec_cmd('rm -f %s' % remote_tar)
568 + def __extract_targetimages(self, ssh, cache_sha_dir):
569 + """Collect target image artifacts from Fuel over SSH/SCP"""
570 + cti_path = os.path.join(REMOTE_CACHE_PATH, TARGETIMAGES)
572 + ssh.scp_get('%s/%s*' % (cti_path, ENVX), local=cache_sha_dir)
574 + def collect_artifacts(self, ssh):
575 + """Collect artifacts from Fuel over SSH/SCP and add them to cache"""
577 + self.__load_fingerprints()
578 + for artifact, func in {
579 + BOOTSTRAPS: self.__extract_bootstraps,
580 + TARGETIMAGES: self.__extract_targetimages
582 + sha = self.__mix_fingerprints(MIRRORS, artifact)
584 + log('WARN: Skip caching, NO fingerprint: %s' % artifact)
586 + local_path = self.__lookup_cache(sha)
588 + log('SHA1 already in cache: %s (%s)' % (str(sha), artifact))
590 + log('New cache SHA1: %s (%s)' % (str(sha), artifact))
591 + cache_sha_dir = os.path.join(self.cache_dir, sha)
592 + exec_cmd('mkdir -p %s' % cache_sha_dir)
593 + func(ssh, cache_sha_dir)
594 + except Exception as ex:
595 + log('Failed to extract artifacts from Fuel: %s' % str(ex))
598 + def extract_targetimages_from_env(env_id):
599 + """Prepare targetimages from env ID for storage in deploy cache
601 + NOTE: This method should be executed locally ON the Fuel Master node.
602 + WARN: This method overwrites targetimages cache on Fuel Master node.
604 + env_n = 'env_%s_' % str(env_id)
605 + cti_path = os.path.join(REMOTE_CACHE_PATH, TARGETIMAGES)
606 + ti_path = os.path.join(NAILGUN_PATH, NAILGUN_TIMAGES_SUBDIR)
608 + exec_cmd('rm -rf %s && mkdir -p %s' % (cti_path, cti_path))
609 + for root, _, files in os.walk(ti_path):
611 + if tif.startswith(env_n):
612 + src = os.path.join(root, tif)
613 + dest = os.path.join(cti_path, tif.replace(env_n, ENVX))
614 + if tif.endswith('.yaml'):
615 + shutil.copy(src, dest)
616 + exec_cmd('sed -i "s|%s|%s|g" %s' %
617 + (env_n, ENVX, dest))
620 + except Exception as ex:
621 + log('Failed to extract targetimages artifacts from env %s: %s' %
622 + (str(env_id), str(ex)))
625 + def install_targetimages_for_env(env_id):
626 + """Install targetimages artifacts for a specific env ID
628 + NOTE: This method should be executed locally ON the Fuel Master node.
630 + env_n = 'env_%s_' % str(env_id)
631 + cti_path = os.path.join(REMOTE_CACHE_PATH, TARGETIMAGES)
632 + ti_path = os.path.join(NAILGUN_PATH, NAILGUN_TIMAGES_SUBDIR)
633 + if not os.path.isdir(cti_path):
634 + log('%s cache dir not found: %s' % (TARGETIMAGES, cti_path))
637 + for root, _, files in os.walk(cti_path):
639 + src = os.path.join(root, tif)
640 + dest = os.path.join(ti_path, tif.replace(ENVX, env_n))
641 + if tif.endswith('.yaml'):
642 + shutil.copy(src, dest)
643 + exec_cmd('sed -i "s|%s|%s|g" %s' %
644 + (ENVX, env_n, dest))
647 + except Exception as ex:
648 + log('Failed to install targetimages for env %s: %s' %
649 + (str(env_id), str(ex)))
650 diff --git a/deploy/deploy_env.py b/deploy/deploy_env.py
651 index aa8c4cb..e9c50bb 100644
652 --- a/deploy/deploy_env.py
653 +++ b/deploy/deploy_env.py
654 @@ -15,6 +15,7 @@ import glob
658 +from deploy_cache import DeployCache
659 from ssh_client import SSHClient
662 @@ -35,7 +36,8 @@ class CloudDeploy(object):
664 def __init__(self, dea, dha, fuel_ip, fuel_username, fuel_password,
665 dea_file, fuel_plugins_conf_dir, work_dir, no_health_check,
666 - deploy_timeout, no_deploy_environment, deploy_log):
667 + deploy_cache_dir, deploy_timeout,
668 + no_deploy_environment, deploy_log):
671 self.fuel_ip = fuel_ip
672 @@ -49,6 +51,8 @@ class CloudDeploy(object):
673 self.fuel_plugins_conf_dir = fuel_plugins_conf_dir
674 self.work_dir = work_dir
675 self.no_health_check = no_health_check
676 + self.deploy_cache = ( DeployCache(deploy_cache_dir)
677 + if deploy_cache_dir else None )
678 self.deploy_timeout = deploy_timeout
679 self.no_deploy_environment = no_deploy_environment
680 self.deploy_log = deploy_log
681 @@ -82,9 +86,14 @@ class CloudDeploy(object):
682 self.work_dir, os.path.basename(self.dea_file)))
683 s.scp_put('%s/common.py' % self.file_dir, self.work_dir)
684 s.scp_put('%s/dea.py' % self.file_dir, self.work_dir)
685 + s.scp_put('%s/deploy_cache.py' % self.file_dir, self.work_dir)
686 for f in glob.glob('%s/cloud/*' % self.file_dir):
687 s.scp_put(f, self.work_dir)
689 + def deploy_cache_collect_artifacts(self):
690 + if self.deploy_cache:
691 + self.deploy_cache.collect_artifacts(self.ssh)
693 def power_off_nodes(self):
694 for node_id in self.node_ids:
695 self.dha.node_power_off(node_id)
696 @@ -283,4 +292,6 @@ class CloudDeploy(object):
698 self.get_put_deploy_log()
700 + self.deploy_cache_collect_artifacts()
703 diff --git a/deploy/install_fuel_master.py b/deploy/install_fuel_master.py
704 index b731c6b..83d31fb 100644
705 --- a/deploy/install_fuel_master.py
706 +++ b/deploy/install_fuel_master.py
711 +from deploy_cache import DeployCache
712 from ssh_client import SSHClient
713 from dha_adapters.libvirt_adapter import LibvirtAdapter
715 @@ -32,7 +33,7 @@ class InstallFuelMaster(object):
717 def __init__(self, dea_file, dha_file, fuel_ip, fuel_username,
718 fuel_password, fuel_node_id, iso_file, work_dir,
719 - fuel_plugins_dir, no_plugins):
720 + deploy_cache_dir, fuel_plugins_dir, no_plugins):
721 self.dea_file = dea_file
722 self.dha = LibvirtAdapter(dha_file)
723 self.fuel_ip = fuel_ip
724 @@ -42,6 +43,8 @@ class InstallFuelMaster(object):
725 self.iso_file = iso_file
726 self.iso_dir = os.path.dirname(self.iso_file)
727 self.work_dir = work_dir
728 + self.deploy_cache = ( DeployCache(deploy_cache_dir)
729 + if deploy_cache_dir else None )
730 self.fuel_plugins_dir = fuel_plugins_dir
731 self.no_plugins = no_plugins
732 self.file_dir = os.path.dirname(os.path.realpath(__file__))
733 @@ -83,6 +86,10 @@ class InstallFuelMaster(object):
734 log('Wait until Fuel menu is up')
735 fuel_menu_pid = self.wait_until_fuel_menu_up()
737 + if self.deploy_cache:
738 + log('Deploy cache: Injecting bootstraps and targetimages')
739 + self.deploy_cache.inject_cache(self.ssh)
741 log('Inject our own astute.yaml and fuel_bootstrap_cli.yaml settings')
742 self.inject_own_astute_and_bootstrap_yaml()