X-Git-Url: https://gerrit.opnfv.org/gerrit/gitweb?a=blobdiff_plain;f=patches%2Fopnfv-fuel%2Fupstream-backports%2F0005-CI-deploy-cache-Store-and-reuse-deploy-artifacts.patch;h=da737cd58cac48f7cbc553556164ee278666a206;hb=2e922579664feeadf722ce11d4f8b92ffffd3c0c;hp=0a20fc3ab29169d40999f26466ab7388b2d8109a;hpb=7e728d2db0fc9c9ada9aeb125947ec367473425d;p=armband.git diff --git a/patches/opnfv-fuel/upstream-backports/0005-CI-deploy-cache-Store-and-reuse-deploy-artifacts.patch b/patches/opnfv-fuel/upstream-backports/0005-CI-deploy-cache-Store-and-reuse-deploy-artifacts.patch index 0a20fc3a..da737cd5 100644 --- a/patches/opnfv-fuel/upstream-backports/0005-CI-deploy-cache-Store-and-reuse-deploy-artifacts.patch +++ b/patches/opnfv-fuel/upstream-backports/0005-CI-deploy-cache-Store-and-reuse-deploy-artifacts.patch @@ -15,28 +15,31 @@ TODO: Use dea interface adapter in target images fingerprinting. TODO: remote fingerprinting TODO: differentiate between bootstraps and targetimages, so we don't end up trying to use one cache artifact type as the other. +TODO: implement sanity checks for bootstrap and target images; +TODO: switch `exec_cmd('mkdir ...')` to `create_dir_if_not_exists`; JIRA: ARMBAND-172 Signed-off-by: Alexandru Avadanii --- - ...p_admin_node.sh-deploy_cache-install-hook.patch | 69 +++++ + ...p_admin_node.sh-deploy_cache-install-hook.patch | 73 +++++ ci/deploy.sh | 14 +- - deploy/cloud/deploy.py | 11 + + deploy/cloud/configure_settings.py | 4 + + deploy/cloud/deployment.py | 12 + deploy/deploy.py | 25 +- - deploy/deploy_cache.py | 319 +++++++++++++++++++++ + deploy/deploy_cache.py | 312 +++++++++++++++++++++ deploy/deploy_env.py | 13 +- deploy/install_fuel_master.py | 9 +- - 7 files changed, 451 insertions(+), 9 deletions(-) + 8 files changed, 453 insertions(+), 9 deletions(-) create mode 100644 build/f_repos/patch/fuel-main/0006-bootstrap_admin_node.sh-deploy_cache-install-hook.patch create mode 100644 deploy/deploy_cache.py diff --git a/build/f_repos/patch/fuel-main/0006-bootstrap_admin_node.sh-deploy_cache-install-hook.patch b/build/f_repos/patch/fuel-main/0006-bootstrap_admin_node.sh-deploy_cache-install-hook.patch new file mode 100644 -index 0000000..d5b7646 +index 0000000..69a4c22 --- /dev/null +++ b/build/f_repos/patch/fuel-main/0006-bootstrap_admin_node.sh-deploy_cache-install-hook.patch -@@ -0,0 +1,69 @@ +@@ -0,0 +1,73 @@ +From: Alexandru Avadanii +Date: Mon, 28 Nov 2016 14:27:48 +0100 +Subject: [PATCH] bootstrap_admin_node.sh: deploy_cache install hook @@ -61,10 +64,10 @@ index 0000000..d5b7646 +--- a/iso/bootstrap_admin_node.sh ++++ b/iso/bootstrap_admin_node.sh +@@ -61,6 +61,8 @@ wget \ -+ + ASTUTE_YAML='/etc/fuel/astute.yaml' + BOOTSTRAP_NODE_CONFIG="/etc/fuel/bootstrap_admin_node.conf" -++OPNFV_CACHE_PATH="/var/lib/opnfv/cache/bootstraps" ++ CUSTOM_REPOS="/root/default_deb_repos.yaml" +++OPNFV_CACHE_PATH="/var/cache/opnfv/bootstraps" ++OPNFV_CACHE_TAR="opnfv-bootstraps-cache.tar" + bs_build_log='/var/log/fuel-bootstrap-image-build.log' + bs_status=0 @@ -77,7 +80,7 @@ index 0000000..d5b7646 + # Update issues messages + update_warn_message="There is an issue connecting to update repository of \ + your distributions of OpenStack. \ -+@@ -500,12 +503,27 @@ set_ui_bootstrap_error () { ++@@ -500,12 +503,31 @@ set_ui_bootstrap_error () { + EOF + } + @@ -85,11 +88,15 @@ index 0000000..d5b7646 ++ if [ -f "${OPNFV_CACHE_PATH}/${OPNFV_CACHE_TAR}" -a \ ++ -f "${OPNFV_CACHE_PATH}/id_rsa.pub" -a \ ++ -f "${OPNFV_CACHE_PATH}/id_rsa" ]; then -++ if cp "${OPNFV_CACHE_PATH}/id_rsa{,.pub}" "~/.ssh/" && \ +++ if cp "${OPNFV_CACHE_PATH}/id_rsa"* "/root/.ssh/" && \ +++ cp "/root/.ssh/id_rsa.pub" "/root/.ssh/authorized_keys" && \ +++ cp "/root/.ssh/id_rsa.pub" "/etc/cobbler/authorized_keys" && \ +++ sed -i -e "s|\"ssh-rsa .*\"|\"$(cat /root/.ssh/id_rsa.pub)\"|g" \ +++ /etc/nailgun/settings.yaml && \ ++ fuel-bootstrap -v --debug import --activate \ ++ "${OPNFV_CACHE_PATH}/${OPNFV_CACHE_TAR}" >>"$bs_build_log" 2>&1; then -++ fuel notify --topic "done" --send "${bs_cache_message}" -++ return 0 +++ fuel notify --topic "done" --send "${bs_cache_message}" +++ return 0 ++ fi ++ fi ++ return 1 @@ -173,49 +180,69 @@ index 081806c..4b1ae0e 100755 fi popd > /dev/null -diff --git a/deploy/cloud/deploy.py b/deploy/cloud/deploy.py -index e00934b..b39e5fc 100644 ---- a/deploy/cloud/deploy.py -+++ b/deploy/cloud/deploy.py -@@ -14,6 +14,7 @@ import io - from dea import DeploymentEnvironmentAdapter - from configure_environment import ConfigureEnvironment - from deployment import Deployment -+from deploy_cache import DeployCache +diff --git a/deploy/cloud/configure_settings.py b/deploy/cloud/configure_settings.py +index b60a60f..4e007e1 100644 +--- a/deploy/cloud/configure_settings.py ++++ b/deploy/cloud/configure_settings.py +@@ -71,5 +71,9 @@ class ConfigureSettings(object): + settings['editable'][plugin]['metadata']['chosen_id'] = orig_dea['editable'][plugin]['metadata']['chosen_id'] + settings['editable'][plugin]['metadata']['versions'][0]['metadata']['plugin_id'] = orig_dea['editable'][plugin]['metadata']['versions'][0]['metadata']['plugin_id'] + ++ # deploy-cache req: pass master id_rsa.pub as authorized key ++ with io.open('/root/.ssh/id_rsa.pub', 'r') as pkey: ++ settings['editable']['operator_user']['authkeys']['value'] = pkey.read() ++ + with io.open(settings_yaml, 'w') as stream: + yaml.dump(settings, stream, default_flow_style=False) +diff --git a/deploy/cloud/deployment.py b/deploy/cloud/deployment.py +index 28bcfdf..b0bfdcc 100644 +--- a/deploy/cloud/deployment.py ++++ b/deploy/cloud/deployment.py +@@ -19,6 +19,8 @@ from common import ( + log, + ) - from common import ( - R, -@@ -61,6 +62,12 @@ class Deploy(object): - config_env.configure_environment() - self.env_id = config_env.env_id ++from deploy_cache import DeployCache ++ + SEARCH_TEXT = '(err)' + LOG_FILE = '/var/log/puppet.log' + GREP_LINES_OF_LEADING_CONTEXT = 100 +@@ -47,6 +49,14 @@ class Deployment(object): + self.pattern = re.compile( + '\d\d\d\d-\d\d-\d\d\s\d\d:\d\d:\d\d') + def deploy_cache_install_targetimages(self): ++ log('Using target images from deploy cache') + DeployCache.install_targetimages_for_env(self.env_id) + + def deploy_cache_extract_targetimages(self): ++ log('Collecting Fuel target image files for deploy cache') + DeployCache.extract_targetimages_from_env(self.env_id) + - def deploy_cloud(self): - dep = Deployment(self.dea, YAML_CONF_DIR, self.env_id, - self.node_roles_dict, self.no_health_check, -@@ -76,8 +83,12 @@ class Deploy(object): - - self.configure_environment() + def collect_error_logs(self): + for node_id, roles_blade in self.node_id_roles_dict.iteritems(): + log_list = [] +@@ -108,6 +118,7 @@ class Deployment(object): + start = time.time() + log('Starting deployment of environment %s' % self.env_id) + self.deploy_cache_install_targetimages() -+ - self.deploy_cloud() - + deploy_id = None + ready = False + timeout = False +@@ -140,6 +151,7 @@ class Deployment(object): + err('Deployment timed out, environment %s is not operational, ' + 'snapshot will not be performed' + % self.env_id) + self.deploy_cache_extract_targetimages() -+ - - def parse_arguments(): - parser = ArgParser(prog='python %s' % __file__) + if ready: + log('Environment %s successfully deployed' + % self.env_id) diff --git a/deploy/deploy.py b/deploy/deploy.py -index 08702d2..1a55361 100755 +index 7648baf..ee3cb7a 100755 --- a/deploy/deploy.py +++ b/deploy/deploy.py -@@ -23,6 +23,7 @@ from dea import DeploymentEnvironmentAdapter +@@ -22,6 +22,7 @@ from dea import DeploymentEnvironmentAdapter from dha import DeploymentHardwareAdapter from install_fuel_master import InstallFuelMaster from deploy_env import CloudDeploy @@ -223,7 +250,7 @@ index 08702d2..1a55361 100755 from execution_environment import ExecutionEnvironment from common import ( -@@ -62,7 +63,8 @@ class AutoDeploy(object): +@@ -61,7 +62,8 @@ class AutoDeploy(object): def __init__(self, no_fuel, fuel_only, no_health_check, cleanup_only, cleanup, storage_dir, pxe_bridge, iso_file, dea_file, dha_file, fuel_plugins_dir, fuel_plugins_conf_dir, @@ -233,7 +260,7 @@ index 08702d2..1a55361 100755 self.no_fuel = no_fuel self.fuel_only = fuel_only self.no_health_check = no_health_check -@@ -76,6 +78,7 @@ class AutoDeploy(object): +@@ -75,6 +77,7 @@ class AutoDeploy(object): self.fuel_plugins_dir = fuel_plugins_dir self.fuel_plugins_conf_dir = fuel_plugins_conf_dir self.no_plugins = no_plugins @@ -241,7 +268,7 @@ index 08702d2..1a55361 100755 self.deploy_timeout = deploy_timeout self.no_deploy_environment = no_deploy_environment self.deploy_log = deploy_log -@@ -117,7 +120,7 @@ class AutoDeploy(object): +@@ -116,7 +119,7 @@ class AutoDeploy(object): self.fuel_username, self.fuel_password, self.dea_file, self.fuel_plugins_conf_dir, WORK_DIR, self.no_health_check, @@ -250,7 +277,7 @@ index 08702d2..1a55361 100755 self.no_deploy_environment, self.deploy_log) with old_dep.ssh: old_dep.check_previous_installation() -@@ -129,6 +132,7 @@ class AutoDeploy(object): +@@ -128,6 +131,7 @@ class AutoDeploy(object): self.fuel_conf['ip'], self.fuel_username, self.fuel_password, self.fuel_node_id, self.iso_file, WORK_DIR, @@ -258,7 +285,7 @@ index 08702d2..1a55361 100755 self.fuel_plugins_dir, self.no_plugins) fuel.install() -@@ -137,6 +141,7 @@ class AutoDeploy(object): +@@ -136,6 +140,7 @@ class AutoDeploy(object): tmp_new_dir = '%s/newiso' % self.tmp_dir try: self.copy(tmp_orig_dir, tmp_new_dir) @@ -266,7 +293,7 @@ index 08702d2..1a55361 100755 self.patch(tmp_new_dir, new_iso) except Exception as e: exec_cmd('fusermount -u %s' % tmp_orig_dir, False) -@@ -157,6 +162,12 @@ class AutoDeploy(object): +@@ -156,6 +161,12 @@ class AutoDeploy(object): delete(tmp_orig_dir) exec_cmd('chmod -R 755 %s' % tmp_new_dir) @@ -279,7 +306,7 @@ index 08702d2..1a55361 100755 def patch(self, tmp_new_dir, new_iso): log('Patching...') patch_dir = '%s/%s' % (CWD, PATCH_DIR) -@@ -219,7 +230,8 @@ class AutoDeploy(object): +@@ -218,7 +229,8 @@ class AutoDeploy(object): dep = CloudDeploy(self.dea, self.dha, self.fuel_conf['ip'], self.fuel_username, self.fuel_password, self.dea_file, self.fuel_plugins_conf_dir, @@ -289,7 +316,7 @@ index 08702d2..1a55361 100755 self.no_deploy_environment, self.deploy_log) return dep.deploy() -@@ -344,6 +356,8 @@ def parse_arguments(): +@@ -343,6 +355,8 @@ def parse_arguments(): help='Fuel Plugins Configuration directory') parser.add_argument('-np', dest='no_plugins', action='store_true', default=False, help='Do not install Fuel Plugins') @@ -298,7 +325,7 @@ index 08702d2..1a55361 100755 parser.add_argument('-dt', dest='deploy_timeout', action='store', default=240, help='Deployment timeout (in minutes) ' '[default: 240]') -@@ -377,6 +391,10 @@ def parse_arguments(): +@@ -376,6 +390,10 @@ def parse_arguments(): for bridge in args.pxe_bridge: check_bridge(bridge, args.dha_file) @@ -309,7 +336,7 @@ index 08702d2..1a55361 100755 kwargs = {'no_fuel': args.no_fuel, 'fuel_only': args.fuel_only, 'no_health_check': args.no_health_check, -@@ -387,6 +405,7 @@ def parse_arguments(): +@@ -386,6 +404,7 @@ def parse_arguments(): 'fuel_plugins_dir': args.fuel_plugins_dir, 'fuel_plugins_conf_dir': args.fuel_plugins_conf_dir, 'no_plugins': args.no_plugins, @@ -319,10 +346,10 @@ index 08702d2..1a55361 100755 'deploy_log': args.deploy_log} diff --git a/deploy/deploy_cache.py b/deploy/deploy_cache.py new file mode 100644 -index 0000000..d7ec1c7 +index 0000000..7df43c6 --- /dev/null +++ b/deploy/deploy_cache.py -@@ -0,0 +1,319 @@ +@@ -0,0 +1,312 @@ +############################################################################### +# Copyright (c) 2016 Enea AB and others. +# Alexandru.Avadanii@enea.com @@ -423,7 +450,7 @@ index 0000000..d7ec1c7 +ISO_BOOTSTRAP_CLI_YAML = '/opnfv/fuel_bootstrap_cli.yaml' + +# OPNFV Deploy Cache path on Fuel Master, where artifacts will be injected -+REMOTE_CACHE_PATH = '/var/lib/opnfv/cache' ++REMOTE_CACHE_PATH = '/var/cache/opnfv' + +# OPNFV Bootstrap Cache tar archive name, to be used by bootstrap_admin_node.sh +BOOTSTRAP_ARCHIVE = 'opnfv-bootstraps-cache.tar' @@ -456,24 +483,15 @@ index 0000000..d7ec1c7 + + def __fingerprint_mirrors(self, chroot_path): + """Collect repo mirror fingerprints""" -+ md5sums = list() -+ # Scan all ISO for deb repo metadata and collect MD5 from Release files -+ for root, _, files in os.walk(chroot_path): -+ for relf in files: -+ if relf == 'Release' and 'binary' not in root: -+ collect_sums = False -+ filepath = os.path.join(root, relf) -+ with open(filepath, "r") as release_file: -+ for line in release_file: -+ if collect_sums: -+ if line.startswith(' '): -+ md5sums += [line[1:33]] -+ else: -+ break -+ elif line.startswith('MD5Sum:'): -+ collect_sums = True -+ sorted_md5sums = json.dumps(md5sums, sort_keys=True) -+ self.fingerprints[MIRRORS] = hashlib.sha1(sorted_md5sums).hexdigest() ++ deb_packages = list() ++ # Scan ISO for deb files (MOS mirror + Ubuntu mirror, no plugins) ++ for repo_dir in ['ubuntu', 'opnfv/nailgun/mirrors/ubuntu']: ++ for _, _, files in os.walk(os.path.join(chroot_path, repo_dir)): ++ for fdeb in files: ++ if fdeb.endswith(".deb"): ++ deb_packages.append(fdeb) ++ sorted_debs = json.dumps(deb_packages, sort_keys=True) ++ self.fingerprints[MIRRORS] = hashlib.sha1(sorted_debs).hexdigest() + + def __fingerprint_bootstrap(self, chroot_path): + """Collect bootstrap image metadata fingerprints""" @@ -555,8 +573,10 @@ index 0000000..d7ec1c7 + for k in RSA_KEYPAIR_FILES: + ssh.scp_get(os.path.join(RSA_KEYPAIR_PATH, k), + local=os.path.join(cache_sha_dir, k)) -+ ssh.exec_cmd('tar cf %s %s/*' % (remote_tar, -+ os.path.join(NAILGUN_PATH, NAILGUN_ACT_BOOTSTRAP_SUBDIR))) ++ ssh.exec_cmd('mkdir -p %s && cd %s && tar cf %s *' % ++ (REMOTE_CACHE_PATH, ++ os.path.join(NAILGUN_PATH, NAILGUN_ACT_BOOTSTRAP_SUBDIR), ++ remote_tar)) + ssh.scp_get(remote_tar, local=local_tar) + ssh.exec_cmd('rm -f %s' % remote_tar) + @@ -643,7 +663,7 @@ index 0000000..d7ec1c7 + log('Failed to install targetimages for env %s: %s' % + (str(env_id), str(ex))) diff --git a/deploy/deploy_env.py b/deploy/deploy_env.py -index 1d2dfeb..2375f51 100644 +index 8afaeb1..318345d 100644 --- a/deploy/deploy_env.py +++ b/deploy/deploy_env.py @@ -15,6 +15,7 @@ import glob @@ -654,7 +674,7 @@ index 1d2dfeb..2375f51 100644 from ssh_client import SSHClient from common import ( -@@ -36,7 +37,8 @@ class CloudDeploy(object): +@@ -35,7 +36,8 @@ class CloudDeploy(object): def __init__(self, dea, dha, fuel_ip, fuel_username, fuel_password, dea_file, fuel_plugins_conf_dir, work_dir, no_health_check, @@ -664,7 +684,7 @@ index 1d2dfeb..2375f51 100644 self.dea = dea self.dha = dha self.fuel_ip = fuel_ip -@@ -50,6 +52,8 @@ class CloudDeploy(object): +@@ -49,6 +51,8 @@ class CloudDeploy(object): self.fuel_plugins_conf_dir = fuel_plugins_conf_dir self.work_dir = work_dir self.no_health_check = no_health_check @@ -673,7 +693,7 @@ index 1d2dfeb..2375f51 100644 self.deploy_timeout = deploy_timeout self.no_deploy_environment = no_deploy_environment self.deploy_log = deploy_log -@@ -83,9 +87,14 @@ class CloudDeploy(object): +@@ -82,9 +86,14 @@ class CloudDeploy(object): self.work_dir, os.path.basename(self.dea_file))) s.scp_put('%s/common.py' % self.file_dir, self.work_dir) s.scp_put('%s/dea.py' % self.file_dir, self.work_dir) @@ -688,7 +708,7 @@ index 1d2dfeb..2375f51 100644 def power_off_nodes(self): for node_id in self.node_ids: self.dha.node_power_off(node_id) -@@ -284,4 +293,6 @@ class CloudDeploy(object): +@@ -281,4 +290,6 @@ class CloudDeploy(object): self.get_put_deploy_log() @@ -696,7 +716,7 @@ index 1d2dfeb..2375f51 100644 + return rc diff --git a/deploy/install_fuel_master.py b/deploy/install_fuel_master.py -index ccc18d3..2615818 100644 +index b731c6b..83d31fb 100644 --- a/deploy/install_fuel_master.py +++ b/deploy/install_fuel_master.py @@ -10,6 +10,7 @@ @@ -707,7 +727,7 @@ index ccc18d3..2615818 100644 from ssh_client import SSHClient from dha_adapters.libvirt_adapter import LibvirtAdapter -@@ -33,7 +34,7 @@ class InstallFuelMaster(object): +@@ -32,7 +33,7 @@ class InstallFuelMaster(object): def __init__(self, dea_file, dha_file, fuel_ip, fuel_username, fuel_password, fuel_node_id, iso_file, work_dir, @@ -716,7 +736,7 @@ index ccc18d3..2615818 100644 self.dea_file = dea_file self.dha = LibvirtAdapter(dha_file) self.fuel_ip = fuel_ip -@@ -43,6 +44,8 @@ class InstallFuelMaster(object): +@@ -42,6 +43,8 @@ class InstallFuelMaster(object): self.iso_file = iso_file self.iso_dir = os.path.dirname(self.iso_file) self.work_dir = work_dir @@ -725,7 +745,7 @@ index ccc18d3..2615818 100644 self.fuel_plugins_dir = fuel_plugins_dir self.no_plugins = no_plugins self.file_dir = os.path.dirname(os.path.realpath(__file__)) -@@ -84,6 +87,10 @@ class InstallFuelMaster(object): +@@ -83,6 +86,10 @@ class InstallFuelMaster(object): log('Wait until Fuel menu is up') fuel_menu_pid = self.wait_until_fuel_menu_up()