TODO: remote fingerprinting
TODO: differentiate between bootstraps and targetimages, so we don't
end up trying to use one cache artifact type as the other.
+TODO: implement sanity checks for bootstrap and target images;
+TODO: switch `exec_cmd('mkdir ...')` to `create_dir_if_not_exists`;
JIRA: ARMBAND-172
Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
---
- ...p_admin_node.sh-deploy_cache-install-hook.patch | 69 +++++
+ ...p_admin_node.sh-deploy_cache-install-hook.patch | 73 +++++
ci/deploy.sh | 14 +-
- deploy/cloud/deploy.py | 11 +
+ deploy/cloud/configure_settings.py | 4 +
+ deploy/cloud/deployment.py | 12 +
deploy/deploy.py | 25 +-
- deploy/deploy_cache.py | 319 +++++++++++++++++++++
+ deploy/deploy_cache.py | 312 +++++++++++++++++++++
deploy/deploy_env.py | 13 +-
deploy/install_fuel_master.py | 9 +-
- 7 files changed, 451 insertions(+), 9 deletions(-)
+ 8 files changed, 453 insertions(+), 9 deletions(-)
create mode 100644 build/f_repos/patch/fuel-main/0006-bootstrap_admin_node.sh-deploy_cache-install-hook.patch
create mode 100644 deploy/deploy_cache.py
diff --git a/build/f_repos/patch/fuel-main/0006-bootstrap_admin_node.sh-deploy_cache-install-hook.patch b/build/f_repos/patch/fuel-main/0006-bootstrap_admin_node.sh-deploy_cache-install-hook.patch
new file mode 100644
-index 0000000..d5b7646
+index 0000000..69a4c22
--- /dev/null
+++ b/build/f_repos/patch/fuel-main/0006-bootstrap_admin_node.sh-deploy_cache-install-hook.patch
-@@ -0,0 +1,69 @@
+@@ -0,0 +1,73 @@
+From: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
+Date: Mon, 28 Nov 2016 14:27:48 +0100
+Subject: [PATCH] bootstrap_admin_node.sh: deploy_cache install hook
+--- a/iso/bootstrap_admin_node.sh
++++ b/iso/bootstrap_admin_node.sh
+@@ -61,6 +61,8 @@ wget \
-+
+ ASTUTE_YAML='/etc/fuel/astute.yaml'
+ BOOTSTRAP_NODE_CONFIG="/etc/fuel/bootstrap_admin_node.conf"
-++OPNFV_CACHE_PATH="/var/lib/opnfv/cache/bootstraps"
++ CUSTOM_REPOS="/root/default_deb_repos.yaml"
+++OPNFV_CACHE_PATH="/var/cache/opnfv/bootstraps"
++OPNFV_CACHE_TAR="opnfv-bootstraps-cache.tar"
+ bs_build_log='/var/log/fuel-bootstrap-image-build.log'
+ bs_status=0
+ # Update issues messages
+ update_warn_message="There is an issue connecting to update repository of \
+ your distributions of OpenStack. \
-+@@ -500,12 +503,27 @@ set_ui_bootstrap_error () {
++@@ -500,12 +503,31 @@ set_ui_bootstrap_error () {
+ EOF
+ }
+
++ if [ -f "${OPNFV_CACHE_PATH}/${OPNFV_CACHE_TAR}" -a \
++ -f "${OPNFV_CACHE_PATH}/id_rsa.pub" -a \
++ -f "${OPNFV_CACHE_PATH}/id_rsa" ]; then
-++ if cp "${OPNFV_CACHE_PATH}/id_rsa{,.pub}" "~/.ssh/" && \
+++ if cp "${OPNFV_CACHE_PATH}/id_rsa"* "/root/.ssh/" && \
+++ cp "/root/.ssh/id_rsa.pub" "/root/.ssh/authorized_keys" && \
+++ cp "/root/.ssh/id_rsa.pub" "/etc/cobbler/authorized_keys" && \
+++ sed -i -e "s|\"ssh-rsa .*\"|\"$(cat /root/.ssh/id_rsa.pub)\"|g" \
+++ /etc/nailgun/settings.yaml && \
++ fuel-bootstrap -v --debug import --activate \
++ "${OPNFV_CACHE_PATH}/${OPNFV_CACHE_TAR}" >>"$bs_build_log" 2>&1; then
-++ fuel notify --topic "done" --send "${bs_cache_message}"
-++ return 0
+++ fuel notify --topic "done" --send "${bs_cache_message}"
+++ return 0
++ fi
++ fi
++ return 1
fi
popd > /dev/null
-diff --git a/deploy/cloud/deploy.py b/deploy/cloud/deploy.py
-index e00934b..b39e5fc 100644
---- a/deploy/cloud/deploy.py
-+++ b/deploy/cloud/deploy.py
-@@ -14,6 +14,7 @@ import io
- from dea import DeploymentEnvironmentAdapter
- from configure_environment import ConfigureEnvironment
- from deployment import Deployment
-+from deploy_cache import DeployCache
+diff --git a/deploy/cloud/configure_settings.py b/deploy/cloud/configure_settings.py
+index b60a60f..4e007e1 100644
+--- a/deploy/cloud/configure_settings.py
++++ b/deploy/cloud/configure_settings.py
+@@ -71,5 +71,9 @@ class ConfigureSettings(object):
+ settings['editable'][plugin]['metadata']['chosen_id'] = orig_dea['editable'][plugin]['metadata']['chosen_id']
+ settings['editable'][plugin]['metadata']['versions'][0]['metadata']['plugin_id'] = orig_dea['editable'][plugin]['metadata']['versions'][0]['metadata']['plugin_id']
+
++ # deploy-cache req: pass master id_rsa.pub as authorized key
++ with io.open('/root/.ssh/id_rsa.pub', 'r') as pkey:
++ settings['editable']['operator_user']['authkeys']['value'] = pkey.read()
++
+ with io.open(settings_yaml, 'w') as stream:
+ yaml.dump(settings, stream, default_flow_style=False)
+diff --git a/deploy/cloud/deployment.py b/deploy/cloud/deployment.py
+index 28bcfdf..b0bfdcc 100644
+--- a/deploy/cloud/deployment.py
++++ b/deploy/cloud/deployment.py
+@@ -19,6 +19,8 @@ from common import (
+ log,
+ )
- from common import (
- R,
-@@ -61,6 +62,12 @@ class Deploy(object):
- config_env.configure_environment()
- self.env_id = config_env.env_id
++from deploy_cache import DeployCache
++
+ SEARCH_TEXT = '(err)'
+ LOG_FILE = '/var/log/puppet.log'
+ GREP_LINES_OF_LEADING_CONTEXT = 100
+@@ -47,6 +49,14 @@ class Deployment(object):
+ self.pattern = re.compile(
+ '\d\d\d\d-\d\d-\d\d\s\d\d:\d\d:\d\d')
+ def deploy_cache_install_targetimages(self):
++ log('Using target images from deploy cache')
+ DeployCache.install_targetimages_for_env(self.env_id)
+
+ def deploy_cache_extract_targetimages(self):
++ log('Collecting Fuel target image files for deploy cache')
+ DeployCache.extract_targetimages_from_env(self.env_id)
+
- def deploy_cloud(self):
- dep = Deployment(self.dea, YAML_CONF_DIR, self.env_id,
- self.node_roles_dict, self.no_health_check,
-@@ -76,8 +83,12 @@ class Deploy(object):
-
- self.configure_environment()
+ def collect_error_logs(self):
+ for node_id, roles_blade in self.node_id_roles_dict.iteritems():
+ log_list = []
+@@ -108,6 +118,7 @@ class Deployment(object):
+ start = time.time()
+ log('Starting deployment of environment %s' % self.env_id)
+ self.deploy_cache_install_targetimages()
-+
- self.deploy_cloud()
-
+ deploy_id = None
+ ready = False
+ timeout = False
+@@ -140,6 +151,7 @@ class Deployment(object):
+ err('Deployment timed out, environment %s is not operational, '
+ 'snapshot will not be performed'
+ % self.env_id)
+ self.deploy_cache_extract_targetimages()
-+
-
- def parse_arguments():
- parser = ArgParser(prog='python %s' % __file__)
+ if ready:
+ log('Environment %s successfully deployed'
+ % self.env_id)
diff --git a/deploy/deploy.py b/deploy/deploy.py
-index 08702d2..1a55361 100755
+index 7648baf..ee3cb7a 100755
--- a/deploy/deploy.py
+++ b/deploy/deploy.py
-@@ -23,6 +23,7 @@ from dea import DeploymentEnvironmentAdapter
+@@ -22,6 +22,7 @@ from dea import DeploymentEnvironmentAdapter
from dha import DeploymentHardwareAdapter
from install_fuel_master import InstallFuelMaster
from deploy_env import CloudDeploy
from execution_environment import ExecutionEnvironment
from common import (
-@@ -62,7 +63,8 @@ class AutoDeploy(object):
+@@ -61,7 +62,8 @@ class AutoDeploy(object):
def __init__(self, no_fuel, fuel_only, no_health_check, cleanup_only,
cleanup, storage_dir, pxe_bridge, iso_file, dea_file,
dha_file, fuel_plugins_dir, fuel_plugins_conf_dir,
self.no_fuel = no_fuel
self.fuel_only = fuel_only
self.no_health_check = no_health_check
-@@ -76,6 +78,7 @@ class AutoDeploy(object):
+@@ -75,6 +77,7 @@ class AutoDeploy(object):
self.fuel_plugins_dir = fuel_plugins_dir
self.fuel_plugins_conf_dir = fuel_plugins_conf_dir
self.no_plugins = no_plugins
self.deploy_timeout = deploy_timeout
self.no_deploy_environment = no_deploy_environment
self.deploy_log = deploy_log
-@@ -117,7 +120,7 @@ class AutoDeploy(object):
+@@ -116,7 +119,7 @@ class AutoDeploy(object):
self.fuel_username, self.fuel_password,
self.dea_file, self.fuel_plugins_conf_dir,
WORK_DIR, self.no_health_check,
self.no_deploy_environment, self.deploy_log)
with old_dep.ssh:
old_dep.check_previous_installation()
-@@ -129,6 +132,7 @@ class AutoDeploy(object):
+@@ -128,6 +131,7 @@ class AutoDeploy(object):
self.fuel_conf['ip'], self.fuel_username,
self.fuel_password, self.fuel_node_id,
self.iso_file, WORK_DIR,
self.fuel_plugins_dir, self.no_plugins)
fuel.install()
-@@ -137,6 +141,7 @@ class AutoDeploy(object):
+@@ -136,6 +140,7 @@ class AutoDeploy(object):
tmp_new_dir = '%s/newiso' % self.tmp_dir
try:
self.copy(tmp_orig_dir, tmp_new_dir)
self.patch(tmp_new_dir, new_iso)
except Exception as e:
exec_cmd('fusermount -u %s' % tmp_orig_dir, False)
-@@ -157,6 +162,12 @@ class AutoDeploy(object):
+@@ -156,6 +161,12 @@ class AutoDeploy(object):
delete(tmp_orig_dir)
exec_cmd('chmod -R 755 %s' % tmp_new_dir)
def patch(self, tmp_new_dir, new_iso):
log('Patching...')
patch_dir = '%s/%s' % (CWD, PATCH_DIR)
-@@ -219,7 +230,8 @@ class AutoDeploy(object):
+@@ -218,7 +229,8 @@ class AutoDeploy(object):
dep = CloudDeploy(self.dea, self.dha, self.fuel_conf['ip'],
self.fuel_username, self.fuel_password,
self.dea_file, self.fuel_plugins_conf_dir,
self.no_deploy_environment, self.deploy_log)
return dep.deploy()
-@@ -344,6 +356,8 @@ def parse_arguments():
+@@ -343,6 +355,8 @@ def parse_arguments():
help='Fuel Plugins Configuration directory')
parser.add_argument('-np', dest='no_plugins', action='store_true',
default=False, help='Do not install Fuel Plugins')
parser.add_argument('-dt', dest='deploy_timeout', action='store',
default=240, help='Deployment timeout (in minutes) '
'[default: 240]')
-@@ -377,6 +391,10 @@ def parse_arguments():
+@@ -376,6 +390,10 @@ def parse_arguments():
for bridge in args.pxe_bridge:
check_bridge(bridge, args.dha_file)
kwargs = {'no_fuel': args.no_fuel, 'fuel_only': args.fuel_only,
'no_health_check': args.no_health_check,
-@@ -387,6 +405,7 @@ def parse_arguments():
+@@ -386,6 +404,7 @@ def parse_arguments():
'fuel_plugins_dir': args.fuel_plugins_dir,
'fuel_plugins_conf_dir': args.fuel_plugins_conf_dir,
'no_plugins': args.no_plugins,
'deploy_log': args.deploy_log}
diff --git a/deploy/deploy_cache.py b/deploy/deploy_cache.py
new file mode 100644
-index 0000000..d7ec1c7
+index 0000000..7df43c6
--- /dev/null
+++ b/deploy/deploy_cache.py
-@@ -0,0 +1,319 @@
+@@ -0,0 +1,312 @@
+###############################################################################
+# Copyright (c) 2016 Enea AB and others.
+# Alexandru.Avadanii@enea.com
+ISO_BOOTSTRAP_CLI_YAML = '/opnfv/fuel_bootstrap_cli.yaml'
+
+# OPNFV Deploy Cache path on Fuel Master, where artifacts will be injected
-+REMOTE_CACHE_PATH = '/var/lib/opnfv/cache'
++REMOTE_CACHE_PATH = '/var/cache/opnfv'
+
+# OPNFV Bootstrap Cache tar archive name, to be used by bootstrap_admin_node.sh
+BOOTSTRAP_ARCHIVE = 'opnfv-bootstraps-cache.tar'
+
+ def __fingerprint_mirrors(self, chroot_path):
+ """Collect repo mirror fingerprints"""
-+ md5sums = list()
-+ # Scan all ISO for deb repo metadata and collect MD5 from Release files
-+ for root, _, files in os.walk(chroot_path):
-+ for relf in files:
-+ if relf == 'Release' and 'binary' not in root:
-+ collect_sums = False
-+ filepath = os.path.join(root, relf)
-+ with open(filepath, "r") as release_file:
-+ for line in release_file:
-+ if collect_sums:
-+ if line.startswith(' '):
-+ md5sums += [line[1:33]]
-+ else:
-+ break
-+ elif line.startswith('MD5Sum:'):
-+ collect_sums = True
-+ sorted_md5sums = json.dumps(md5sums, sort_keys=True)
-+ self.fingerprints[MIRRORS] = hashlib.sha1(sorted_md5sums).hexdigest()
++ deb_packages = list()
++ # Scan ISO for deb files (MOS mirror + Ubuntu mirror, no plugins)
++ for repo_dir in ['ubuntu', 'opnfv/nailgun/mirrors/ubuntu']:
++ for _, _, files in os.walk(os.path.join(chroot_path, repo_dir)):
++ for fdeb in files:
++ if fdeb.endswith(".deb"):
++ deb_packages.append(fdeb)
++ sorted_debs = json.dumps(deb_packages, sort_keys=True)
++ self.fingerprints[MIRRORS] = hashlib.sha1(sorted_debs).hexdigest()
+
+ def __fingerprint_bootstrap(self, chroot_path):
+ """Collect bootstrap image metadata fingerprints"""
+ for k in RSA_KEYPAIR_FILES:
+ ssh.scp_get(os.path.join(RSA_KEYPAIR_PATH, k),
+ local=os.path.join(cache_sha_dir, k))
-+ ssh.exec_cmd('tar cf %s %s/*' % (remote_tar,
-+ os.path.join(NAILGUN_PATH, NAILGUN_ACT_BOOTSTRAP_SUBDIR)))
++ ssh.exec_cmd('mkdir -p %s && cd %s && tar cf %s *' %
++ (REMOTE_CACHE_PATH,
++ os.path.join(NAILGUN_PATH, NAILGUN_ACT_BOOTSTRAP_SUBDIR),
++ remote_tar))
+ ssh.scp_get(remote_tar, local=local_tar)
+ ssh.exec_cmd('rm -f %s' % remote_tar)
+
+ log('Failed to install targetimages for env %s: %s' %
+ (str(env_id), str(ex)))
diff --git a/deploy/deploy_env.py b/deploy/deploy_env.py
-index 1d2dfeb..2375f51 100644
+index 8afaeb1..318345d 100644
--- a/deploy/deploy_env.py
+++ b/deploy/deploy_env.py
@@ -15,6 +15,7 @@ import glob
from ssh_client import SSHClient
from common import (
-@@ -36,7 +37,8 @@ class CloudDeploy(object):
+@@ -35,7 +36,8 @@ class CloudDeploy(object):
def __init__(self, dea, dha, fuel_ip, fuel_username, fuel_password,
dea_file, fuel_plugins_conf_dir, work_dir, no_health_check,
self.dea = dea
self.dha = dha
self.fuel_ip = fuel_ip
-@@ -50,6 +52,8 @@ class CloudDeploy(object):
+@@ -49,6 +51,8 @@ class CloudDeploy(object):
self.fuel_plugins_conf_dir = fuel_plugins_conf_dir
self.work_dir = work_dir
self.no_health_check = no_health_check
self.deploy_timeout = deploy_timeout
self.no_deploy_environment = no_deploy_environment
self.deploy_log = deploy_log
-@@ -83,9 +87,14 @@ class CloudDeploy(object):
+@@ -82,9 +86,14 @@ class CloudDeploy(object):
self.work_dir, os.path.basename(self.dea_file)))
s.scp_put('%s/common.py' % self.file_dir, self.work_dir)
s.scp_put('%s/dea.py' % self.file_dir, self.work_dir)
def power_off_nodes(self):
for node_id in self.node_ids:
self.dha.node_power_off(node_id)
-@@ -284,4 +293,6 @@ class CloudDeploy(object):
+@@ -281,4 +290,6 @@ class CloudDeploy(object):
self.get_put_deploy_log()
+
return rc
diff --git a/deploy/install_fuel_master.py b/deploy/install_fuel_master.py
-index ccc18d3..2615818 100644
+index b731c6b..83d31fb 100644
--- a/deploy/install_fuel_master.py
+++ b/deploy/install_fuel_master.py
@@ -10,6 +10,7 @@
from ssh_client import SSHClient
from dha_adapters.libvirt_adapter import LibvirtAdapter
-@@ -33,7 +34,7 @@ class InstallFuelMaster(object):
+@@ -32,7 +33,7 @@ class InstallFuelMaster(object):
def __init__(self, dea_file, dha_file, fuel_ip, fuel_username,
fuel_password, fuel_node_id, iso_file, work_dir,
self.dea_file = dea_file
self.dha = LibvirtAdapter(dha_file)
self.fuel_ip = fuel_ip
-@@ -43,6 +44,8 @@ class InstallFuelMaster(object):
+@@ -42,6 +43,8 @@ class InstallFuelMaster(object):
self.iso_file = iso_file
self.iso_dir = os.path.dirname(self.iso_file)
self.work_dir = work_dir
self.fuel_plugins_dir = fuel_plugins_dir
self.no_plugins = no_plugins
self.file_dir = os.path.dirname(os.path.realpath(__file__))
-@@ -84,6 +87,10 @@ class InstallFuelMaster(object):
+@@ -83,6 +86,10 @@ class InstallFuelMaster(object):
log('Wait until Fuel menu is up')
fuel_menu_pid = self.wait_until_fuel_menu_up()
Code Review / armband.git / blobdiff