deploy/deploy.py

   1 #!/usr/bin/python
   2 ###############################################################################
   3 # Copyright (c) 2015 Ericsson AB and others.
   4 # szilard.cserey@ericsson.com
   5 # All rights reserved. This program and the accompanying materials
   6 # are made available under the terms of the Apache License, Version 2.0
   7 # which accompanies this distribution, and is available at
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 ###############################################################################
  10
  11
  12 import os
  13 import io
  14 import re
  15 import sys
  16 import yaml
  17 import errno
  18 import signal
  19 import netaddr
  20
  21 from dea import DeploymentEnvironmentAdapter
  22 from dha import DeploymentHardwareAdapter
  23 from install_fuel_master import InstallFuelMaster
  24 from deploy_env import CloudDeploy
  25 from execution_environment import ExecutionEnvironment
  26
  27 from common import (
  28     log,
  29     exec_cmd,
  30     err,
  31     warn,
  32     check_file_exists,
  33     check_dir_exists,
  34     create_dir_if_not_exists,
  35     delete,
  36     check_if_root,
  37     ArgParser,
  38 )
  39
  40 FUEL_VM = 'fuel'
  41 PATCH_DIR = 'fuel_patch'
  42 WORK_DIR = '~/deploy'
  43 CWD = os.getcwd()
  44 MOUNT_STATE_VAR = 'AUTODEPLOY_ISO_MOUNTED'
  45
  46
  47 class cd:
  48
  49     def __init__(self, new_path):
  50         self.new_path = os.path.expanduser(new_path)
  51
  52     def __enter__(self):
  53         self.saved_path = CWD
  54         os.chdir(self.new_path)
  55
  56     def __exit__(self, etype, value, traceback):
  57         os.chdir(self.saved_path)
  58
  59
  60 class AutoDeploy(object):
  61
  62     def __init__(self, no_fuel, fuel_only, no_health_check, cleanup_only,
  63                  cleanup, storage_dir, pxe_bridge, iso_file, dea_file,
  64                  dha_file, fuel_plugins_dir, fuel_plugins_conf_dir,
  65                  no_plugins, deploy_timeout, no_deploy_environment, deploy_log):
  66         self.no_fuel = no_fuel
  67         self.fuel_only = fuel_only
  68         self.no_health_check = no_health_check
  69         self.cleanup_only = cleanup_only
  70         self.cleanup = cleanup
  71         self.storage_dir = storage_dir
  72         self.pxe_bridge = pxe_bridge
  73         self.iso_file = iso_file
  74         self.dea_file = dea_file
  75         self.dha_file = dha_file
  76         self.fuel_plugins_dir = fuel_plugins_dir
  77         self.fuel_plugins_conf_dir = fuel_plugins_conf_dir
  78         self.no_plugins = no_plugins
  79         self.deploy_timeout = deploy_timeout
  80         self.no_deploy_environment = no_deploy_environment
  81         self.deploy_log = deploy_log
  82         self.dea = (DeploymentEnvironmentAdapter(dea_file)
  83                     if not cleanup_only else None)
  84         self.dha = DeploymentHardwareAdapter(dha_file)
  85         self.fuel_conf = {}
  86         self.fuel_node_id = self.dha.get_fuel_node_id()
  87         self.fuel_username, self.fuel_password = self.dha.get_fuel_access()
  88         self.tmp_dir = None
  89
  90     def modify_ip(self, ip_addr, index, val):
  91         ip_str = str(netaddr.IPAddress(ip_addr))
  92         decimal_list = map(int, ip_str.split('.'))
  93         decimal_list[index] = val
  94         return '.'.join(map(str, decimal_list))
  95
  96     def collect_fuel_info(self):
  97         self.fuel_conf['ip'] = self.dea.get_fuel_ip()
  98         self.fuel_conf['gw'] = self.dea.get_fuel_gateway()
  99         self.fuel_conf['dns1'] = self.dea.get_fuel_dns()
 100         self.fuel_conf['netmask'] = self.dea.get_fuel_netmask()
 101         self.fuel_conf['hostname'] = self.dea.get_fuel_hostname()
 102         self.fuel_conf['showmenu'] = 'yes'
 103
 104     def install_fuel_master(self):
 105         log('Install Fuel Master')
 106         new_iso = ('%s/deploy-%s'
 107                    % (self.tmp_dir, os.path.basename(self.iso_file)))
 108         self.patch_iso(new_iso)
 109         self.iso_file = new_iso
 110         self.install_iso()
 111
 112     def install_iso(self):
 113         fuel = InstallFuelMaster(self.dea_file, self.dha_file,
 114                                  self.fuel_conf['ip'], self.fuel_username,
 115                                  self.fuel_password, self.fuel_node_id,
 116                                  self.iso_file, WORK_DIR,
 117                                  self.fuel_plugins_dir, self.no_plugins)
 118         fuel.install()
 119
 120     def patch_iso(self, new_iso):
 121         tmp_orig_dir = '%s/origiso' % self.tmp_dir
 122         tmp_new_dir = '%s/newiso' % self.tmp_dir
 123         try:
 124             self.copy(tmp_orig_dir, tmp_new_dir)
 125             self.patch(tmp_new_dir, new_iso)
 126         except Exception as e:
 127             exec_cmd('fusermount -u %s' % tmp_orig_dir, False)
 128             os.environ.pop(MOUNT_STATE_VAR, None)
 129             delete(self.tmp_dir)
 130             err(e)
 131
 132     def copy(self, tmp_orig_dir, tmp_new_dir):
 133         log('Copying...')
 134         os.makedirs(tmp_orig_dir)
 135         os.makedirs(tmp_new_dir)
 136         exec_cmd('fuseiso %s %s' % (self.iso_file, tmp_orig_dir))
 137         os.environ[MOUNT_STATE_VAR] = tmp_orig_dir
 138         with cd(tmp_orig_dir):
 139             exec_cmd('find . | cpio -pd %s' % tmp_new_dir)
 140         exec_cmd('fusermount -u %s' % tmp_orig_dir)
 141         os.environ.pop(MOUNT_STATE_VAR, None)
 142         delete(tmp_orig_dir)
 143         exec_cmd('chmod -R 755 %s' % tmp_new_dir)
 144
 145     def patch(self, tmp_new_dir, new_iso):
 146         log('Patching...')
 147         patch_dir = '%s/%s' % (CWD, PATCH_DIR)
 148         ks_path = '%s/ks.cfg.patch' % patch_dir
 149
 150         with cd(tmp_new_dir):
 151             exec_cmd('cat %s | patch -p0' % ks_path)
 152             delete('.rr_moved')
 153             isolinux = 'isolinux/isolinux.cfg'
 154             log('isolinux.cfg before: %s'
 155                 % exec_cmd('grep ip= %s' % isolinux))
 156             self.update_fuel_isolinux(isolinux)
 157             log('isolinux.cfg after: %s'
 158                 % exec_cmd('grep ip= %s' % isolinux))
 159
 160             iso_label = self.parse_iso_volume_label(self.iso_file)
 161             log('Volume label: %s' % iso_label)
 162
 163             iso_linux_bin = 'isolinux/isolinux.bin'
 164             exec_cmd('mkisofs -quiet -r -J -R -b %s '
 165                      '-no-emul-boot -boot-load-size 4 '
 166                      '-boot-info-table -hide-rr-moved '
 167                      '-joliet-long '
 168                      '-x "lost+found:" -V %s -o %s .'
 169                      % (iso_linux_bin, iso_label, new_iso))
 170
 171         delete(tmp_new_dir)
 172
 173     def update_fuel_isolinux(self, file):
 174         with io.open(file) as f:
 175             data = f.read()
 176         for key, val in self.fuel_conf.iteritems():
 177             # skip replacing these keys, as the format is different
 178             if key in ['ip', 'gw', 'netmask', 'hostname']:
 179                 continue
 180
 181             pattern = r'%s=[^ ]\S+' % key
 182             replace = '%s=%s' % (key, val)
 183             data = re.sub(pattern, replace, data)
 184
 185         # process networking parameters
 186         ip = ':'.join([self.fuel_conf['ip'],
 187                       '',
 188                       self.fuel_conf['gw'],
 189                       self.fuel_conf['netmask'],
 190                       self.fuel_conf['hostname'],
 191                       'eth0:off:::'])
 192
 193         data = re.sub(r'ip=[^ ]\S+', 'ip=%s' % ip, data)
 194
 195         with io.open(file, 'w') as f:
 196             f.write(data)
 197
 198     def parse_iso_volume_label(self, iso_filename):
 199         label_line = exec_cmd('isoinfo -d -i %s | grep -i "Volume id: "' % iso_filename)
 200         # cut leading text: 'Volume id: '
 201         return label_line[11:]
 202
 203     def deploy_env(self):
 204         dep = CloudDeploy(self.dea, self.dha, self.fuel_conf['ip'],
 205                           self.fuel_username, self.fuel_password,
 206                           self.dea_file, self.fuel_plugins_conf_dir,
 207                           WORK_DIR, self.no_health_check, self.deploy_timeout,
 208                           self.no_deploy_environment, self.deploy_log)
 209         return dep.deploy()
 210
 211     def setup_execution_environment(self):
 212         exec_env = ExecutionEnvironment(self.storage_dir, self.pxe_bridge,
 213                                         self.dha_file, self.dea)
 214         exec_env.setup_environment()
 215
 216     def cleanup_execution_environment(self):
 217         exec_env = ExecutionEnvironment(self.storage_dir, self.pxe_bridge,
 218                                         self.dha_file, self.dea)
 219         exec_env.cleanup_environment()
 220
 221     def create_tmp_dir(self):
 222         self.tmp_dir = '%s/fueltmp' % CWD
 223         delete(self.tmp_dir)
 224         create_dir_if_not_exists(self.tmp_dir)
 225
 226     def deploy(self):
 227         self.collect_fuel_info()
 228         if not self.no_fuel:
 229             self.setup_execution_environment()
 230             self.create_tmp_dir()
 231             self.install_fuel_master()
 232         if not self.fuel_only:
 233             return self.deploy_env()
 234         # Exit status
 235         return 0
 236
 237     def run(self):
 238         check_if_root()
 239         if self.cleanup_only:
 240             self.cleanup_execution_environment()
 241         else:
 242             deploy_success = self.deploy()
 243             if self.cleanup:
 244                 self.cleanup_execution_environment()
 245             return deploy_success
 246         # Exit status
 247         return 0
 248
 249
 250 def check_bridge(pxe_bridge, dha_path):
 251     # Assume that bridges on remote nodes exists, we could ssh but
 252     # the remote user might not have a login shell.
 253     if os.environ.get('LIBVIRT_DEFAULT_URI'):
 254         return
 255
 256     with io.open(dha_path) as yaml_file:
 257         dha_struct = yaml.load(yaml_file)
 258     if dha_struct['adapter'] != 'libvirt':
 259         log('Using Linux Bridge %s for booting up the Fuel Master VM'
 260             % pxe_bridge)
 261         r = exec_cmd('ip link show %s' % pxe_bridge)
 262         if pxe_bridge in r and 'state DOWN' in r:
 263             err('Linux Bridge {0} is not Active, bring'
 264                 ' it UP first: [ip link set dev {0} up]'.format(pxe_bridge))
 265
 266
 267 def check_fuel_plugins_dir(dir):
 268     msg = None
 269     if not dir:
 270         msg = 'Fuel Plugins Directory not specified!'
 271     elif not os.path.isdir(dir):
 272         msg = 'Fuel Plugins Directory does not exist!'
 273     elif not os.listdir(dir):
 274         msg = 'Fuel Plugins Directory is empty!'
 275     if msg:
 276         warn('%s No external plugins will be installed!' % msg)
 277
 278
 279 def parse_arguments():
 280     parser = ArgParser(prog='python %s' % __file__)
 281     parser.add_argument('-nf', dest='no_fuel', action='store_true',
 282                         default=False,
 283                         help='Do not install Fuel Master (and Node VMs when '
 284                              'using libvirt)')
 285     parser.add_argument('-nh', dest='no_health_check', action='store_true',
 286                         default=False,
 287                         help='Don\'t run health check after deployment')
 288     parser.add_argument('-fo', dest='fuel_only', action='store_true',
 289                         default=False,
 290                         help='Install Fuel Master only (and Node VMs when '
 291                              'using libvirt)')
 292     parser.add_argument('-co', dest='cleanup_only', action='store_true',
 293                         default=False,
 294                         help='Cleanup VMs and Virtual Networks according to '
 295                              'what is defined in DHA')
 296     parser.add_argument('-c', dest='cleanup', action='store_true',
 297                         default=False,
 298                         help='Cleanup after deploy')
 299     if {'-iso', '-dea', '-dha', '-h'}.intersection(sys.argv):
 300         parser.add_argument('-iso', dest='iso_file', action='store', nargs='?',
 301                             default='%s/OPNFV.iso' % CWD,
 302                             help='ISO File [default: OPNFV.iso]')
 303         parser.add_argument('-dea', dest='dea_file', action='store', nargs='?',
 304                             default='%s/dea.yaml' % CWD,
 305                             help='Deployment Environment Adapter: dea.yaml')
 306         parser.add_argument('-dha', dest='dha_file', action='store', nargs='?',
 307                             default='%s/dha.yaml' % CWD,
 308                             help='Deployment Hardware Adapter: dha.yaml')
 309     else:
 310         parser.add_argument('iso_file', action='store', nargs='?',
 311                             default='%s/OPNFV.iso' % CWD,
 312                             help='ISO File [default: OPNFV.iso]')
 313         parser.add_argument('dea_file', action='store', nargs='?',
 314                             default='%s/dea.yaml' % CWD,
 315                             help='Deployment Environment Adapter: dea.yaml')
 316         parser.add_argument('dha_file', action='store', nargs='?',
 317                             default='%s/dha.yaml' % CWD,
 318                             help='Deployment Hardware Adapter: dha.yaml')
 319     parser.add_argument('-s', dest='storage_dir', action='store',
 320                         default='%s/images' % CWD,
 321                         help='Storage Directory [default: images]')
 322     parser.add_argument('-b', dest='pxe_bridge', action='append',
 323                         default=[],
 324                         help='Linux Bridge for booting up the Fuel Master VM '
 325                              '[default: pxebr]')
 326     parser.add_argument('-p', dest='fuel_plugins_dir', action='store',
 327                         help='Fuel Plugins directory')
 328     parser.add_argument('-pc', dest='fuel_plugins_conf_dir', action='store',
 329                         help='Fuel Plugins Configuration directory')
 330     parser.add_argument('-np', dest='no_plugins', action='store_true',
 331                         default=False, help='Do not install Fuel Plugins')
 332     parser.add_argument('-dt', dest='deploy_timeout', action='store',
 333                         default=240, help='Deployment timeout (in minutes) '
 334                         '[default: 240]')
 335     parser.add_argument('-nde', dest='no_deploy_environment',
 336                         action='store_true', default=False,
 337                         help=('Do not launch environment deployment'))
 338     parser.add_argument('-log', dest='deploy_log',
 339                         action='store', default='../ci/.',
 340                         help=('Path and name of the deployment log archive'))
 341
 342     args = parser.parse_args()
 343     log(args)
 344
 345     if not args.pxe_bridge:
 346         args.pxe_bridge = ['pxebr']
 347
 348     check_file_exists(args.dha_file)
 349
 350     check_dir_exists(os.path.dirname(args.deploy_log))
 351
 352     if not args.cleanup_only:
 353         check_file_exists(args.dea_file)
 354         check_fuel_plugins_dir(args.fuel_plugins_dir)
 355
 356     iso_abs_path = os.path.abspath(args.iso_file)
 357     if not args.no_fuel and not args.cleanup_only:
 358         log('Using OPNFV ISO file: %s' % iso_abs_path)
 359         check_file_exists(iso_abs_path)
 360         log('Using image directory: %s' % args.storage_dir)
 361         create_dir_if_not_exists(args.storage_dir)
 362         for bridge in args.pxe_bridge:
 363             check_bridge(bridge, args.dha_file)
 364
 365
 366     kwargs = {'no_fuel': args.no_fuel, 'fuel_only': args.fuel_only,
 367               'no_health_check': args.no_health_check,
 368               'cleanup_only': args.cleanup_only, 'cleanup': args.cleanup,
 369               'storage_dir': args.storage_dir, 'pxe_bridge': args.pxe_bridge,
 370               'iso_file': iso_abs_path, 'dea_file': args.dea_file,
 371               'dha_file': args.dha_file,
 372               'fuel_plugins_dir': args.fuel_plugins_dir,
 373               'fuel_plugins_conf_dir': args.fuel_plugins_conf_dir,
 374               'no_plugins': args.no_plugins,
 375               'deploy_timeout': args.deploy_timeout,
 376               'no_deploy_environment': args.no_deploy_environment,
 377               'deploy_log': args.deploy_log}
 378     return kwargs
 379
 380
 381 def handle_signals(signal_num, frame):
 382     signal.signal(signal.SIGINT, signal.SIG_IGN)
 383     signal.signal(signal.SIGTERM, signal.SIG_IGN)
 384
 385     log('Caught signal %s, cleaning up and exiting.' % signal_num)
 386
 387     mount_point = os.environ.get(MOUNT_STATE_VAR)
 388     if mount_point:
 389         log('Unmounting ISO from "%s"' % mount_point)
 390         # Prevent 'Device or resource busy' errors when unmounting
 391         os.chdir('/')
 392         exec_cmd('fusermount -u %s' % mount_point, True)
 393         # Be nice and remove our environment variable, even though the OS would
 394         # would clean it up anyway
 395         os.environ.pop(MOUNT_STATE_VAR)
 396
 397     sys.exit(1)
 398
 399
 400 def main():
 401     signal.signal(signal.SIGINT, handle_signals)
 402     signal.signal(signal.SIGTERM, handle_signals)
 403     kwargs = parse_arguments()
 404     d = AutoDeploy(**kwargs)
 405     sys.exit(d.run())
 406
 407 if __name__ == '__main__':
 408     main()