deploy/deploy.py

   1 #!/usr/bin/python
   2 ###############################################################################
   3 # Copyright (c) 2015 Ericsson AB and others.
   4 # szilard.cserey@ericsson.com
   5 # All rights reserved. This program and the accompanying materials
   6 # are made available under the terms of the Apache License, Version 2.0
   7 # which accompanies this distribution, and is available at
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 ###############################################################################
  10
  11
  12 import os
  13 import io
  14 import re
  15 import sys
  16 import yaml
  17 import errno
  18 import signal
  19 import netaddr
  20
  21 from dea import DeploymentEnvironmentAdapter
  22 from dha import DeploymentHardwareAdapter
  23 from install_fuel_master import InstallFuelMaster
  24 from deploy_env import CloudDeploy
  25 from execution_environment import ExecutionEnvironment
  26
  27 from common import (
  28     log,
  29     exec_cmd,
  30     err,
  31     warn,
  32     check_file_exists,
  33     check_dir_exists,
  34     create_dir_if_not_exists,
  35     delete,
  36     check_if_root,
  37     ArgParser,
  38 )
  39
  40 FUEL_VM = 'fuel'
  41 PATCH_DIR = 'fuel_patch'
  42 WORK_DIR = '~/deploy'
  43 CWD = os.getcwd()
  44 MOUNT_STATE_VAR = 'AUTODEPLOY_ISO_MOUNTED'
  45
  46
  47 class cd:
  48
  49     def __init__(self, new_path):
  50         self.new_path = os.path.expanduser(new_path)
  51
  52     def __enter__(self):
  53         self.saved_path = CWD
  54         os.chdir(self.new_path)
  55
  56     def __exit__(self, etype, value, traceback):
  57         os.chdir(self.saved_path)
  58
  59
  60 class AutoDeploy(object):
  61
  62     def __init__(self, no_fuel, fuel_only, no_health_check, cleanup_only,
  63                  cleanup, storage_dir, pxe_bridge, iso_file, dea_file,
  64                  dha_file, fuel_plugins_dir, fuel_plugins_conf_dir,
  65                  no_plugins, deploy_timeout, no_deploy_environment, deploy_log):
  66         self.no_fuel = no_fuel
  67         self.fuel_only = fuel_only
  68         self.no_health_check = no_health_check
  69         self.cleanup_only = cleanup_only
  70         self.cleanup = cleanup
  71         self.storage_dir = storage_dir
  72         self.pxe_bridge = pxe_bridge
  73         self.iso_file = iso_file
  74         self.dea_file = dea_file
  75         self.dha_file = dha_file
  76         self.fuel_plugins_dir = fuel_plugins_dir
  77         self.fuel_plugins_conf_dir = fuel_plugins_conf_dir
  78         self.no_plugins = no_plugins
  79         self.deploy_timeout = deploy_timeout
  80         self.no_deploy_environment = no_deploy_environment
  81         self.deploy_log = deploy_log
  82         self.dea = (DeploymentEnvironmentAdapter(dea_file)
  83                     if not cleanup_only else None)
  84         self.dha = DeploymentHardwareAdapter(dha_file)
  85         self.fuel_conf = {}
  86         self.fuel_node_id = self.dha.get_fuel_node_id()
  87         self.fuel_username, self.fuel_password = self.dha.get_fuel_access()
  88         self.tmp_dir = None
  89
  90     def modify_ip(self, ip_addr, index, val):
  91         ip_str = str(netaddr.IPAddress(ip_addr))
  92         decimal_list = map(int, ip_str.split('.'))
  93         decimal_list[index] = val
  94         return '.'.join(map(str, decimal_list))
  95
  96     def collect_fuel_info(self):
  97         self.fuel_conf['ip'] = self.dea.get_fuel_ip()
  98         self.fuel_conf['gw'] = self.dea.get_fuel_gateway()
  99         self.fuel_conf['dns1'] = self.dea.get_fuel_dns()
 100         self.fuel_conf['netmask'] = self.dea.get_fuel_netmask()
 101         self.fuel_conf['hostname'] = self.dea.get_fuel_hostname()
 102         self.fuel_conf['showmenu'] = 'yes'
 103
 104     def install_fuel_master(self):
 105         log('Install Fuel Master')
 106         new_iso = ('%s/deploy-%s'
 107                    % (self.tmp_dir, os.path.basename(self.iso_file)))
 108         self.patch_iso(new_iso)
 109         self.iso_file = new_iso
 110         self.install_iso()
 111
 112     def delete_old_fuel_env(self):
 113         log('Delete old Fuel Master environments if present')
 114         try:
 115             old_dep = CloudDeploy(self.dea, self.dha, self.fuel_conf['ip'],
 116                                   self.fuel_username, self.fuel_password,
 117                                   self.dea_file, self.fuel_plugins_conf_dir,
 118                                   WORK_DIR, self.no_health_check,
 119                                   self.deploy_timeout,
 120                                   self.no_deploy_environment, self.deploy_log)
 121             with old_dep.ssh:
 122                 old_dep.check_previous_installation()
 123         except Exception as e:
 124             log('Could not delete old env: %s' % str(e))
 125
 126     def install_iso(self):
 127         fuel = InstallFuelMaster(self.dea_file, self.dha_file,
 128                                  self.fuel_conf['ip'], self.fuel_username,
 129                                  self.fuel_password, self.fuel_node_id,
 130                                  self.iso_file, WORK_DIR,
 131                                  self.fuel_plugins_dir, self.no_plugins)
 132         fuel.install()
 133
 134     def patch_iso(self, new_iso):
 135         tmp_orig_dir = '%s/origiso' % self.tmp_dir
 136         tmp_new_dir = '%s/newiso' % self.tmp_dir
 137         try:
 138             self.copy(tmp_orig_dir, tmp_new_dir)
 139             self.patch(tmp_new_dir, new_iso)
 140         except Exception as e:
 141             exec_cmd('fusermount -u %s' % tmp_orig_dir, False)
 142             os.environ.pop(MOUNT_STATE_VAR, None)
 143             delete(self.tmp_dir)
 144             err(e)
 145
 146     def copy(self, tmp_orig_dir, tmp_new_dir):
 147         log('Copying...')
 148         os.makedirs(tmp_orig_dir)
 149         os.makedirs(tmp_new_dir)
 150         exec_cmd('fuseiso %s %s' % (self.iso_file, tmp_orig_dir))
 151         os.environ[MOUNT_STATE_VAR] = tmp_orig_dir
 152         with cd(tmp_orig_dir):
 153             exec_cmd('find . | cpio -pd %s' % tmp_new_dir)
 154         exec_cmd('fusermount -u %s' % tmp_orig_dir)
 155         os.environ.pop(MOUNT_STATE_VAR, None)
 156         delete(tmp_orig_dir)
 157         exec_cmd('chmod -R 755 %s' % tmp_new_dir)
 158
 159     def patch(self, tmp_new_dir, new_iso):
 160         log('Patching...')
 161         patch_dir = '%s/%s' % (CWD, PATCH_DIR)
 162         ks_path = '%s/ks.cfg.patch' % patch_dir
 163
 164         with cd(tmp_new_dir):
 165             exec_cmd('cat %s | patch -p0' % ks_path)
 166             delete('.rr_moved')
 167             isolinux = 'isolinux/isolinux.cfg'
 168             log('isolinux.cfg before: %s'
 169                 % exec_cmd('grep ip= %s' % isolinux))
 170             self.update_fuel_isolinux(isolinux)
 171             log('isolinux.cfg after: %s'
 172                 % exec_cmd('grep ip= %s' % isolinux))
 173
 174             iso_label = self.parse_iso_volume_label(self.iso_file)
 175             log('Volume label: %s' % iso_label)
 176
 177             iso_linux_bin = 'isolinux/isolinux.bin'
 178             exec_cmd('mkisofs -quiet -r -J -R -b %s '
 179                      '-no-emul-boot -boot-load-size 4 '
 180                      '-boot-info-table -hide-rr-moved '
 181                      '-joliet-long '
 182                      '-x "lost+found:" -V %s -o %s .'
 183                      % (iso_linux_bin, iso_label, new_iso))
 184
 185         delete(tmp_new_dir)
 186
 187     def update_fuel_isolinux(self, file):
 188         with io.open(file) as f:
 189             data = f.read()
 190         for key, val in self.fuel_conf.iteritems():
 191             # skip replacing these keys, as the format is different
 192             if key in ['ip', 'gw', 'netmask', 'hostname']:
 193                 continue
 194
 195             pattern = r'%s=[^ ]\S+' % key
 196             replace = '%s=%s' % (key, val)
 197             data = re.sub(pattern, replace, data)
 198
 199         # process networking parameters
 200         ip = ':'.join([self.fuel_conf['ip'],
 201                       '',
 202                       self.fuel_conf['gw'],
 203                       self.fuel_conf['netmask'],
 204                       self.fuel_conf['hostname'],
 205                       'eth0:off:::'])
 206
 207         data = re.sub(r'ip=[^ ]\S+', 'ip=%s' % ip, data)
 208
 209         with io.open(file, 'w') as f:
 210             f.write(data)
 211
 212     def parse_iso_volume_label(self, iso_filename):
 213         label_line = exec_cmd('isoinfo -d -i %s | grep -i "Volume id: "' % iso_filename)
 214         # cut leading text: 'Volume id: '
 215         return label_line[11:]
 216
 217     def deploy_env(self):
 218         dep = CloudDeploy(self.dea, self.dha, self.fuel_conf['ip'],
 219                           self.fuel_username, self.fuel_password,
 220                           self.dea_file, self.fuel_plugins_conf_dir,
 221                           WORK_DIR, self.no_health_check, self.deploy_timeout,
 222                           self.no_deploy_environment, self.deploy_log)
 223         return dep.deploy()
 224
 225     def setup_execution_environment(self):
 226         exec_env = ExecutionEnvironment(self.storage_dir, self.pxe_bridge,
 227                                         self.dha_file, self.dea)
 228         exec_env.setup_environment()
 229
 230     def cleanup_execution_environment(self):
 231         exec_env = ExecutionEnvironment(self.storage_dir, self.pxe_bridge,
 232                                         self.dha_file, self.dea)
 233         exec_env.cleanup_environment()
 234
 235     def create_tmp_dir(self):
 236         self.tmp_dir = '%s/fueltmp' % CWD
 237         delete(self.tmp_dir)
 238         create_dir_if_not_exists(self.tmp_dir)
 239
 240     def deploy(self):
 241         self.collect_fuel_info()
 242         if not self.no_fuel:
 243             self.delete_old_fuel_env()
 244             self.setup_execution_environment()
 245             self.create_tmp_dir()
 246             self.install_fuel_master()
 247         if not self.fuel_only:
 248             return self.deploy_env()
 249         # Exit status
 250         return 0
 251
 252     def run(self):
 253         check_if_root()
 254         if self.cleanup_only:
 255             self.cleanup_execution_environment()
 256         else:
 257             deploy_success = self.deploy()
 258             if self.cleanup:
 259                 self.cleanup_execution_environment()
 260             return deploy_success
 261         # Exit status
 262         return 0
 263
 264
 265 def check_bridge(pxe_bridge, dha_path):
 266     # Assume that bridges on remote nodes exists, we could ssh but
 267     # the remote user might not have a login shell.
 268     if os.environ.get('LIBVIRT_DEFAULT_URI'):
 269         return
 270
 271     with io.open(dha_path) as yaml_file:
 272         dha_struct = yaml.load(yaml_file)
 273     if dha_struct['adapter'] != 'libvirt':
 274         log('Using Linux Bridge %s for booting up the Fuel Master VM'
 275             % pxe_bridge)
 276         r = exec_cmd('ip link show %s' % pxe_bridge)
 277         if pxe_bridge in r and 'state DOWN' in r:
 278             err('Linux Bridge {0} is not Active, bring'
 279                 ' it UP first: [ip link set dev {0} up]'.format(pxe_bridge))
 280
 281
 282 def check_fuel_plugins_dir(dir):
 283     msg = None
 284     if not dir:
 285         msg = 'Fuel Plugins Directory not specified!'
 286     elif not os.path.isdir(dir):
 287         msg = 'Fuel Plugins Directory does not exist!'
 288     elif not os.listdir(dir):
 289         msg = 'Fuel Plugins Directory is empty!'
 290     if msg:
 291         warn('%s No external plugins will be installed!' % msg)
 292
 293
 294 def parse_arguments():
 295     parser = ArgParser(prog='python %s' % __file__)
 296     parser.add_argument('-nf', dest='no_fuel', action='store_true',
 297                         default=False,
 298                         help='Do not install Fuel Master (and Node VMs when '
 299                              'using libvirt)')
 300     parser.add_argument('-nh', dest='no_health_check', action='store_true',
 301                         default=False,
 302                         help='Don\'t run health check after deployment')
 303     parser.add_argument('-fo', dest='fuel_only', action='store_true',
 304                         default=False,
 305                         help='Install Fuel Master only (and Node VMs when '
 306                              'using libvirt)')
 307     parser.add_argument('-co', dest='cleanup_only', action='store_true',
 308                         default=False,
 309                         help='Cleanup VMs and Virtual Networks according to '
 310                              'what is defined in DHA')
 311     parser.add_argument('-c', dest='cleanup', action='store_true',
 312                         default=False,
 313                         help='Cleanup after deploy')
 314     if {'-iso', '-dea', '-dha', '-h'}.intersection(sys.argv):
 315         parser.add_argument('-iso', dest='iso_file', action='store', nargs='?',
 316                             default='%s/OPNFV.iso' % CWD,
 317                             help='ISO File [default: OPNFV.iso]')
 318         parser.add_argument('-dea', dest='dea_file', action='store', nargs='?',
 319                             default='%s/dea.yaml' % CWD,
 320                             help='Deployment Environment Adapter: dea.yaml')
 321         parser.add_argument('-dha', dest='dha_file', action='store', nargs='?',
 322                             default='%s/dha.yaml' % CWD,
 323                             help='Deployment Hardware Adapter: dha.yaml')
 324     else:
 325         parser.add_argument('iso_file', action='store', nargs='?',
 326                             default='%s/OPNFV.iso' % CWD,
 327                             help='ISO File [default: OPNFV.iso]')
 328         parser.add_argument('dea_file', action='store', nargs='?',
 329                             default='%s/dea.yaml' % CWD,
 330                             help='Deployment Environment Adapter: dea.yaml')
 331         parser.add_argument('dha_file', action='store', nargs='?',
 332                             default='%s/dha.yaml' % CWD,
 333                             help='Deployment Hardware Adapter: dha.yaml')
 334     parser.add_argument('-s', dest='storage_dir', action='store',
 335                         default='%s/images' % CWD,
 336                         help='Storage Directory [default: images]')
 337     parser.add_argument('-b', dest='pxe_bridge', action='append',
 338                         default=[],
 339                         help='Linux Bridge for booting up the Fuel Master VM '
 340                              '[default: pxebr]')
 341     parser.add_argument('-p', dest='fuel_plugins_dir', action='store',
 342                         help='Fuel Plugins directory')
 343     parser.add_argument('-pc', dest='fuel_plugins_conf_dir', action='store',
 344                         help='Fuel Plugins Configuration directory')
 345     parser.add_argument('-np', dest='no_plugins', action='store_true',
 346                         default=False, help='Do not install Fuel Plugins')
 347     parser.add_argument('-dt', dest='deploy_timeout', action='store',
 348                         default=240, help='Deployment timeout (in minutes) '
 349                         '[default: 240]')
 350     parser.add_argument('-nde', dest='no_deploy_environment',
 351                         action='store_true', default=False,
 352                         help=('Do not launch environment deployment'))
 353     parser.add_argument('-log', dest='deploy_log',
 354                         action='store', default='../ci/.',
 355                         help=('Path and name of the deployment log archive'))
 356
 357     args = parser.parse_args()
 358     log(args)
 359
 360     if not args.pxe_bridge:
 361         args.pxe_bridge = ['pxebr']
 362
 363     check_file_exists(args.dha_file)
 364
 365     check_dir_exists(os.path.dirname(args.deploy_log))
 366
 367     if not args.cleanup_only:
 368         check_file_exists(args.dea_file)
 369         check_fuel_plugins_dir(args.fuel_plugins_dir)
 370
 371     iso_abs_path = os.path.abspath(args.iso_file)
 372     if not args.no_fuel and not args.cleanup_only:
 373         log('Using OPNFV ISO file: %s' % iso_abs_path)
 374         check_file_exists(iso_abs_path)
 375         log('Using image directory: %s' % args.storage_dir)
 376         create_dir_if_not_exists(args.storage_dir)
 377         for bridge in args.pxe_bridge:
 378             check_bridge(bridge, args.dha_file)
 379
 380
 381     kwargs = {'no_fuel': args.no_fuel, 'fuel_only': args.fuel_only,
 382               'no_health_check': args.no_health_check,
 383               'cleanup_only': args.cleanup_only, 'cleanup': args.cleanup,
 384               'storage_dir': args.storage_dir, 'pxe_bridge': args.pxe_bridge,
 385               'iso_file': iso_abs_path, 'dea_file': args.dea_file,
 386               'dha_file': args.dha_file,
 387               'fuel_plugins_dir': args.fuel_plugins_dir,
 388               'fuel_plugins_conf_dir': args.fuel_plugins_conf_dir,
 389               'no_plugins': args.no_plugins,
 390               'deploy_timeout': args.deploy_timeout,
 391               'no_deploy_environment': args.no_deploy_environment,
 392               'deploy_log': args.deploy_log}
 393     return kwargs
 394
 395
 396 def handle_signals(signal_num, frame):
 397     signal.signal(signal.SIGINT, signal.SIG_IGN)
 398     signal.signal(signal.SIGTERM, signal.SIG_IGN)
 399
 400     log('Caught signal %s, cleaning up and exiting.' % signal_num)
 401
 402     mount_point = os.environ.get(MOUNT_STATE_VAR)
 403     if mount_point:
 404         log('Unmounting ISO from "%s"' % mount_point)
 405         # Prevent 'Device or resource busy' errors when unmounting
 406         os.chdir('/')
 407         exec_cmd('fusermount -u %s' % mount_point, True)
 408         # Be nice and remove our environment variable, even though the OS would
 409         # would clean it up anyway
 410         os.environ.pop(MOUNT_STATE_VAR)
 411
 412     sys.exit(1)
 413
 414
 415 def main():
 416     signal.signal(signal.SIGINT, handle_signals)
 417     signal.signal(signal.SIGTERM, handle_signals)
 418     kwargs = parse_arguments()
 419     d = AutoDeploy(**kwargs)
 420     sys.exit(d.run())
 421
 422 if __name__ == '__main__':
 423     main()