deploy/deploy.py

   1 #!/usr/bin/python
   2 ###############################################################################
   3 # Copyright (c) 2015 Ericsson AB and others.
   4 # szilard.cserey@ericsson.com
   5 # All rights reserved. This program and the accompanying materials
   6 # are made available under the terms of the Apache License, Version 2.0
   7 # which accompanies this distribution, and is available at
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 ###############################################################################
  10
  11
  12 import os
  13 import io
  14 import re
  15 import sys
  16 import yaml
  17 import signal
  18 import netaddr
  19
  20 from dea import DeploymentEnvironmentAdapter
  21 from dha import DeploymentHardwareAdapter
  22 from install_fuel_master import InstallFuelMaster
  23 from deploy_env import CloudDeploy
  24 from execution_environment import ExecutionEnvironment
  25
  26 from common import (
  27     log,
  28     exec_cmd,
  29     err,
  30     warn,
  31     check_file_exists,
  32     check_dir_exists,
  33     create_dir_if_not_exists,
  34     delete,
  35     check_if_root,
  36     ArgParser,
  37 )
  38
  39 FUEL_VM = 'fuel'
  40 PATCH_DIR = 'fuel_patch'
  41 WORK_DIR = '~/deploy'
  42 CWD = os.getcwd()
  43 MOUNT_STATE_VAR = 'AUTODEPLOY_ISO_MOUNTED'
  44
  45
  46 class cd:
  47
  48     def __init__(self, new_path):
  49         self.new_path = os.path.expanduser(new_path)
  50
  51     def __enter__(self):
  52         self.saved_path = CWD
  53         os.chdir(self.new_path)
  54
  55     def __exit__(self, etype, value, traceback):
  56         os.chdir(self.saved_path)
  57
  58
  59 class AutoDeploy(object):
  60
  61     def __init__(self, no_fuel, fuel_only, no_health_check, cleanup_only,
  62                  cleanup, storage_dir, pxe_bridge, iso_file, dea_file,
  63                  dha_file, fuel_plugins_dir, fuel_plugins_conf_dir,
  64                  no_plugins, deploy_timeout, no_deploy_environment, deploy_log):
  65         self.no_fuel = no_fuel
  66         self.fuel_only = fuel_only
  67         self.no_health_check = no_health_check
  68         self.cleanup_only = cleanup_only
  69         self.cleanup = cleanup
  70         self.storage_dir = storage_dir
  71         self.pxe_bridge = pxe_bridge
  72         self.iso_file = iso_file
  73         self.dea_file = dea_file
  74         self.dha_file = dha_file
  75         self.fuel_plugins_dir = fuel_plugins_dir
  76         self.fuel_plugins_conf_dir = fuel_plugins_conf_dir
  77         self.no_plugins = no_plugins
  78         self.deploy_timeout = deploy_timeout
  79         self.no_deploy_environment = no_deploy_environment
  80         self.deploy_log = deploy_log
  81         self.dea = (DeploymentEnvironmentAdapter(dea_file)
  82                     if not cleanup_only else None)
  83         self.dha = DeploymentHardwareAdapter(dha_file)
  84         self.fuel_conf = {}
  85         self.fuel_node_id = self.dha.get_fuel_node_id()
  86         self.fuel_username, self.fuel_password = self.dha.get_fuel_access()
  87         self.tmp_dir = None
  88
  89     def modify_ip(self, ip_addr, index, val):
  90         ip_str = str(netaddr.IPAddress(ip_addr))
  91         decimal_list = map(int, ip_str.split('.'))
  92         decimal_list[index] = val
  93         return '.'.join(map(str, decimal_list))
  94
  95     def collect_fuel_info(self):
  96         self.fuel_conf['ip'] = self.dea.get_fuel_ip()
  97         self.fuel_conf['gw'] = self.dea.get_fuel_gateway()
  98         self.fuel_conf['dns1'] = self.dea.get_fuel_dns()
  99         self.fuel_conf['netmask'] = self.dea.get_fuel_netmask()
 100         self.fuel_conf['hostname'] = self.dea.get_fuel_hostname()
 101         self.fuel_conf['showmenu'] = 'yes'
 102
 103     def install_fuel_master(self):
 104         log('Install Fuel Master')
 105         new_iso = ('%s/deploy-%s'
 106                    % (self.tmp_dir, os.path.basename(self.iso_file)))
 107         self.patch_iso(new_iso)
 108         self.iso_file = new_iso
 109         self.install_iso()
 110
 111     def delete_old_fuel_env(self):
 112         log('Delete old Fuel Master environments if present')
 113         try:
 114             old_dep = CloudDeploy(self.dea, self.dha, self.fuel_conf['ip'],
 115                                   self.fuel_username, self.fuel_password,
 116                                   self.dea_file, self.fuel_plugins_conf_dir,
 117                                   WORK_DIR, self.no_health_check,
 118                                   self.deploy_timeout,
 119                                   self.no_deploy_environment, self.deploy_log)
 120             with old_dep.ssh:
 121                 old_dep.check_previous_installation()
 122         except Exception as e:
 123             log('Could not delete old env: %s' % str(e))
 124
 125     def install_iso(self):
 126         fuel = InstallFuelMaster(self.dea_file, self.dha_file,
 127                                  self.fuel_conf['ip'], self.fuel_username,
 128                                  self.fuel_password, self.fuel_node_id,
 129                                  self.iso_file, WORK_DIR,
 130                                  self.fuel_plugins_dir, self.no_plugins)
 131         fuel.install()
 132
 133     def patch_iso(self, new_iso):
 134         tmp_orig_dir = '%s/origiso' % self.tmp_dir
 135         tmp_new_dir = '%s/newiso' % self.tmp_dir
 136         try:
 137             self.copy(tmp_orig_dir, tmp_new_dir)
 138             self.patch(tmp_new_dir, new_iso)
 139         except Exception as e:
 140             exec_cmd('fusermount -u %s' % tmp_orig_dir, False)
 141             os.environ.pop(MOUNT_STATE_VAR, None)
 142             delete(self.tmp_dir)
 143             err(e)
 144
 145     def copy(self, tmp_orig_dir, tmp_new_dir):
 146         log('Copying...')
 147         os.makedirs(tmp_orig_dir)
 148         os.makedirs(tmp_new_dir)
 149         exec_cmd('fuseiso %s %s' % (self.iso_file, tmp_orig_dir))
 150         os.environ[MOUNT_STATE_VAR] = tmp_orig_dir
 151         with cd(tmp_orig_dir):
 152             exec_cmd('find . | cpio -pd %s' % tmp_new_dir)
 153         exec_cmd('fusermount -u %s' % tmp_orig_dir)
 154         os.environ.pop(MOUNT_STATE_VAR, None)
 155         delete(tmp_orig_dir)
 156         exec_cmd('chmod -R 755 %s' % tmp_new_dir)
 157
 158     def patch(self, tmp_new_dir, new_iso):
 159         log('Patching...')
 160         patch_dir = '%s/%s' % (CWD, PATCH_DIR)
 161         ks_path = '%s/ks.cfg.patch' % patch_dir
 162
 163         with cd(tmp_new_dir):
 164             exec_cmd('cat %s | patch -p0' % ks_path)
 165             delete('.rr_moved')
 166             isolinux = 'isolinux/isolinux.cfg'
 167             log('isolinux.cfg before: %s'
 168                 % exec_cmd('grep ip= %s' % isolinux))
 169             self.update_fuel_isolinux(isolinux)
 170             log('isolinux.cfg after: %s'
 171                 % exec_cmd('grep ip= %s' % isolinux))
 172
 173             iso_label = self.parse_iso_volume_label(self.iso_file)
 174             log('Volume label: %s' % iso_label)
 175
 176             iso_linux_bin = 'isolinux/isolinux.bin'
 177             exec_cmd('mkisofs -quiet -r -J -R -b %s '
 178                      '-no-emul-boot -boot-load-size 4 '
 179                      '-boot-info-table -hide-rr-moved '
 180                      '-joliet-long '
 181                      '-x "lost+found:" -V %s -o %s .'
 182                      % (iso_linux_bin, iso_label, new_iso))
 183
 184         delete(tmp_new_dir)
 185
 186     def update_fuel_isolinux(self, file):
 187         with io.open(file) as f:
 188             data = f.read()
 189         for key, val in self.fuel_conf.iteritems():
 190             # skip replacing these keys, as the format is different
 191             if key in ['ip', 'gw', 'netmask', 'hostname']:
 192                 continue
 193
 194             pattern = r'%s=[^ ]\S+' % key
 195             replace = '%s=%s' % (key, val)
 196             data = re.sub(pattern, replace, data)
 197
 198         # process networking parameters
 199         ip = ':'.join([self.fuel_conf['ip'],
 200                       '',
 201                       self.fuel_conf['gw'],
 202                       self.fuel_conf['netmask'],
 203                       self.fuel_conf['hostname'],
 204                       'eth0:off:::'])
 205
 206         data = re.sub(r'ip=[^ ]\S+', 'ip=%s' % ip, data)
 207
 208         with io.open(file, 'w') as f:
 209             f.write(data)
 210
 211     def parse_iso_volume_label(self, iso_filename):
 212         label_line = exec_cmd('isoinfo -d -i %s | grep -i "Volume id: "' % iso_filename)
 213         # cut leading text: 'Volume id: '
 214         return label_line[11:]
 215
 216     def deploy_env(self):
 217         dep = CloudDeploy(self.dea, self.dha, self.fuel_conf['ip'],
 218                           self.fuel_username, self.fuel_password,
 219                           self.dea_file, self.fuel_plugins_conf_dir,
 220                           WORK_DIR, self.no_health_check, self.deploy_timeout,
 221                           self.no_deploy_environment, self.deploy_log)
 222         return dep.deploy()
 223
 224     def setup_execution_environment(self):
 225         exec_env = ExecutionEnvironment(self.storage_dir, self.pxe_bridge,
 226                                         self.dha_file, self.dea)
 227         exec_env.setup_environment()
 228
 229     def cleanup_execution_environment(self):
 230         exec_env = ExecutionEnvironment(self.storage_dir, self.pxe_bridge,
 231                                         self.dha_file, self.dea)
 232         exec_env.cleanup_environment()
 233
 234     def create_tmp_dir(self):
 235         self.tmp_dir = '%s/fueltmp' % CWD
 236         delete(self.tmp_dir)
 237         create_dir_if_not_exists(self.tmp_dir)
 238
 239     def deploy(self):
 240         self.collect_fuel_info()
 241         if not self.no_fuel:
 242             self.delete_old_fuel_env()
 243             self.setup_execution_environment()
 244             self.create_tmp_dir()
 245             self.install_fuel_master()
 246         if not self.fuel_only:
 247             return self.deploy_env()
 248         # Exit status
 249         return 0
 250
 251     def run(self):
 252         check_if_root()
 253         if self.cleanup_only:
 254             self.cleanup_execution_environment()
 255         else:
 256             deploy_success = self.deploy()
 257             if self.cleanup:
 258                 self.cleanup_execution_environment()
 259             return deploy_success
 260         # Exit status
 261         return 0
 262
 263
 264 def check_bridge(pxe_bridge, dha_path):
 265     # Assume that bridges on remote nodes exists, we could ssh but
 266     # the remote user might not have a login shell.
 267     if os.environ.get('LIBVIRT_DEFAULT_URI'):
 268         return
 269
 270     with io.open(dha_path) as yaml_file:
 271         dha_struct = yaml.load(yaml_file)
 272     if dha_struct['adapter'] != 'libvirt':
 273         log('Using Linux Bridge %s for booting up the Fuel Master VM'
 274             % pxe_bridge)
 275         r = exec_cmd('ip link show %s' % pxe_bridge)
 276         if pxe_bridge in r and 'state DOWN' in r:
 277             err('Linux Bridge {0} is not Active, bring'
 278                 ' it UP first: [ip link set dev {0} up]'.format(pxe_bridge))
 279
 280
 281 def check_fuel_plugins_dir(dir):
 282     msg = None
 283     if not dir:
 284         msg = 'Fuel Plugins Directory not specified!'
 285     elif not os.path.isdir(dir):
 286         msg = 'Fuel Plugins Directory does not exist!'
 287     elif not os.listdir(dir):
 288         msg = 'Fuel Plugins Directory is empty!'
 289     if msg:
 290         warn('%s No external plugins will be installed!' % msg)
 291
 292
 293 def parse_arguments():
 294     parser = ArgParser(prog='python %s' % __file__)
 295     parser.add_argument('-nf', dest='no_fuel', action='store_true',
 296                         default=False,
 297                         help='Do not install Fuel Master (and Node VMs when '
 298                              'using libvirt)')
 299     parser.add_argument('-nh', dest='no_health_check', action='store_true',
 300                         default=False,
 301                         help='Don\'t run health check after deployment')
 302     parser.add_argument('-fo', dest='fuel_only', action='store_true',
 303                         default=False,
 304                         help='Install Fuel Master only (and Node VMs when '
 305                              'using libvirt)')
 306     parser.add_argument('-co', dest='cleanup_only', action='store_true',
 307                         default=False,
 308                         help='Cleanup VMs and Virtual Networks according to '
 309                              'what is defined in DHA')
 310     parser.add_argument('-c', dest='cleanup', action='store_true',
 311                         default=False,
 312                         help='Cleanup after deploy')
 313     if {'-iso', '-dea', '-dha', '-h'}.intersection(sys.argv):
 314         parser.add_argument('-iso', dest='iso_file', action='store', nargs='?',
 315                             default='%s/OPNFV.iso' % CWD,
 316                             help='ISO File [default: OPNFV.iso]')
 317         parser.add_argument('-dea', dest='dea_file', action='store', nargs='?',
 318                             default='%s/dea.yaml' % CWD,
 319                             help='Deployment Environment Adapter: dea.yaml')
 320         parser.add_argument('-dha', dest='dha_file', action='store', nargs='?',
 321                             default='%s/dha.yaml' % CWD,
 322                             help='Deployment Hardware Adapter: dha.yaml')
 323     else:
 324         parser.add_argument('iso_file', action='store', nargs='?',
 325                             default='%s/OPNFV.iso' % CWD,
 326                             help='ISO File [default: OPNFV.iso]')
 327         parser.add_argument('dea_file', action='store', nargs='?',
 328                             default='%s/dea.yaml' % CWD,
 329                             help='Deployment Environment Adapter: dea.yaml')
 330         parser.add_argument('dha_file', action='store', nargs='?',
 331                             default='%s/dha.yaml' % CWD,
 332                             help='Deployment Hardware Adapter: dha.yaml')
 333     parser.add_argument('-s', dest='storage_dir', action='store',
 334                         default='%s/images' % CWD,
 335                         help='Storage Directory [default: images]')
 336     parser.add_argument('-b', dest='pxe_bridge', action='append',
 337                         default=[],
 338                         help='Linux Bridge for booting up the Fuel Master VM '
 339                              '[default: pxebr]')
 340     parser.add_argument('-p', dest='fuel_plugins_dir', action='store',
 341                         help='Fuel Plugins directory')
 342     parser.add_argument('-pc', dest='fuel_plugins_conf_dir', action='store',
 343                         help='Fuel Plugins Configuration directory')
 344     parser.add_argument('-np', dest='no_plugins', action='store_true',
 345                         default=False, help='Do not install Fuel Plugins')
 346     parser.add_argument('-dt', dest='deploy_timeout', action='store',
 347                         default=240, help='Deployment timeout (in minutes) '
 348                         '[default: 240]')
 349     parser.add_argument('-nde', dest='no_deploy_environment',
 350                         action='store_true', default=False,
 351                         help=('Do not launch environment deployment'))
 352     parser.add_argument('-log', dest='deploy_log',
 353                         action='store', default='../ci/.',
 354                         help=('Path and name of the deployment log archive'))
 355
 356     args = parser.parse_args()
 357     log(args)
 358
 359     if not args.pxe_bridge:
 360         args.pxe_bridge = ['pxebr']
 361
 362     check_file_exists(args.dha_file)
 363
 364     check_dir_exists(os.path.dirname(args.deploy_log))
 365
 366     if not args.cleanup_only:
 367         check_file_exists(args.dea_file)
 368         check_fuel_plugins_dir(args.fuel_plugins_dir)
 369
 370     iso_abs_path = os.path.abspath(args.iso_file)
 371     if not args.no_fuel and not args.cleanup_only:
 372         log('Using OPNFV ISO file: %s' % iso_abs_path)
 373         check_file_exists(iso_abs_path)
 374         log('Using image directory: %s' % args.storage_dir)
 375         create_dir_if_not_exists(args.storage_dir)
 376         for bridge in args.pxe_bridge:
 377             check_bridge(bridge, args.dha_file)
 378
 379
 380     kwargs = {'no_fuel': args.no_fuel, 'fuel_only': args.fuel_only,
 381               'no_health_check': args.no_health_check,
 382               'cleanup_only': args.cleanup_only, 'cleanup': args.cleanup,
 383               'storage_dir': args.storage_dir, 'pxe_bridge': args.pxe_bridge,
 384               'iso_file': iso_abs_path, 'dea_file': args.dea_file,
 385               'dha_file': args.dha_file,
 386               'fuel_plugins_dir': args.fuel_plugins_dir,
 387               'fuel_plugins_conf_dir': args.fuel_plugins_conf_dir,
 388               'no_plugins': args.no_plugins,
 389               'deploy_timeout': args.deploy_timeout,
 390               'no_deploy_environment': args.no_deploy_environment,
 391               'deploy_log': args.deploy_log}
 392     return kwargs
 393
 394
 395 def handle_signals(signal_num, frame):
 396     signal.signal(signal.SIGINT, signal.SIG_IGN)
 397     signal.signal(signal.SIGTERM, signal.SIG_IGN)
 398
 399     log('Caught signal %s, cleaning up and exiting.' % signal_num)
 400
 401     mount_point = os.environ.get(MOUNT_STATE_VAR)
 402     if mount_point:
 403         log('Unmounting ISO from "%s"' % mount_point)
 404         # Prevent 'Device or resource busy' errors when unmounting
 405         os.chdir('/')
 406         exec_cmd('fusermount -u %s' % mount_point, True)
 407         # Be nice and remove our environment variable, even though the OS would
 408         # would clean it up anyway
 409         os.environ.pop(MOUNT_STATE_VAR)
 410
 411     sys.exit(1)
 412
 413
 414 def main():
 415     signal.signal(signal.SIGINT, handle_signals)
 416     signal.signal(signal.SIGTERM, handle_signals)
 417     kwargs = parse_arguments()
 418     d = AutoDeploy(**kwargs)
 419     sys.exit(d.run())
 420
 421 if __name__ == '__main__':
 422     main()