deploy/deploy.py

   1 #!/usr/bin/python
   2 ###############################################################################
   3 # Copyright (c) 2015 Ericsson AB and others.
   4 # szilard.cserey@ericsson.com
   5 # All rights reserved. This program and the accompanying materials
   6 # are made available under the terms of the Apache License, Version 2.0
   7 # which accompanies this distribution, and is available at
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 ###############################################################################
  10
  11
  12 import os
  13 import io
  14 import re
  15 import sys
  16 import yaml
  17 import errno
  18 import signal
  19 import netaddr
  20
  21 from dea import DeploymentEnvironmentAdapter
  22 from dha import DeploymentHardwareAdapter
  23 from install_fuel_master import InstallFuelMaster
  24 from deploy_env import CloudDeploy
  25 from execution_environment import ExecutionEnvironment
  26
  27 from common import (
  28     log,
  29     exec_cmd,
  30     err,
  31     warn,
  32     check_file_exists,
  33     check_dir_exists,
  34     create_dir_if_not_exists,
  35     delete,
  36     check_if_root,
  37     ArgParser,
  38 )
  39
  40 FUEL_VM = 'fuel'
  41 PATCH_DIR = 'fuel_patch'
  42 WORK_DIR = '~/deploy'
  43 CWD = os.getcwd()
  44 MOUNT_STATE_VAR = 'AUTODEPLOY_ISO_MOUNTED'
  45
  46
  47 class cd:
  48
  49     def __init__(self, new_path):
  50         self.new_path = os.path.expanduser(new_path)
  51
  52     def __enter__(self):
  53         self.saved_path = CWD
  54         os.chdir(self.new_path)
  55
  56     def __exit__(self, etype, value, traceback):
  57         os.chdir(self.saved_path)
  58
  59
  60 class AutoDeploy(object):
  61
  62     def __init__(self, no_fuel, fuel_only, no_health_check, cleanup_only,
  63                  cleanup, storage_dir, pxe_bridge, iso_file, dea_file,
  64                  dha_file, fuel_plugins_dir, fuel_plugins_conf_dir,
  65                  no_plugins, deploy_timeout, no_deploy_environment, deploy_log):
  66         self.no_fuel = no_fuel
  67         self.fuel_only = fuel_only
  68         self.no_health_check = no_health_check
  69         self.cleanup_only = cleanup_only
  70         self.cleanup = cleanup
  71         self.storage_dir = storage_dir
  72         self.pxe_bridge = pxe_bridge
  73         self.iso_file = iso_file
  74         self.dea_file = dea_file
  75         self.dha_file = dha_file
  76         self.fuel_plugins_dir = fuel_plugins_dir
  77         self.fuel_plugins_conf_dir = fuel_plugins_conf_dir
  78         self.no_plugins = no_plugins
  79         self.deploy_timeout = deploy_timeout
  80         self.no_deploy_environment = no_deploy_environment
  81         self.deploy_log = deploy_log
  82         self.dea = (DeploymentEnvironmentAdapter(dea_file)
  83                     if not cleanup_only else None)
  84         self.dha = DeploymentHardwareAdapter(dha_file)
  85         self.fuel_conf = {}
  86         self.fuel_node_id = self.dha.get_fuel_node_id()
  87         self.fuel_username, self.fuel_password = self.dha.get_fuel_access()
  88         self.tmp_dir = None
  89
  90     def modify_ip(self, ip_addr, index, val):
  91         ip_str = str(netaddr.IPAddress(ip_addr))
  92         decimal_list = map(int, ip_str.split('.'))
  93         decimal_list[index] = val
  94         return '.'.join(map(str, decimal_list))
  95
  96     def collect_fuel_info(self):
  97         self.fuel_conf['ip'] = self.dea.get_fuel_ip()
  98         self.fuel_conf['gw'] = self.dea.get_fuel_gateway()
  99         self.fuel_conf['dns1'] = self.dea.get_fuel_dns()
 100         self.fuel_conf['netmask'] = self.dea.get_fuel_netmask()
 101         self.fuel_conf['hostname'] = self.dea.get_fuel_hostname()
 102         self.fuel_conf['showmenu'] = 'yes'
 103
 104     def install_fuel_master(self):
 105         log('Install Fuel Master')
 106         new_iso = ('%s/deploy-%s'
 107                    % (self.tmp_dir, os.path.basename(self.iso_file)))
 108         self.patch_iso(new_iso)
 109         self.iso_file = new_iso
 110         self.install_iso()
 111
 112     def install_iso(self):
 113         fuel = InstallFuelMaster(self.dea_file, self.dha_file,
 114                                  self.fuel_conf['ip'], self.fuel_username,
 115                                  self.fuel_password, self.fuel_node_id,
 116                                  self.iso_file, WORK_DIR,
 117                                  self.fuel_plugins_dir, self.no_plugins)
 118         fuel.install()
 119
 120     def patch_iso(self, new_iso):
 121         tmp_orig_dir = '%s/origiso' % self.tmp_dir
 122         tmp_new_dir = '%s/newiso' % self.tmp_dir
 123         try:
 124             self.copy(tmp_orig_dir, tmp_new_dir)
 125             self.patch(tmp_new_dir, new_iso)
 126         except Exception as e:
 127             exec_cmd('fusermount -u %s' % tmp_orig_dir, False)
 128             os.environ.pop(MOUNT_STATE_VAR, None)
 129             delete(self.tmp_dir)
 130             err(e)
 131
 132     def copy(self, tmp_orig_dir, tmp_new_dir):
 133         log('Copying...')
 134         os.makedirs(tmp_orig_dir)
 135         os.makedirs(tmp_new_dir)
 136         exec_cmd('fuseiso %s %s' % (self.iso_file, tmp_orig_dir))
 137         os.environ[MOUNT_STATE_VAR] = tmp_orig_dir
 138         with cd(tmp_orig_dir):
 139             exec_cmd('find . | cpio -pd %s' % tmp_new_dir)
 140         exec_cmd('fusermount -u %s' % tmp_orig_dir)
 141         os.environ.pop(MOUNT_STATE_VAR, None)
 142         delete(tmp_orig_dir)
 143         exec_cmd('chmod -R 755 %s' % tmp_new_dir)
 144
 145     def patch(self, tmp_new_dir, new_iso):
 146         log('Patching...')
 147         patch_dir = '%s/%s' % (CWD, PATCH_DIR)
 148         ks_path = '%s/ks.cfg.patch' % patch_dir
 149
 150         with cd(tmp_new_dir):
 151             exec_cmd('cat %s | patch -p0' % ks_path)
 152             delete('.rr_moved')
 153             isolinux = 'isolinux/isolinux.cfg'
 154             log('isolinux.cfg before: %s'
 155                 % exec_cmd('grep ip= %s' % isolinux))
 156             self.update_fuel_isolinux(isolinux)
 157             log('isolinux.cfg after: %s'
 158                 % exec_cmd('grep ip= %s' % isolinux))
 159
 160             iso_label = self.parse_iso_volume_label(self.iso_file)
 161             log('Volume label: %s' % iso_label)
 162
 163             iso_linux_bin = 'isolinux/isolinux.bin'
 164             exec_cmd('mkisofs -quiet -r -J -R -b %s '
 165                      '-no-emul-boot -boot-load-size 4 '
 166                      '-boot-info-table -hide-rr-moved '
 167                      '-x "lost+found:" -V %s -o %s .'
 168                      % (iso_linux_bin, iso_label, new_iso))
 169
 170         delete(tmp_new_dir)
 171
 172     def update_fuel_isolinux(self, file):
 173         with io.open(file) as f:
 174             data = f.read()
 175         for key, val in self.fuel_conf.iteritems():
 176             # skip replacing these keys, as the format is different
 177             if key in ['ip', 'gw', 'netmask', 'hostname']:
 178                 continue
 179
 180             pattern = r'%s=[^ ]\S+' % key
 181             replace = '%s=%s' % (key, val)
 182             data = re.sub(pattern, replace, data)
 183
 184         # process networking parameters
 185         ip = ':'.join([self.fuel_conf['ip'],
 186                       '',
 187                       self.fuel_conf['gw'],
 188                       self.fuel_conf['netmask'],
 189                       self.fuel_conf['hostname'],
 190                       'eth0:off:::'])
 191
 192         data = re.sub(r'ip=[^ ]\S+', 'ip=%s' % ip, data)
 193
 194         with io.open(file, 'w') as f:
 195             f.write(data)
 196
 197     def parse_iso_volume_label(self, iso_filename):
 198         label_line = exec_cmd('isoinfo -d -i %s | grep -i "Volume id: "' % iso_filename)
 199         # cut leading text: 'Volume id: '
 200         return label_line[11:]
 201
 202     def deploy_env(self):
 203         dep = CloudDeploy(self.dea, self.dha, self.fuel_conf['ip'],
 204                           self.fuel_username, self.fuel_password,
 205                           self.dea_file, self.fuel_plugins_conf_dir,
 206                           WORK_DIR, self.no_health_check, self.deploy_timeout,
 207                           self.no_deploy_environment, self.deploy_log)
 208         return dep.deploy()
 209
 210     def setup_execution_environment(self):
 211         exec_env = ExecutionEnvironment(self.storage_dir, self.pxe_bridge,
 212                                         self.dha_file, self.dea)
 213         exec_env.setup_environment()
 214
 215     def cleanup_execution_environment(self):
 216         exec_env = ExecutionEnvironment(self.storage_dir, self.pxe_bridge,
 217                                         self.dha_file, self.dea)
 218         exec_env.cleanup_environment()
 219
 220     def create_tmp_dir(self):
 221         self.tmp_dir = '%s/fueltmp' % CWD
 222         delete(self.tmp_dir)
 223         create_dir_if_not_exists(self.tmp_dir)
 224
 225     def deploy(self):
 226         self.collect_fuel_info()
 227         if not self.no_fuel:
 228             self.setup_execution_environment()
 229             self.create_tmp_dir()
 230             self.install_fuel_master()
 231         if not self.fuel_only:
 232             return self.deploy_env()
 233         # Exit status
 234         return 0
 235
 236     def run(self):
 237         check_if_root()
 238         if self.cleanup_only:
 239             self.cleanup_execution_environment()
 240         else:
 241             deploy_success = self.deploy()
 242             if self.cleanup:
 243                 self.cleanup_execution_environment()
 244             return deploy_success
 245         # Exit status
 246         return 0
 247
 248
 249 def check_bridge(pxe_bridge, dha_path):
 250     # Assume that bridges on remote nodes exists, we could ssh but
 251     # the remote user might not have a login shell.
 252     if os.environ.get('LIBVIRT_DEFAULT_URI'):
 253         return
 254
 255     with io.open(dha_path) as yaml_file:
 256         dha_struct = yaml.load(yaml_file)
 257     if dha_struct['adapter'] != 'libvirt':
 258         log('Using Linux Bridge %s for booting up the Fuel Master VM'
 259             % pxe_bridge)
 260         r = exec_cmd('ip link show %s' % pxe_bridge)
 261         if pxe_bridge in r and 'state DOWN' in r:
 262             err('Linux Bridge {0} is not Active, bring'
 263                 ' it UP first: [ip link set dev {0} up]'.format(pxe_bridge))
 264
 265
 266 def check_fuel_plugins_dir(dir):
 267     msg = None
 268     if not dir:
 269         msg = 'Fuel Plugins Directory not specified!'
 270     elif not os.path.isdir(dir):
 271         msg = 'Fuel Plugins Directory does not exist!'
 272     elif not os.listdir(dir):
 273         msg = 'Fuel Plugins Directory is empty!'
 274     if msg:
 275         warn('%s No external plugins will be installed!' % msg)
 276
 277
 278 def parse_arguments():
 279     parser = ArgParser(prog='python %s' % __file__)
 280     parser.add_argument('-nf', dest='no_fuel', action='store_true',
 281                         default=False,
 282                         help='Do not install Fuel Master (and Node VMs when '
 283                              'using libvirt)')
 284     parser.add_argument('-nh', dest='no_health_check', action='store_true',
 285                         default=False,
 286                         help='Don\'t run health check after deployment')
 287     parser.add_argument('-fo', dest='fuel_only', action='store_true',
 288                         default=False,
 289                         help='Install Fuel Master only (and Node VMs when '
 290                              'using libvirt)')
 291     parser.add_argument('-co', dest='cleanup_only', action='store_true',
 292                         default=False,
 293                         help='Cleanup VMs and Virtual Networks according to '
 294                              'what is defined in DHA')
 295     parser.add_argument('-c', dest='cleanup', action='store_true',
 296                         default=False,
 297                         help='Cleanup after deploy')
 298     if {'-iso', '-dea', '-dha', '-h'}.intersection(sys.argv):
 299         parser.add_argument('-iso', dest='iso_file', action='store', nargs='?',
 300                             default='%s/OPNFV.iso' % CWD,
 301                             help='ISO File [default: OPNFV.iso]')
 302         parser.add_argument('-dea', dest='dea_file', action='store', nargs='?',
 303                             default='%s/dea.yaml' % CWD,
 304                             help='Deployment Environment Adapter: dea.yaml')
 305         parser.add_argument('-dha', dest='dha_file', action='store', nargs='?',
 306                             default='%s/dha.yaml' % CWD,
 307                             help='Deployment Hardware Adapter: dha.yaml')
 308     else:
 309         parser.add_argument('iso_file', action='store', nargs='?',
 310                             default='%s/OPNFV.iso' % CWD,
 311                             help='ISO File [default: OPNFV.iso]')
 312         parser.add_argument('dea_file', action='store', nargs='?',
 313                             default='%s/dea.yaml' % CWD,
 314                             help='Deployment Environment Adapter: dea.yaml')
 315         parser.add_argument('dha_file', action='store', nargs='?',
 316                             default='%s/dha.yaml' % CWD,
 317                             help='Deployment Hardware Adapter: dha.yaml')
 318     parser.add_argument('-s', dest='storage_dir', action='store',
 319                         default='%s/images' % CWD,
 320                         help='Storage Directory [default: images]')
 321     parser.add_argument('-b', dest='pxe_bridge', action='append',
 322                         default=[],
 323                         help='Linux Bridge for booting up the Fuel Master VM '
 324                              '[default: pxebr]')
 325     parser.add_argument('-p', dest='fuel_plugins_dir', action='store',
 326                         help='Fuel Plugins directory')
 327     parser.add_argument('-pc', dest='fuel_plugins_conf_dir', action='store',
 328                         help='Fuel Plugins Configuration directory')
 329     parser.add_argument('-np', dest='no_plugins', action='store_true',
 330                         default=False, help='Do not install Fuel Plugins')
 331     parser.add_argument('-dt', dest='deploy_timeout', action='store',
 332                         default=240, help='Deployment timeout (in minutes) '
 333                         '[default: 240]')
 334     parser.add_argument('-nde', dest='no_deploy_environment',
 335                         action='store_true', default=False,
 336                         help=('Do not launch environment deployment'))
 337     parser.add_argument('-log', dest='deploy_log',
 338                         action='store', default='../ci/.',
 339                         help=('Path and name of the deployment log archive'))
 340
 341     args = parser.parse_args()
 342     log(args)
 343
 344     if not args.pxe_bridge:
 345         args.pxe_bridge = ['pxebr']
 346
 347     check_file_exists(args.dha_file)
 348
 349     check_dir_exists(os.path.dirname(args.deploy_log))
 350
 351     if not args.cleanup_only:
 352         check_file_exists(args.dea_file)
 353         check_fuel_plugins_dir(args.fuel_plugins_dir)
 354
 355     iso_abs_path = os.path.abspath(args.iso_file)
 356     if not args.no_fuel and not args.cleanup_only:
 357         log('Using OPNFV ISO file: %s' % iso_abs_path)
 358         check_file_exists(iso_abs_path)
 359         log('Using image directory: %s' % args.storage_dir)
 360         create_dir_if_not_exists(args.storage_dir)
 361         for bridge in args.pxe_bridge:
 362             check_bridge(bridge, args.dha_file)
 363
 364
 365     kwargs = {'no_fuel': args.no_fuel, 'fuel_only': args.fuel_only,
 366               'no_health_check': args.no_health_check,
 367               'cleanup_only': args.cleanup_only, 'cleanup': args.cleanup,
 368               'storage_dir': args.storage_dir, 'pxe_bridge': args.pxe_bridge,
 369               'iso_file': iso_abs_path, 'dea_file': args.dea_file,
 370               'dha_file': args.dha_file,
 371               'fuel_plugins_dir': args.fuel_plugins_dir,
 372               'fuel_plugins_conf_dir': args.fuel_plugins_conf_dir,
 373               'no_plugins': args.no_plugins,
 374               'deploy_timeout': args.deploy_timeout,
 375               'no_deploy_environment': args.no_deploy_environment,
 376               'deploy_log': args.deploy_log}
 377     return kwargs
 378
 379
 380 def handle_signals(signal_num, frame):
 381     signal.signal(signal.SIGINT, signal.SIG_IGN)
 382     signal.signal(signal.SIGTERM, signal.SIG_IGN)
 383
 384     log('Caught signal %s, cleaning up and exiting.' % signal_num)
 385
 386     mount_point = os.environ.get(MOUNT_STATE_VAR)
 387     if mount_point:
 388         log('Unmounting ISO from "%s"' % mount_point)
 389         # Prevent 'Device or resource busy' errors when unmounting
 390         os.chdir('/')
 391         exec_cmd('fusermount -u %s' % mount_point, True)
 392         # Be nice and remove our environment variable, even though the OS would
 393         # would clean it up anyway
 394         os.environ.pop(MOUNT_STATE_VAR)
 395
 396     sys.exit(1)
 397
 398
 399 def main():
 400     signal.signal(signal.SIGINT, handle_signals)
 401     signal.signal(signal.SIGTERM, handle_signals)
 402     kwargs = parse_arguments()
 403     d = AutoDeploy(**kwargs)
 404     sys.exit(d.run())
 405
 406 if __name__ == '__main__':
 407     main()