deploy/deploy_env.py

   1 ###############################################################################
   2 # Copyright (c) 2015 Ericsson AB and others.
   3 # szilard.cserey@ericsson.com
   4 # All rights reserved. This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 # http://www.apache.org/licenses/LICENSE-2.0
   8 ###############################################################################
   9
  10
  11 import os
  12 import io
  13 import yaml
  14 import glob
  15 import time
  16 import shutil
  17
  18 from ssh_client import SSHClient
  19
  20 from common import (
  21     err,
  22     log,
  23     exec_cmd,
  24     parse,
  25     N,
  26     E,
  27     R,
  28     delete,
  29 )
  30
  31 CLOUD_DEPLOY_FILE = 'deploy.py'
  32 BLADE_RESTART_TIMES = 3
  33
  34
  35 class CloudDeploy(object):
  36
  37     def __init__(self, dea, dha, fuel_ip, fuel_username, fuel_password,
  38                  dea_file, fuel_plugins_conf_dir, work_dir, no_health_check,
  39                  deploy_timeout, no_deploy_environment, deploy_log):
  40         self.dea = dea
  41         self.dha = dha
  42         self.fuel_ip = fuel_ip
  43         self.fuel_username = fuel_username
  44         self.fuel_password = fuel_password
  45         self.dea_file = dea_file
  46         self.updated_dea_file = (
  47             '%s/.%s' % (os.path.dirname(self.dea_file),
  48             os.path.basename(self.dea_file)))
  49         shutil.copy2(self.dea_file, self.updated_dea_file)
  50         self.fuel_plugins_conf_dir = fuel_plugins_conf_dir
  51         self.work_dir = work_dir
  52         self.no_health_check = no_health_check
  53         self.deploy_timeout = deploy_timeout
  54         self.no_deploy_environment = no_deploy_environment
  55         self.deploy_log = deploy_log
  56         self.file_dir = os.path.dirname(os.path.realpath(__file__))
  57         self.ssh = SSHClient(self.fuel_ip, self.fuel_username,
  58                              self.fuel_password)
  59         self.node_ids = self.dha.get_node_ids()
  60         self.wanted_release = self.dea.get_property('wanted_release')
  61         self.blade_node_dict = {}
  62         self.macs_per_blade = {}
  63
  64     def merge_plugin_config_files_to_dea_file(self):
  65         plugins_conf_dir = (
  66             self.fuel_plugins_conf_dir if self.fuel_plugins_conf_dir
  67             else '%s/plugins_conf' % os.path.dirname(self.dea_file))
  68         if os.path.isdir(plugins_conf_dir):
  69             with io.open(self.updated_dea_file) as stream:
  70                 updated_dea = yaml.load(stream)
  71             for plugin_file in glob.glob('%s/*.yaml' % plugins_conf_dir):
  72                 with io.open(plugin_file) as stream:
  73                     plugin_conf = yaml.load(stream)
  74                 updated_dea['settings']['editable'].update(plugin_conf)
  75             with io.open(self.updated_dea_file, 'w') as stream:
  76                 yaml.dump(updated_dea, stream, default_flow_style=False)
  77
  78     def upload_cloud_deployment_files(self):
  79         with self.ssh as s:
  80             s.exec_cmd('rm -rf %s' % self.work_dir, False)
  81             s.exec_cmd('mkdir %s' % self.work_dir)
  82             s.scp_put(self.updated_dea_file, '%s/%s' % (
  83                 self.work_dir, os.path.basename(self.dea_file)))
  84             s.scp_put('%s/common.py' % self.file_dir, self.work_dir)
  85             s.scp_put('%s/dea.py' % self.file_dir, self.work_dir)
  86             for f in glob.glob('%s/cloud/*' % self.file_dir):
  87                 s.scp_put(f, self.work_dir)
  88
  89     def power_off_nodes(self):
  90         for node_id in self.node_ids:
  91             self.dha.node_power_off(node_id)
  92
  93     def power_on_nodes(self):
  94         for node_id in self.node_ids:
  95             self.dha.node_power_on(node_id)
  96
  97     def set_boot_order(self, boot_order_list):
  98         for node_id in self.node_ids:
  99             self.dha.node_set_boot_order(node_id, boot_order_list[:])
 100
 101     def get_mac_addresses(self):
 102         self.macs_per_blade = {}
 103         for node_id in self.node_ids:
 104             self.macs_per_blade[node_id] = self.dha.get_node_pxe_mac(node_id)
 105
 106     def run_cloud_deploy(self, deploy_app):
 107         log('START CLOUD DEPLOYMENT')
 108         deploy_app = '%s/%s' % (self.work_dir, deploy_app)
 109         dea_file = '%s/%s' % (self.work_dir, os.path.basename(self.dea_file))
 110         with self.ssh as s:
 111             status = s.run('python %s %s %s %s %s' % (
 112                 deploy_app,
 113                 ('-nh' if self.no_health_check else ''),
 114                 ('-dt %s' %
 115                  self.deploy_timeout if self.deploy_timeout else ''),
 116                 ('-nde' if self.no_deploy_environment else ''),
 117                 dea_file))
 118         return status
 119
 120     def check_supported_release(self):
 121         log('Check supported release: %s' % self.wanted_release)
 122         found = False
 123         release_list = parse(self.ssh.exec_cmd('fuel release -l'))
 124         for release in release_list:
 125             if release[R['name']] == self.wanted_release:
 126                 found = True
 127                 break
 128         if not found:
 129             err('This Fuel does not contain the following release: %s'
 130                 % self.wanted_release)
 131
 132     def check_previous_installation(self):
 133         log('Check previous installation')
 134         env_list = parse(self.ssh.exec_cmd('fuel env list'))
 135         if env_list:
 136             self.cleanup_fuel_environments(env_list)
 137             node_list = parse(self.ssh.exec_cmd('fuel node list'))
 138             if node_list:
 139                 self.cleanup_fuel_nodes(node_list)
 140
 141     def cleanup_fuel_environments(self, env_list):
 142         WAIT_LOOP = 60
 143         SLEEP_TIME = 10
 144         for env in env_list:
 145             log('Deleting environment %s' % env[E['id']])
 146             self.ssh.exec_cmd('fuel env --env %s --delete --force'
 147                               % env[E['id']])
 148         all_env_erased = False
 149         for i in range(WAIT_LOOP):
 150             env_list = parse(self.ssh.exec_cmd('fuel env list'))
 151             if env_list:
 152                 time.sleep(SLEEP_TIME)
 153             else:
 154                 all_env_erased = True
 155                 break
 156         if not all_env_erased:
 157             err('Could not erase these environments %s'
 158                 % [(env[E['id']], env[E['status']]) for env in env_list])
 159
 160     def cleanup_fuel_nodes(self, node_list):
 161         for node in node_list:
 162             if node[N['status']] == 'discover':
 163                 log('Deleting node %s' % node[N['id']])
 164                 self.ssh.exec_cmd('fuel node --node-id %s --delete-from-db '
 165                                   '--force' % node[N['id']])
 166                 self.ssh.exec_cmd('cobbler system remove --name node-%s'
 167                                   % node[N['id']], False)
 168
 169     def check_prerequisites(self):
 170         log('Check prerequisites')
 171         with self.ssh:
 172             self.check_supported_release()
 173             self.check_previous_installation()
 174
 175     def wait_for_discovered_blades(self):
 176         log('Wait for discovered blades')
 177         discovered_macs = []
 178         restart_times = BLADE_RESTART_TIMES
 179
 180         for blade in self.node_ids:
 181             self.blade_node_dict[blade] = None
 182
 183         with self.ssh:
 184             all_discovered = self.discovery_waiting_loop(discovered_macs)
 185
 186         while not all_discovered and restart_times != 0:
 187             restart_times -= 1
 188             for blade in self.get_not_discovered_blades():
 189                 self.dha.node_reset(blade)
 190             with self.ssh:
 191                 all_discovered = self.discovery_waiting_loop(discovered_macs)
 192
 193         if not all_discovered:
 194             err('Not all blades have been discovered: %s'
 195                 % self.not_discovered_blades_summary())
 196
 197         with io.open(self.updated_dea_file) as stream:
 198             updated_dea = yaml.load(stream)
 199         updated_dea.update({'blade_node_map': self.blade_node_dict})
 200         with io.open(self.updated_dea_file, 'w') as stream:
 201             yaml.dump(updated_dea, stream, default_flow_style=False)
 202
 203     def discovery_waiting_loop(self, discovered_macs):
 204         WAIT_LOOP = 360
 205         SLEEP_TIME = 10
 206         all_discovered = False
 207         for i in range(WAIT_LOOP):
 208             node_list = parse(self.ssh.exec_cmd('fuel node list'))
 209             if node_list:
 210                 self.node_discovery(node_list, discovered_macs)
 211             if self.all_blades_discovered():
 212                 all_discovered = True
 213                 break
 214             else:
 215                 time.sleep(SLEEP_TIME)
 216         return all_discovered
 217
 218     def node_discovery(self, node_list, discovered_macs):
 219         for node in node_list:
 220             if (node[N['status']] == 'discover' and
 221                 (node[N['online']] == 'True' or
 222                 node[N['online']] == '1') and
 223                 node[N['mac']] not in discovered_macs):
 224                 discovered_macs.append(node[N['mac']])
 225                 blade = self.find_mac_in_dict(node[N['mac']])
 226                 if blade:
 227                     log('Blade %s discovered as Node %s with MAC %s'
 228                         % (blade, node[N['id']], node[N['mac']]))
 229                     self.blade_node_dict[blade] = int(node[N['id']])
 230
 231     def find_mac_in_dict(self, mac):
 232         for blade, mac_list in self.macs_per_blade.iteritems():
 233             if mac in mac_list:
 234                 return blade
 235
 236     def all_blades_discovered(self):
 237         for blade, node_id in self.blade_node_dict.iteritems():
 238             if not node_id:
 239                 return False
 240         return True
 241
 242     def not_discovered_blades_summary(self):
 243         summary = ''
 244         for blade, node_id in self.blade_node_dict.iteritems():
 245             if not node_id:
 246                 summary += '\n[blade %s]' % blade
 247         return summary
 248
 249     def get_not_discovered_blades(self):
 250         not_discovered_blades = []
 251         for blade, node_id in self.blade_node_dict.iteritems():
 252             if not node_id:
 253                 not_discovered_blades.append(blade)
 254         return not_discovered_blades
 255
 256     def set_boot_order_nodes(self):
 257         self.power_off_nodes()
 258         self.set_boot_order(['pxe', 'disk'])
 259         self.power_on_nodes()
 260
 261     def get_put_deploy_log(self):
 262         with self.ssh as s:
 263             s.scp_get("deploy-*", local=self.deploy_log)
 264
 265     def deploy(self):
 266
 267         self.set_boot_order_nodes()
 268
 269         self.check_prerequisites()
 270
 271         self.get_mac_addresses()
 272
 273         self.wait_for_discovered_blades()
 274
 275         self.merge_plugin_config_files_to_dea_file()
 276
 277         self.upload_cloud_deployment_files()
 278
 279         delete(self.updated_dea_file)
 280
 281         rc = self.run_cloud_deploy(CLOUD_DEPLOY_FILE)
 282
 283         self.get_put_deploy_log()
 284
 285         return rc