deploy/deploy_env.py

   1 ###############################################################################
   2 # Copyright (c) 2015 Ericsson AB and others.
   3 # szilard.cserey@ericsson.com
   4 # All rights reserved. This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 # http://www.apache.org/licenses/LICENSE-2.0
   8 ###############################################################################
   9
  10
  11 import os
  12 import io
  13 import yaml
  14 import glob
  15 import time
  16 import shutil
  17
  18 from ssh_client import SSHClient
  19
  20 from common import (
  21     err,
  22     log,
  23     parse,
  24     N,
  25     E,
  26     R,
  27     delete,
  28 )
  29
  30 CLOUD_DEPLOY_FILE = 'deploy.py'
  31 BLADE_RESTART_TIMES = 3
  32
  33
  34 class CloudDeploy(object):
  35
  36     def __init__(self, dea, dha, fuel_ip, fuel_username, fuel_password,
  37                  dea_file, fuel_plugins_conf_dir, work_dir, no_health_check,
  38                  deploy_timeout, no_deploy_environment, deploy_log):
  39         self.dea = dea
  40         self.dha = dha
  41         self.fuel_ip = fuel_ip
  42         self.fuel_username = fuel_username
  43         self.fuel_password = fuel_password
  44         self.dea_file = dea_file
  45         self.updated_dea_file = (
  46             '%s/.%s' % (os.path.dirname(self.dea_file),
  47             os.path.basename(self.dea_file)))
  48         shutil.copy2(self.dea_file, self.updated_dea_file)
  49         self.fuel_plugins_conf_dir = fuel_plugins_conf_dir
  50         self.work_dir = work_dir
  51         self.no_health_check = no_health_check
  52         self.deploy_timeout = deploy_timeout
  53         self.no_deploy_environment = no_deploy_environment
  54         self.deploy_log = deploy_log
  55         self.file_dir = os.path.dirname(os.path.realpath(__file__))
  56         self.ssh = SSHClient(self.fuel_ip, self.fuel_username,
  57                              self.fuel_password)
  58         self.node_ids = self.dha.get_node_ids()
  59         self.wanted_release = self.dea.get_property('wanted_release')
  60         self.blade_node_dict = {}
  61         self.macs_per_blade = {}
  62
  63     def merge_plugin_config_files_to_dea_file(self):
  64         plugins_conf_dir = (
  65             self.fuel_plugins_conf_dir if self.fuel_plugins_conf_dir
  66             else '%s/plugins_conf' % os.path.dirname(self.dea_file))
  67         if os.path.isdir(plugins_conf_dir):
  68             with io.open(self.updated_dea_file) as stream:
  69                 updated_dea = yaml.load(stream)
  70             for plugin_file in glob.glob('%s/*.yaml' % plugins_conf_dir):
  71                 with io.open(plugin_file) as stream:
  72                     plugin_conf = yaml.load(stream)
  73                 updated_dea['settings']['editable'].update(plugin_conf)
  74             with io.open(self.updated_dea_file, 'w') as stream:
  75                 yaml.dump(updated_dea, stream, default_flow_style=False)
  76
  77     def upload_cloud_deployment_files(self):
  78         with self.ssh as s:
  79             s.exec_cmd('rm -rf %s' % self.work_dir, False)
  80             s.exec_cmd('mkdir %s' % self.work_dir)
  81             s.scp_put(self.updated_dea_file, '%s/%s' % (
  82                 self.work_dir, os.path.basename(self.dea_file)))
  83             s.scp_put('%s/common.py' % self.file_dir, self.work_dir)
  84             s.scp_put('%s/dea.py' % self.file_dir, self.work_dir)
  85             for f in glob.glob('%s/cloud/*' % self.file_dir):
  86                 s.scp_put(f, self.work_dir)
  87
  88     def power_off_nodes(self):
  89         for node_id in self.node_ids:
  90             self.dha.node_power_off(node_id)
  91
  92     def power_on_nodes(self):
  93         for node_id in self.node_ids:
  94             self.dha.node_power_on(node_id)
  95
  96     def set_boot_order(self, boot_order_list):
  97         for node_id in self.node_ids:
  98             self.dha.node_set_boot_order(node_id, boot_order_list[:])
  99
 100     def get_mac_addresses(self):
 101         self.macs_per_blade = {}
 102         for node_id in self.node_ids:
 103             self.macs_per_blade[node_id] = self.dha.get_node_pxe_mac(node_id)
 104
 105     def run_cloud_deploy(self, deploy_app):
 106         log('START CLOUD DEPLOYMENT')
 107         deploy_app = '%s/%s' % (self.work_dir, deploy_app)
 108         dea_file = '%s/%s' % (self.work_dir, os.path.basename(self.dea_file))
 109         with self.ssh as s:
 110             status = s.run('python %s %s %s %s %s' % (
 111                 deploy_app,
 112                 ('-nh' if self.no_health_check else ''),
 113                 ('-dt %s' %
 114                  self.deploy_timeout if self.deploy_timeout else ''),
 115                 ('-nde' if self.no_deploy_environment else ''),
 116                 dea_file))
 117         return status
 118
 119     def check_supported_release(self):
 120         log('Check supported release: %s' % self.wanted_release)
 121         found = False
 122         release_list = parse(self.ssh.exec_cmd('fuel release -l'))
 123         for release in release_list:
 124             if release[R['name']] == self.wanted_release:
 125                 found = True
 126                 break
 127         if not found:
 128             err('This Fuel does not contain the following release: %s'
 129                 % self.wanted_release)
 130
 131     def check_previous_installation(self):
 132         log('Check previous installation')
 133         env_list = parse(self.ssh.exec_cmd('fuel env list'))
 134         if env_list:
 135             self.cleanup_fuel_environments(env_list)
 136             node_list = parse(self.ssh.exec_cmd('fuel node list'))
 137             if node_list:
 138                 self.cleanup_fuel_nodes(node_list)
 139
 140     def cleanup_fuel_environments(self, env_list):
 141         WAIT_LOOP = 60
 142         SLEEP_TIME = 10
 143         for env in env_list:
 144             log('Deleting environment %s' % env[E['id']])
 145             self.ssh.exec_cmd('fuel env --env %s --delete --force'
 146                               % env[E['id']])
 147         all_env_erased = False
 148         for i in range(WAIT_LOOP):
 149             env_list = parse(self.ssh.exec_cmd('fuel env list'))
 150             if env_list:
 151                 time.sleep(SLEEP_TIME)
 152             else:
 153                 all_env_erased = True
 154                 break
 155         if not all_env_erased:
 156             log('Could not erase these environments %s'
 157                 % [(env[E['id']], env[E['status']]) for env in env_list])
 158
 159     def cleanup_fuel_nodes(self, node_list):
 160         for node in node_list:
 161             if node[N['status']] == 'discover':
 162                 log('Deleting node %s' % node[N['id']])
 163                 self.ssh.exec_cmd('fuel node --node-id %s --delete-from-db '
 164                                   '--force' % node[N['id']])
 165                 self.ssh.exec_cmd('cobbler system remove --name node-%s'
 166                                   % node[N['id']], False)
 167
 168     def check_prerequisites(self):
 169         log('Check prerequisites')
 170         with self.ssh:
 171             self.check_supported_release()
 172             self.check_previous_installation()
 173
 174     def wait_for_discovered_blades(self):
 175         log('Wait for discovered blades')
 176         discovered_macs = []
 177         restart_times = BLADE_RESTART_TIMES
 178
 179         for blade in self.node_ids:
 180             self.blade_node_dict[blade] = None
 181
 182         with self.ssh:
 183             all_discovered = self.discovery_waiting_loop(discovered_macs)
 184
 185         while not all_discovered and restart_times != 0:
 186             restart_times -= 1
 187             for blade in self.get_not_discovered_blades():
 188                 self.dha.node_reset(blade)
 189             with self.ssh:
 190                 all_discovered = self.discovery_waiting_loop(discovered_macs)
 191
 192         if not all_discovered:
 193             err('Not all blades have been discovered: %s'
 194                 % self.not_discovered_blades_summary())
 195
 196         with io.open(self.updated_dea_file) as stream:
 197             updated_dea = yaml.load(stream)
 198         updated_dea.update({'blade_node_map': self.blade_node_dict})
 199         with io.open(self.updated_dea_file, 'w') as stream:
 200             yaml.dump(updated_dea, stream, default_flow_style=False)
 201
 202     def discovery_waiting_loop(self, discovered_macs):
 203         WAIT_LOOP = 360
 204         SLEEP_TIME = 10
 205         all_discovered = False
 206         for i in range(WAIT_LOOP):
 207             node_list = parse(self.ssh.exec_cmd('fuel node list'))
 208             if node_list:
 209                 self.node_discovery(node_list, discovered_macs)
 210             if self.all_blades_discovered():
 211                 all_discovered = True
 212                 break
 213             else:
 214                 time.sleep(SLEEP_TIME)
 215         return all_discovered
 216
 217     def node_discovery(self, node_list, discovered_macs):
 218         for node in node_list:
 219             if (node[N['status']] == 'discover' and
 220                 (node[N['online']] == 'True' or
 221                 node[N['online']] == '1') and
 222                 node[N['mac']] not in discovered_macs):
 223                 discovered_macs.append(node[N['mac']])
 224                 blade = self.find_mac_in_dict(node[N['mac']])
 225                 if blade:
 226                     log('Blade %s discovered as Node %s with MAC %s'
 227                         % (blade, node[N['id']], node[N['mac']]))
 228                     self.blade_node_dict[blade] = int(node[N['id']])
 229
 230     def find_mac_in_dict(self, mac):
 231         for blade, mac_list in self.macs_per_blade.iteritems():
 232             if mac in mac_list:
 233                 return blade
 234
 235     def all_blades_discovered(self):
 236         for blade, node_id in self.blade_node_dict.iteritems():
 237             if not node_id:
 238                 return False
 239         return True
 240
 241     def not_discovered_blades_summary(self):
 242         summary = ''
 243         for blade, node_id in self.blade_node_dict.iteritems():
 244             if not node_id:
 245                 summary += '\n[blade %s]' % blade
 246         return summary
 247
 248     def get_not_discovered_blades(self):
 249         not_discovered_blades = []
 250         for blade, node_id in self.blade_node_dict.iteritems():
 251             if not node_id:
 252                 not_discovered_blades.append(blade)
 253         return not_discovered_blades
 254
 255     def set_boot_order_nodes(self):
 256         self.power_off_nodes()
 257         self.set_boot_order(['pxe', 'disk'])
 258         self.power_on_nodes()
 259
 260     def get_put_deploy_log(self):
 261         with self.ssh as s:
 262             s.scp_get("deploy-*", local=self.deploy_log)
 263
 264     def deploy(self):
 265
 266         self.set_boot_order_nodes()
 267
 268         self.check_prerequisites()
 269
 270         self.get_mac_addresses()
 271
 272         self.wait_for_discovered_blades()
 273
 274         self.merge_plugin_config_files_to_dea_file()
 275
 276         self.upload_cloud_deployment_files()
 277
 278         delete(self.updated_dea_file)
 279
 280         rc = self.run_cloud_deploy(CLOUD_DEPLOY_FILE)
 281
 282         self.get_put_deploy_log()
 283
 284         return rc