fuel/deploy/deploy_env.py

   1 ###############################################################################
   2 # Copyright (c) 2015 Ericsson AB and others.
   3 # szilard.cserey@ericsson.com
   4 # All rights reserved. This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 # http://www.apache.org/licenses/LICENSE-2.0
   8 ###############################################################################
   9
  10
  11 import os
  12 import io
  13 import yaml
  14 import glob
  15 import time
  16
  17 from ssh_client import SSHClient
  18 import common
  19
  20 exec_cmd = common.exec_cmd
  21 err = common.err
  22 check_file_exists = common.check_file_exists
  23 log = common.log
  24 parse = common.parse
  25 commafy = common.commafy
  26 N = common.N
  27 E = common.E
  28 R = common.R
  29 RO = common.RO
  30
  31 CLOUD_DEPLOY_FILE = 'deploy.py'
  32 BLADE_RESTART_TIMES = 3
  33
  34
  35 class CloudDeploy(object):
  36
  37     def __init__(self, dea, dha, fuel_ip, fuel_username, fuel_password,
  38                  dea_file, work_dir, no_health_check):
  39         self.dea = dea
  40         self.dha = dha
  41         self.fuel_ip = fuel_ip
  42         self.fuel_username = fuel_username
  43         self.fuel_password = fuel_password
  44         self.dea_file = dea_file
  45         self.work_dir = work_dir
  46         self.no_health_check = no_health_check
  47         self.file_dir = os.path.dirname(os.path.realpath(__file__))
  48         self.ssh = SSHClient(self.fuel_ip, self.fuel_username,
  49                              self.fuel_password)
  50         self.blade_node_file = '%s/blade_node.yaml' % self.file_dir
  51         self.node_ids = self.dha.get_node_ids()
  52         self.wanted_release = self.dea.get_property('wanted_release')
  53         self.blade_node_dict = {}
  54         self.macs_per_blade = {}
  55
  56     def upload_cloud_deployment_files(self):
  57         with self.ssh as s:
  58             s.exec_cmd('rm -rf %s' % self.work_dir, False)
  59             s.exec_cmd('mkdir %s' % self.work_dir)
  60             s.scp_put(self.dea_file, self.work_dir)
  61             s.scp_put(self.blade_node_file, self.work_dir)
  62             s.scp_put('%s/common.py' % self.file_dir, self.work_dir)
  63             s.scp_put('%s/dea.py' % self.file_dir, self.work_dir)
  64             for f in glob.glob('%s/cloud/*' % self.file_dir):
  65                 s.scp_put(f, self.work_dir)
  66
  67     def power_off_nodes(self):
  68         for node_id in self.node_ids:
  69             self.dha.node_power_off(node_id)
  70
  71     def power_on_nodes(self):
  72         for node_id in self.node_ids:
  73             self.dha.node_power_on(node_id)
  74
  75     def set_boot_order(self, boot_order_list):
  76         for node_id in self.node_ids:
  77             self.dha.node_set_boot_order(node_id, boot_order_list[:])
  78
  79     def get_mac_addresses(self):
  80         self.macs_per_blade = {}
  81         for node_id in self.node_ids:
  82             self.macs_per_blade[node_id] = self.dha.get_node_pxe_mac(node_id)
  83
  84     def run_cloud_deploy(self, deploy_app):
  85         log('START CLOUD DEPLOYMENT')
  86         deploy_app = '%s/%s' % (self.work_dir, deploy_app)
  87         dea_file = '%s/%s' % (self.work_dir, os.path.basename(self.dea_file))
  88         blade_node_file = '%s/%s' % (
  89             self.work_dir, os.path.basename(self.blade_node_file))
  90         with self.ssh as s:
  91             status = s.run('python %s %s %s %s'
  92                            % (('-nh' if self.no_health_check else ''),
  93                               deploy_app, dea_file, blade_node_file))
  94         return status
  95
  96     def check_supported_release(self):
  97         log('Check supported release: %s' % self.wanted_release)
  98         found = False
  99         release_list = parse(self.ssh.exec_cmd('fuel release -l'))
 100         for release in release_list:
 101             if release[R['name']] == self.wanted_release:
 102                 found = True
 103                 break
 104         if not found:
 105             err('This Fuel does not contain the following release: %s'
 106                 % self.wanted_release)
 107
 108     def check_previous_installation(self):
 109         log('Check previous installation')
 110         env_list = parse(self.ssh.exec_cmd('fuel env list'))
 111         if env_list:
 112             self.cleanup_fuel_environments(env_list)
 113             node_list = parse(self.ssh.exec_cmd('fuel node list'))
 114             if node_list:
 115                 self.cleanup_fuel_nodes(node_list)
 116
 117     def cleanup_fuel_environments(self, env_list):
 118         WAIT_LOOP = 60
 119         SLEEP_TIME = 10
 120         for env in env_list:
 121             log('Deleting environment %s' % env[E['id']])
 122             self.ssh.exec_cmd('fuel env --env %s --delete --force'
 123                               % env[E['id']])
 124         all_env_erased = False
 125         for i in range(WAIT_LOOP):
 126             env_list = parse(self.ssh.exec_cmd('fuel env list'))
 127             if env_list:
 128                 time.sleep(SLEEP_TIME)
 129             else:
 130                 all_env_erased = True
 131                 break
 132         if not all_env_erased:
 133             err('Could not erase these environments %s'
 134                 % [(env[E['id']], env[E['status']]) for env in env_list])
 135
 136     def cleanup_fuel_nodes(self, node_list):
 137         for node in node_list:
 138             if node[N['status']] == 'discover':
 139                 log('Deleting node %s' % node[N['id']])
 140                 self.ssh.exec_cmd('fuel node --node-id %s --delete-from-db '
 141                                   '--force' % node[N['id']])
 142                 self.ssh.exec_cmd('cobbler system remove --name node-%s'
 143                                   % node[N['id']], False)
 144
 145     def check_prerequisites(self):
 146         log('Check prerequisites')
 147         with self.ssh:
 148             self.check_supported_release()
 149             self.check_previous_installation()
 150
 151     def wait_for_discovered_blades(self):
 152         log('Wait for discovered blades')
 153         discovered_macs = []
 154         restart_times = BLADE_RESTART_TIMES
 155
 156         for blade in self.node_ids:
 157             self.blade_node_dict[blade] = None
 158
 159         with self.ssh:
 160             all_discovered = self.discovery_waiting_loop(discovered_macs)
 161
 162         while not all_discovered and restart_times != 0:
 163             restart_times -= 1
 164             for blade in self.get_not_discovered_blades():
 165                 self.dha.node_reset(blade)
 166             with self.ssh:
 167                 all_discovered = self.discovery_waiting_loop(discovered_macs)
 168
 169         if not all_discovered:
 170             err('Not all blades have been discovered: %s'
 171                 % self.not_discovered_blades_summary())
 172
 173         with io.open(self.blade_node_file, 'w') as stream:
 174             yaml.dump(self.blade_node_dict, stream, default_flow_style=False)
 175
 176     def discovery_waiting_loop(self, discovered_macs):
 177         WAIT_LOOP = 360
 178         SLEEP_TIME = 10
 179         all_discovered = False
 180         for i in range(WAIT_LOOP):
 181             node_list = parse(self.ssh.exec_cmd('fuel node list'))
 182             if node_list:
 183                 self.node_discovery(node_list, discovered_macs)
 184             if self.all_blades_discovered():
 185                 all_discovered = True
 186                 break
 187             else:
 188                 time.sleep(SLEEP_TIME)
 189         return all_discovered
 190
 191     def node_discovery(self, node_list, discovered_macs):
 192         for node in node_list:
 193             if (node[N['status']] == 'discover' and
 194                 node[N['online']] == 'True' and
 195                 node[N['mac']] not in discovered_macs):
 196                 discovered_macs.append(node[N['mac']])
 197                 blade = self.find_mac_in_dict(node[N['mac']])
 198                 if blade:
 199                     log('Blade %s discovered as Node %s with MAC %s'
 200                         % (blade, node[N['id']], node[N['mac']]))
 201                     self.blade_node_dict[blade] = node[N['id']]
 202
 203     def find_mac_in_dict(self, mac):
 204         for blade, mac_list in self.macs_per_blade.iteritems():
 205             if mac in mac_list:
 206                 return blade
 207
 208     def all_blades_discovered(self):
 209         for blade, node_id in self.blade_node_dict.iteritems():
 210             if not node_id:
 211                 return False
 212         return True
 213
 214     def not_discovered_blades_summary(self):
 215         summary = ''
 216         for blade, node_id in self.blade_node_dict.iteritems():
 217             if not node_id:
 218                 summary += '\n[blade %s]' % blade
 219         return summary
 220
 221     def get_not_discovered_blades(self):
 222         not_discovered_blades = []
 223         for blade, node_id in self.blade_node_dict.iteritems():
 224             if not node_id:
 225                 not_discovered_blades.append(blade)
 226         return not_discovered_blades
 227
 228     def set_boot_order_nodes(self):
 229         self.power_off_nodes()
 230         self.set_boot_order(['pxe', 'disk'])
 231         self.power_on_nodes()
 232
 233     def deploy(self):
 234
 235         self.set_boot_order_nodes()
 236
 237         self.check_prerequisites()
 238
 239         self.get_mac_addresses()
 240
 241         self.wait_for_discovered_blades()
 242
 243         self.upload_cloud_deployment_files()
 244
 245         return self.run_cloud_deploy(CLOUD_DEPLOY_FILE)