fuel/deploy/deploy_env.py

   1 ###############################################################################
   2 # Copyright (c) 2015 Ericsson AB and others.
   3 # szilard.cserey@ericsson.com
   4 # All rights reserved. This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 # http://www.apache.org/licenses/LICENSE-2.0
   8 ###############################################################################
   9
  10
  11 import os
  12 import io
  13 import yaml
  14 import glob
  15 import time
  16
  17 from ssh_client import SSHClient
  18 import common
  19
  20 exec_cmd = common.exec_cmd
  21 err = common.err
  22 check_file_exists = common.check_file_exists
  23 log = common.log
  24 parse = common.parse
  25 commafy = common.commafy
  26 N = common.N
  27 E = common.E
  28 R = common.R
  29 RO = common.RO
  30
  31 CLOUD_DEPLOY_FILE = 'deploy.py'
  32 BLADE_RESTART_TIMES = 3
  33
  34
  35 class CloudDeploy(object):
  36
  37     def __init__(self, dea, dha, fuel_ip, fuel_username, fuel_password,
  38                  dea_file, work_dir, no_health_check):
  39         self.dea = dea
  40         self.dha = dha
  41         self.fuel_ip = fuel_ip
  42         self.fuel_username = fuel_username
  43         self.fuel_password = fuel_password
  44         self.dea_file = dea_file
  45         self.work_dir = work_dir
  46         self.no_health_check = no_health_check
  47         self.file_dir = os.path.dirname(os.path.realpath(__file__))
  48         self.ssh = SSHClient(self.fuel_ip, self.fuel_username,
  49                              self.fuel_password)
  50         self.blade_node_file = '%s/blade_node.yaml' % self.file_dir
  51         self.node_ids = self.dha.get_node_ids()
  52         self.wanted_release = self.dea.get_property('wanted_release')
  53         self.blade_node_dict = {}
  54         self.macs_per_blade = {}
  55
  56     def upload_cloud_deployment_files(self):
  57         with self.ssh as s:
  58             s.exec_cmd('rm -rf %s' % self.work_dir, False)
  59             s.exec_cmd('mkdir %s' % self.work_dir)
  60             s.scp_put(self.dea_file, self.work_dir)
  61             s.scp_put(self.blade_node_file, self.work_dir)
  62             s.scp_put('%s/common.py' % self.file_dir, self.work_dir)
  63             s.scp_put('%s/dea.py' % self.file_dir, self.work_dir)
  64             for f in glob.glob('%s/cloud/*' % self.file_dir):
  65                 s.scp_put(f, self.work_dir)
  66
  67     def power_off_nodes(self):
  68         for node_id in self.node_ids:
  69             self.dha.node_power_off(node_id)
  70
  71     def power_on_nodes(self):
  72         for node_id in self.node_ids:
  73             self.dha.node_power_on(node_id)
  74
  75     def set_boot_order(self, boot_order_list):
  76         for node_id in self.node_ids:
  77             self.dha.node_set_boot_order(node_id, boot_order_list[:])
  78
  79     def get_mac_addresses(self):
  80         self.macs_per_blade = {}
  81         for node_id in self.node_ids:
  82             self.macs_per_blade[node_id] = self.dha.get_node_pxe_mac(node_id)
  83
  84     def run_cloud_deploy(self, deploy_app):
  85         log('START CLOUD DEPLOYMENT')
  86         deploy_app = '%s/%s' % (self.work_dir, deploy_app)
  87         dea_file = '%s/%s' % (self.work_dir, os.path.basename(self.dea_file))
  88         blade_node_file = '%s/%s' % (
  89             self.work_dir, os.path.basename(self.blade_node_file))
  90         with self.ssh as s:
  91             status = s.run(
  92                 'python %s %s %s %s' % (
  93                     deploy_app, ('-nh' if self.no_health_check else ''),
  94                     dea_file, blade_node_file))
  95         return status
  96
  97     def check_supported_release(self):
  98         log('Check supported release: %s' % self.wanted_release)
  99         found = False
 100         release_list = parse(self.ssh.exec_cmd('fuel release -l'))
 101         for release in release_list:
 102             if release[R['name']] == self.wanted_release:
 103                 found = True
 104                 break
 105         if not found:
 106             err('This Fuel does not contain the following release: %s'
 107                 % self.wanted_release)
 108
 109     def check_previous_installation(self):
 110         log('Check previous installation')
 111         env_list = parse(self.ssh.exec_cmd('fuel env list'))
 112         if env_list:
 113             self.cleanup_fuel_environments(env_list)
 114             node_list = parse(self.ssh.exec_cmd('fuel node list'))
 115             if node_list:
 116                 self.cleanup_fuel_nodes(node_list)
 117
 118     def cleanup_fuel_environments(self, env_list):
 119         WAIT_LOOP = 60
 120         SLEEP_TIME = 10
 121         for env in env_list:
 122             log('Deleting environment %s' % env[E['id']])
 123             self.ssh.exec_cmd('fuel env --env %s --delete --force'
 124                               % env[E['id']])
 125         all_env_erased = False
 126         for i in range(WAIT_LOOP):
 127             env_list = parse(self.ssh.exec_cmd('fuel env list'))
 128             if env_list:
 129                 time.sleep(SLEEP_TIME)
 130             else:
 131                 all_env_erased = True
 132                 break
 133         if not all_env_erased:
 134             err('Could not erase these environments %s'
 135                 % [(env[E['id']], env[E['status']]) for env in env_list])
 136
 137     def cleanup_fuel_nodes(self, node_list):
 138         for node in node_list:
 139             if node[N['status']] == 'discover':
 140                 log('Deleting node %s' % node[N['id']])
 141                 self.ssh.exec_cmd('fuel node --node-id %s --delete-from-db '
 142                                   '--force' % node[N['id']])
 143                 self.ssh.exec_cmd('cobbler system remove --name node-%s'
 144                                   % node[N['id']], False)
 145
 146     def check_prerequisites(self):
 147         log('Check prerequisites')
 148         with self.ssh:
 149             self.check_supported_release()
 150             self.check_previous_installation()
 151
 152     def wait_for_discovered_blades(self):
 153         log('Wait for discovered blades')
 154         discovered_macs = []
 155         restart_times = BLADE_RESTART_TIMES
 156
 157         for blade in self.node_ids:
 158             self.blade_node_dict[blade] = None
 159
 160         with self.ssh:
 161             all_discovered = self.discovery_waiting_loop(discovered_macs)
 162
 163         while not all_discovered and restart_times != 0:
 164             restart_times -= 1
 165             for blade in self.get_not_discovered_blades():
 166                 self.dha.node_reset(blade)
 167             with self.ssh:
 168                 all_discovered = self.discovery_waiting_loop(discovered_macs)
 169
 170         if not all_discovered:
 171             err('Not all blades have been discovered: %s'
 172                 % self.not_discovered_blades_summary())
 173
 174         with io.open(self.blade_node_file, 'w') as stream:
 175             yaml.dump(self.blade_node_dict, stream, default_flow_style=False)
 176
 177     def discovery_waiting_loop(self, discovered_macs):
 178         WAIT_LOOP = 360
 179         SLEEP_TIME = 10
 180         all_discovered = False
 181         for i in range(WAIT_LOOP):
 182             node_list = parse(self.ssh.exec_cmd('fuel node list'))
 183             if node_list:
 184                 self.node_discovery(node_list, discovered_macs)
 185             if self.all_blades_discovered():
 186                 all_discovered = True
 187                 break
 188             else:
 189                 time.sleep(SLEEP_TIME)
 190         return all_discovered
 191
 192     def node_discovery(self, node_list, discovered_macs):
 193         for node in node_list:
 194             if (node[N['status']] == 'discover' and
 195                 node[N['online']] == 'True' and
 196                 node[N['mac']] not in discovered_macs):
 197                 discovered_macs.append(node[N['mac']])
 198                 blade = self.find_mac_in_dict(node[N['mac']])
 199                 if blade:
 200                     log('Blade %s discovered as Node %s with MAC %s'
 201                         % (blade, node[N['id']], node[N['mac']]))
 202                     self.blade_node_dict[blade] = node[N['id']]
 203
 204     def find_mac_in_dict(self, mac):
 205         for blade, mac_list in self.macs_per_blade.iteritems():
 206             if mac in mac_list:
 207                 return blade
 208
 209     def all_blades_discovered(self):
 210         for blade, node_id in self.blade_node_dict.iteritems():
 211             if not node_id:
 212                 return False
 213         return True
 214
 215     def not_discovered_blades_summary(self):
 216         summary = ''
 217         for blade, node_id in self.blade_node_dict.iteritems():
 218             if not node_id:
 219                 summary += '\n[blade %s]' % blade
 220         return summary
 221
 222     def get_not_discovered_blades(self):
 223         not_discovered_blades = []
 224         for blade, node_id in self.blade_node_dict.iteritems():
 225             if not node_id:
 226                 not_discovered_blades.append(blade)
 227         return not_discovered_blades
 228
 229     def set_boot_order_nodes(self):
 230         self.power_off_nodes()
 231         self.set_boot_order(['pxe', 'disk'])
 232         self.power_on_nodes()
 233
 234     def deploy(self):
 235
 236         self.set_boot_order_nodes()
 237
 238         self.check_prerequisites()
 239
 240         self.get_mac_addresses()
 241
 242         self.wait_for_discovered_blades()
 243
 244         self.upload_cloud_deployment_files()
 245
 246         return self.run_cloud_deploy(CLOUD_DEPLOY_FILE)