deploy/deploy-config.py

   1 #!/usr/bin/python
   2 ###############################################################################
   3 # Copyright (c) 2015 Ericsson AB and others.
   4 # jonas.bjurel@ericsson.com
   5 # All rights reserved. This program and the accompanying materials
   6 # are made available under the terms of the Apache License, Version 2.0
   7 # which accompanies this distribution, and is available at
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 ###############################################################################
  10
  11 ###############################################################################
  12 # Description
  13 # This script constructs the final deployment dea.yaml and dha.yaml files
  14 # The dea.yaml get's constructed from (in reverse priority):
  15 # 1) dea-base
  16 # 2) dea-pod-override
  17 # 3) deployment-scenario dea-override-config section
  18 #
  19 # The dha.yaml get's constructed from (in reverse priority):
  20 # 1) pod dha
  21 # 2) deployment-scenario dha-override-config section
  22 ###############################################################################
  23
  24
  25 import os
  26 import yaml
  27 import sys
  28 import urllib2
  29 import calendar
  30 import time
  31 import collections
  32 import hashlib
  33
  34 from functools import reduce
  35 from operator import or_
  36 from common import (
  37     log,
  38     exec_cmd,
  39     err,
  40     warn,
  41     check_file_exists,
  42     create_dir_if_not_exists,
  43     delete,
  44     check_if_root,
  45     ArgParser,
  46 )
  47
  48
  49 def parse_arguments():
  50     parser = ArgParser(prog='python %s' % __file__)
  51     parser.add_argument('-dha', dest='dha_uri', action='store',
  52                         default=False,
  53                         help='dha configuration file FQDN URI',
  54                         required=True)
  55     parser.add_argument('-deab', dest='dea_base_uri', action='store',
  56                         default=False,
  57                         help='dea base configuration FQDN URI',
  58                         required=True)
  59     parser.add_argument('-deao', dest='dea_pod_override_uri',
  60                         action='store',
  61                         default=False,
  62                         help='dea POD override configuration FQDN URI',
  63                         required=True)
  64     parser.add_argument('-scenario-base-uri',
  65                         dest='scenario_base_uri',
  66                         action='store',
  67                         default=False,
  68                         help='Deployment scenario base directory URI',
  69                         required=True)
  70     parser.add_argument('-scenario', dest='scenario', action='store',
  71                         default=False,
  72                         help=('Deployment scenario short-name (priority), '
  73                               'or base file name (in the absense of a '
  74                               'shortname defenition)'),
  75                         required=True)
  76
  77     parser.add_argument('-plugins', dest='plugins_uri', action='store',
  78                         default=False,
  79                         help='Plugin configurations directory URI',
  80                         required=True)
  81     parser.add_argument('-output', dest='output_path', action='store',
  82                         default=False,
  83                         help='Local path for resulting output configuration files',
  84                         required=True)
  85     args = parser.parse_args()
  86     kwargs = {'dha_uri': args.dha_uri,
  87               'dea_base_uri': args.dea_base_uri,
  88               'dea_pod_override_uri': args.dea_pod_override_uri,
  89               'scenario_base_uri': args.scenario_base_uri,
  90               'scenario': args.scenario,
  91               'plugins_uri': args.plugins_uri,
  92               'output_path': args.output_path}
  93     return kwargs
  94
  95
  96 def warning(msg):
  97     red = '\033[0;31m'
  98     NC = '\033[0m'
  99     print('%(red)s WARNING: %(msg)s %(NC)s' % {'red': red,
 100                                                'msg': msg,
 101                                                'NC': NC})
 102
 103
 104 def setup_yaml():
 105     represent_dict_order = lambda self, data: self.represent_mapping('tag:yaml.org,2002:map', data.items())
 106     yaml.add_representer(collections.OrderedDict, represent_dict_order)
 107
 108
 109 def sha_uri(uri):
 110     response = urllib2.urlopen(uri)
 111     data = response.read()
 112     sha1 = hashlib.sha1()
 113     sha1.update(data)
 114     return sha1.hexdigest()
 115
 116
 117 def merge_fuel_plugin_version_list(list1, list2):
 118     final_list = []
 119     # When the plugin version in not there in list1 it will
 120     # not be copied
 121     for e_l1 in list1:
 122         plugin_version = e_l1.get('metadata', {}).get('plugin_version')
 123         plugin_version_found = False
 124         for e_l2 in list2:
 125             if plugin_version == e_l2.get('metadata', {}).get('plugin_version'):
 126                 final_list.append(dict(merge_dicts(e_l1, e_l2)))
 127                 plugin_version_found = True
 128         if not plugin_version_found:
 129             final_list.append(e_l1)
 130     return final_list
 131
 132
 133 def merge_networks(list_1, list_2):
 134     new_nets = {x.get('name'): x for x in list_2}
 135
 136     return [new_nets.get(net.get('name'), net) for net in list_1]
 137
 138
 139 def merge_dicts(dict1, dict2):
 140     for k in set(dict1).union(dict2):
 141         if k in dict1 and k in dict2:
 142             if isinstance(dict1[k], dict) and isinstance(dict2[k], dict):
 143                 yield (k, dict(merge_dicts(dict1[k], dict2[k])))
 144                 continue
 145             if isinstance(dict1[k], list) and isinstance(dict2[k], list):
 146                 if k == 'versions':
 147                     yield (k,
 148                            merge_fuel_plugin_version_list(dict1[k], dict2[k]))
 149                     continue
 150                 if k == 'networks':
 151                     yield (k,
 152                            merge_networks(dict1[k], dict2[k]))
 153                     continue
 154
 155             # If one of the values is not a dict nor a list,
 156             # you can't continue merging it.
 157             # Value from second dict overrides one in first if exists.
 158         if k in dict2:
 159             yield (k, dict2[k])
 160         else:
 161             yield (k, dict1[k])
 162
 163
 164 def get_node_ifaces_and_trans(nodes, nid):
 165     for node in nodes:
 166         if node['id'] == nid:
 167             if 'transformations' in node and 'interfaces' in node:
 168                 return (node['interfaces'], node['transformations'])
 169             else:
 170                 return None
 171
 172     return None
 173
 174
 175 class DeployConfig(object):
 176     def __init__(self):
 177         self.kwargs = parse_arguments()
 178         self.dea_conf = dict()
 179         self.dea_metadata = dict()
 180         self.dea_pod_ovr_metadata = dict()
 181         self.dea_pod_ovr_nodes = None
 182         self.scenario_metadata = dict()
 183         self.modules = []
 184         self.module_uris = []
 185         self.module_titles = []
 186         self.module_versions = []
 187         self.module_createds = []
 188         self.module_shas = []
 189         self.module_comments = []
 190         self.dha_pod_conf = dict()
 191         self.dha_metadata = dict()
 192
 193     def process_dea_base(self):
 194         # Generate final dea.yaml by merging following config files/fragments in reverse priority order:
 195         # "dea-base", "dea-pod-override", "deplyment-scenario/module-config-override"
 196         # and "deployment-scenario/dea-override"
 197         print('Generating final dea.yaml configuration....')
 198
 199         # Fetch dea-base, extract and purge meta-data
 200         print('Parsing dea-base from: ' + self.kwargs["dea_base_uri"] + "....")
 201         response = urllib2.urlopen(self.kwargs["dea_base_uri"])
 202         dea_conf = yaml.load(response.read())
 203
 204         dea_metadata = dict()
 205         dea_metadata['title'] = dea_conf['dea-base-config-metadata']['title']
 206         dea_metadata['version'] = dea_conf['dea-base-config-metadata']['version']
 207         dea_metadata['created'] = dea_conf['dea-base-config-metadata']['created']
 208         dea_metadata['sha'] = sha_uri(self.kwargs["dea_base_uri"])
 209         dea_metadata['comment'] = dea_conf['dea-base-config-metadata']['comment']
 210         self.dea_metadata = dea_metadata
 211         dea_conf.pop('dea-base-config-metadata')
 212         self.dea_conf = dea_conf
 213
 214     def process_dea_pod_override(self):
 215         # Fetch dea-pod-override, extract and purge meta-data, merge with previous dea data structure
 216         print('Parsing the dea-pod-override from: ' + self.kwargs["dea_pod_override_uri"] + "....")
 217         response = urllib2.urlopen(self.kwargs["dea_pod_override_uri"])
 218         dea_pod_override_conf = yaml.load(response.read())
 219
 220         if dea_pod_override_conf:
 221             metadata = dict()
 222             metadata['title'] = dea_pod_override_conf['dea-pod-override-config-metadata']['title']
 223             metadata['version'] = dea_pod_override_conf['dea-pod-override-config-metadata']['version']
 224             metadata['created'] = dea_pod_override_conf['dea-pod-override-config-metadata']['created']
 225             metadata['sha'] = sha_uri(self.kwargs["dea_pod_override_uri"])
 226             metadata['comment'] = dea_pod_override_conf['dea-pod-override-config-metadata']['comment']
 227             self.dea_pod_ovr_metadata = metadata
 228
 229             print('Merging dea-base and dea-pod-override configuration ....')
 230             dea_pod_override_conf.pop('dea-pod-override-config-metadata')
 231
 232             # Copy the list of original nodes, which holds info on their transformations
 233             if 'nodes' in dea_pod_override_conf:
 234                 self.dea_pod_ovr_nodes = list(dea_pod_override_conf['nodes'])
 235             if dea_pod_override_conf:
 236                 self.dea_conf = dict(merge_dicts(self.dea_conf, dea_pod_override_conf))
 237
 238     def get_scenario_uri(self):
 239         response = urllib2.urlopen(self.kwargs["scenario_base_uri"] + "/scenario.yaml")
 240         scenario_short_translation_conf = yaml.load(response.read())
 241         if self.kwargs["scenario"] in scenario_short_translation_conf:
 242             scenario_uri = (self.kwargs["scenario_base_uri"]
 243                             + "/"
 244                             + scenario_short_translation_conf[self.kwargs["scenario"]]['configfile'])
 245         else:
 246             scenario_uri = self.kwargs["scenario_base_uri"] + "/" + self.kwargs["scenario"]
 247
 248         return scenario_uri
 249
 250     def get_scenario_config(self):
 251         self.scenario_metadata['uri'] = self.get_scenario_uri()
 252         response = urllib2.urlopen(self.scenario_metadata['uri'])
 253         return yaml.load(response.read())
 254
 255     def process_modules(self):
 256         scenario_conf = self.get_scenario_config()
 257         if scenario_conf["stack-extensions"]:
 258             for module in scenario_conf["stack-extensions"]:
 259                 print('Loading configuration for module: '
 260                       + module["module"]
 261                       + ' and merging it to final dea.yaml configuration....')
 262                 response = urllib2.urlopen(self.kwargs["plugins_uri"]
 263                                            + '/'
 264                                            + module["module-config-name"]
 265                                            + '_'
 266                                            + module["module-config-version"]
 267                                            + '.yaml')
 268                 module_conf = yaml.load(response.read())
 269                 self.modules.append(module["module"])
 270                 self.module_uris.append(self.kwargs["plugins_uri"]
 271                                         + '/'
 272                                         + module["module-config-name"]
 273                                         + '_'
 274                                         + module["module-config-version"]
 275                                         + '.yaml')
 276                 self.module_titles.append(str(module_conf['plugin-config-metadata']['title']))
 277                 self.module_versions.append(str(module_conf['plugin-config-metadata']['version']))
 278                 self.module_createds.append(str(module_conf['plugin-config-metadata']['created']))
 279                 self.module_shas.append(sha_uri(self.kwargs["plugins_uri"]
 280                                                 + '/'
 281                                                 + module["module-config-name"]
 282                                                 + '_'
 283                                                 + module["module-config-version"]
 284                                                 + '.yaml'))
 285                 self.module_comments.append(str(module_conf['plugin-config-metadata']['comment']))
 286                 module_conf.pop('plugin-config-metadata')
 287                 self.dea_conf['settings']['editable'].update(module_conf)
 288
 289                 scenario_module_override_conf = module.get('module-config-override')
 290                 if scenario_module_override_conf:
 291                     dea_scenario_module_override_conf = {}
 292                     dea_scenario_module_override_conf['settings'] = {}
 293                     dea_scenario_module_override_conf['settings']['editable'] = {}
 294                     dea_scenario_module_override_conf['settings']['editable'][module["module"]] = scenario_module_override_conf
 295                     self.dea_conf = dict(merge_dicts(self.dea_conf, dea_scenario_module_override_conf))
 296
 297     def process_scenario_config(self):
 298         # Fetch deployment-scenario, extract and purge meta-data, merge deployment-scenario/
 299         # dea-override-configith previous dea data structure
 300         print('Parsing deployment-scenario from: ' + self.kwargs["scenario"] + "....")
 301
 302         scenario_conf = self.get_scenario_config()
 303
 304         metadata = dict()
 305         if scenario_conf:
 306             metadata['title'] = scenario_conf['deployment-scenario-metadata']['title']
 307             metadata['version'] = scenario_conf['deployment-scenario-metadata']['version']
 308             metadata['created'] = scenario_conf['deployment-scenario-metadata']['created']
 309             metadata['sha'] = sha_uri(self.scenario_metadata['uri'])
 310             metadata['comment'] = scenario_conf['deployment-scenario-metadata']['comment']
 311             self.scenario_metadata = metadata
 312             scenario_conf.pop('deployment-scenario-metadata')
 313         else:
 314             print("Deployment scenario file not found or is empty")
 315             print("Cannot continue, exiting ....")
 316             sys.exit(1)
 317
 318         dea_scenario_override_conf = scenario_conf["dea-override-config"]
 319         if dea_scenario_override_conf:
 320             print('Merging dea-base-, dea-pod-override- and deployment-scenario '
 321                   'configuration into final dea.yaml configuration....')
 322             self.dea_conf = dict(merge_dicts(self.dea_conf, dea_scenario_override_conf))
 323
 324         self.process_modules()
 325
 326         # Fetch plugin-configuration configuration files, extract and purge meta-data,
 327         # merge/append with previous dea data structure, override plugin-configuration with
 328         # deploy-scenario/module-config-override
 329
 330         if self.dea_pod_ovr_nodes:
 331             for node in self.dea_conf['nodes']:
 332                 data = get_node_ifaces_and_trans(self.dea_pod_ovr_nodes, node['id'])
 333                 if data:
 334                     print("Honoring original interfaces and transformations for "
 335                           "node %d to %s, %s" % (node['id'], data[0], data[1]))
 336                     node['interfaces'] = data[0]
 337                     node['transformations'] = data[1]
 338
 339     def dump_dea_config(self):
 340         # Dump final dea.yaml including configuration management meta-data to argument provided
 341         # directory
 342         path = self.kwargs["output_path"]
 343         if not os.path.exists(path):
 344             os.makedirs(path)
 345         print('Dumping final dea.yaml to ' + path + '/dea.yaml....')
 346         with open(path + '/dea.yaml', "w") as f:
 347             f.write("\n".join([("title: DEA.yaml file automatically generated from the "
 348                                 'configuration files stated in the "configuration-files" '
 349                                 "fragment below"),
 350                                "version: " + str(calendar.timegm(time.gmtime())),
 351                                "created: " + time.strftime("%d/%m/%Y %H:%M:%S"),
 352                                "comment: none\n"]))
 353
 354             f.write("\n".join(["configuration-files:",
 355                                "  dea-base:",
 356                                "    uri: " + self.kwargs["dea_base_uri"],
 357                                "    title: " + self.dea_metadata['title'],
 358                                "    version: " + self.dea_metadata['version'],
 359                                "    created: " + self.dea_metadata['created'],
 360                                "    sha1: " + sha_uri(self.kwargs["dea_base_uri"]),
 361                                "    comment: " + self.dea_metadata['comment'] + "\n"]))
 362
 363             f.write("\n".join(["  pod-override:",
 364                                "    uri: " + self.kwargs["dea_pod_override_uri"],
 365                                "    title: " + self.dea_pod_ovr_metadata['title'],
 366                                "    version: " + self.dea_pod_ovr_metadata['version'],
 367                                "    created: " + self.dea_pod_ovr_metadata['created'],
 368                                "    sha1: " + self.dea_pod_ovr_metadata['sha'],
 369                                "    comment: " + self.dea_pod_ovr_metadata['comment'] + "\n"]))
 370
 371             f.write("\n".join(["  deployment-scenario:",
 372                                "    uri: " + self.scenario_metadata['uri'],
 373                                "    title: " + self.scenario_metadata['title'],
 374                                "    version: " + self.scenario_metadata['version'],
 375                                "    created: " + self.scenario_metadata['created'],
 376                                "    sha1: " + self.scenario_metadata['sha'],
 377                                "    comment: " + self.scenario_metadata['comment'] + "\n"]))
 378
 379             f.write("  plugin-modules:\n")
 380             for k, _ in enumerate(self.modules):
 381                 f.write("\n".join(["  - module: " + self.modules[k],
 382                                    "    uri: " + self.module_uris[k],
 383                                    "    title: " + self.module_titles[k],
 384                                    "    version: " + self.module_versions[k],
 385                                    "    created: " + self.module_createds[k],
 386                                    "    sha-1: " + self.module_shas[k],
 387                                    "    comment: " + self.module_comments[k] + "\n"]))
 388
 389             yaml.dump(self.dea_conf, f, default_flow_style=False)
 390
 391     def process_dha_pod_config(self):
 392         # Load POD dha and override it with "deployment-scenario/dha-override-config" section
 393         print('Generating final dha.yaml configuration....')
 394         print('Parsing dha-pod yaml configuration....')
 395         response = urllib2.urlopen(self.kwargs["dha_uri"])
 396         dha_pod_conf = yaml.load(response.read())
 397
 398         dha_metadata = dict()
 399         dha_metadata['title'] = dha_pod_conf['dha-pod-config-metadata']['title']
 400         dha_metadata['version'] = dha_pod_conf['dha-pod-config-metadata']['version']
 401         dha_metadata['created'] = dha_pod_conf['dha-pod-config-metadata']['created']
 402         dha_metadata['sha'] = sha_uri(self.kwargs["dha_uri"])
 403         dha_metadata['comment'] = dha_pod_conf['dha-pod-config-metadata']['comment']
 404         self.dha_metadata = dha_metadata
 405         dha_pod_conf.pop('dha-pod-config-metadata')
 406         self.dha_pod_conf = dha_pod_conf
 407
 408         scenario_conf = self.get_scenario_config()
 409         dha_scenario_override_conf = scenario_conf["dha-override-config"]
 410         # Only virtual deploy scenarios can override dha.yaml since there
 411         # is no way to programatically override a physical environment:
 412         # wireing, IPMI set-up, etc.
 413         # For Physical environments, dha.yaml overrides will be silently ignored
 414         if dha_scenario_override_conf and (dha_pod_conf['adapter'] == 'libvirt'
 415                                            or dha_pod_conf['adapter'] == 'esxi'
 416                                            or dha_pod_conf['adapter'] == 'vbox'):
 417             print('Merging dha-pod and deployment-scenario override information to final dha.yaml configuration....')
 418             self.dha_pod_conf = dict(merge_dicts(self.dha_pod_conf, dha_scenario_override_conf))
 419
 420     def dump_dha_config(self):
 421         # Dump final dha.yaml to argument provided directory
 422         path = self.kwargs["output_path"]
 423         print('Dumping final dha.yaml to ' + path + '/dha.yaml....')
 424         with open(path + '/dha.yaml', "w") as f:
 425             f.write("\n".join([("title: DHA.yaml file automatically generated from "
 426                                 "the configuration files stated in the "
 427                                 '"configuration-files" fragment below'),
 428                                "version: " + str(calendar.timegm(time.gmtime())),
 429                                "created: " + time.strftime("%d/%m/%Y %H:%M:%S"),
 430                                "comment: none\n"]))
 431
 432             f.write("configuration-files:\n")
 433
 434             f.write("\n".join(["  dha-pod-configuration:",
 435                                "    uri: " + self.kwargs["dha_uri"],
 436                                "    title: " + self.dha_metadata['title'],
 437                                "    version: " + self.dha_metadata['version'],
 438                                "    created: " + self.dha_metadata['created'],
 439                                "    sha-1: " + self.dha_metadata['sha'],
 440                                "    comment: " + self.dha_metadata['comment'] + "\n"]))
 441
 442             f.write("\n".join(["  deployment-scenario:",
 443                                "    uri: " + self.scenario_metadata['uri'],
 444                                "    title: " + self.scenario_metadata['title'],
 445                                "    version: " + self.scenario_metadata['version'],
 446                                "    created: " + self.scenario_metadata['created'],
 447                                "    sha-1: " + self.scenario_metadata['sha'],
 448                                "    comment: " + self.scenario_metadata['comment'] + "\n"]))
 449
 450             yaml.dump(self.dha_pod_conf, f, default_flow_style=False)
 451
 452
 453 def main():
 454     setup_yaml()
 455
 456     deploy_config = DeployConfig()
 457     deploy_config.process_dea_base()
 458     deploy_config.process_dea_pod_override()
 459     deploy_config.process_scenario_config()
 460     deploy_config.dump_dea_config()
 461
 462     deploy_config.process_dha_pod_config()
 463     deploy_config.dump_dha_config()
 464
 465
 466 if __name__ == '__main__':
 467     main()