deploy/deploy-config.py

   1 #!/usr/bin/python
   2 ###############################################################################
   3 # Copyright (c) 2015 Ericsson AB and others.
   4 # jonas.bjurel@ericsson.com
   5 # All rights reserved. This program and the accompanying materials
   6 # are made available under the terms of the Apache License, Version 2.0
   7 # which accompanies this distribution, and is available at
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 ###############################################################################
  10
  11 ###############################################################################
  12 # Description
  13 # This script constructs the final deployment dea.yaml and dha.yaml files
  14 # The dea.yaml get's constructed from (in reverse priority):
  15 # 1) dea-base
  16 # 2) dea-pod-override
  17 # 3) deployment-scenario dea-override-config section
  18 #
  19 # The dha.yaml get's constructed from (in reverse priority):
  20 # 1) pod dha
  21 # 2) deployment-scenario dha-override-config section
  22 ###############################################################################
  23
  24
  25 import os
  26 import yaml
  27 import sys
  28 import urllib2
  29 import calendar
  30 import time
  31 import collections
  32 import hashlib
  33
  34 from common import (
  35     ArgParser,
  36 )
  37
  38
  39 def parse_arguments():
  40     parser = ArgParser(prog='python %s' % __file__)
  41     parser.add_argument('-dha', dest='dha_uri', action='store',
  42                         default=False,
  43                         help='dha configuration file FQDN URI',
  44                         required=True)
  45     parser.add_argument('-deab', dest='dea_base_uri', action='store',
  46                         default=False,
  47                         help='dea base configuration FQDN URI',
  48                         required=True)
  49     parser.add_argument('-deao', dest='dea_pod_override_uri',
  50                         action='store',
  51                         default=False,
  52                         help='dea POD override configuration FQDN URI',
  53                         required=True)
  54     parser.add_argument('-scenario-base-uri',
  55                         dest='scenario_base_uri',
  56                         action='store',
  57                         default=False,
  58                         help='Deployment scenario base directory URI',
  59                         required=True)
  60     parser.add_argument('-scenario', dest='scenario', action='store',
  61                         default=False,
  62                         help=('Deployment scenario short-name (priority), '
  63                               'or base file name (in the absense of a '
  64                               'shortname defenition)'),
  65                         required=True)
  66
  67     parser.add_argument('-plugins', dest='plugins_uri', action='store',
  68                         default=False,
  69                         help='Plugin configurations directory URI',
  70                         required=True)
  71     parser.add_argument('-output', dest='output_path', action='store',
  72                         default=False,
  73                         help='Local path for resulting output configuration files',
  74                         required=True)
  75     args = parser.parse_args()
  76     kwargs = {'dha_uri': args.dha_uri,
  77               'dea_base_uri': args.dea_base_uri,
  78               'dea_pod_override_uri': args.dea_pod_override_uri,
  79               'scenario_base_uri': args.scenario_base_uri,
  80               'scenario': args.scenario,
  81               'plugins_uri': args.plugins_uri,
  82               'output_path': args.output_path}
  83     return kwargs
  84
  85
  86 def warning(msg):
  87     red = '\033[0;31m'
  88     NC = '\033[0m'
  89     print('%(red)s WARNING: %(msg)s %(NC)s' % {'red': red,
  90                                                'msg': msg,
  91                                                'NC': NC})
  92
  93
  94 def setup_yaml():
  95     represent_dict_order = lambda self, data: self.represent_mapping('tag:yaml.org,2002:map', data.items())
  96     yaml.add_representer(collections.OrderedDict, represent_dict_order)
  97
  98
  99 def sha_uri(uri):
 100     response = urllib2.urlopen(uri)
 101     data = response.read()
 102     sha1 = hashlib.sha1()
 103     sha1.update(data)
 104     return sha1.hexdigest()
 105
 106
 107 def merge_fuel_plugin_version_list(list1, list2):
 108     final_list = []
 109     # When the plugin version in not there in list1 it will
 110     # not be copied
 111     for e_l1 in list1:
 112         plugin_version = e_l1.get('metadata', {}).get('plugin_version')
 113         plugin_version_found = False
 114         for e_l2 in list2:
 115             if plugin_version == e_l2.get('metadata', {}).get('plugin_version'):
 116                 final_list.append(dict(merge_dicts(e_l1, e_l2)))
 117                 plugin_version_found = True
 118         if not plugin_version_found:
 119             final_list.append(e_l1)
 120     return final_list
 121
 122
 123 def merge_networks(list_1, list_2):
 124     new_nets = {x.get('name'): x for x in list_2}
 125
 126     return [new_nets.get(net.get('name'), net) for net in list_1]
 127
 128
 129 def merge_dicts(dict1, dict2):
 130     for k in set(dict1).union(dict2):
 131         if k in dict1 and k in dict2:
 132             if isinstance(dict1[k], dict) and isinstance(dict2[k], dict):
 133                 yield (k, dict(merge_dicts(dict1[k], dict2[k])))
 134                 continue
 135             if isinstance(dict1[k], list) and isinstance(dict2[k], list):
 136                 if k == 'versions':
 137                     yield (k,
 138                            merge_fuel_plugin_version_list(dict1[k], dict2[k]))
 139                     continue
 140                 if k == 'networks':
 141                     yield (k,
 142                            merge_networks(dict1[k], dict2[k]))
 143                     continue
 144
 145             # If one of the values is not a dict nor a list,
 146             # you can't continue merging it.
 147             # Value from second dict overrides one in first if exists.
 148         if k in dict2:
 149             yield (k, dict2[k])
 150         else:
 151             yield (k, dict1[k])
 152
 153
 154 def get_node_ifaces_and_trans(nodes, nid):
 155     for node in nodes:
 156         if node['id'] == nid:
 157             if 'transformations' in node and 'interfaces' in node:
 158                 return (node['interfaces'], node['transformations'])
 159             else:
 160                 return None
 161
 162     return None
 163
 164
 165 class DeployConfig(object):
 166     def __init__(self):
 167         self.kwargs = parse_arguments()
 168         self.dea_conf = dict()
 169         self.dea_metadata = dict()
 170         self.dea_pod_ovr_metadata = dict()
 171         self.dea_pod_ovr_nodes = None
 172         self.scenario_metadata = dict()
 173         self.modules = []
 174         self.module_uris = []
 175         self.module_titles = []
 176         self.module_versions = []
 177         self.module_createds = []
 178         self.module_shas = []
 179         self.module_comments = []
 180         self.dha_pod_conf = dict()
 181         self.dha_metadata = dict()
 182
 183     def process_dea_base(self):
 184         # Generate final dea.yaml by merging following config files/fragments in reverse priority order:
 185         # "dea-base", "dea-pod-override", "deplyment-scenario/module-config-override"
 186         # and "deployment-scenario/dea-override"
 187         print('Generating final dea.yaml configuration....')
 188
 189         # Fetch dea-base, extract and purge meta-data
 190         print('Parsing dea-base from: ' + self.kwargs["dea_base_uri"] + "....")
 191         response = urllib2.urlopen(self.kwargs["dea_base_uri"])
 192         dea_conf = yaml.load(response.read())
 193
 194         dea_metadata = dict()
 195         dea_metadata['title'] = dea_conf['dea-base-config-metadata']['title']
 196         dea_metadata['version'] = dea_conf['dea-base-config-metadata']['version']
 197         dea_metadata['created'] = dea_conf['dea-base-config-metadata']['created']
 198         dea_metadata['sha'] = sha_uri(self.kwargs["dea_base_uri"])
 199         dea_metadata['comment'] = dea_conf['dea-base-config-metadata']['comment']
 200         self.dea_metadata = dea_metadata
 201         dea_conf.pop('dea-base-config-metadata')
 202         self.dea_conf = dea_conf
 203
 204     def process_dea_pod_override(self):
 205         # Fetch dea-pod-override, extract and purge meta-data, merge with previous dea data structure
 206         print('Parsing the dea-pod-override from: ' + self.kwargs["dea_pod_override_uri"] + "....")
 207         response = urllib2.urlopen(self.kwargs["dea_pod_override_uri"])
 208         dea_pod_override_conf = yaml.load(response.read())
 209
 210         if dea_pod_override_conf:
 211             metadata = dict()
 212             metadata['title'] = dea_pod_override_conf['dea-pod-override-config-metadata']['title']
 213             metadata['version'] = dea_pod_override_conf['dea-pod-override-config-metadata']['version']
 214             metadata['created'] = dea_pod_override_conf['dea-pod-override-config-metadata']['created']
 215             metadata['sha'] = sha_uri(self.kwargs["dea_pod_override_uri"])
 216             metadata['comment'] = dea_pod_override_conf['dea-pod-override-config-metadata']['comment']
 217             self.dea_pod_ovr_metadata = metadata
 218
 219             print('Merging dea-base and dea-pod-override configuration ....')
 220             dea_pod_override_conf.pop('dea-pod-override-config-metadata')
 221
 222             # Copy the list of original nodes, which holds info on their transformations
 223             if 'nodes' in dea_pod_override_conf:
 224                 self.dea_pod_ovr_nodes = list(dea_pod_override_conf['nodes'])
 225             if dea_pod_override_conf:
 226                 self.dea_conf = dict(merge_dicts(self.dea_conf, dea_pod_override_conf))
 227
 228     def get_scenario_uri(self):
 229         response = urllib2.urlopen(self.kwargs["scenario_base_uri"] + "/scenario.yaml")
 230         scenario_short_translation_conf = yaml.load(response.read())
 231         if self.kwargs["scenario"] in scenario_short_translation_conf:
 232             scenario_uri = (self.kwargs["scenario_base_uri"]
 233                             + "/"
 234                             + scenario_short_translation_conf[self.kwargs["scenario"]]['configfile'])
 235         else:
 236             scenario_uri = self.kwargs["scenario_base_uri"] + "/" + self.kwargs["scenario"]
 237
 238         return scenario_uri
 239
 240     def get_scenario_config(self):
 241         self.scenario_metadata['uri'] = self.get_scenario_uri()
 242         response = urllib2.urlopen(self.scenario_metadata['uri'])
 243         return yaml.load(response.read())
 244
 245     def process_modules(self):
 246         scenario_conf = self.get_scenario_config()
 247         if scenario_conf["stack-extensions"]:
 248             for module in scenario_conf["stack-extensions"]:
 249                 print('Loading configuration for module: '
 250                       + module["module"]
 251                       + ' and merging it to final dea.yaml configuration....')
 252                 response = urllib2.urlopen(self.kwargs["plugins_uri"]
 253                                            + '/'
 254                                            + module["module-config-name"]
 255                                            + '_'
 256                                            + module["module-config-version"]
 257                                            + '.yaml')
 258                 module_conf = yaml.load(response.read())
 259                 self.modules.append(module["module"])
 260                 self.module_uris.append(self.kwargs["plugins_uri"]
 261                                         + '/'
 262                                         + module["module-config-name"]
 263                                         + '_'
 264                                         + module["module-config-version"]
 265                                         + '.yaml')
 266                 self.module_titles.append(str(module_conf['plugin-config-metadata']['title']))
 267                 self.module_versions.append(str(module_conf['plugin-config-metadata']['version']))
 268                 self.module_createds.append(str(module_conf['plugin-config-metadata']['created']))
 269                 self.module_shas.append(sha_uri(self.kwargs["plugins_uri"]
 270                                                 + '/'
 271                                                 + module["module-config-name"]
 272                                                 + '_'
 273                                                 + module["module-config-version"]
 274                                                 + '.yaml'))
 275                 self.module_comments.append(str(module_conf['plugin-config-metadata']['comment']))
 276                 module_conf.pop('plugin-config-metadata')
 277                 self.dea_conf['settings']['editable'].update(module_conf)
 278
 279                 scenario_module_override_conf = module.get('module-config-override')
 280                 if scenario_module_override_conf:
 281                     dea_scenario_module_override_conf = {}
 282                     dea_scenario_module_override_conf['settings'] = {}
 283                     dea_scenario_module_override_conf['settings']['editable'] = {}
 284                     dea_scenario_module_override_conf['settings']['editable'][module["module"]] = scenario_module_override_conf
 285                     self.dea_conf = dict(merge_dicts(self.dea_conf, dea_scenario_module_override_conf))
 286
 287     def process_scenario_config(self):
 288         # Fetch deployment-scenario, extract and purge meta-data, merge deployment-scenario/
 289         # dea-override-configith previous dea data structure
 290         print('Parsing deployment-scenario from: ' + self.kwargs["scenario"] + "....")
 291
 292         scenario_conf = self.get_scenario_config()
 293
 294         metadata = dict()
 295         if scenario_conf:
 296             metadata['title'] = scenario_conf['deployment-scenario-metadata']['title']
 297             metadata['version'] = scenario_conf['deployment-scenario-metadata']['version']
 298             metadata['created'] = scenario_conf['deployment-scenario-metadata']['created']
 299             metadata['sha'] = sha_uri(self.scenario_metadata['uri'])
 300             metadata['comment'] = scenario_conf['deployment-scenario-metadata']['comment']
 301             self.scenario_metadata = metadata
 302             scenario_conf.pop('deployment-scenario-metadata')
 303         else:
 304             print("Deployment scenario file not found or is empty")
 305             print("Cannot continue, exiting ....")
 306             sys.exit(1)
 307
 308         dea_scenario_override_conf = scenario_conf["dea-override-config"]
 309         if dea_scenario_override_conf:
 310             print('Merging dea-base-, dea-pod-override- and deployment-scenario '
 311                   'configuration into final dea.yaml configuration....')
 312             self.dea_conf = dict(merge_dicts(self.dea_conf, dea_scenario_override_conf))
 313
 314         self.process_modules()
 315
 316         # Fetch plugin-configuration configuration files, extract and purge meta-data,
 317         # merge/append with previous dea data structure, override plugin-configuration with
 318         # deploy-scenario/module-config-override
 319
 320         if self.dea_pod_ovr_nodes:
 321             for node in self.dea_conf['nodes']:
 322                 data = get_node_ifaces_and_trans(self.dea_pod_ovr_nodes, node['id'])
 323                 if data:
 324                     print("Honoring original interfaces and transformations for "
 325                           "node %d to %s, %s" % (node['id'], data[0], data[1]))
 326                     node['interfaces'] = data[0]
 327                     node['transformations'] = data[1]
 328
 329     def dump_dea_config(self):
 330         # Dump final dea.yaml including configuration management meta-data to argument provided
 331         # directory
 332         path = self.kwargs["output_path"]
 333         if not os.path.exists(path):
 334             os.makedirs(path)
 335         print('Dumping final dea.yaml to ' + path + '/dea.yaml....')
 336         with open(path + '/dea.yaml', "w") as f:
 337             f.write("\n".join([("title: DEA.yaml file automatically generated from the "
 338                                 'configuration files stated in the "configuration-files" '
 339                                 "fragment below"),
 340                                "version: " + str(calendar.timegm(time.gmtime())),
 341                                "created: " + time.strftime("%d/%m/%Y %H:%M:%S"),
 342                                "comment: none\n"]))
 343
 344             f.write("\n".join(["configuration-files:",
 345                                "  dea-base:",
 346                                "    uri: " + self.kwargs["dea_base_uri"],
 347                                "    title: " + str(self.dea_metadata['title']),
 348                                "    version: " + str(self.dea_metadata['version']),
 349                                "    created: " + str(self.dea_metadata['created']),
 350                                "    sha1: " + sha_uri(self.kwargs["dea_base_uri"]),
 351                                "    comment: " + str(self.dea_metadata['comment']) + "\n"]))
 352
 353             f.write("\n".join(["  pod-override:",
 354                                "    uri: " + self.kwargs["dea_pod_override_uri"],
 355                                "    title: " + str(self.dea_pod_ovr_metadata['title']),
 356                                "    version: " + str(self.dea_pod_ovr_metadata['version']),
 357                                "    created: " + str(self.dea_pod_ovr_metadata['created']),
 358                                "    sha1: " + self.dea_pod_ovr_metadata['sha'],
 359                                "    comment: " + str(self.dea_pod_ovr_metadata['comment']) + "\n"]))
 360
 361             f.write("\n".join(["  deployment-scenario:",
 362                                "    uri: " + self.scenario_metadata['uri'],
 363                                "    title: " + str(self.scenario_metadata['title']),
 364                                "    version: " + str(self.scenario_metadata['version']),
 365                                "    created: " + str(self.scenario_metadata['created']),
 366                                "    sha1: " + self.scenario_metadata['sha'],
 367                                "    comment: " + str(self.scenario_metadata['comment']) + "\n"]))
 368
 369             f.write("  plugin-modules:\n")
 370             for k, _ in enumerate(self.modules):
 371                 f.write("\n".join(["  - module: " + self.modules[k],
 372                                    "    uri: " + self.module_uris[k],
 373                                    "    title: " + str(self.module_titles[k]),
 374                                    "    version: " + str(self.module_versions[k]),
 375                                    "    created: " + str(self.module_createds[k]),
 376                                    "    sha-1: " + self.module_shas[k],
 377                                    "    comment: " + str(self.module_comments[k]) + "\n"]))
 378
 379             yaml.dump(self.dea_conf, f, default_flow_style=False)
 380
 381     def process_dha_pod_config(self):
 382         # Load POD dha and override it with "deployment-scenario/dha-override-config" section
 383         print('Generating final dha.yaml configuration....')
 384         print('Parsing dha-pod yaml configuration....')
 385         response = urllib2.urlopen(self.kwargs["dha_uri"])
 386         dha_pod_conf = yaml.load(response.read())
 387
 388         dha_metadata = dict()
 389         dha_metadata['title'] = dha_pod_conf['dha-pod-config-metadata']['title']
 390         dha_metadata['version'] = dha_pod_conf['dha-pod-config-metadata']['version']
 391         dha_metadata['created'] = dha_pod_conf['dha-pod-config-metadata']['created']
 392         dha_metadata['sha'] = sha_uri(self.kwargs["dha_uri"])
 393         dha_metadata['comment'] = dha_pod_conf['dha-pod-config-metadata']['comment']
 394         self.dha_metadata = dha_metadata
 395         dha_pod_conf.pop('dha-pod-config-metadata')
 396         self.dha_pod_conf = dha_pod_conf
 397
 398         scenario_conf = self.get_scenario_config()
 399         dha_scenario_override_conf = scenario_conf["dha-override-config"]
 400         # Only virtual deploy scenarios can override dha.yaml since there
 401         # is no way to programatically override a physical environment:
 402         # wireing, IPMI set-up, etc.
 403         # For Physical environments, dha.yaml overrides will be silently ignored
 404         if dha_scenario_override_conf and (dha_pod_conf['adapter'] == 'libvirt'
 405                                            or dha_pod_conf['adapter'] == 'esxi'
 406                                            or dha_pod_conf['adapter'] == 'vbox'):
 407             print('Merging dha-pod and deployment-scenario override information to final dha.yaml configuration....')
 408             self.dha_pod_conf = dict(merge_dicts(self.dha_pod_conf, dha_scenario_override_conf))
 409
 410     def dump_dha_config(self):
 411         # Dump final dha.yaml to argument provided directory
 412         path = self.kwargs["output_path"]
 413         print('Dumping final dha.yaml to ' + path + '/dha.yaml....')
 414         with open(path + '/dha.yaml', "w") as f:
 415             f.write("\n".join([("title: DHA.yaml file automatically generated from "
 416                                 "the configuration files stated in the "
 417                                 '"configuration-files" fragment below'),
 418                                "version: " + str(calendar.timegm(time.gmtime())),
 419                                "created: " + time.strftime("%d/%m/%Y %H:%M:%S"),
 420                                "comment: none\n"]))
 421
 422             f.write("configuration-files:\n")
 423
 424             f.write("\n".join(["  dha-pod-configuration:",
 425                                "    uri: " + self.kwargs["dha_uri"],
 426                                "    title: " + str(self.dha_metadata['title']),
 427                                "    version: " + str(self.dha_metadata['version']),
 428                                "    created: " + str(self.dha_metadata['created']),
 429                                "    sha-1: " + self.dha_metadata['sha'],
 430                                "    comment: " + str(self.dha_metadata['comment']) + "\n"]))
 431
 432             f.write("\n".join(["  deployment-scenario:",
 433                                "    uri: " + self.scenario_metadata['uri'],
 434                                "    title: " + str(self.scenario_metadata['title']),
 435                                "    version: " + str(self.scenario_metadata['version']),
 436                                "    created: " + str(self.scenario_metadata['created']),
 437                                "    sha-1: " + self.scenario_metadata['sha'],
 438                                "    comment: " + str(self.scenario_metadata['comment']) + "\n"]))
 439
 440             yaml.dump(self.dha_pod_conf, f, default_flow_style=False)
 441
 442
 443 def main():
 444     setup_yaml()
 445
 446     deploy_config = DeployConfig()
 447     deploy_config.process_dea_base()
 448     deploy_config.process_dea_pod_override()
 449     deploy_config.process_scenario_config()
 450     deploy_config.dump_dea_config()
 451
 452     deploy_config.process_dha_pod_config()
 453     deploy_config.dump_dha_config()
 454
 455
 456 if __name__ == '__main__':
 457     main()