deploy/deploy-config.py

   1 #!/usr/bin/python
   2 ###############################################################################
   3 # Copyright (c) 2015 Ericsson AB and others.
   4 # jonas.bjurel@ericsson.com
   5 # All rights reserved. This program and the accompanying materials
   6 # are made available under the terms of the Apache License, Version 2.0
   7 # which accompanies this distribution, and is available at
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 ###############################################################################
  10
  11 ###############################################################################
  12 # Description
  13 # This script constructs the final deployment dea.yaml and dha.yaml files
  14 # The dea.yaml get's constructed from (in reverse priority):
  15 # 1) dea-base
  16 # 2) dea-pod-override
  17 # 3) deployment-scenario dea-override-config section
  18 #
  19 # The dha.yaml get's constructed from (in reverse priority):
  20 # 1) pod dha
  21 # 2) deployment-scenario dha-override-config section
  22 ###############################################################################
  23
  24
  25 import os
  26 import yaml
  27 import sys
  28 import urllib2
  29 import calendar
  30 import time
  31 import collections
  32 import hashlib
  33
  34 from functools import reduce
  35 from operator import or_
  36 from common import (
  37     log,
  38     exec_cmd,
  39     err,
  40     warn,
  41     check_file_exists,
  42     create_dir_if_not_exists,
  43     delete,
  44     check_if_root,
  45     ArgParser,
  46 )
  47
  48
  49 def parse_arguments():
  50     parser = ArgParser(prog='python %s' % __file__)
  51     parser.add_argument('-dha', dest='dha_uri', action='store',
  52                         default=False,
  53                         help='dha configuration file FQDN URI',
  54                         required=True)
  55     parser.add_argument('-deab', dest='dea_base_uri', action='store',
  56                         default=False,
  57                         help='dea base configuration FQDN URI',
  58                         required=True)
  59     parser.add_argument('-deao', dest='dea_pod_override_uri',
  60                         action='store',
  61                         default=False,
  62                         help='dea POD override configuration FQDN URI',
  63                         required=True)
  64     parser.add_argument('-scenario-base-uri',
  65                         dest='scenario_base_uri',
  66                         action='store',
  67                         default=False,
  68                         help='Deployment scenario base directory URI',
  69                         required=True)
  70     parser.add_argument('-scenario', dest='scenario', action='store',
  71                         default=False,
  72                         help=('Deployment scenario short-name (priority),'
  73                               'or base file name (in the absense of a'
  74                               'shortname defenition)'),
  75                         required=True)
  76
  77     parser.add_argument('-plugins', dest='plugins_uri', action='store',
  78                         default=False,
  79                         help='Plugin configurations directory URI',
  80                         required=True)
  81     parser.add_argument('-output', dest='output_path', action='store',
  82                         default=False,
  83                         help='Local path for resulting output configuration files',
  84                         required=True)
  85     args = parser.parse_args()
  86     log(args)
  87     kwargs = {'dha_uri': args.dha_uri,
  88               'dea_base_uri': args.dea_base_uri,
  89               'dea_pod_override_uri': args.dea_pod_override_uri,
  90               'scenario_base_uri': args.scenario_base_uri,
  91               'scenario': args.scenario,
  92               'plugins_uri': args.plugins_uri,
  93               'output_path': args.output_path}
  94     return kwargs
  95
  96
  97 def warning(msg):
  98     red = '\033[0;31m'
  99     NC = '\033[0m'
 100     print('%(red)s WARNING: %(msg)s %(NC)s' % {'red': red,
 101                                                'msg': msg,
 102                                                'NC': NC})
 103
 104
 105 def setup_yaml():
 106     represent_dict_order = lambda self, data: self.represent_mapping('tag:yaml.org,2002:map', data.items())
 107     yaml.add_representer(collections.OrderedDict, represent_dict_order)
 108
 109
 110 def sha_uri(uri):
 111     response = urllib2.urlopen(uri)
 112     data = response.read()
 113     sha1 = hashlib.sha1()
 114     sha1.update(data)
 115     return sha1.hexdigest()
 116
 117
 118 def merge_fuel_plugin_version_list(list1, list2):
 119     final_list = []
 120     # When the plugin version in not there in list1 it will
 121     # not be copied
 122     for e_l1 in list1:
 123         plugin_version = e_l1.get('metadata', {}).get('plugin_version')
 124         plugin_version_found = False
 125         for e_l2 in list2:
 126             if plugin_version == e_l2.get('metadata', {}).get('plugin_version'):
 127                 final_list.append(dict(merge_dicts(e_l1, e_l2)))
 128                 plugin_version_found = True
 129         if not plugin_version_found:
 130             final_list.append(e_l1)
 131     return final_list
 132
 133
 134 def merge_networks(list_1, list_2):
 135     new_nets = {x.get('name'): x for x in list_2}
 136
 137     return [new_nets.get(net.get('name'), net) for net in list_1]
 138
 139
 140 def merge_dicts(dict1, dict2):
 141     for k in set(dict1).union(dict2):
 142         if k in dict1 and k in dict2:
 143             if isinstance(dict1[k], dict) and isinstance(dict2[k], dict):
 144                 yield (k, dict(merge_dicts(dict1[k], dict2[k])))
 145                 continue
 146             if isinstance(dict1[k], list) and isinstance(dict2[k], list):
 147                 if k == 'versions':
 148                     yield (k,
 149                            merge_fuel_plugin_version_list(dict1[k], dict2[k]))
 150                     continue
 151                 if k == 'networks':
 152                     yield (k,
 153                            merge_networks(dict1[k], dict2[k]))
 154                     continue
 155
 156             # If one of the values is not a dict nor a list,
 157             # you can't continue merging it.
 158             # Value from second dict overrides one in first if exists.
 159         if k in dict2:
 160             yield (k, dict2[k])
 161         else:
 162             yield (k, dict1[k])
 163
 164
 165 def get_node_ifaces_and_trans(nodes, nid):
 166     for node in nodes:
 167         if node['id'] == nid:
 168             if 'transformations' in node and 'interfaces' in node:
 169                 return (node['interfaces'], node['transformations'])
 170             else:
 171                 return None
 172
 173     return None
 174
 175
 176 def main():
 177     setup_yaml()
 178     kwargs = parse_arguments()
 179
 180     # Generate final dea.yaml by merging following config files/fragments in revers priority order:
 181     # "dea-base", "dea-pod-override", "deplyment-scenario/module-config-override"
 182     # and "deployment-scenario/dea-override"
 183     print('Generating final dea.yaml configuration....')
 184
 185     # Fetch dea-base, extract and purge meta-data
 186     print('Parsing dea-base from: ' + kwargs["dea_base_uri"] + "....")
 187     response = urllib2.urlopen(kwargs["dea_base_uri"])
 188     dea_base_conf = yaml.load(response.read())
 189     dea_base_title = dea_base_conf['dea-base-config-metadata']['title']
 190     dea_base_version = dea_base_conf['dea-base-config-metadata']['version']
 191     dea_base_creation = dea_base_conf['dea-base-config-metadata']['created']
 192     dea_base_sha = sha_uri(kwargs["dea_base_uri"])
 193     dea_base_comment = dea_base_conf['dea-base-config-metadata']['comment']
 194     dea_base_conf.pop('dea-base-config-metadata')
 195     final_dea_conf = dea_base_conf
 196     dea_pod_override_nodes = None
 197
 198     # Fetch dea-pod-override, extract and purge meta-data, merge with previous dea data structure
 199     print('Parsing the dea-pod-override from: ' + kwargs["dea_pod_override_uri"] + "....")
 200     response = urllib2.urlopen(kwargs["dea_pod_override_uri"])
 201     dea_pod_override_conf = yaml.load(response.read())
 202     if dea_pod_override_conf:
 203         dea_pod_title = dea_pod_override_conf['dea-pod-override-config-metadata']['title']
 204         dea_pod_version = dea_pod_override_conf['dea-pod-override-config-metadata']['version']
 205         dea_pod_creation = dea_pod_override_conf['dea-pod-override-config-metadata']['created']
 206         dea_pod_sha = sha_uri(kwargs["dea_pod_override_uri"])
 207         dea_pod_comment = dea_pod_override_conf['dea-pod-override-config-metadata']['comment']
 208         print('Merging dea-base and dea-pod-override configuration ....')
 209         dea_pod_override_conf.pop('dea-pod-override-config-metadata')
 210         # Copy the list of original nodes, which holds info on their transformations
 211         if 'nodes' in dea_pod_override_conf:
 212             dea_pod_override_nodes = list(dea_pod_override_conf['nodes'])
 213         if dea_pod_override_conf:
 214             final_dea_conf = dict(merge_dicts(final_dea_conf, dea_pod_override_conf))
 215
 216     # Fetch deployment-scenario, extract and purge meta-data, merge deployment-scenario/
 217     # dea-override-configith previous dea data structure
 218     print('Parsing deployment-scenario from: ' + kwargs["scenario"] + "....")
 219
 220     response = urllib2.urlopen(kwargs["scenario_base_uri"] + "/scenario.yaml")
 221     scenario_short_translation_conf = yaml.load(response.read())
 222     if kwargs["scenario"] in scenario_short_translation_conf:
 223         scenario_uri = (kwargs["scenario_base_uri"]
 224                         + "/"
 225                         + scenario_short_translation_conf[kwargs["scenario"]]['configfile'])
 226     else:
 227         scenario_uri = kwargs["scenario_base_uri"] + "/" + kwargs["scenario"]
 228     response = urllib2.urlopen(scenario_uri)
 229     deploy_scenario_conf = yaml.load(response.read())
 230
 231     if deploy_scenario_conf:
 232         deploy_scenario_title = deploy_scenario_conf['deployment-scenario-metadata']['title']
 233         deploy_scenario_version = deploy_scenario_conf['deployment-scenario-metadata']['version']
 234         deploy_scenario_creation = deploy_scenario_conf['deployment-scenario-metadata']['created']
 235         deploy_scenario_sha = sha_uri(scenario_uri)
 236         deploy_scenario_comment = deploy_scenario_conf['deployment-scenario-metadata']['comment']
 237         deploy_scenario_conf.pop('deployment-scenario-metadata')
 238     else:
 239         print("Deployment scenario file not found or is empty")
 240         print("Cannot continue, exiting ....")
 241         sys.exit(1)
 242
 243     dea_scenario_override_conf = deploy_scenario_conf["dea-override-config"]
 244     if dea_scenario_override_conf:
 245         print('Merging dea-base-, dea-pod-override- and deployment-scenario '
 246              'configuration into final dea.yaml configuration....')
 247         final_dea_conf = dict(merge_dicts(final_dea_conf, dea_scenario_override_conf))
 248
 249     # Fetch plugin-configuration configuration files, extract and purge meta-data,
 250     # merge/append with previous dea data structure, override plugin-configuration with
 251     # deploy-scenario/module-config-override
 252     modules = []
 253     module_uris = []
 254     module_titles = []
 255     module_versions = []
 256     module_creations = []
 257     module_shas = []
 258     module_comments = []
 259     if deploy_scenario_conf["stack-extensions"]:
 260         for module in deploy_scenario_conf["stack-extensions"]:
 261             print('Loading configuration for module: '
 262                   + module["module"]
 263                   + ' and merging it to final dea.yaml configuration....')
 264             response = urllib2.urlopen(kwargs["plugins_uri"]
 265                                        + '/'
 266                                        + module["module-config-name"]
 267                                        + '_'
 268                                        + module["module-config-version"]
 269                                        + '.yaml')
 270             module_conf = yaml.load(response.read())
 271             modules.append(module["module"])
 272             module_uris.append(kwargs["plugins_uri"]
 273                                + '/'
 274                                + module["module-config-name"]
 275                                + '_'
 276                                + module["module-config-version"]
 277                                + '.yaml')
 278             module_titles.append(str(module_conf['plugin-config-metadata']['title']))
 279             module_versions.append(str(module_conf['plugin-config-metadata']['version']))
 280             module_creations.append(str(module_conf['plugin-config-metadata']['created']))
 281             module_shas.append(sha_uri(kwargs["plugins_uri"]
 282                                        + '/'
 283                                        + module["module-config-name"]
 284                                        + '_'
 285                                        + module["module-config-version"]
 286                                        + '.yaml'))
 287             module_comments.append(str(module_conf['plugin-config-metadata']['comment']))
 288             module_conf.pop('plugin-config-metadata')
 289             final_dea_conf['settings']['editable'].update(module_conf)
 290             scenario_module_override_conf = module.get('module-config-override')
 291             if scenario_module_override_conf:
 292                 dea_scenario_module_override_conf = {}
 293                 dea_scenario_module_override_conf['settings'] = {}
 294                 dea_scenario_module_override_conf['settings']['editable'] = {}
 295                 dea_scenario_module_override_conf['settings']['editable'][module["module"]] = scenario_module_override_conf
 296                 final_dea_conf = dict(merge_dicts(final_dea_conf, dea_scenario_module_override_conf))
 297
 298     if dea_pod_override_nodes:
 299         for node in final_dea_conf['nodes']:
 300             data = get_node_ifaces_and_trans(dea_pod_override_nodes, node['id'])
 301             if data:
 302                 print ("Honoring original interfaces and transformations for "
 303                         "node %d to %s, %s" % (node['id'], data[0], data[1]))
 304                 node['interfaces'] = data[0]
 305                 node['transformations'] = data[1]
 306
 307     # Dump final dea.yaml including configuration management meta-data to argument provided
 308     # directory
 309     if not os.path.exists(kwargs["output_path"]):
 310         os.makedirs(kwargs["output_path"])
 311     print('Dumping final dea.yaml to ' + kwargs["output_path"] + '/dea.yaml....')
 312     with open(kwargs["output_path"] + '/dea.yaml', "w") as f:
 313         f.write("\n".join([("title: DEA.yaml file automatically generated from the"
 314                             'configuration files stated in the "configuration-files"'
 315                             "fragment below"),
 316                            "version: " + str(calendar.timegm(time.gmtime())),
 317                            "created: " + str(time.strftime("%d/%m/%Y")) + " "
 318                            + str(time.strftime("%H:%M:%S")),
 319                            "comment: none\n"]))
 320
 321         f.write("\n".join(["configuration-files:",
 322                            "  dea-base:",
 323                            "    uri: " +  kwargs["dea_base_uri"],
 324                            "    title: " + str(dea_base_title),
 325                            "    version: " + str(dea_base_version),
 326                            "    created: " + str(dea_base_creation),
 327                            "    sha1: " + str(dea_base_sha),
 328                            "    comment: " + str(dea_base_comment) + "\n"]))
 329
 330         f.write("\n".join(["  pod-override:",
 331                            "    uri: " + kwargs["dea_pod_override_uri"],
 332                            "    title: " + str(dea_pod_title),
 333                            "    version: " + str(dea_pod_version),
 334                            "    created: " + str(dea_pod_creation),
 335                            "    sha1: " + str(dea_pod_sha),
 336                            "    comment: " + str(dea_pod_comment) + "\n"]))
 337
 338         f.write("\n".join(["  deployment-scenario:",
 339                            "    uri: " + str(scenario_uri),
 340                            "    title: " + str(deploy_scenario_title),
 341                            "    version: " + str(deploy_scenario_version),
 342                            "    created: " + str(deploy_scenario_creation),
 343                            "    sha1: " + str(deploy_scenario_sha),
 344                            "    comment: " + str(deploy_scenario_comment) + "\n"]))
 345
 346         f.write("  plugin-modules:\n")
 347         for k, _ in enumerate(modules):
 348             f.write("\n".join(["  - module: " + modules[k],
 349                                "    uri: " + module_uris[k],
 350                                "    title: " + module_titles[k],
 351                                "    version: " + module_versions[k],
 352                                "    created: " + module_creations[k],
 353                                "    sha-1: " + module_shas[k],
 354                                "    comment: " + module_comments[k] + "\n"]))
 355
 356         yaml.dump(final_dea_conf, f, default_flow_style=False)
 357
 358     # Load POD dha and override it with "deployment-scenario/dha-override-config" section
 359     print('Generating final dha.yaml configuration....')
 360     print('Parsing dha-pod yaml configuration....')
 361     response = urllib2.urlopen(kwargs["dha_uri"])
 362     dha_pod_conf = yaml.load(response.read())
 363     dha_pod_title = dha_pod_conf['dha-pod-config-metadata']['title']
 364     dha_pod_version = dha_pod_conf['dha-pod-config-metadata']['version']
 365     dha_pod_creation = dha_pod_conf['dha-pod-config-metadata']['created']
 366     dha_pod_sha = sha_uri(kwargs["dha_uri"])
 367     dha_pod_comment = dha_pod_conf['dha-pod-config-metadata']['comment']
 368     dha_pod_conf.pop('dha-pod-config-metadata')
 369     final_dha_conf = dha_pod_conf
 370
 371     dha_scenario_override_conf = deploy_scenario_conf["dha-override-config"]
 372     # Only virtual deploy scenarios can override dha.yaml since there
 373     # is no way to programatically override a physical environment:
 374     # wireing, IPMI set-up, etc.
 375     # For Physical environments, dha.yaml overrides will be silently ignored
 376     if dha_scenario_override_conf and (final_dha_conf['adapter'] == 'libvirt'
 377                                        or final_dha_conf['adapter'] == 'esxi'
 378                                        or final_dha_conf['adapter'] == 'vbox'):
 379         print('Merging dha-pod and deployment-scenario override information to final dha.yaml configuration....')
 380         final_dha_conf = dict(merge_dicts(final_dha_conf, dha_scenario_override_conf))
 381
 382     # Dump final dha.yaml to argument provided directory
 383     print('Dumping final dha.yaml to ' + kwargs["output_path"] + '/dha.yaml....')
 384     with open(kwargs["output_path"] + '/dha.yaml', "w") as f:
 385         f.write("\n".join([("title: DHA.yaml file automatically generated from"
 386                             "the configuration files stated in the"
 387                             '"configuration-files" fragment below'),
 388                            "version: " + str(calendar.timegm(time.gmtime())),
 389                            "created: " + str(time.strftime("%d/%m/%Y")) + " "
 390                            + str(time.strftime("%H:%M:%S")),
 391                            "comment: none\n"]))
 392
 393         f.write("configuration-files:\n")
 394
 395         f.write("\n".join(["  dha-pod-configuration:",
 396                            "    uri: " + kwargs["dha_uri"],
 397                            "    title: " + str(dha_pod_title),
 398                            "    version: " + str(dha_pod_version),
 399                            "    created: " + str(dha_pod_creation),
 400                            "    sha-1: " + str(dha_pod_sha),
 401                            "    comment: " + str(dha_pod_comment) + "\n"]))
 402
 403         f.write("\n".join(["  deployment-scenario:",
 404                            "    uri: " + str(scenario_uri),
 405                            "    title: " + str(deploy_scenario_title),
 406                            "    version: " + str(deploy_scenario_version),
 407                            "    created: " + str(deploy_scenario_creation),
 408                            "    sha-1: " + str(deploy_scenario_sha),
 409                            "    comment: " + str(deploy_scenario_comment) + "\n"]))
 410
 411         yaml.dump(final_dha_conf, f, default_flow_style=False)
 412
 413 if __name__ == '__main__':
 414     main()