deploy/deploy-config.py

   1 #!/usr/bin/python
   2 ###############################################################################
   3 # Copyright (c) 2015 Ericsson AB and others.
   4 # jonas.bjurel@ericsson.com
   5 # All rights reserved. This program and the accompanying materials
   6 # are made available under the terms of the Apache License, Version 2.0
   7 # which accompanies this distribution, and is available at
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 ###############################################################################
  10
  11 ###############################################################################
  12 # Description
  13 # This script constructs the final deployment dea.yaml and dha.yaml files
  14 # The dea.yaml get's constructed from (in reverse priority):
  15 # 1) dea-base
  16 # 2) dea-pod-override
  17 # 3) deployment-scenario dea-override-config section
  18 #
  19 # The dha.yaml get's constructed from (in reverse priority):
  20 # 1) pod dha
  21 # 2) deployment-scenario dha-override-config section
  22 ###############################################################################
  23
  24
  25 import os
  26 import yaml
  27 import sys
  28 import urllib2
  29 import calendar
  30 import time
  31 import collections
  32 import hashlib
  33
  34 from functools import reduce
  35 from operator import or_
  36 from common import (
  37     log,
  38     exec_cmd,
  39     err,
  40     warn,
  41     check_file_exists,
  42     create_dir_if_not_exists,
  43     delete,
  44     check_if_root,
  45     ArgParser,
  46 )
  47
  48
  49 def parse_arguments():
  50     parser = ArgParser(prog='python %s' % __file__)
  51     parser.add_argument('-dha', dest='dha_uri', action='store',
  52                         default=False,
  53                         help='dha configuration file FQDN URI',
  54                         required=True)
  55     parser.add_argument('-deab', dest='dea_base_uri', action='store',
  56                         default=False,
  57                         help='dea base configuration FQDN URI',
  58                         required=True)
  59     parser.add_argument('-deao', dest='dea_pod_override_uri',
  60                         action='store',
  61                         default=False,
  62                         help='dea POD override configuration FQDN URI',
  63                         required=True)
  64     parser.add_argument('-scenario-base-uri',
  65                         dest='scenario_base_uri',
  66                         action='store',
  67                         default=False,
  68                         help='Deployment scenario base directory URI',
  69                         required=True)
  70     parser.add_argument('-scenario', dest='scenario', action='store',
  71                         default=False,
  72                         help=('Deployment scenario short-name (priority),'
  73                               'or base file name (in the absense of a'
  74                               'shortname defenition)'),
  75                         required=True)
  76
  77     parser.add_argument('-plugins', dest='plugins_uri', action='store',
  78                         default=False,
  79                         help='Plugin configurations directory URI',
  80                         required=True)
  81     parser.add_argument('-output', dest='output_path', action='store',
  82                         default=False,
  83                         help='Local path for resulting output configuration files',
  84                         required=True)
  85     args = parser.parse_args()
  86     log(args)
  87     kwargs = {'dha_uri': args.dha_uri,
  88               'dea_base_uri': args.dea_base_uri,
  89               'dea_pod_override_uri': args.dea_pod_override_uri,
  90               'scenario_base_uri': args.scenario_base_uri,
  91               'scenario': args.scenario,
  92               'plugins_uri': args.plugins_uri,
  93               'output_path': args.output_path}
  94     return kwargs
  95
  96
  97 def warning(msg):
  98     red = '\033[0;31m'
  99     NC = '\033[0m'
 100     print('%(red)s WARNING: %(msg)s %(NC)s' % {'red': red,
 101                                                'msg': msg,
 102                                                'NC': NC})
 103
 104
 105 def setup_yaml():
 106     represent_dict_order = lambda self, data: self.represent_mapping('tag:yaml.org,2002:map', data.items())
 107     yaml.add_representer(collections.OrderedDict, represent_dict_order)
 108
 109
 110 def sha_uri(uri):
 111     response = urllib2.urlopen(uri)
 112     data = response.read()
 113     sha1 = hashlib.sha1()
 114     sha1.update(data)
 115     return sha1.hexdigest()
 116
 117
 118 def merge_fuel_plugin_version_list(list1, list2):
 119     final_list = []
 120     # When the plugin version in not there in list1 it will
 121     # not be copied
 122     for e_l1 in list1:
 123         plugin_version = e_l1.get('metadata', {}).get('plugin_version')
 124         plugin_version_found = False
 125         for e_l2 in list2:
 126             if plugin_version == e_l2.get('metadata', {}).get('plugin_version'):
 127                 final_list.append(dict(merge_dicts(e_l1, e_l2)))
 128                 plugin_version_found = True
 129         if not plugin_version_found:
 130             final_list.append(e_l1)
 131     return final_list
 132
 133
 134 def merge_networks(list_1, list_2):
 135     new_nets = {x.get('name'): x for x in list_2}
 136
 137     return [new_nets.get(net.get('name'), net) for net in list_1]
 138
 139
 140 def merge_dicts(dict1, dict2):
 141     for k in set(dict1).union(dict2):
 142         if k in dict1 and k in dict2:
 143             if isinstance(dict1[k], dict) and isinstance(dict2[k], dict):
 144                 yield (k, dict(merge_dicts(dict1[k], dict2[k])))
 145                 continue
 146             if isinstance(dict1[k], list) and isinstance(dict2[k], list):
 147                 if k == 'versions':
 148                     yield (k,
 149                            merge_fuel_plugin_version_list(dict1[k], dict2[k]))
 150                     continue
 151                 if k == 'networks':
 152                     yield (k,
 153                            merge_networks(dict1[k], dict2[k]))
 154                     continue
 155
 156             # If one of the values is not a dict nor a list,
 157             # you can't continue merging it.
 158             # Value from second dict overrides one in first if exists.
 159         if k in dict2:
 160             yield (k, dict2[k])
 161         else:
 162             yield (k, dict1[k])
 163
 164
 165 setup_yaml()
 166 kwargs = parse_arguments()
 167
 168 # Generate final dea.yaml by merging following config files/fragments in revers priority order:
 169 # "dea-base", "dea-pod-override", "deplyment-scenario/module-config-override"
 170 # and "deployment-scenario/dea-override"
 171 print('Generating final dea.yaml configuration....')
 172
 173 # Fetch dea-base, extract and purge meta-data
 174 print('Parsing dea-base from: ' + kwargs["dea_base_uri"] + "....")
 175 response = urllib2.urlopen(kwargs["dea_base_uri"])
 176 dea_base_conf = yaml.load(response.read())
 177 dea_base_title = dea_base_conf['dea-base-config-metadata']['title']
 178 dea_base_version = dea_base_conf['dea-base-config-metadata']['version']
 179 dea_base_creation = dea_base_conf['dea-base-config-metadata']['created']
 180 dea_base_sha = sha_uri(kwargs["dea_base_uri"])
 181 dea_base_comment = dea_base_conf['dea-base-config-metadata']['comment']
 182 dea_base_conf.pop('dea-base-config-metadata')
 183 final_dea_conf = dea_base_conf
 184 dea_pod_override_nodes = None
 185
 186 # Fetch dea-pod-override, extract and purge meta-data, merge with previous dea data structure
 187 print('Parsing the dea-pod-override from: ' + kwargs["dea_pod_override_uri"] + "....")
 188 response = urllib2.urlopen(kwargs["dea_pod_override_uri"])
 189 dea_pod_override_conf = yaml.load(response.read())
 190 if dea_pod_override_conf:
 191     dea_pod_title = dea_pod_override_conf['dea-pod-override-config-metadata']['title']
 192     dea_pod_version = dea_pod_override_conf['dea-pod-override-config-metadata']['version']
 193     dea_pod_creation = dea_pod_override_conf['dea-pod-override-config-metadata']['created']
 194     dea_pod_sha = sha_uri(kwargs["dea_pod_override_uri"])
 195     dea_pod_comment = dea_pod_override_conf['dea-pod-override-config-metadata']['comment']
 196     print('Merging dea-base and dea-pod-override configuration ....')
 197     dea_pod_override_conf.pop('dea-pod-override-config-metadata')
 198     # Copy the list of original nodes, which holds info on their transformations
 199     if 'nodes' in dea_pod_override_conf:
 200         dea_pod_override_nodes = list(dea_pod_override_conf['nodes'])
 201     if dea_pod_override_conf:
 202         final_dea_conf = dict(merge_dicts(final_dea_conf, dea_pod_override_conf))
 203
 204 # Fetch deployment-scenario, extract and purge meta-data, merge deployment-scenario/
 205 # dea-override-configith previous dea data structure
 206 print('Parsing deployment-scenario from: ' + kwargs["scenario"] + "....")
 207
 208 response = urllib2.urlopen(kwargs["scenario_base_uri"] + "/scenario.yaml")
 209 scenario_short_translation_conf = yaml.load(response.read())
 210 if kwargs["scenario"] in scenario_short_translation_conf:
 211     scenario_uri = (kwargs["scenario_base_uri"]
 212                     + "/"
 213                     + scenario_short_translation_conf[kwargs["scenario"]]['configfile'])
 214 else:
 215     scenario_uri = kwargs["scenario_base_uri"] + "/" + kwargs["scenario"]
 216 response = urllib2.urlopen(scenario_uri)
 217 deploy_scenario_conf = yaml.load(response.read())
 218
 219 if deploy_scenario_conf:
 220     deploy_scenario_title = deploy_scenario_conf['deployment-scenario-metadata']['title']
 221     deploy_scenario_version = deploy_scenario_conf['deployment-scenario-metadata']['version']
 222     deploy_scenario_creation = deploy_scenario_conf['deployment-scenario-metadata']['created']
 223     deploy_scenario_sha = sha_uri(scenario_uri)
 224     deploy_scenario_comment = deploy_scenario_conf['deployment-scenario-metadata']['comment']
 225     deploy_scenario_conf.pop('deployment-scenario-metadata')
 226 else:
 227     print("Deployment scenario file not found or is empty")
 228     print("Cannot continue, exiting ....")
 229     sys.exit(1)
 230
 231 dea_scenario_override_conf = deploy_scenario_conf["dea-override-config"]
 232 if dea_scenario_override_conf:
 233     print('Merging dea-base-, dea-pod-override- and deployment-scenario '
 234           'configuration into final dea.yaml configuration....')
 235     final_dea_conf = dict(merge_dicts(final_dea_conf, dea_scenario_override_conf))
 236
 237 # Fetch plugin-configuration configuration files, extract and purge meta-data,
 238 # merge/append with previous dea data structure, override plugin-configuration with
 239 # deploy-scenario/module-config-override
 240 modules = []
 241 module_uris = []
 242 module_titles = []
 243 module_versions = []
 244 module_creations = []
 245 module_shas = []
 246 module_comments = []
 247 if deploy_scenario_conf["stack-extensions"]:
 248     for module in deploy_scenario_conf["stack-extensions"]:
 249         print('Loading configuration for module: '
 250               + module["module"]
 251               + ' and merging it to final dea.yaml configuration....')
 252         response = urllib2.urlopen(kwargs["plugins_uri"]
 253                                    + '/'
 254                                    + module["module-config-name"]
 255                                    + '_'
 256                                    + module["module-config-version"]
 257                                    + '.yaml')
 258         module_conf = yaml.load(response.read())
 259         modules.append(module["module"])
 260         module_uris.append(kwargs["plugins_uri"]
 261                            + '/'
 262                            + module["module-config-name"]
 263                            + '_'
 264                            + module["module-config-version"]
 265                            + '.yaml')
 266         module_titles.append(str(module_conf['plugin-config-metadata']['title']))
 267         module_versions.append(str(module_conf['plugin-config-metadata']['version']))
 268         module_creations.append(str(module_conf['plugin-config-metadata']['created']))
 269         module_shas.append(sha_uri(kwargs["plugins_uri"]
 270                                    + '/'
 271                                    + module["module-config-name"]
 272                                    + '_'
 273                                    + module["module-config-version"]
 274                                    + '.yaml'))
 275         module_comments.append(str(module_conf['plugin-config-metadata']['comment']))
 276         module_conf.pop('plugin-config-metadata')
 277         final_dea_conf['settings']['editable'].update(module_conf)
 278         scenario_module_override_conf = module.get('module-config-override')
 279         if scenario_module_override_conf:
 280             dea_scenario_module_override_conf = {}
 281             dea_scenario_module_override_conf['settings'] = {}
 282             dea_scenario_module_override_conf['settings']['editable'] = {}
 283             dea_scenario_module_override_conf['settings']['editable'][module["module"]] = scenario_module_override_conf
 284             final_dea_conf = dict(merge_dicts(final_dea_conf, dea_scenario_module_override_conf))
 285
 286 def get_node_ifaces_and_trans(nodes, nid):
 287     for node in nodes:
 288         if node['id'] == nid:
 289             if 'transformations' in node and 'interfaces' in node:
 290                 return (node['interfaces'], node['transformations'])
 291             else:
 292                 return None
 293
 294     return None
 295
 296 if dea_pod_override_nodes:
 297     for node in final_dea_conf['nodes']:
 298        data = get_node_ifaces_and_trans(dea_pod_override_nodes, node['id'])
 299        if data:
 300            print ("Honoring original interfaces and transformations for "
 301                   "node %d to %s, %s" % (node['id'], data[0], data[1]))
 302            node['interfaces'] = data[0]
 303            node['transformations'] = data[1]
 304
 305 # Dump final dea.yaml including configuration management meta-data to argument provided
 306 # directory
 307 if not os.path.exists(kwargs["output_path"]):
 308     os.makedirs(kwargs["output_path"])
 309 print('Dumping final dea.yaml to ' + kwargs["output_path"] + '/dea.yaml....')
 310 with open(kwargs["output_path"] + '/dea.yaml', "w") as f:
 311     f.write("\n".join([("title: DEA.yaml file automatically generated from the"
 312                         'configuration files stated in the "configuration-files"'
 313                         "fragment below"),
 314                        "version: " + str(calendar.timegm(time.gmtime())),
 315                        "created: " + str(time.strftime("%d/%m/%Y")) + " "
 316                        + str(time.strftime("%H:%M:%S")),
 317                        "comment: none\n"]))
 318
 319     f.write("\n".join(["configuration-files:",
 320                        "  dea-base:",
 321                        "    uri: " +  kwargs["dea_base_uri"],
 322                        "    title: " + str(dea_base_title),
 323                        "    version: " + str(dea_base_version),
 324                        "    created: " + str(dea_base_creation),
 325                        "    sha1: " + str(dea_base_sha),
 326                        "    comment: " + str(dea_base_comment) + "\n"]))
 327
 328     f.write("\n".join(["  pod-override:",
 329                        "    uri: " + kwargs["dea_pod_override_uri"],
 330                        "    title: " + str(dea_pod_title),
 331                        "    version: " + str(dea_pod_version),
 332                        "    created: " + str(dea_pod_creation),
 333                        "    sha1: " + str(dea_pod_sha),
 334                        "    comment: " + str(dea_pod_comment) + "\n"]))
 335
 336     f.write("\n".join(["  deployment-scenario:",
 337                        "    uri: " + str(scenario_uri),
 338                        "    title: " + str(deploy_scenario_title),
 339                        "    version: " + str(deploy_scenario_version),
 340                        "    created: " + str(deploy_scenario_creation),
 341                        "    sha1: " + str(deploy_scenario_sha),
 342                        "    comment: " + str(deploy_scenario_comment) + "\n"]))
 343
 344     f.write("  plugin-modules:\n")
 345     for k, _ in enumerate(modules):
 346         f.write("\n".join(["  - module: " + modules[k],
 347                            "    uri: " + module_uris[k],
 348                            "    title: " + module_titles[k],
 349                            "    version: " + module_versions[k],
 350                            "    created: " + module_creations[k],
 351                            "    sha-1: " + module_shas[k],
 352                            "    comment: " + module_comments[k] + "\n"]))
 353
 354     yaml.dump(final_dea_conf, f, default_flow_style=False)
 355
 356 # Load POD dha and override it with "deployment-scenario/dha-override-config" section
 357 print('Generating final dha.yaml configuration....')
 358 print('Parsing dha-pod yaml configuration....')
 359 response = urllib2.urlopen(kwargs["dha_uri"])
 360 dha_pod_conf = yaml.load(response.read())
 361 dha_pod_title = dha_pod_conf['dha-pod-config-metadata']['title']
 362 dha_pod_version = dha_pod_conf['dha-pod-config-metadata']['version']
 363 dha_pod_creation = dha_pod_conf['dha-pod-config-metadata']['created']
 364 dha_pod_sha = sha_uri(kwargs["dha_uri"])
 365 dha_pod_comment = dha_pod_conf['dha-pod-config-metadata']['comment']
 366 dha_pod_conf.pop('dha-pod-config-metadata')
 367 final_dha_conf = dha_pod_conf
 368
 369 dha_scenario_override_conf = deploy_scenario_conf["dha-override-config"]
 370 # Only virtual deploy scenarios can override dha.yaml since there
 371 # is no way to programatically override a physical environment:
 372 # wireing, IPMI set-up, etc.
 373 # For Physical environments, dha.yaml overrides will be silently ignored
 374 if dha_scenario_override_conf and (final_dha_conf['adapter'] == 'libvirt'
 375                                    or final_dha_conf['adapter'] == 'esxi'
 376                                    or final_dha_conf['adapter'] == 'vbox'):
 377     print('Merging dha-pod and deployment-scenario override information to final dha.yaml configuration....')
 378     final_dha_conf = dict(merge_dicts(final_dha_conf, dha_scenario_override_conf))
 379
 380 # Dump final dha.yaml to argument provided directory
 381 print('Dumping final dha.yaml to ' + kwargs["output_path"] + '/dha.yaml....')
 382 with open(kwargs["output_path"] + '/dha.yaml', "w") as f:
 383     f.write("\n".join([("title: DHA.yaml file automatically generated from"
 384                         "the configuration files stated in the"
 385                         '"configuration-files" fragment below'),
 386                        "version: " + str(calendar.timegm(time.gmtime())),
 387                        "created: " + str(time.strftime("%d/%m/%Y")) + " "
 388                        + str(time.strftime("%H:%M:%S")),
 389                        "comment: none\n"]))
 390
 391     f.write("configuration-files:\n")
 392
 393     f.write("\n".join(["  dha-pod-configuration:",
 394                        "    uri: " + kwargs["dha_uri"],
 395                        "    title: " + str(dha_pod_title),
 396                        "    version: " + str(dha_pod_version),
 397                        "    created: " + str(dha_pod_creation),
 398                        "    sha-1: " + str(dha_pod_sha),
 399                        "    comment: " + str(dha_pod_comment) + "\n"]))
 400
 401     f.write("\n".join(["  deployment-scenario:",
 402                        "    uri: " + str(scenario_uri),
 403                        "    title: " + str(deploy_scenario_title),
 404                        "    version: " + str(deploy_scenario_version),
 405                        "    created: " + str(deploy_scenario_creation),
 406                        "    sha-1: " + str(deploy_scenario_sha),
 407                        "    comment: " + str(deploy_scenario_comment) + "\n"]))
 408
 409     yaml.dump(final_dha_conf, f, default_flow_style=False)