deploy/deploy-config.py

   1 #!/usr/bin/python
   2 ###############################################################################
   3 # Copyright (c) 2015 Ericsson AB and others.
   4 # jonas.bjurel@ericsson.com
   5 # All rights reserved. This program and the accompanying materials
   6 # are made available under the terms of the Apache License, Version 2.0
   7 # which accompanies this distribution, and is available at
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 ###############################################################################
  10
  11 ###############################################################################
  12 # Description
  13 # This script constructs the final deployment dea.yaml and dha.yaml files
  14 # The dea.yaml get's constructed from (in reverse priority):
  15 # 1) dea-base
  16 # 2) dea-pod-override
  17 # 3) deployment-scenario dea-override-config section
  18 #
  19 # The dha.yaml get's constructed from (in reverse priority):
  20 # 1) pod dha
  21 # 2) deployment-scenario dha-override-config section
  22 ###############################################################################
  23
  24
  25 import os
  26 import yaml
  27 import sys
  28 import urllib2
  29 import calendar
  30 import time
  31 import collections
  32 import hashlib
  33
  34 from functools import reduce
  35 from operator import or_
  36 from common import (
  37     log,
  38     exec_cmd,
  39     err,
  40     warn,
  41     check_file_exists,
  42     create_dir_if_not_exists,
  43     delete,
  44     check_if_root,
  45     ArgParser,
  46 )
  47
  48
  49 def parse_arguments():
  50     parser = ArgParser(prog='python %s' % __file__)
  51     parser.add_argument('-dha', dest='dha_uri', action='store',
  52                         default=False,
  53                         help='dha configuration file FQDN URI',
  54                         required=True)
  55     parser.add_argument('-deab', dest='dea_base_uri', action='store',
  56                         default=False,
  57                         help='dea base configuration FQDN URI',
  58                         required=True)
  59     parser.add_argument('-deao', dest='dea_pod_override_uri',
  60                         action='store',
  61                         default=False,
  62                         help='dea POD override configuration FQDN URI',
  63                         required=True)
  64     parser.add_argument('-scenario-base-uri',
  65                         dest='scenario_base_uri',
  66                         action='store',
  67                         default=False,
  68                         help='Deployment scenario base directory URI',
  69                         required=True)
  70     parser.add_argument('-scenario', dest='scenario', action='store',
  71                         default=False,
  72                         help=('Deployment scenario short-name (priority),'
  73                               'or base file name (in the absense of a'
  74                               'shortname defenition)'),
  75                         required=True)
  76
  77     parser.add_argument('-plugins', dest='plugins_uri', action='store',
  78                         default=False,
  79                         help='Plugin configurations directory URI',
  80                         required=True)
  81     parser.add_argument('-output', dest='output_path', action='store',
  82                         default=False,
  83                         help='Local path for resulting output configuration files',
  84                         required=True)
  85     args = parser.parse_args()
  86     log(args)
  87     kwargs = {'dha_uri': args.dha_uri,
  88               'dea_base_uri': args.dea_base_uri,
  89               'dea_pod_override_uri': args.dea_pod_override_uri,
  90               'scenario_base_uri': args.scenario_base_uri,
  91               'scenario': args.scenario,
  92               'plugins_uri': args.plugins_uri,
  93               'output_path': args.output_path}
  94     return kwargs
  95
  96
  97 def warning(msg):
  98     red = '\033[0;31m'
  99     NC = '\033[0m'
 100     print('%(red)s WARNING: %(msg)s %(NC)s' % {'red': red,
 101                                                'msg': msg,
 102                                                'NC': NC})
 103
 104
 105 def setup_yaml():
 106     represent_dict_order = lambda self, data: self.represent_mapping('tag:yaml.org,2002:map', data.items())
 107     yaml.add_representer(collections.OrderedDict, represent_dict_order)
 108
 109
 110 def sha_uri(uri):
 111     response = urllib2.urlopen(uri)
 112     data = response.read()
 113     sha1 = hashlib.sha1()
 114     sha1.update(data)
 115     return sha1.hexdigest()
 116
 117
 118 def merge_fuel_plugin_version_list(list1, list2):
 119     final_list = []
 120     # When the plugin version in not there in list1 it will
 121     # not be copied
 122     for e_l1 in list1:
 123         plugin_version = e_l1.get('metadata', {}).get('plugin_version')
 124         plugin_version_found = False
 125         for e_l2 in list2:
 126             if plugin_version == e_l2.get('metadata', {}).get('plugin_version'):
 127                 final_list.append(dict(merge_dicts(e_l1, e_l2)))
 128                 plugin_version_found = True
 129         if not plugin_version_found:
 130             final_list.append(e_l1)
 131     return final_list
 132
 133
 134 def merge_networks(list_1, list_2):
 135     new_nets = {x.get('name'): x for x in list_2}
 136
 137     return [new_nets.get(net.get('name'), net) for net in list_1]
 138
 139
 140
 141 def merge_dicts(dict1, dict2):
 142     for k in set(dict1).union(dict2):
 143         if k in dict1 and k in dict2:
 144             if isinstance(dict1[k], dict) and isinstance(dict2[k], dict):
 145                 yield (k, dict(merge_dicts(dict1[k], dict2[k])))
 146                 continue
 147             if isinstance(dict1[k], list) and isinstance(dict2[k], list):
 148                 if k == 'versions':
 149                     yield (k,
 150                            merge_fuel_plugin_version_list(dict1[k], dict2[k]))
 151                     continue
 152                 if k == 'networks':
 153                     yield (k,
 154                            merge_networks(dict1[k], dict2[k]))
 155                     continue
 156
 157             # If one of the values is not a dict nor a list,
 158             # you can't continue merging it.
 159             # Value from second dict overrides one in first if exists.
 160         if k in dict2:
 161             yield (k, dict2[k])
 162         else:
 163             yield (k, dict1[k])
 164
 165
 166 setup_yaml()
 167 kwargs = parse_arguments()
 168
 169 # Generate final dea.yaml by merging following config files/fragments in revers priority order:
 170 # "dea-base", "dea-pod-override", "deplyment-scenario/module-config-override"
 171 # and "deployment-scenario/dea-override"
 172 print('Generating final dea.yaml configuration....')
 173
 174 # Fetch dea-base, extract and purge meta-data
 175 print('Parsing dea-base from: ' + kwargs["dea_base_uri"] + "....")
 176 response = urllib2.urlopen(kwargs["dea_base_uri"])
 177 dea_base_conf = yaml.load(response.read())
 178 dea_base_title = dea_base_conf['dea-base-config-metadata']['title']
 179 dea_base_version = dea_base_conf['dea-base-config-metadata']['version']
 180 dea_base_creation = dea_base_conf['dea-base-config-metadata']['created']
 181 dea_base_sha = sha_uri(kwargs["dea_base_uri"])
 182 dea_base_comment = dea_base_conf['dea-base-config-metadata']['comment']
 183 dea_base_conf.pop('dea-base-config-metadata')
 184 final_dea_conf = dea_base_conf
 185 dea_pod_override_nodes = None
 186
 187 # Fetch dea-pod-override, extract and purge meta-data, merge with previous dea data structure
 188 print('Parsing the dea-pod-override from: ' + kwargs["dea_pod_override_uri"] + "....")
 189 response = urllib2.urlopen(kwargs["dea_pod_override_uri"])
 190 dea_pod_override_conf = yaml.load(response.read())
 191 if dea_pod_override_conf:
 192     dea_pod_title = dea_pod_override_conf['dea-pod-override-config-metadata']['title']
 193     dea_pod_version = dea_pod_override_conf['dea-pod-override-config-metadata']['version']
 194     dea_pod_creation = dea_pod_override_conf['dea-pod-override-config-metadata']['created']
 195     dea_pod_sha = sha_uri(kwargs["dea_pod_override_uri"])
 196     dea_pod_comment = dea_pod_override_conf['dea-pod-override-config-metadata']['comment']
 197     print('Merging dea-base and dea-pod-override configuration ....')
 198     dea_pod_override_conf.pop('dea-pod-override-config-metadata')
 199     # Copy the list of original nodes, which holds info on their transformations
 200     if 'nodes' in dea_pod_override_conf:
 201         dea_pod_override_nodes = list(dea_pod_override_conf['nodes'])
 202     if dea_pod_override_conf:
 203         final_dea_conf = dict(merge_dicts(final_dea_conf, dea_pod_override_conf))
 204
 205 # Fetch deployment-scenario, extract and purge meta-data, merge deployment-scenario/
 206 # dea-override-configith previous dea data structure
 207 print('Parsing deployment-scenario from: ' + kwargs["scenario"] + "....")
 208
 209 response = urllib2.urlopen(kwargs["scenario_base_uri"] + "/scenario.yaml")
 210 scenario_short_translation_conf = yaml.load(response.read())
 211 if kwargs["scenario"] in scenario_short_translation_conf:
 212     scenario_uri = (kwargs["scenario_base_uri"]
 213                     + "/"
 214                     + scenario_short_translation_conf[kwargs["scenario"]]['configfile'])
 215 else:
 216     scenario_uri = kwargs["scenario_base_uri"] + "/" + kwargs["scenario"]
 217 response = urllib2.urlopen(scenario_uri)
 218 deploy_scenario_conf = yaml.load(response.read())
 219
 220 if deploy_scenario_conf:
 221     deploy_scenario_title = deploy_scenario_conf['deployment-scenario-metadata']['title']
 222     deploy_scenario_version = deploy_scenario_conf['deployment-scenario-metadata']['version']
 223     deploy_scenario_creation = deploy_scenario_conf['deployment-scenario-metadata']['created']
 224     deploy_scenario_sha = sha_uri(scenario_uri)
 225     deploy_scenario_comment = deploy_scenario_conf['deployment-scenario-metadata']['comment']
 226     deploy_scenario_conf.pop('deployment-scenario-metadata')
 227 else:
 228     print("Deployment scenario file not found or is empty")
 229     print("Cannot continue, exiting ....")
 230     sys.exit(1)
 231
 232 dea_scenario_override_conf = deploy_scenario_conf["dea-override-config"]
 233 if dea_scenario_override_conf:
 234     print('Merging dea-base-, dea-pod-override- and deployment-scenario '
 235           'configuration into final dea.yaml configuration....')
 236     final_dea_conf = dict(merge_dicts(final_dea_conf, dea_scenario_override_conf))
 237
 238 # Fetch plugin-configuration configuration files, extract and purge meta-data,
 239 # merge/append with previous dea data structure, override plugin-configuration with
 240 # deploy-scenario/module-config-override
 241 modules = []
 242 module_uris = []
 243 module_titles = []
 244 module_versions = []
 245 module_creations = []
 246 module_shas = []
 247 module_comments = []
 248 if deploy_scenario_conf["stack-extensions"]:
 249     for module in deploy_scenario_conf["stack-extensions"]:
 250         print('Loading configuration for module: '
 251               + module["module"]
 252               + ' and merging it to final dea.yaml configuration....')
 253         response = urllib2.urlopen(kwargs["plugins_uri"]
 254                                    + '/'
 255                                    + module["module-config-name"]
 256                                    + '_'
 257                                    + module["module-config-version"]
 258                                    + '.yaml')
 259         module_conf = yaml.load(response.read())
 260         modules.append(module["module"])
 261         module_uris.append(kwargs["plugins_uri"]
 262                            + '/'
 263                            + module["module-config-name"]
 264                            + '_'
 265                            + module["module-config-version"]
 266                            + '.yaml')
 267         module_titles.append(str(module_conf['plugin-config-metadata']['title']))
 268         module_versions.append(str(module_conf['plugin-config-metadata']['version']))
 269         module_creations.append(str(module_conf['plugin-config-metadata']['created']))
 270         module_shas.append(sha_uri(kwargs["plugins_uri"]
 271                                    + '/'
 272                                    + module["module-config-name"]
 273                                    + '_'
 274                                    + module["module-config-version"]
 275                                    + '.yaml'))
 276         module_comments.append(str(module_conf['plugin-config-metadata']['comment']))
 277         module_conf.pop('plugin-config-metadata')
 278         final_dea_conf['settings']['editable'].update(module_conf)
 279         scenario_module_override_conf = module.get('module-config-override')
 280         if scenario_module_override_conf:
 281             dea_scenario_module_override_conf = {}
 282             dea_scenario_module_override_conf['settings'] = {}
 283             dea_scenario_module_override_conf['settings']['editable'] = {}
 284             dea_scenario_module_override_conf['settings']['editable'][module["module"]] = scenario_module_override_conf
 285             final_dea_conf = dict(merge_dicts(final_dea_conf, dea_scenario_module_override_conf))
 286
 287 def get_node_ifaces_and_trans(nodes, nid):
 288     for node in nodes:
 289         if node['id'] == nid:
 290             if 'transformations' in node and 'interfaces' in node:
 291                 return (node['interfaces'], node['transformations'])
 292             else:
 293                 return None
 294
 295     return None
 296
 297 if dea_pod_override_nodes:
 298     for node in final_dea_conf['nodes']:
 299        data = get_node_ifaces_and_trans(dea_pod_override_nodes, node['id'])
 300        if data:
 301            print ("Honoring original interfaces and transformations for "
 302                   "node %d to %s, %s" % (node['id'], data[0], data[1]))
 303            node['interfaces'] = data[0]
 304            node['transformations'] = data[1]
 305
 306 # Dump final dea.yaml including configuration management meta-data to argument provided
 307 # directory
 308 if not os.path.exists(kwargs["output_path"]):
 309     os.makedirs(kwargs["output_path"])
 310 print('Dumping final dea.yaml to ' + kwargs["output_path"] + '/dea.yaml....')
 311 with open(kwargs["output_path"] + '/dea.yaml', "w") as f:
 312     f.write("\n".join([("title: DEA.yaml file automatically generated from the"
 313                         'configuration files stated in the "configuration-files"'
 314                         "fragment below"),
 315                        "version: " + str(calendar.timegm(time.gmtime())),
 316                        "created: " + str(time.strftime("%d/%m/%Y")) + " "
 317                        + str(time.strftime("%H:%M:%S")),
 318                        "comment: none\n"]))
 319
 320     f.write("\n".join(["configuration-files:",
 321                        "  dea-base:",
 322                        "    uri: " +  kwargs["dea_base_uri"],
 323                        "    title: " + str(dea_base_title),
 324                        "    version: " + str(dea_base_version),
 325                        "    created: " + str(dea_base_creation),
 326                        "    sha1: " + str(dea_base_sha),
 327                        "    comment: " + str(dea_base_comment) + "\n"]))
 328
 329     f.write("\n".join(["  pod-override:",
 330                        "    uri: " + kwargs["dea_pod_override_uri"],
 331                        "    title: " + str(dea_pod_title),
 332                        "    version: " + str(dea_pod_version),
 333                        "    created: " + str(dea_pod_creation),
 334                        "    sha1: " + str(dea_pod_sha),
 335                        "    comment: " + str(dea_pod_comment) + "\n"]))
 336
 337     f.write("\n".join(["  deployment-scenario:",
 338                        "    uri: " + str(scenario_uri),
 339                        "    title: " + str(deploy_scenario_title),
 340                        "    version: " + str(deploy_scenario_version),
 341                        "    created: " + str(deploy_scenario_creation),
 342                        "    sha1: " + str(deploy_scenario_sha),
 343                        "    comment: " + str(deploy_scenario_comment) + "\n"]))
 344
 345     f.write("  plugin-modules:\n")
 346     for k, _ in enumerate(modules):
 347         f.write("\n".join(["  - module: " + modules[k],
 348                            "    uri: " + module_uris[k],
 349                            "    title: " + module_titles[k],
 350                            "    version: " + module_versions[k],
 351                            "    created: " + module_creations[k],
 352                            "    sha-1: " + module_shas[k],
 353                            "    comment: " + module_comments[k] + "\n"]))
 354
 355     yaml.dump(final_dea_conf, f, default_flow_style=False)
 356
 357 # Load POD dha and override it with "deployment-scenario/dha-override-config" section
 358 print('Generating final dha.yaml configuration....')
 359 print('Parsing dha-pod yaml configuration....')
 360 response = urllib2.urlopen(kwargs["dha_uri"])
 361 dha_pod_conf = yaml.load(response.read())
 362 dha_pod_title = dha_pod_conf['dha-pod-config-metadata']['title']
 363 dha_pod_version = dha_pod_conf['dha-pod-config-metadata']['version']
 364 dha_pod_creation = dha_pod_conf['dha-pod-config-metadata']['created']
 365 dha_pod_sha = sha_uri(kwargs["dha_uri"])
 366 dha_pod_comment = dha_pod_conf['dha-pod-config-metadata']['comment']
 367 dha_pod_conf.pop('dha-pod-config-metadata')
 368 final_dha_conf = dha_pod_conf
 369
 370 dha_scenario_override_conf = deploy_scenario_conf["dha-override-config"]
 371 # Only virtual deploy scenarios can override dha.yaml since there
 372 # is no way to programatically override a physical environment:
 373 # wireing, IPMI set-up, etc.
 374 # For Physical environments, dha.yaml overrides will be silently ignored
 375 if dha_scenario_override_conf and (final_dha_conf['adapter'] == 'libvirt'
 376                                    or final_dha_conf['adapter'] == 'esxi'
 377                                    or final_dha_conf['adapter'] == 'vbox'):
 378     print('Merging dha-pod and deployment-scenario override information to final dha.yaml configuration....')
 379     final_dha_conf = dict(merge_dicts(final_dha_conf, dha_scenario_override_conf))
 380
 381 # Dump final dha.yaml to argument provided directory
 382 print('Dumping final dha.yaml to ' + kwargs["output_path"] + '/dha.yaml....')
 383 with open(kwargs["output_path"] + '/dha.yaml', "w") as f:
 384     f.write("\n".join([("title: DHA.yaml file automatically generated from"
 385                         "the configuration files stated in the"
 386                         '"configuration-files" fragment below'),
 387                        "version: " + str(calendar.timegm(time.gmtime())),
 388                        "created: " + str(time.strftime("%d/%m/%Y")) + " "
 389                        + str(time.strftime("%H:%M:%S")),
 390                        "comment: none\n"]))
 391
 392     f.write("configuration-files:\n")
 393
 394     f.write("\n".join(["  dha-pod-configuration:",
 395                        "    uri: " + kwargs["dha_uri"],
 396                        "    title: " + str(dha_pod_title),
 397                        "    version: " + str(dha_pod_version),
 398                        "    created: " + str(dha_pod_creation),
 399                        "    sha-1: " + str(dha_pod_sha),
 400                        "    comment: " + str(dha_pod_comment) + "\n"]))
 401
 402     f.write("\n".join(["  deployment-scenario:",
 403                        "    uri: " + str(scenario_uri),
 404                        "    title: " + str(deploy_scenario_title),
 405                        "    version: " + str(deploy_scenario_version),
 406                        "    created: " + str(deploy_scenario_creation),
 407                        "    sha-1: " + str(deploy_scenario_sha),
 408                        "    comment: " + str(deploy_scenario_comment) + "\n"]))
 409
 410     yaml.dump(final_dha_conf, f, default_flow_style=False)