deploy/deploy-config.py

   1 #!/usr/bin/python
   2 ###############################################################################
   3 # Copyright (c) 2015 Ericsson AB and others.
   4 # jonas.bjurel@ericsson.com
   5 # All rights reserved. This program and the accompanying materials
   6 # are made available under the terms of the Apache License, Version 2.0
   7 # which accompanies this distribution, and is available at
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 ###############################################################################
  10
  11 ###############################################################################
  12 # Description
  13 # This script constructs the final deployment dea.yaml and dha.yaml files
  14 # The dea.yaml get's constructed from (in reverse priority):
  15 # 1) dea-base
  16 # 2) dea-pod-override
  17 # 3) deployment-scenario dea-override-config section
  18 #
  19 # The dha.yaml get's constructed from (in reverse priority):
  20 # 1) pod dha
  21 # 2) deployment-scenario dha-override-config section
  22 ###############################################################################
  23
  24
  25 import os
  26 import yaml
  27 import sys
  28 import urllib2
  29 import calendar
  30 import time
  31 import collections
  32 import hashlib
  33
  34 from functools import reduce
  35 from operator import or_
  36 from common import (
  37     log,
  38     exec_cmd,
  39     err,
  40     warn,
  41     check_file_exists,
  42     create_dir_if_not_exists,
  43     delete,
  44     check_if_root,
  45     ArgParser,
  46 )
  47
  48
  49 def parse_arguments():
  50     parser = ArgParser(prog='python %s' % __file__)
  51     parser.add_argument('-dha', dest='dha_uri', action='store',
  52                         default=False,
  53                         help='dha configuration file FQDN URI',
  54                         required=True)
  55     parser.add_argument('-deab', dest='dea_base_uri', action='store',
  56                         default=False,
  57                         help='dea base configuration FQDN URI',
  58                         required=True)
  59     parser.add_argument('-deao', dest='dea_pod_override_uri',
  60                         action='store',
  61                         default=False,
  62                         help='dea POD override configuration FQDN URI',
  63                         required=True)
  64     parser.add_argument('-scenario-base-uri',
  65                         dest='scenario_base_uri',
  66                         action='store',
  67                         default=False,
  68                         help='Deployment scenario base directory URI',
  69                         required=True)
  70     parser.add_argument('-scenario', dest='scenario', action='store',
  71                         default=False,
  72                         help=('Deployment scenario short-name (priority),'
  73                               'or base file name (in the absense of a'
  74                               'shortname defenition)'),
  75                         required=True)
  76
  77     parser.add_argument('-plugins', dest='plugins_uri', action='store',
  78                         default=False,
  79                         help='Plugin configurations directory URI',
  80                         required=True)
  81     parser.add_argument('-output', dest='output_path', action='store',
  82                         default=False,
  83                         help='Local path for resulting output configuration files',
  84                         required=True)
  85     args = parser.parse_args()
  86     log(args)
  87     kwargs = {'dha_uri': args.dha_uri,
  88               'dea_base_uri': args.dea_base_uri,
  89               'dea_pod_override_uri': args.dea_pod_override_uri,
  90               'scenario_base_uri': args.scenario_base_uri,
  91               'scenario': args.scenario,
  92               'plugins_uri': args.plugins_uri,
  93               'output_path': args.output_path}
  94     return kwargs
  95
  96
  97 def warning(msg):
  98     red = '\033[0;31m'
  99     NC = '\033[0m'
 100     print('%(red)s WARNING: %(msg)s %(NC)s' % {'red': red,
 101                                                'msg': msg,
 102                                                'NC': NC})
 103
 104
 105 def setup_yaml():
 106     represent_dict_order = lambda self, data: self.represent_mapping('tag:yaml.org,2002:map', data.items())
 107     yaml.add_representer(collections.OrderedDict, represent_dict_order)
 108
 109
 110 def sha_uri(uri):
 111     response = urllib2.urlopen(uri)
 112     data = response.read()
 113     sha1 = hashlib.sha1()
 114     sha1.update(data)
 115     return sha1.hexdigest()
 116
 117
 118 def merge_fuel_plugin_version_list(list1, list2):
 119     final_list = []
 120     # When the plugin version in not there in list1 it will
 121     # not be copied
 122     for e_l1 in list1:
 123         plugin_version = e_l1.get('metadata', {}).get('plugin_version')
 124         plugin_version_found = False
 125         for e_l2 in list2:
 126             if plugin_version == e_l2.get('metadata', {}).get('plugin_version'):
 127                 final_list.append(dict(merge_dicts(e_l1, e_l2)))
 128                 plugin_version_found = True
 129         if not plugin_version_found:
 130             final_list.append(e_l1)
 131     return final_list
 132
 133
 134 def merge_networks(list_1, list_2):
 135     new_nets = {x.get('name'): x for x in list_2}
 136
 137     return [new_nets.get(net.get('name'), net) for net in list_1]
 138
 139
 140
 141 def merge_dicts(dict1, dict2):
 142     for k in set(dict1).union(dict2):
 143         if k in dict1 and k in dict2:
 144             if isinstance(dict1[k], dict) and isinstance(dict2[k], dict):
 145                 yield (k, dict(merge_dicts(dict1[k], dict2[k])))
 146             elif isinstance(dict1[k], list) and isinstance(dict2[k], list):
 147                 if k == 'versions':
 148                     yield (k,
 149                            merge_fuel_plugin_version_list(dict1[k], dict2[k]))
 150                 if k == 'networks':
 151                     yield (k,
 152                            merge_networks(dict1[k], dict2[k]))
 153             else:
 154                 # If one of the values is not a dict nor a list,
 155                 # you can't continue merging it.
 156                 # Value from second dict overrides one in first and we move on.
 157                 yield (k, dict2[k])
 158         elif k in dict1:
 159             yield (k, dict1[k])
 160         else:
 161             yield (k, dict2[k])
 162
 163
 164 setup_yaml()
 165 kwargs = parse_arguments()
 166
 167 # Generate final dea.yaml by merging following config files/fragments in revers priority order:
 168 # "dea-base", "dea-pod-override", "deplyment-scenario/module-config-override"
 169 # and "deployment-scenario/dea-override"
 170 print('Generating final dea.yaml configuration....')
 171
 172 # Fetch dea-base, extract and purge meta-data
 173 print('Parsing dea-base from: ' + kwargs["dea_base_uri"] + "....")
 174 response = urllib2.urlopen(kwargs["dea_base_uri"])
 175 dea_base_conf = yaml.load(response.read())
 176 dea_base_title = dea_base_conf['dea-base-config-metadata']['title']
 177 dea_base_version = dea_base_conf['dea-base-config-metadata']['version']
 178 dea_base_creation = dea_base_conf['dea-base-config-metadata']['created']
 179 dea_base_sha = sha_uri(kwargs["dea_base_uri"])
 180 dea_base_comment = dea_base_conf['dea-base-config-metadata']['comment']
 181 dea_base_conf.pop('dea-base-config-metadata')
 182 final_dea_conf = dea_base_conf
 183
 184 # Fetch dea-pod-override, extract and purge meta-data, merge with previous dea data structure
 185 print('Parsing the dea-pod-override from: ' + kwargs["dea_pod_override_uri"] + "....")
 186 response = urllib2.urlopen(kwargs["dea_pod_override_uri"])
 187 dea_pod_override_conf = yaml.load(response.read())
 188 if dea_pod_override_conf:
 189     dea_pod_title = dea_pod_override_conf['dea-pod-override-config-metadata']['title']
 190     dea_pod_version = dea_pod_override_conf['dea-pod-override-config-metadata']['version']
 191     dea_pod_creation = dea_pod_override_conf['dea-pod-override-config-metadata']['created']
 192     dea_pod_sha = sha_uri(kwargs["dea_pod_override_uri"])
 193     dea_pod_comment = dea_pod_override_conf['dea-pod-override-config-metadata']['comment']
 194     print('Merging dea-base and dea-pod-override configuration ....')
 195     dea_pod_override_conf.pop('dea-pod-override-config-metadata')
 196     if dea_pod_override_conf:
 197         final_dea_conf = dict(merge_dicts(final_dea_conf, dea_pod_override_conf))
 198
 199 # Fetch deployment-scenario, extract and purge meta-data, merge deployment-scenario/
 200 # dea-override-configith previous dea data structure
 201 print('Parsing deployment-scenario from: ' + kwargs["scenario"] + "....")
 202
 203 response = urllib2.urlopen(kwargs["scenario_base_uri"] + "/scenario.yaml")
 204 scenario_short_translation_conf = yaml.load(response.read())
 205 if kwargs["scenario"] in scenario_short_translation_conf:
 206     scenario_uri = (kwargs["scenario_base_uri"]
 207                     + "/"
 208                     + scenario_short_translation_conf[kwargs["scenario"]]['configfile'])
 209 else:
 210     scenario_uri = kwargs["scenario_base_uri"] + "/" + kwargs["scenario"]
 211 response = urllib2.urlopen(scenario_uri)
 212 deploy_scenario_conf = yaml.load(response.read())
 213
 214 if deploy_scenario_conf:
 215     deploy_scenario_title = deploy_scenario_conf['deployment-scenario-metadata']['title']
 216     deploy_scenario_version = deploy_scenario_conf['deployment-scenario-metadata']['version']
 217     deploy_scenario_creation = deploy_scenario_conf['deployment-scenario-metadata']['created']
 218     deploy_scenario_sha = sha_uri(scenario_uri)
 219     deploy_scenario_comment = deploy_scenario_conf['deployment-scenario-metadata']['comment']
 220     deploy_scenario_conf.pop('deployment-scenario-metadata')
 221 else:
 222     print("Deployment scenario file not found or is empty")
 223     print("Cannot continue, exiting ....")
 224     sys.exit(1)
 225
 226 dea_scenario_override_conf = deploy_scenario_conf["dea-override-config"]
 227 if dea_scenario_override_conf:
 228     print('Merging dea-base-, dea-pod-override- and deployment-scenario '
 229           'configuration into final dea.yaml configuration....')
 230     final_dea_conf = dict(merge_dicts(final_dea_conf, dea_scenario_override_conf))
 231
 232 # Fetch plugin-configuration configuration files, extract and purge meta-data,
 233 # merge/append with previous dea data structure, override plugin-configuration with
 234 # deploy-scenario/module-config-override
 235 modules = []
 236 module_uris = []
 237 module_titles = []
 238 module_versions = []
 239 module_creations = []
 240 module_shas = []
 241 module_comments = []
 242 if deploy_scenario_conf["stack-extensions"]:
 243     for module in deploy_scenario_conf["stack-extensions"]:
 244         print('Loading configuration for module: '
 245               + module["module"]
 246               + ' and merging it to final dea.yaml configuration....')
 247         response = urllib2.urlopen(kwargs["plugins_uri"]
 248                                    + '/'
 249                                    + module["module-config-name"]
 250                                    + '_'
 251                                    + module["module-config-version"]
 252                                    + '.yaml')
 253         module_conf = yaml.load(response.read())
 254         modules.append(module["module"])
 255         module_uris.append(kwargs["plugins_uri"]
 256                            + '/'
 257                            + module["module-config-name"]
 258                            + '_'
 259                            + module["module-config-version"]
 260                            + '.yaml')
 261         module_titles.append(str(module_conf['plugin-config-metadata']['title']))
 262         module_versions.append(str(module_conf['plugin-config-metadata']['version']))
 263         module_creations.append(str(module_conf['plugin-config-metadata']['created']))
 264         module_shas.append(sha_uri(kwargs["plugins_uri"]
 265                                    + '/'
 266                                    + module["module-config-name"]
 267                                    + '_'
 268                                    + module["module-config-version"]
 269                                    + '.yaml'))
 270         module_comments.append(str(module_conf['plugin-config-metadata']['comment']))
 271         module_conf.pop('plugin-config-metadata')
 272         final_dea_conf['settings']['editable'].update(module_conf)
 273         scenario_module_override_conf = module.get('module-config-override')
 274         if scenario_module_override_conf:
 275             dea_scenario_module_override_conf = {}
 276             dea_scenario_module_override_conf['settings'] = {}
 277             dea_scenario_module_override_conf['settings']['editable'] = {}
 278             dea_scenario_module_override_conf['settings']['editable'][module["module"]] = scenario_module_override_conf
 279             final_dea_conf = dict(merge_dicts(final_dea_conf, dea_scenario_module_override_conf))
 280
 281 # Dump final dea.yaml including configuration management meta-data to argument provided
 282 # directory
 283 if not os.path.exists(kwargs["output_path"]):
 284     os.makedirs(kwargs["output_path"])
 285 print('Dumping final dea.yaml to ' + kwargs["output_path"] + '/dea.yaml....')
 286 with open(kwargs["output_path"] + '/dea.yaml', "w") as f:
 287     f.write("\n".join([("title: DEA.yaml file automatically generated from the"
 288                         'configuration files stated in the "configuration-files"'
 289                         "fragment below"),
 290                        "version: " + str(calendar.timegm(time.gmtime())),
 291                        "created: " + str(time.strftime("%d/%m/%Y")) + " "
 292                        + str(time.strftime("%H:%M:%S")),
 293                        "comment: none\n"]))
 294
 295     f.write("\n".join(["configuration-files:",
 296                        "  dea-base:",
 297                        "    uri: " +  kwargs["dea_base_uri"],
 298                        "    title: " + str(dea_base_title),
 299                        "    version: " + str(dea_base_version),
 300                        "    created: " + str(dea_base_creation),
 301                        "    sha1: " + str(dea_base_sha),
 302                        "    comment: " + str(dea_base_comment) + "\n"]))
 303
 304     f.write("\n".join(["  pod-override:",
 305                        "    uri: " + kwargs["dea_pod_override_uri"],
 306                        "    title: " + str(dea_pod_title),
 307                        "    version: " + str(dea_pod_version),
 308                        "    created: " + str(dea_pod_creation),
 309                        "    sha1: " + str(dea_pod_sha),
 310                        "    comment: " + str(dea_pod_comment) + "\n"]))
 311
 312     f.write("\n".join(["  deployment-scenario:",
 313                        "    uri: " + str(scenario_uri),
 314                        "    title: " + str(deploy_scenario_title),
 315                        "    version: " + str(deploy_scenario_version),
 316                        "    created: " + str(deploy_scenario_creation),
 317                        "    sha1: " + str(deploy_scenario_sha),
 318                        "    comment: " + str(deploy_scenario_comment) + "\n"]))
 319
 320     f.write("  plugin-modules:\n")
 321     for k, _ in enumerate(modules):
 322         f.write("\n".join(["  - module: " + modules[k],
 323                            "    uri: " + module_uris[k],
 324                            "    title: " + module_titles[k],
 325                            "    version: " + module_versions[k],
 326                            "    created: " + module_creations[k],
 327                            "    sha-1: " + module_shas[k],
 328                            "    comment: " + module_comments[k] + "\n"]))
 329
 330     yaml.dump(final_dea_conf, f, default_flow_style=False)
 331
 332 # Load POD dha and override it with "deployment-scenario/dha-override-config" section
 333 print('Generating final dha.yaml configuration....')
 334 print('Parsing dha-pod yaml configuration....')
 335 response = urllib2.urlopen(kwargs["dha_uri"])
 336 dha_pod_conf = yaml.load(response.read())
 337 dha_pod_title = dha_pod_conf['dha-pod-config-metadata']['title']
 338 dha_pod_version = dha_pod_conf['dha-pod-config-metadata']['version']
 339 dha_pod_creation = dha_pod_conf['dha-pod-config-metadata']['created']
 340 dha_pod_sha = sha_uri(kwargs["dha_uri"])
 341 dha_pod_comment = dha_pod_conf['dha-pod-config-metadata']['comment']
 342 dha_pod_conf.pop('dha-pod-config-metadata')
 343 final_dha_conf = dha_pod_conf
 344
 345 dha_scenario_override_conf = deploy_scenario_conf["dha-override-config"]
 346 # Only virtual deploy scenarios can override dha.yaml since there
 347 # is no way to programatically override a physical environment:
 348 # wireing, IPMI set-up, etc.
 349 # For Physical environments, dha.yaml overrides will be silently ignored
 350 if dha_scenario_override_conf and (final_dha_conf['adapter'] == 'libvirt'
 351                                    or final_dha_conf['adapter'] == 'esxi'
 352                                    or final_dha_conf['adapter'] == 'vbox'):
 353     print('Merging dha-pod and deployment-scenario override information to final dha.yaml configuration....')
 354     final_dha_conf = dict(merge_dicts(final_dha_conf, dha_scenario_override_conf))
 355
 356 # Dump final dha.yaml to argument provided directory
 357 print('Dumping final dha.yaml to ' + kwargs["output_path"] + '/dha.yaml....')
 358 with open(kwargs["output_path"] + '/dha.yaml', "w") as f:
 359     f.write("\n".join([("title: DHA.yaml file automatically generated from"
 360                         "the configuration files stated in the"
 361                         '"configuration-files" fragment below'),
 362                        "version: " + str(calendar.timegm(time.gmtime())),
 363                        "created: " + str(time.strftime("%d/%m/%Y")) + " "
 364                        + str(time.strftime("%H:%M:%S")),
 365                        "comment: none\n"]))
 366
 367     f.write("configuration-files:\n")
 368
 369     f.write("\n".join(["  dha-pod-configuration:",
 370                        "    uri: " + kwargs["dha_uri"],
 371                        "    title: " + str(dha_pod_title),
 372                        "    version: " + str(dha_pod_version),
 373                        "    created: " + str(dha_pod_creation),
 374                        "    sha-1: " + str(dha_pod_sha),
 375                        "    comment: " + str(dha_pod_comment) + "\n"]))
 376
 377     f.write("\n".join(["  deployment-scenario:",
 378                        "    uri: " + str(scenario_uri),
 379                        "    title: " + str(deploy_scenario_title),
 380                        "    version: " + str(deploy_scenario_version),
 381                        "    created: " + str(deploy_scenario_creation),
 382                        "    sha-1: " + str(deploy_scenario_sha),
 383                        "    comment: " + str(deploy_scenario_comment) + "\n"]))
 384
 385     yaml.dump(final_dha_conf, f, default_flow_style=False)