deploy/deploy-config.py

   1 #!/usr/bin/python
   2 ###############################################################################
   3 # Copyright (c) 2015 Ericsson AB and others.
   4 # jonas.bjurel@ericsson.com
   5 # All rights reserved. This program and the accompanying materials
   6 # are made available under the terms of the Apache License, Version 2.0
   7 # which accompanies this distribution, and is available at
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 ###############################################################################
  10
  11 ###############################################################################
  12 # Description
  13 # This script constructs the final deployment dea.yaml and dha.yaml files
  14 # The dea.yaml get's constructed from (in reverse priority):
  15 # 1) dea-base
  16 # 2) dea-pod-override
  17 # 3) deployment-scenario dea-override-config section
  18 #
  19 # The dha.yaml get's constructed from (in reverse priority):
  20 # 1) pod dha
  21 # 2) deployment-scenario dha-override-config section
  22 ###############################################################################
  23
  24
  25 import os
  26 import yaml
  27 import sys
  28 import urllib2
  29 import calendar
  30 import time
  31 import collections
  32 import hashlib
  33
  34 from functools import reduce
  35 from operator import or_
  36 from common import (
  37     log,
  38     exec_cmd,
  39     err,
  40     warn,
  41     check_file_exists,
  42     create_dir_if_not_exists,
  43     delete,
  44     check_if_root,
  45     ArgParser,
  46 )
  47
  48
  49 def parse_arguments():
  50     parser = ArgParser(prog='python %s' % __file__)
  51     parser.add_argument('-dha', dest='dha_uri', action='store',
  52                         default=False,
  53                         help='dha configuration file FQDN URI',
  54                         required=True)
  55     parser.add_argument('-deab', dest='dea_base_uri', action='store',
  56                         default=False,
  57                         help='dea base configuration FQDN URI',
  58                         required=True)
  59     parser.add_argument('-deao', dest='dea_pod_override_uri',
  60                         action='store',
  61                         default=False,
  62                         help='dea POD override configuration FQDN URI',
  63                         required=True)
  64     parser.add_argument('-scenario-base-uri',
  65                         dest='scenario_base_uri',
  66                         action='store',
  67                         default=False,
  68                         help='Deployment scenario base directory URI',
  69                         required=True)
  70     parser.add_argument('-scenario', dest='scenario', action='store',
  71                         default=False,
  72                         help=('Deployment scenario short-name (priority),'
  73                               'or base file name (in the absense of a'
  74                               'shortname defenition)'),
  75                         required=True)
  76
  77     parser.add_argument('-plugins', dest='plugins_uri', action='store',
  78                         default=False,
  79                         help='Plugin configurations directory URI',
  80                         required=True)
  81     parser.add_argument('-output', dest='output_path', action='store',
  82                         default=False,
  83                         help='Local path for resulting output configuration files',
  84                         required=True)
  85     args = parser.parse_args()
  86     log(args)
  87     kwargs = {'dha_uri': args.dha_uri,
  88               'dea_base_uri': args.dea_base_uri,
  89               'dea_pod_override_uri': args.dea_pod_override_uri,
  90               'scenario_base_uri': args.scenario_base_uri,
  91               'scenario': args.scenario,
  92               'plugins_uri': args.plugins_uri,
  93               'output_path': args.output_path}
  94     return kwargs
  95
  96
  97 def warning(msg):
  98     red = '\033[0;31m'
  99     NC = '\033[0m'
 100     print('%(red)s WARNING: %(msg)s %(NC)s' % {'red': red,
 101                                                'msg': msg,
 102                                                'NC': NC})
 103
 104
 105 def setup_yaml():
 106     represent_dict_order = lambda self, data: self.represent_mapping('tag:yaml.org,2002:map', data.items())
 107     yaml.add_representer(collections.OrderedDict, represent_dict_order)
 108
 109
 110 def sha_uri(uri):
 111     response = urllib2.urlopen(uri)
 112     data = response.read()
 113     sha1 = hashlib.sha1()
 114     sha1.update(data)
 115     return sha1.hexdigest()
 116
 117
 118 def merge_fuel_plugin_version_list(list1, list2):
 119     final_list = []
 120     # When the plugin version in not there in list1 it will
 121     # not be copied
 122     for e_l1 in list1:
 123         plugin_version = e_l1.get('metadata', {}).get('plugin_version')
 124         plugin_version_found = False
 125         for e_l2 in list2:
 126             if plugin_version == e_l2.get('metadata', {}).get('plugin_version'):
 127                 final_list.append(dict(merge_dicts(e_l1, e_l2)))
 128                 plugin_version_found = True
 129         if not plugin_version_found:
 130             final_list.append(e_l1)
 131     return final_list
 132
 133
 134 def merge_lists(list1, list2):
 135     if list1 and list2:
 136         if isinstance(list1[0], dict):
 137             if 'plugin_version' in list1[0].get('metadata', {}):
 138                 return merge_fuel_plugin_version_list(list1, list2)
 139             else:
 140                 warning("Lists with dictionary inside are not mergeable! "
 141                         "List2 will overwrite List1. "
 142                         "List1: %s\nList2: %s"
 143                         % (list1, list2))
 144                 return list2
 145         else:
 146             return list2
 147     elif list1:
 148         return list1
 149     else:
 150         return list2
 151
 152
 153 def merge_dicts(dict1, dict2):
 154     for k in set(dict1).union(dict2):
 155         if k in dict1 and k in dict2:
 156             if isinstance(dict1[k], dict) and isinstance(dict2[k], dict):
 157                 yield (k, dict(merge_dicts(dict1[k], dict2[k])))
 158             elif isinstance(dict1[k], list) and isinstance(dict2[k], list):
 159                 yield (k, merge_lists(dict1[k], dict2[k]))
 160             else:
 161                 # If one of the values is not a dict nor a list,
 162                 # you can't continue merging it.
 163                 # Value from second dict overrides one in first and we move on.
 164                 yield (k, dict2[k])
 165         elif k in dict1:
 166             yield (k, dict1[k])
 167         else:
 168             yield (k, dict2[k])
 169
 170
 171 setup_yaml()
 172 kwargs = parse_arguments()
 173
 174 # Generate final dea.yaml by merging following config files/fragments in revers priority order:
 175 # "dea-base", "dea-pod-override", "deplyment-scenario/module-config-override"
 176 # and "deployment-scenario/dea-override"
 177 print('Generating final dea.yaml configuration....')
 178
 179 # Fetch dea-base, extract and purge meta-data
 180 print('Parsing dea-base from: ' + kwargs["dea_base_uri"] + "....")
 181 response = urllib2.urlopen(kwargs["dea_base_uri"])
 182 dea_base_conf = yaml.load(response.read())
 183 dea_base_title = dea_base_conf['dea-base-config-metadata']['title']
 184 dea_base_version = dea_base_conf['dea-base-config-metadata']['version']
 185 dea_base_creation = dea_base_conf['dea-base-config-metadata']['created']
 186 dea_base_sha = sha_uri(kwargs["dea_base_uri"])
 187 dea_base_comment = dea_base_conf['dea-base-config-metadata']['comment']
 188 dea_base_conf.pop('dea-base-config-metadata')
 189 final_dea_conf = dea_base_conf
 190
 191 # Fetch dea-pod-override, extract and purge meta-data, merge with previous dea data structure
 192 print('Parsing the dea-pod-override from: ' + kwargs["dea_pod_override_uri"] + "....")
 193 response = urllib2.urlopen(kwargs["dea_pod_override_uri"])
 194 dea_pod_override_conf = yaml.load(response.read())
 195 if dea_pod_override_conf:
 196     dea_pod_title = dea_pod_override_conf['dea-pod-override-config-metadata']['title']
 197     dea_pod_version = dea_pod_override_conf['dea-pod-override-config-metadata']['version']
 198     dea_pod_creation = dea_pod_override_conf['dea-pod-override-config-metadata']['created']
 199     dea_pod_sha = sha_uri(kwargs["dea_pod_override_uri"])
 200     dea_pod_comment = dea_pod_override_conf['dea-pod-override-config-metadata']['comment']
 201     print('Merging dea-base and dea-pod-override configuration ....')
 202     dea_pod_override_conf.pop('dea-pod-override-config-metadata')
 203     if dea_pod_override_conf:
 204         final_dea_conf = dict(merge_dicts(final_dea_conf, dea_pod_override_conf))
 205
 206 # Fetch deployment-scenario, extract and purge meta-data, merge deployment-scenario/
 207 # dea-override-configith previous dea data structure
 208 print('Parsing deployment-scenario from: ' + kwargs["scenario"] + "....")
 209
 210 response = urllib2.urlopen(kwargs["scenario_base_uri"] + "/scenario.yaml")
 211 scenario_short_translation_conf = yaml.load(response.read())
 212 if kwargs["scenario"] in scenario_short_translation_conf:
 213     scenario_uri = (kwargs["scenario_base_uri"]
 214                     + "/"
 215                     + scenario_short_translation_conf[kwargs["scenario"]]['configfile'])
 216 else:
 217     scenario_uri = kwargs["scenario_base_uri"] + "/" + kwargs["scenario"]
 218 response = urllib2.urlopen(scenario_uri)
 219 deploy_scenario_conf = yaml.load(response.read())
 220
 221 if deploy_scenario_conf:
 222     deploy_scenario_title = deploy_scenario_conf['deployment-scenario-metadata']['title']
 223     deploy_scenario_version = deploy_scenario_conf['deployment-scenario-metadata']['version']
 224     deploy_scenario_creation = deploy_scenario_conf['deployment-scenario-metadata']['created']
 225     deploy_scenario_sha = sha_uri(scenario_uri)
 226     deploy_scenario_comment = deploy_scenario_conf['deployment-scenario-metadata']['comment']
 227     deploy_scenario_conf.pop('deployment-scenario-metadata')
 228 else:
 229     print("Deployment scenario file not found or is empty")
 230     print("Cannot continue, exiting ....")
 231     sys.exit(1)
 232
 233 dea_scenario_override_conf = deploy_scenario_conf["dea-override-config"]
 234 if dea_scenario_override_conf:
 235     print('Merging dea-base-, dea-pod-override- and deployment-scenario '
 236           'configuration into final dea.yaml configuration....')
 237     final_dea_conf = dict(merge_dicts(final_dea_conf, dea_scenario_override_conf))
 238
 239 # Fetch plugin-configuration configuration files, extract and purge meta-data,
 240 # merge/append with previous dea data structure, override plugin-configuration with
 241 # deploy-scenario/module-config-override
 242 modules = []
 243 module_uris = []
 244 module_titles = []
 245 module_versions = []
 246 module_creations = []
 247 module_shas = []
 248 module_comments = []
 249 if deploy_scenario_conf["stack-extensions"]:
 250     for module in deploy_scenario_conf["stack-extensions"]:
 251         print('Loading configuration for module: '
 252               + module["module"]
 253               + ' and merging it to final dea.yaml configuration....')
 254         response = urllib2.urlopen(kwargs["plugins_uri"]
 255                                    + '/'
 256                                    + module["module-config-name"]
 257                                    + '_'
 258                                    + module["module-config-version"]
 259                                    + '.yaml')
 260         module_conf = yaml.load(response.read())
 261         modules.append(module["module"])
 262         module_uris.append(kwargs["plugins_uri"]
 263                            + '/'
 264                            + module["module-config-name"]
 265                            + '_'
 266                            + module["module-config-version"]
 267                            + '.yaml')
 268         module_titles.append(str(module_conf['plugin-config-metadata']['title']))
 269         module_versions.append(str(module_conf['plugin-config-metadata']['version']))
 270         module_creations.append(str(module_conf['plugin-config-metadata']['created']))
 271         module_shas.append(sha_uri(kwargs["plugins_uri"]
 272                                    + '/'
 273                                    + module["module-config-name"]
 274                                    + '_'
 275                                    + module["module-config-version"]
 276                                    + '.yaml'))
 277         module_comments.append(str(module_conf['plugin-config-metadata']['comment']))
 278         module_conf.pop('plugin-config-metadata')
 279         final_dea_conf['settings']['editable'].update(module_conf)
 280         scenario_module_override_conf = module.get('module-config-override')
 281         if scenario_module_override_conf:
 282             dea_scenario_module_override_conf = {}
 283             dea_scenario_module_override_conf['settings'] = {}
 284             dea_scenario_module_override_conf['settings']['editable'] = {}
 285             dea_scenario_module_override_conf['settings']['editable'][module["module"]] = scenario_module_override_conf
 286             final_dea_conf = dict(merge_dicts(final_dea_conf, dea_scenario_module_override_conf))
 287
 288 # Dump final dea.yaml including configuration management meta-data to argument provided
 289 # directory
 290 if not os.path.exists(kwargs["output_path"]):
 291     os.makedirs(kwargs["output_path"])
 292 print('Dumping final dea.yaml to ' + kwargs["output_path"] + '/dea.yaml....')
 293 with open(kwargs["output_path"] + '/dea.yaml', "w") as f:
 294     f.write("\n".join([("title: DEA.yaml file automatically generated from the"
 295                         'configuration files stated in the "configuration-files"'
 296                         "fragment below"),
 297                        "version: " + str(calendar.timegm(time.gmtime())),
 298                        "created: " + str(time.strftime("%d/%m/%Y")) + " "
 299                        + str(time.strftime("%H:%M:%S")),
 300                        "comment: none\n"]))
 301
 302     f.write("\n".join(["configuration-files:",
 303                        "  dea-base:",
 304                        "    uri: " +  kwargs["dea_base_uri"],
 305                        "    title: " + str(dea_base_title),
 306                        "    version: " + str(dea_base_version),
 307                        "    created: " + str(dea_base_creation),
 308                        "    sha1: " + str(dea_base_sha),
 309                        "    comment: " + str(dea_base_comment) + "\n"]))
 310
 311     f.write("\n".join(["  pod-override:",
 312                        "    uri: " + kwargs["dea_pod_override_uri"],
 313                        "    title: " + str(dea_pod_title),
 314                        "    version: " + str(dea_pod_version),
 315                        "    created: " + str(dea_pod_creation),
 316                        "    sha1: " + str(dea_pod_sha),
 317                        "    comment: " + str(dea_pod_comment) + "\n"]))
 318
 319     f.write("\n".join(["  deployment-scenario:",
 320                        "    uri: " + str(scenario_uri),
 321                        "    title: " + str(deploy_scenario_title),
 322                        "    version: " + str(deploy_scenario_version),
 323                        "    created: " + str(deploy_scenario_creation),
 324                        "    sha1: " + str(deploy_scenario_sha),
 325                        "    comment: " + str(deploy_scenario_comment) + "\n"]))
 326
 327     f.write("  plugin-modules:\n")
 328     for k, _ in enumerate(modules):
 329         f.write("\n".join(["  - module: " + modules[k],
 330                            "    uri: " + module_uris[k],
 331                            "    title: " + module_titles[k],
 332                            "    version: " + module_versions[k],
 333                            "    created: " + module_creations[k],
 334                            "    sha-1: " + module_shas[k],
 335                            "    comment: " + module_comments[k] + "\n"]))
 336
 337     yaml.dump(final_dea_conf, f, default_flow_style=False)
 338
 339 # Load POD dha and override it with "deployment-scenario/dha-override-config" section
 340 print('Generating final dha.yaml configuration....')
 341 print('Parsing dha-pod yaml configuration....')
 342 response = urllib2.urlopen(kwargs["dha_uri"])
 343 dha_pod_conf = yaml.load(response.read())
 344 dha_pod_title = dha_pod_conf['dha-pod-config-metadata']['title']
 345 dha_pod_version = dha_pod_conf['dha-pod-config-metadata']['version']
 346 dha_pod_creation = dha_pod_conf['dha-pod-config-metadata']['created']
 347 dha_pod_sha = sha_uri(kwargs["dha_uri"])
 348 dha_pod_comment = dha_pod_conf['dha-pod-config-metadata']['comment']
 349 dha_pod_conf.pop('dha-pod-config-metadata')
 350 final_dha_conf = dha_pod_conf
 351
 352 dha_scenario_override_conf = deploy_scenario_conf["dha-override-config"]
 353 # Only virtual deploy scenarios can override dha.yaml since there
 354 # is no way to programatically override a physical environment:
 355 # wireing, IPMI set-up, etc.
 356 # For Physical environments, dha.yaml overrides will be silently ignored
 357 if dha_scenario_override_conf and (final_dha_conf['adapter'] == 'libvirt'
 358                                    or final_dha_conf['adapter'] == 'esxi'
 359                                    or final_dha_conf['adapter'] == 'vbox'):
 360     print('Merging dha-pod and deployment-scenario override information to final dha.yaml configuration....')
 361     final_dha_conf = dict(merge_dicts(final_dha_conf, dha_scenario_override_conf))
 362
 363 # Dump final dha.yaml to argument provided directory
 364 print('Dumping final dha.yaml to ' + kwargs["output_path"] + '/dha.yaml....')
 365 with open(kwargs["output_path"] + '/dha.yaml', "w") as f:
 366     f.write("\n".join([("title: DHA.yaml file automatically generated from"
 367                         "the configuration files stated in the"
 368                         '"configuration-files" fragment below'),
 369                        "version: " + str(calendar.timegm(time.gmtime())),
 370                        "created: " + str(time.strftime("%d/%m/%Y")) + " "
 371                        + str(time.strftime("%H:%M:%S")),
 372                        "comment: none\n"]))
 373
 374     f.write("configuration-files:\n")
 375
 376     f.write("\n".join(["  dha-pod-configuration:",
 377                        "    uri: " + kwargs["dha_uri"],
 378                        "    title: " + str(dha_pod_title),
 379                        "    version: " + str(dha_pod_version),
 380                        "    created: " + str(dha_pod_creation),
 381                        "    sha-1: " + str(dha_pod_sha),
 382                        "    comment: " + str(dha_pod_comment) + "\n"]))
 383
 384     f.write("\n".join(["  deployment-scenario:",
 385                        "    uri: " + str(scenario_uri),
 386                        "    title: " + str(deploy_scenario_title),
 387                        "    version: " + str(deploy_scenario_version),
 388                        "    created: " + str(deploy_scenario_creation),
 389                        "    sha-1: " + str(deploy_scenario_sha),
 390                        "    comment: " + str(deploy_scenario_comment) + "\n"]))
 391
 392     yaml.dump(final_dha_conf, f, default_flow_style=False)