deploy/deploy-config.py

   1 #!/usr/bin/python
   2 ###############################################################################
   3 # Copyright (c) 2015 Ericsson AB and others.
   4 # jonas.bjurel@ericsson.com
   5 # All rights reserved. This program and the accompanying materials
   6 # are made available under the terms of the Apache License, Version 2.0
   7 # which accompanies this distribution, and is available at
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 ###############################################################################
  10
  11 ###############################################################################
  12 # Description
  13 # This script constructs the final deployment dea.yaml and dha.yaml files
  14 # The dea.yaml get's constructed from (in reverse priority):
  15 # 1) dea-base
  16 # 2) dea-pod-override
  17 # 3) deployment-scenario dea-override-config section
  18 #
  19 # The dha.yaml get's constructed from (in reverse priority):
  20 # 1) pod dha
  21 # 2) deployment-scenario dha-override-config section
  22 ###############################################################################
  23
  24
  25 import os
  26 import yaml
  27 import sys
  28 import urllib2
  29 import calendar
  30 import time
  31 import collections
  32 import hashlib
  33
  34 from functools import reduce
  35 from operator import or_
  36 from common import (
  37     log,
  38     exec_cmd,
  39     err,
  40     warn,
  41     check_file_exists,
  42     create_dir_if_not_exists,
  43     delete,
  44     check_if_root,
  45     ArgParser,
  46 )
  47
  48
  49 def parse_arguments():
  50     parser = ArgParser(prog='python %s' % __file__)
  51     parser.add_argument('-dha', dest='dha_uri', action='store',
  52                         default=False,
  53                         help='dha configuration file FQDN URI',
  54                         required=True)
  55     parser.add_argument('-deab', dest='dea_base_uri', action='store',
  56                         default=False,
  57                         help='dea base configuration FQDN URI',
  58                         required=True)
  59     parser.add_argument('-deao', dest='dea_pod_override_uri',
  60                         action='store',
  61                         default=False,
  62                         help='dea POD override configuration FQDN URI',
  63                         required=True)
  64     parser.add_argument('-scenario-base-uri',
  65                         dest='scenario_base_uri',
  66                         action='store',
  67                         default=False,
  68                         help='Deployment scenario base directory URI',
  69                         required=True)
  70     parser.add_argument('-scenario', dest='scenario', action='store',
  71                         default=False,
  72                         help=('Deployment scenario short-name (priority),'
  73                               'or base file name (in the absense of a'
  74                               'shortname defenition)'),
  75                         required=True)
  76
  77     parser.add_argument('-plugins', dest='plugins_uri', action='store',
  78                         default=False,
  79                         help='Plugin configurations directory URI',
  80                         required=True)
  81     parser.add_argument('-output', dest='output_path', action='store',
  82                         default=False,
  83                         help='Local path for resulting output configuration files',
  84                         required=True)
  85     args = parser.parse_args()
  86     log(args)
  87     kwargs = {'dha_uri': args.dha_uri,
  88               'dea_base_uri': args.dea_base_uri,
  89               'dea_pod_override_uri': args.dea_pod_override_uri,
  90               'scenario_base_uri': args.scenario_base_uri,
  91               'scenario': args.scenario,
  92               'plugins_uri': args.plugins_uri,
  93               'output_path': args.output_path}
  94     return kwargs
  95
  96
  97 def warning(msg):
  98     red = '\033[0;31m'
  99     NC = '\033[0m'
 100     print('%(red)s WARNING: %(msg)s %(NC)s' % {'red': red,
 101                                                'msg': msg,
 102                                                'NC': NC})
 103
 104
 105 def setup_yaml():
 106     represent_dict_order = lambda self, data: self.represent_mapping('tag:yaml.org,2002:map', data.items())
 107     yaml.add_representer(collections.OrderedDict, represent_dict_order)
 108
 109
 110 def sha_uri(uri):
 111     response = urllib2.urlopen(uri)
 112     data = response.read()
 113     sha1 = hashlib.sha1()
 114     sha1.update(data)
 115     return sha1.hexdigest()
 116
 117
 118 def merge_fuel_plugin_version_list(list1, list2):
 119     final_list = []
 120     # When the plugin version in not there in list1 it will
 121     # not be copied
 122     for e_l1 in list1:
 123         plugin_version = e_l1.get('metadata', {}).get('plugin_version')
 124         plugin_version_found = False
 125         for e_l2 in list2:
 126             if plugin_version == e_l2.get('metadata', {}).get('plugin_version'):
 127                 final_list.append(dict(merge_dicts(e_l1, e_l2)))
 128                 plugin_version_found = True
 129         if not plugin_version_found:
 130             final_list.append(e_l1)
 131     return final_list
 132
 133
 134 def merge_networks(list_1, list_2):
 135     new_nets = {x.get('name'): x for x in list_2}
 136
 137     return [new_nets.get(net.get('name'), net) for net in list_1]
 138
 139
 140
 141 def merge_dicts(dict1, dict2):
 142     for k in set(dict1).union(dict2):
 143         if k in dict1 and k in dict2:
 144             if isinstance(dict1[k], dict) and isinstance(dict2[k], dict):
 145                 yield (k, dict(merge_dicts(dict1[k], dict2[k])))
 146                 continue
 147             if isinstance(dict1[k], list) and isinstance(dict2[k], list):
 148                 if k == 'versions':
 149                     yield (k,
 150                            merge_fuel_plugin_version_list(dict1[k], dict2[k]))
 151                     continue
 152                 if k == 'networks':
 153                     yield (k,
 154                            merge_networks(dict1[k], dict2[k]))
 155                     continue
 156
 157             # If one of the values is not a dict nor a list,
 158             # you can't continue merging it.
 159             # Value from second dict overrides one in first if exists.
 160         if k in dict2:
 161             yield (k, dict2[k])
 162         else:
 163             yield (k, dict1[k])
 164
 165
 166 setup_yaml()
 167 kwargs = parse_arguments()
 168
 169 # Generate final dea.yaml by merging following config files/fragments in revers priority order:
 170 # "dea-base", "dea-pod-override", "deplyment-scenario/module-config-override"
 171 # and "deployment-scenario/dea-override"
 172 print('Generating final dea.yaml configuration....')
 173
 174 # Fetch dea-base, extract and purge meta-data
 175 print('Parsing dea-base from: ' + kwargs["dea_base_uri"] + "....")
 176 response = urllib2.urlopen(kwargs["dea_base_uri"])
 177 dea_base_conf = yaml.load(response.read())
 178 dea_base_title = dea_base_conf['dea-base-config-metadata']['title']
 179 dea_base_version = dea_base_conf['dea-base-config-metadata']['version']
 180 dea_base_creation = dea_base_conf['dea-base-config-metadata']['created']
 181 dea_base_sha = sha_uri(kwargs["dea_base_uri"])
 182 dea_base_comment = dea_base_conf['dea-base-config-metadata']['comment']
 183 dea_base_conf.pop('dea-base-config-metadata')
 184 final_dea_conf = dea_base_conf
 185
 186 # Fetch dea-pod-override, extract and purge meta-data, merge with previous dea data structure
 187 print('Parsing the dea-pod-override from: ' + kwargs["dea_pod_override_uri"] + "....")
 188 response = urllib2.urlopen(kwargs["dea_pod_override_uri"])
 189 dea_pod_override_conf = yaml.load(response.read())
 190 if dea_pod_override_conf:
 191     dea_pod_title = dea_pod_override_conf['dea-pod-override-config-metadata']['title']
 192     dea_pod_version = dea_pod_override_conf['dea-pod-override-config-metadata']['version']
 193     dea_pod_creation = dea_pod_override_conf['dea-pod-override-config-metadata']['created']
 194     dea_pod_sha = sha_uri(kwargs["dea_pod_override_uri"])
 195     dea_pod_comment = dea_pod_override_conf['dea-pod-override-config-metadata']['comment']
 196     print('Merging dea-base and dea-pod-override configuration ....')
 197     dea_pod_override_conf.pop('dea-pod-override-config-metadata')
 198     if dea_pod_override_conf:
 199         final_dea_conf = dict(merge_dicts(final_dea_conf, dea_pod_override_conf))
 200
 201 # Fetch deployment-scenario, extract and purge meta-data, merge deployment-scenario/
 202 # dea-override-configith previous dea data structure
 203 print('Parsing deployment-scenario from: ' + kwargs["scenario"] + "....")
 204
 205 response = urllib2.urlopen(kwargs["scenario_base_uri"] + "/scenario.yaml")
 206 scenario_short_translation_conf = yaml.load(response.read())
 207 if kwargs["scenario"] in scenario_short_translation_conf:
 208     scenario_uri = (kwargs["scenario_base_uri"]
 209                     + "/"
 210                     + scenario_short_translation_conf[kwargs["scenario"]]['configfile'])
 211 else:
 212     scenario_uri = kwargs["scenario_base_uri"] + "/" + kwargs["scenario"]
 213 response = urllib2.urlopen(scenario_uri)
 214 deploy_scenario_conf = yaml.load(response.read())
 215
 216 if deploy_scenario_conf:
 217     deploy_scenario_title = deploy_scenario_conf['deployment-scenario-metadata']['title']
 218     deploy_scenario_version = deploy_scenario_conf['deployment-scenario-metadata']['version']
 219     deploy_scenario_creation = deploy_scenario_conf['deployment-scenario-metadata']['created']
 220     deploy_scenario_sha = sha_uri(scenario_uri)
 221     deploy_scenario_comment = deploy_scenario_conf['deployment-scenario-metadata']['comment']
 222     deploy_scenario_conf.pop('deployment-scenario-metadata')
 223 else:
 224     print("Deployment scenario file not found or is empty")
 225     print("Cannot continue, exiting ....")
 226     sys.exit(1)
 227
 228 dea_scenario_override_conf = deploy_scenario_conf["dea-override-config"]
 229 if dea_scenario_override_conf:
 230     print('Merging dea-base-, dea-pod-override- and deployment-scenario '
 231           'configuration into final dea.yaml configuration....')
 232     final_dea_conf = dict(merge_dicts(final_dea_conf, dea_scenario_override_conf))
 233
 234 # Fetch plugin-configuration configuration files, extract and purge meta-data,
 235 # merge/append with previous dea data structure, override plugin-configuration with
 236 # deploy-scenario/module-config-override
 237 modules = []
 238 module_uris = []
 239 module_titles = []
 240 module_versions = []
 241 module_creations = []
 242 module_shas = []
 243 module_comments = []
 244 if deploy_scenario_conf["stack-extensions"]:
 245     for module in deploy_scenario_conf["stack-extensions"]:
 246         print('Loading configuration for module: '
 247               + module["module"]
 248               + ' and merging it to final dea.yaml configuration....')
 249         response = urllib2.urlopen(kwargs["plugins_uri"]
 250                                    + '/'
 251                                    + module["module-config-name"]
 252                                    + '_'
 253                                    + module["module-config-version"]
 254                                    + '.yaml')
 255         module_conf = yaml.load(response.read())
 256         modules.append(module["module"])
 257         module_uris.append(kwargs["plugins_uri"]
 258                            + '/'
 259                            + module["module-config-name"]
 260                            + '_'
 261                            + module["module-config-version"]
 262                            + '.yaml')
 263         module_titles.append(str(module_conf['plugin-config-metadata']['title']))
 264         module_versions.append(str(module_conf['plugin-config-metadata']['version']))
 265         module_creations.append(str(module_conf['plugin-config-metadata']['created']))
 266         module_shas.append(sha_uri(kwargs["plugins_uri"]
 267                                    + '/'
 268                                    + module["module-config-name"]
 269                                    + '_'
 270                                    + module["module-config-version"]
 271                                    + '.yaml'))
 272         module_comments.append(str(module_conf['plugin-config-metadata']['comment']))
 273         module_conf.pop('plugin-config-metadata')
 274         final_dea_conf['settings']['editable'].update(module_conf)
 275         scenario_module_override_conf = module.get('module-config-override')
 276         if scenario_module_override_conf:
 277             dea_scenario_module_override_conf = {}
 278             dea_scenario_module_override_conf['settings'] = {}
 279             dea_scenario_module_override_conf['settings']['editable'] = {}
 280             dea_scenario_module_override_conf['settings']['editable'][module["module"]] = scenario_module_override_conf
 281             final_dea_conf = dict(merge_dicts(final_dea_conf, dea_scenario_module_override_conf))
 282
 283 # Dump final dea.yaml including configuration management meta-data to argument provided
 284 # directory
 285 if not os.path.exists(kwargs["output_path"]):
 286     os.makedirs(kwargs["output_path"])
 287 print('Dumping final dea.yaml to ' + kwargs["output_path"] + '/dea.yaml....')
 288 with open(kwargs["output_path"] + '/dea.yaml', "w") as f:
 289     f.write("\n".join([("title: DEA.yaml file automatically generated from the"
 290                         'configuration files stated in the "configuration-files"'
 291                         "fragment below"),
 292                        "version: " + str(calendar.timegm(time.gmtime())),
 293                        "created: " + str(time.strftime("%d/%m/%Y")) + " "
 294                        + str(time.strftime("%H:%M:%S")),
 295                        "comment: none\n"]))
 296
 297     f.write("\n".join(["configuration-files:",
 298                        "  dea-base:",
 299                        "    uri: " +  kwargs["dea_base_uri"],
 300                        "    title: " + str(dea_base_title),
 301                        "    version: " + str(dea_base_version),
 302                        "    created: " + str(dea_base_creation),
 303                        "    sha1: " + str(dea_base_sha),
 304                        "    comment: " + str(dea_base_comment) + "\n"]))
 305
 306     f.write("\n".join(["  pod-override:",
 307                        "    uri: " + kwargs["dea_pod_override_uri"],
 308                        "    title: " + str(dea_pod_title),
 309                        "    version: " + str(dea_pod_version),
 310                        "    created: " + str(dea_pod_creation),
 311                        "    sha1: " + str(dea_pod_sha),
 312                        "    comment: " + str(dea_pod_comment) + "\n"]))
 313
 314     f.write("\n".join(["  deployment-scenario:",
 315                        "    uri: " + str(scenario_uri),
 316                        "    title: " + str(deploy_scenario_title),
 317                        "    version: " + str(deploy_scenario_version),
 318                        "    created: " + str(deploy_scenario_creation),
 319                        "    sha1: " + str(deploy_scenario_sha),
 320                        "    comment: " + str(deploy_scenario_comment) + "\n"]))
 321
 322     f.write("  plugin-modules:\n")
 323     for k, _ in enumerate(modules):
 324         f.write("\n".join(["  - module: " + modules[k],
 325                            "    uri: " + module_uris[k],
 326                            "    title: " + module_titles[k],
 327                            "    version: " + module_versions[k],
 328                            "    created: " + module_creations[k],
 329                            "    sha-1: " + module_shas[k],
 330                            "    comment: " + module_comments[k] + "\n"]))
 331
 332     yaml.dump(final_dea_conf, f, default_flow_style=False)
 333
 334 # Load POD dha and override it with "deployment-scenario/dha-override-config" section
 335 print('Generating final dha.yaml configuration....')
 336 print('Parsing dha-pod yaml configuration....')
 337 response = urllib2.urlopen(kwargs["dha_uri"])
 338 dha_pod_conf = yaml.load(response.read())
 339 dha_pod_title = dha_pod_conf['dha-pod-config-metadata']['title']
 340 dha_pod_version = dha_pod_conf['dha-pod-config-metadata']['version']
 341 dha_pod_creation = dha_pod_conf['dha-pod-config-metadata']['created']
 342 dha_pod_sha = sha_uri(kwargs["dha_uri"])
 343 dha_pod_comment = dha_pod_conf['dha-pod-config-metadata']['comment']
 344 dha_pod_conf.pop('dha-pod-config-metadata')
 345 final_dha_conf = dha_pod_conf
 346
 347 dha_scenario_override_conf = deploy_scenario_conf["dha-override-config"]
 348 # Only virtual deploy scenarios can override dha.yaml since there
 349 # is no way to programatically override a physical environment:
 350 # wireing, IPMI set-up, etc.
 351 # For Physical environments, dha.yaml overrides will be silently ignored
 352 if dha_scenario_override_conf and (final_dha_conf['adapter'] == 'libvirt'
 353                                    or final_dha_conf['adapter'] == 'esxi'
 354                                    or final_dha_conf['adapter'] == 'vbox'):
 355     print('Merging dha-pod and deployment-scenario override information to final dha.yaml configuration....')
 356     final_dha_conf = dict(merge_dicts(final_dha_conf, dha_scenario_override_conf))
 357
 358 # Dump final dha.yaml to argument provided directory
 359 print('Dumping final dha.yaml to ' + kwargs["output_path"] + '/dha.yaml....')
 360 with open(kwargs["output_path"] + '/dha.yaml', "w") as f:
 361     f.write("\n".join([("title: DHA.yaml file automatically generated from"
 362                         "the configuration files stated in the"
 363                         '"configuration-files" fragment below'),
 364                        "version: " + str(calendar.timegm(time.gmtime())),
 365                        "created: " + str(time.strftime("%d/%m/%Y")) + " "
 366                        + str(time.strftime("%H:%M:%S")),
 367                        "comment: none\n"]))
 368
 369     f.write("configuration-files:\n")
 370
 371     f.write("\n".join(["  dha-pod-configuration:",
 372                        "    uri: " + kwargs["dha_uri"],
 373                        "    title: " + str(dha_pod_title),
 374                        "    version: " + str(dha_pod_version),
 375                        "    created: " + str(dha_pod_creation),
 376                        "    sha-1: " + str(dha_pod_sha),
 377                        "    comment: " + str(dha_pod_comment) + "\n"]))
 378
 379     f.write("\n".join(["  deployment-scenario:",
 380                        "    uri: " + str(scenario_uri),
 381                        "    title: " + str(deploy_scenario_title),
 382                        "    version: " + str(deploy_scenario_version),
 383                        "    created: " + str(deploy_scenario_creation),
 384                        "    sha-1: " + str(deploy_scenario_sha),
 385                        "    comment: " + str(deploy_scenario_comment) + "\n"]))
 386
 387     yaml.dump(final_dha_conf, f, default_flow_style=False)