src/workflow/resource_bundle_workflow.py

   1 ##############################################################################
   2 # Copyright (c) 2018 Parker Berberian, Sawyer Bergeron, and others.
   3 #
   4 # All rights reserved. This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 # http://www.apache.org/licenses/LICENSE-2.0
   8 ##############################################################################
   9
  10
  11 from django.conf import settings
  12 from django.forms import formset_factory
  13 from django.core.exceptions import ValidationError
  14
  15 from typing import List
  16
  17 import re
  18 import json
  19 from xml.dom import minidom
  20 import traceback
  21
  22 from workflow.models import WorkflowStep
  23 from account.models import Lab
  24 from workflow.forms import (
  25     HardwareDefinitionForm,
  26     NetworkDefinitionForm,
  27     ResourceMetaForm,
  28     HostSoftwareDefinitionForm,
  29 )
  30 from resource_inventory.models import (
  31     ResourceTemplate,
  32     ResourceConfiguration,
  33     InterfaceConfiguration,
  34     Network,
  35     NetworkConnection,
  36     Image,
  37 )
  38 from dashboard.exceptions import (
  39     InvalidVlanConfigurationException,
  40     NetworkExistsException,
  41     ResourceAvailabilityException
  42 )
  43
  44 import logging
  45 logger = logging.getLogger(__name__)
  46
  47
  48 class Define_Hardware(WorkflowStep):
  49
  50     template = 'resource/steps/define_hardware.html'
  51     title = "Define Hardware"
  52     description = "Choose the type and amount of machines you want"
  53     short_title = "hosts"
  54
  55     def __init__(self, *args, **kwargs):
  56         self.form = None
  57         super().__init__(*args, **kwargs)
  58
  59     def get_context(self):
  60         context = super(Define_Hardware, self).get_context()
  61         user = self.repo_get(self.repo.SESSION_USER)
  62         context['form'] = self.form or HardwareDefinitionForm(user)
  63         return context
  64
  65     def update_models(self, data):
  66         data = data['filter_field']
  67         models = self.repo_get(self.repo.RESOURCE_TEMPLATE_MODELS, {})
  68         models['resources'] = []  # This will always clear existing data when this step changes
  69         models['connections'] = []
  70         models['interfaces'] = {}
  71         if "template" not in models:
  72             template = ResourceTemplate.objects.create(temporary=True)
  73             models['template'] = template
  74
  75         resource_data = data['resource']
  76
  77         new_template = models['template']
  78
  79         public_network = Network.objects.create(name="public", bundle=new_template, is_public=True)
  80
  81         all_networks = {public_network.id: public_network}
  82
  83         for resource_template_dict in resource_data.values():
  84             id = resource_template_dict['id']
  85             old_template = ResourceTemplate.objects.get(id=id)
  86
  87             # instantiate genericHost and store in repo
  88             for _ in range(0, resource_template_dict['count']):
  89                 resource_configs = old_template.resourceConfigurations.all()
  90                 for config in resource_configs:
  91                     # need to save now for connections to refer to it later
  92                     new_config = ResourceConfiguration.objects.create(
  93                         profile=config.profile,
  94                         image=config.image,
  95                         name=config.name,
  96                         template=new_template)
  97
  98                     for interface_config in config.interface_configs.all():
  99                         new_interface_config = InterfaceConfiguration.objects.create(
 100                             profile=interface_config.profile,
 101                             resource_config=new_config)
 102
 103                         for connection in interface_config.connections.all():
 104                             network = None
 105                             if connection.network.is_public:
 106                                 network = public_network
 107                             else:
 108                                 # check if network is known
 109                                 if connection.network.id not in all_networks:
 110                                     # create matching one
 111                                     new_network = Network(
 112                                         name=connection.network.name + "_" + str(new_config.id),
 113                                         bundle=new_template,
 114                                         is_public=False)
 115                                     new_network.save()
 116
 117                                     all_networks[connection.network.id] = new_network
 118
 119                                 network = all_networks[connection.network.id]
 120
 121                             new_connection = NetworkConnection(
 122                                 network=network,
 123                                 vlan_is_tagged=connection.vlan_is_tagged)
 124
 125                             new_interface_config.save()  # can't do later because M2M on next line
 126                             new_connection.save()
 127
 128                             new_interface_config.connections.add(new_connection)
 129
 130                         unique_resource_ref = new_config.name + "_" + str(new_config.id)
 131                         if unique_resource_ref not in models['interfaces']:
 132                             models['interfaces'][unique_resource_ref] = []
 133                         models['interfaces'][unique_resource_ref].append(interface_config)
 134
 135                     models['resources'].append(new_config)
 136
 137             models['networks'] = all_networks
 138
 139         # add selected lab to models
 140         for lab_dict in data['lab'].values():
 141             if lab_dict['selected']:
 142                 models['template'].lab = Lab.objects.get(lab_user__id=lab_dict['id'])
 143                 models['template'].save()
 144                 break  # if somehow we get two 'true' labs, we only use one
 145
 146         # return to repo
 147         self.repo_put(self.repo.RESOURCE_TEMPLATE_MODELS, models)
 148
 149     def update_confirmation(self):
 150         confirm = self.repo_get(self.repo.CONFIRMATION, {})
 151         if "template" not in confirm:
 152             confirm['template'] = {}
 153         confirm['template']['resources'] = []
 154         models = self.repo_get(self.repo.RESOURCE_TEMPLATE_MODELS, {})
 155         if 'template' in models:
 156             for resource in models['template'].getConfigs():
 157                 host_dict = {"name": resource.name, "profile": resource.profile.name}
 158                 confirm['template']['resources'].append(host_dict)
 159         if "template" in models:
 160             confirm['template']['lab'] = models['template'].lab.lab_user.username
 161         self.repo_put(self.repo.CONFIRMATION, confirm)
 162
 163     def post(self, post_data, user):
 164         try:
 165             user = self.repo_get(self.repo.SESSION_USER)
 166             self.form = HardwareDefinitionForm(user, post_data)
 167             if self.form.is_valid():
 168                 self.update_models(self.form.cleaned_data)
 169                 self.update_confirmation()
 170                 self.set_valid("Step Completed")
 171             else:
 172                 self.set_invalid("Please complete the fields highlighted in red to continue")
 173         except Exception as e:
 174             print("Caught exception: " + str(e))
 175             traceback.print_exc()
 176             self.form = None
 177             self.set_invalid("Please select a lab.")
 178
 179
 180 class Define_Software(WorkflowStep):
 181     template = 'config_bundle/steps/define_software.html'
 182     title = "Pick Software"
 183     description = "Choose the opnfv and image of your machines"
 184     short_title = "host config"
 185
 186     def build_filter_data(self, hosts_data):
 187         """
 188         Build list of Images to filter out.
 189
 190         returns a 2D array of images to exclude
 191         based on the ordering of the passed
 192         hosts_data
 193         """
 194
 195         filter_data = []
 196         user = self.repo_get(self.repo.SESSION_USER)
 197         lab = self.repo_get(self.repo.RESOURCE_TEMPLATE_MODELS)['template'].lab
 198         for i, host_data in enumerate(hosts_data):
 199             host = ResourceConfiguration.objects.get(pk=host_data['host_id'])
 200             wrong_owner = Image.objects.exclude(owner=user).exclude(public=True)
 201             wrong_host = Image.objects.exclude(architecture=host.profile.architecture)
 202             wrong_lab = Image.objects.exclude(from_lab=lab)
 203             excluded_images = wrong_owner | wrong_host | wrong_lab
 204             filter_data.append([])
 205             for image in excluded_images:
 206                 filter_data[i].append(image.pk)
 207         return filter_data
 208
 209     def create_hostformset(self, hostlist, data=None):
 210         hosts_initial = []
 211         configs = self.repo_get(self.repo.RESOURCE_TEMPLATE_MODELS, {}).get("resources")
 212         if configs:
 213             for i in range(len(configs)):
 214                 default_name = 'laas-node'
 215                 if i > 0:
 216                     default_name = default_name + "-" + str(i + 1)
 217                 hosts_initial.append({
 218                     'host_id': configs[i].id,
 219                     'host_name': default_name,
 220                     'headnode': False,
 221                     'image': configs[i].image
 222                 })
 223         else:
 224             for host in hostlist:
 225                 hosts_initial.append({
 226                     'host_id': host.id,
 227                     'host_name': host.name
 228                 })
 229
 230         HostFormset = formset_factory(HostSoftwareDefinitionForm, extra=0)
 231         filter_data = self.build_filter_data(hosts_initial)
 232
 233         class SpecialHostFormset(HostFormset):
 234             def get_form_kwargs(self, index):
 235                 kwargs = super(SpecialHostFormset, self).get_form_kwargs(index)
 236                 if index is not None:
 237                     kwargs['imageQS'] = Image.objects.exclude(pk__in=filter_data[index])
 238                 return kwargs
 239
 240         if data:
 241             return SpecialHostFormset(data, initial=hosts_initial)
 242         return SpecialHostFormset(initial=hosts_initial)
 243
 244     def get_host_list(self, grb=None):
 245         return self.repo_get(self.repo.RESOURCE_TEMPLATE_MODELS).get("resources")
 246
 247     def get_context(self):
 248         context = super(Define_Software, self).get_context()
 249
 250         context["formset"] = self.create_hostformset(self.get_host_list())
 251
 252         return context
 253
 254     def post(self, post_data, user):
 255         hosts = self.get_host_list()
 256         formset = self.create_hostformset(hosts, data=post_data)
 257         has_headnode = False
 258         if formset.is_valid():
 259             for i, form in enumerate(formset):
 260                 host = hosts[i]
 261                 image = form.cleaned_data['image']
 262                 hostname = form.cleaned_data['host_name']
 263                 headnode = form.cleaned_data['headnode']
 264                 if headnode:
 265                     has_headnode = True
 266                 host.is_head_node = headnode
 267                 host.name = hostname
 268                 host.image = image
 269                 # RFC921: They must start with a letter, end with a letter or digit and have only letters or digits or hyphen as interior characters
 270                 if bool(re.match("^[A-Za-z0-9-]*$", hostname)) is False:
 271                     self.set_invalid("Device names must only contain alphanumeric characters and dashes.")
 272                     return
 273                 if not hostname[0].isalpha() or not hostname[-1].isalnum():
 274                     self.set_invalid("Device names must start with a letter and end with a letter or digit.")
 275                     return
 276                 for j in range(i):
 277                     if j != i and hostname == hosts[j].name:
 278                         self.set_invalid("Devices must have unique names. Please try again.")
 279                         return
 280                 host.save()
 281
 282             if not has_headnode and len(hosts) > 0:
 283                 self.set_invalid("No headnode. Please set a headnode.")
 284                 return
 285
 286             self.set_valid("Completed")
 287         else:
 288             self.set_invalid("Please complete all fields.")
 289
 290
 291 class Define_Nets(WorkflowStep):
 292     template = 'resource/steps/pod_definition.html'
 293     title = "Define Networks"
 294     description = "Use the tool below to draw the network topology of your POD"
 295     short_title = "networking"
 296     form = NetworkDefinitionForm
 297
 298     def get_vlans(self):
 299         vlans = self.repo_get(self.repo.VLANS)
 300         if vlans:
 301             return vlans
 302         # try to grab some vlans from lab
 303         models = self.repo_get(self.repo.RESOURCE_TEMPLATE_MODELS, {})
 304         if "bundle" not in models:
 305             return None
 306         lab = models['bundle'].lab
 307         if lab is None or lab.vlan_manager is None:
 308             return None
 309         try:
 310             vlans = lab.vlan_manager.get_vlans(count=lab.vlan_manager.block_size)
 311             self.repo_put(self.repo.VLANS, vlans)
 312             return vlans
 313         except Exception:
 314             return None
 315
 316     def make_mx_network_dict(self, network):
 317         return {
 318             'id': network.id,
 319             'name': network.name,
 320             'public': network.is_public
 321         }
 322
 323     def make_mx_resource_dict(self, resource_config):
 324         resource_dict = {
 325             'id': resource_config.id,
 326             'interfaces': [],
 327             'value': {
 328                 'name': resource_config.name,
 329                 'id': resource_config.id,
 330                 'description': resource_config.profile.description
 331             }
 332         }
 333
 334         for interface_config in resource_config.interface_configs.all():
 335             connections = []
 336             for connection in interface_config.connections.all():
 337                 connections.append({'tagged': connection.vlan_is_tagged, 'network': connection.network.id})
 338
 339             interface_dict = {
 340                 "id": interface_config.id,
 341                 "name": interface_config.profile.name,
 342                 "description": "speed: " + str(interface_config.profile.speed) + "M\ntype: " + interface_config.profile.nic_type,
 343                 "connections": connections
 344             }
 345
 346             resource_dict['interfaces'].append(interface_dict)
 347
 348         return resource_dict
 349
 350     def make_mx_host_dict(self, generic_host):
 351         host = {
 352             'id': generic_host.profile.name,
 353             'interfaces': [],
 354             'value': {
 355                 "name": generic_host.profile.name,
 356                 "description": generic_host.profile.description
 357             }
 358         }
 359         for iface in generic_host.profile.interfaceprofile.all():
 360             host['interfaces'].append({
 361                 "name": iface.name,
 362                 "description": "speed: " + str(iface.speed) + "M\ntype: " + iface.nic_type
 363             })
 364         return host
 365
 366     # first step guards this one, so can't get here without at least empty
 367     # models being populated by step one
 368     def get_context(self):
 369         context = super(Define_Nets, self).get_context()
 370         context.update({
 371             'form': NetworkDefinitionForm(),
 372             'debug': settings.DEBUG,
 373             'resources': {},
 374             'networks': {},
 375             'vlans': [],
 376             # remove others
 377             'hosts': [],
 378             'added_hosts': [],
 379             'removed_hosts': []
 380         })
 381
 382         models = self.repo_get(self.repo.RESOURCE_TEMPLATE_MODELS)  # infallible, guarded by prior step
 383         for resource in models['resources']:
 384             d = self.make_mx_resource_dict(resource)
 385             context['resources'][d['id']] = d
 386
 387         for network in models['networks'].values():
 388             d = self.make_mx_network_dict(network)
 389             context['networks'][d['id']] = d
 390
 391         return context
 392
 393     def post(self, post_data, user):
 394         try:
 395             xmlData = post_data.get("xml")
 396             self.updateModels(xmlData)
 397             # update model with xml
 398             self.set_valid("Networks applied successfully")
 399         except ResourceAvailabilityException:
 400             self.set_invalid("Public network not availble")
 401         except Exception as e:
 402             traceback.print_exc()
 403             self.set_invalid("An error occurred when applying networks: " + str(e))
 404
 405     def resetNetworks(self, networks: List[Network]):  # potentially just pass template here?
 406         for network in networks:
 407             network.delete()
 408
 409     def updateModels(self, xmlData):
 410         models = self.repo_get(self.repo.RESOURCE_TEMPLATE_MODELS, {})
 411         given_hosts = None
 412         interfaces = None
 413         networks = None
 414         try:
 415             given_hosts, interfaces, networks = self.parseXml(xmlData)
 416         except Exception as e:
 417             print("tried to parse Xml, got exception instead:")
 418             print(e)
 419
 420         existing_rconfig_list = models.get("resources", [])
 421         existing_rconfigs = {}  # maps id to host
 422         for rconfig in existing_rconfig_list:
 423             existing_rconfigs["host_" + str(rconfig.id)] = rconfig
 424
 425         bundle = models.get("template")  # hard fail if not in repo
 426
 427         self.resetNetworks(models['networks'].values())
 428         models['networks'] = {}
 429
 430         for net_id, net in networks.items():
 431             network = Network.objects.create(
 432                 name=net['name'],
 433                 bundle=bundle,
 434                 is_public=net['public'])
 435
 436             models['networks'][net_id] = network
 437             network.save()
 438
 439         for hostid, given_host in given_hosts.items():
 440             for ifaceId in given_host['interfaces']:
 441                 iface = interfaces[ifaceId]
 442
 443                 iface_config = InterfaceConfiguration.objects.get(id=iface['config_id'])
 444                 if iface_config.resource_config.template.id != bundle.id:
 445                     raise ValidationError("User does not own the template they are editing")
 446
 447                 for connection in iface['connections']:
 448                     network_id = connection['network']
 449                     net = models['networks'][network_id]
 450                     connection = NetworkConnection(vlan_is_tagged=connection['tagged'], network=net)
 451                     connection.save()
 452                     iface_config.connections.add(connection)
 453                     iface_config.save()
 454         self.repo_put(self.repo.RESOURCE_TEMPLATE_MODELS, models)
 455
 456     def decomposeXml(self, xmlString):
 457         """
 458         Translate XML into useable data.
 459
 460         This function takes in an xml doc from our front end
 461         and returns dictionaries that map cellIds to the xml
 462         nodes themselves. There is no unpacking of the
 463         xml objects, just grouping and organizing
 464         """
 465         connections = {}
 466         networks = {}
 467         hosts = {}
 468         interfaces = {}
 469         network_ports = {}
 470
 471         xmlDom = minidom.parseString(xmlString)
 472         root = xmlDom.documentElement.firstChild
 473         for cell in root.childNodes:
 474             cellId = cell.getAttribute('id')
 475             group = cellId.split("_")[0]
 476             parentGroup = cell.getAttribute("parent").split("_")[0]
 477             # place cell into correct group
 478
 479             if cell.getAttribute("edge"):
 480                 connections[cellId] = cell
 481
 482             elif "network" in group:
 483                 networks[cellId] = cell
 484
 485             elif "host" in group:
 486                 hosts[cellId] = cell
 487
 488             elif "host" in parentGroup:
 489                 interfaces[cellId] = cell
 490
 491             # make network ports also map to thier network
 492             elif "network" in parentGroup:
 493                 network_ports[cellId] = cell.getAttribute("parent")  # maps port ID to net ID
 494
 495         return connections, networks, hosts, interfaces, network_ports
 496
 497     # serialize and deserialize xml from mxGraph
 498     def parseXml(self, xmlString):
 499         networks = {}  # maps net name to network object
 500         hosts = {}  # cotains id -> hosts, each containing interfaces, referencing networks
 501         interfaces = {}  # maps id -> interface
 502         untagged_ifaces = set()  # used to check vlan config
 503         network_names = set()  # used to check network names
 504         xml_connections, xml_nets, xml_hosts, xml_ifaces, xml_ports = self.decomposeXml(xmlString)
 505
 506         # parse Hosts
 507         for cellId, cell in xml_hosts.items():
 508             cell_json_str = cell.getAttribute("value")
 509             cell_json = json.loads(cell_json_str)
 510             host = {"interfaces": [], "name": cellId, "hostname": cell_json['name']}
 511             hosts[cellId] = host
 512
 513         # parse networks
 514         for cellId, cell in xml_nets.items():
 515             escaped_json_str = cell.getAttribute("value")
 516             json_str = escaped_json_str.replace('&quot;', '"')
 517             net_info = json.loads(json_str)
 518             net_name = net_info['name']
 519             public = net_info['public']
 520             if net_name in network_names:
 521                 raise NetworkExistsException("Non unique network name found")
 522             network = {"name": net_name, "public": public, "id": cellId}
 523             networks[cellId] = network
 524             network_names.add(net_name)
 525
 526         # parse interfaces
 527         for cellId, cell in xml_ifaces.items():
 528             parentId = cell.getAttribute('parent')
 529             cell_json_str = cell.getAttribute("value")
 530             cell_json = json.loads(cell_json_str)
 531             iface = {"graph_id": cellId, "connections": [], "config_id": cell_json['id'], "profile_name": cell_json['name']}
 532             hosts[parentId]['interfaces'].append(cellId)
 533             interfaces[cellId] = iface
 534
 535         # parse connections
 536         for cellId, cell in xml_connections.items():
 537             escaped_json_str = cell.getAttribute("value")
 538             json_str = escaped_json_str.replace('&quot;', '"')
 539             attributes = json.loads(json_str)
 540             tagged = attributes['tagged']
 541             interface = None
 542             network = None
 543             src = cell.getAttribute("source")
 544             tgt = cell.getAttribute("target")
 545             if src in interfaces:
 546                 interface = interfaces[src]
 547                 network = networks[xml_ports[tgt]]
 548             else:
 549                 interface = interfaces[tgt]
 550                 network = networks[xml_ports[src]]
 551
 552             if not tagged:
 553                 if interface['config_id'] in untagged_ifaces:
 554                     raise InvalidVlanConfigurationException("More than one untagged vlan on an interface")
 555                 untagged_ifaces.add(interface['config_id'])
 556
 557             # add connection to interface
 558             interface['connections'].append({"tagged": tagged, "network": network['id']})
 559
 560         return hosts, interfaces, networks
 561
 562
 563 class Resource_Meta_Info(WorkflowStep):
 564     template = 'resource/steps/meta_info.html'
 565     title = "Extra Info"
 566     description = "Please fill out the rest of the information about your resource"
 567     short_title = "pod info"
 568
 569     def update_confirmation(self):
 570         confirm = self.repo_get(self.repo.CONFIRMATION, {})
 571         if "template" not in confirm:
 572             confirm['template'] = {}
 573         models = self.repo_get(self.repo.RESOURCE_TEMPLATE_MODELS, {})
 574         if "template" in models:
 575             confirm['template']['description'] = models['template'].description
 576             confirm['template']['name'] = models['template'].name
 577         self.repo_put(self.repo.CONFIRMATION, confirm)
 578
 579     def get_context(self):
 580         context = super(Resource_Meta_Info, self).get_context()
 581         name = ""
 582         desc = ""
 583         models = self.repo_get(self.repo.RESOURCE_TEMPLATE_MODELS, None)
 584         bundle = models['template']
 585         if bundle:
 586             name = bundle.name
 587             desc = bundle.description
 588         context['form'] = ResourceMetaForm(initial={"bundle_name": name, "bundle_description": desc})
 589         return context
 590
 591     def post(self, post_data, user):
 592         form = ResourceMetaForm(post_data)
 593         if form.is_valid():
 594             models = self.repo_get(self.repo.RESOURCE_TEMPLATE_MODELS, {})
 595             name = form.cleaned_data['bundle_name']
 596             desc = form.cleaned_data['bundle_description']
 597             bundle = models['template']  # infallible
 598             bundle.name = name
 599             bundle.description = desc
 600             bundle.save()
 601             self.repo_put(self.repo.RESOURCE_TEMPLATE_MODELS, models)
 602             confirm = self.repo_get(self.repo.CONFIRMATION)
 603             if "resource" not in confirm:
 604                 confirm['resource'] = {}
 605             confirm_info = confirm['resource']
 606             confirm_info["name"] = name
 607             tmp = desc
 608             if len(tmp) > 60:
 609                 tmp = tmp[:60] + "..."
 610             confirm_info["description"] = tmp
 611             self.repo_put(self.repo.CONFIRMATION, confirm)
 612             self.set_valid("Step Completed")
 613         else:
 614             self.set_invalid("Please complete all fields.")