src/workflow/resource_bundle_workflow.py

   1 ##############################################################################
   2 # Copyright (c) 2018 Parker Berberian, Sawyer Bergeron, and others.
   3 #
   4 # All rights reserved. This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 # http://www.apache.org/licenses/LICENSE-2.0
   8 ##############################################################################
   9
  10
  11 from django.conf import settings
  12 from django.forms import formset_factory
  13 from django.core.exceptions import ValidationError
  14
  15 from typing import List
  16
  17 import json
  18 from xml.dom import minidom
  19 import traceback
  20
  21 from workflow.models import WorkflowStep
  22 from account.models import Lab
  23 from workflow.forms import (
  24     HardwareDefinitionForm,
  25     NetworkDefinitionForm,
  26     ResourceMetaForm,
  27     HostSoftwareDefinitionForm,
  28 )
  29 from resource_inventory.models import (
  30     ResourceTemplate,
  31     ResourceConfiguration,
  32     InterfaceConfiguration,
  33     Network,
  34     NetworkConnection,
  35     Image,
  36 )
  37 from dashboard.exceptions import (
  38     InvalidVlanConfigurationException,
  39     NetworkExistsException,
  40     ResourceAvailabilityException
  41 )
  42
  43 import logging
  44 logger = logging.getLogger(__name__)
  45
  46
  47 class Define_Hardware(WorkflowStep):
  48
  49     template = 'resource/steps/define_hardware.html'
  50     title = "Define Hardware"
  51     description = "Choose the type and amount of machines you want"
  52     short_title = "hosts"
  53
  54     def __init__(self, *args, **kwargs):
  55         self.form = None
  56         super().__init__(*args, **kwargs)
  57
  58     def get_context(self):
  59         context = super(Define_Hardware, self).get_context()
  60         user = self.repo_get(self.repo.SESSION_USER)
  61         context['form'] = self.form or HardwareDefinitionForm(user)
  62         return context
  63
  64     def update_models(self, data):
  65         data = data['filter_field']
  66         models = self.repo_get(self.repo.RESOURCE_TEMPLATE_MODELS, {})
  67         models['resources'] = []  # This will always clear existing data when this step changes
  68         models['connections'] = []
  69         models['interfaces'] = {}
  70         if "template" not in models:
  71             template = ResourceTemplate.objects.create(temporary=True)
  72             models['template'] = template
  73
  74         resource_data = data['resource']
  75
  76         new_template = models['template']
  77
  78         public_network = Network.objects.create(name="public", bundle=new_template, is_public=True)
  79
  80         all_networks = {public_network.id: public_network}
  81
  82         for resource_template_dict in resource_data.values():
  83             id = resource_template_dict['id']
  84             old_template = ResourceTemplate.objects.get(id=id)
  85
  86             # instantiate genericHost and store in repo
  87             for _ in range(0, resource_template_dict['count']):
  88                 resource_configs = old_template.resourceConfigurations.all()
  89                 for config in resource_configs:
  90                     # need to save now for connections to refer to it later
  91                     new_config = ResourceConfiguration.objects.create(
  92                         profile=config.profile,
  93                         image=config.image,
  94                         name=config.name,
  95                         template=new_template)
  96
  97                     for interface_config in config.interface_configs.all():
  98                         new_interface_config = InterfaceConfiguration.objects.create(
  99                             profile=interface_config.profile,
 100                             resource_config=new_config)
 101
 102                         for connection in interface_config.connections.all():
 103                             network = None
 104                             if connection.network.is_public:
 105                                 network = public_network
 106                             else:
 107                                 # check if network is known
 108                                 if connection.network.id not in all_networks:
 109                                     # create matching one
 110                                     new_network = Network(
 111                                         name=connection.network.name + "_" + str(new_config.id),
 112                                         bundle=new_template,
 113                                         is_public=False)
 114                                     new_network.save()
 115
 116                                     all_networks[connection.network.id] = new_network
 117
 118                                 network = all_networks[connection.network.id]
 119
 120                             new_connection = NetworkConnection(
 121                                 network=network,
 122                                 vlan_is_tagged=connection.vlan_is_tagged)
 123
 124                             new_interface_config.save()  # can't do later because M2M on next line
 125                             new_connection.save()
 126
 127                             new_interface_config.connections.add(new_connection)
 128
 129                         unique_resource_ref = new_config.name + "_" + str(new_config.id)
 130                         if unique_resource_ref not in models['interfaces']:
 131                             models['interfaces'][unique_resource_ref] = []
 132                         models['interfaces'][unique_resource_ref].append(interface_config)
 133
 134                     models['resources'].append(new_config)
 135
 136             models['networks'] = all_networks
 137
 138         # add selected lab to models
 139         for lab_dict in data['lab'].values():
 140             if lab_dict['selected']:
 141                 models['template'].lab = Lab.objects.get(lab_user__id=lab_dict['id'])
 142                 models['template'].save()
 143                 break  # if somehow we get two 'true' labs, we only use one
 144
 145         # return to repo
 146         self.repo_put(self.repo.RESOURCE_TEMPLATE_MODELS, models)
 147
 148     def update_confirmation(self):
 149         confirm = self.repo_get(self.repo.CONFIRMATION, {})
 150         if "template" not in confirm:
 151             confirm['template'] = {}
 152         confirm['template']['resources'] = []
 153         models = self.repo_get(self.repo.RESOURCE_TEMPLATE_MODELS, {})
 154         if 'template' in models:
 155             for resource in models['template'].getConfigs():
 156                 host_dict = {"name": resource.name, "profile": resource.profile.name}
 157                 confirm['template']['resources'].append(host_dict)
 158         if "template" in models:
 159             confirm['template']['lab'] = models['template'].lab.lab_user.username
 160         self.repo_put(self.repo.CONFIRMATION, confirm)
 161
 162     def post(self, post_data, user):
 163         try:
 164             user = self.repo_get(self.repo.SESSION_USER)
 165             self.form = HardwareDefinitionForm(user, post_data)
 166             if self.form.is_valid():
 167                 self.update_models(self.form.cleaned_data)
 168                 self.update_confirmation()
 169                 self.set_valid("Step Completed")
 170             else:
 171                 self.set_invalid("Please complete the fields highlighted in red to continue")
 172         except Exception as e:
 173             print("Caught exception: " + str(e))
 174             traceback.print_exc()
 175             self.set_invalid(str(e))
 176
 177
 178 class Define_Software(WorkflowStep):
 179     template = 'config_bundle/steps/define_software.html'
 180     title = "Pick Software"
 181     description = "Choose the opnfv and image of your machines"
 182     short_title = "host config"
 183
 184     def build_filter_data(self, hosts_data):
 185         """
 186         Build list of Images to filter out.
 187
 188         returns a 2D array of images to exclude
 189         based on the ordering of the passed
 190         hosts_data
 191         """
 192
 193         filter_data = []
 194         user = self.repo_get(self.repo.SESSION_USER)
 195         lab = self.repo_get(self.repo.RESOURCE_TEMPLATE_MODELS)['template'].lab
 196         for i, host_data in enumerate(hosts_data):
 197             host = ResourceConfiguration.objects.get(pk=host_data['host_id'])
 198             wrong_owner = Image.objects.exclude(owner=user).exclude(public=True)
 199             wrong_host = Image.objects.exclude(architecture=host.profile.architecture)
 200             wrong_lab = Image.objects.exclude(from_lab=lab)
 201             excluded_images = wrong_owner | wrong_host | wrong_lab
 202             filter_data.append([])
 203             for image in excluded_images:
 204                 filter_data[i].append(image.pk)
 205         return filter_data
 206
 207     def create_hostformset(self, hostlist, data=None):
 208         hosts_initial = []
 209         configs = self.repo_get(self.repo.RESOURCE_TEMPLATE_MODELS, {}).get("resources")
 210         if configs:
 211             for config in configs:
 212                 hosts_initial.append({
 213                     'host_id': config.id,
 214                     'host_name': config.name,
 215                     'headnode': config.is_head_node,
 216                     'image': config.image
 217                 })
 218         else:
 219             for host in hostlist:
 220                 hosts_initial.append({
 221                     'host_id': host.id,
 222                     'host_name': host.name
 223                 })
 224
 225         HostFormset = formset_factory(HostSoftwareDefinitionForm, extra=0)
 226         filter_data = self.build_filter_data(hosts_initial)
 227
 228         class SpecialHostFormset(HostFormset):
 229             def get_form_kwargs(self, index):
 230                 kwargs = super(SpecialHostFormset, self).get_form_kwargs(index)
 231                 if index is not None:
 232                     kwargs['imageQS'] = Image.objects.exclude(pk__in=filter_data[index])
 233                 return kwargs
 234
 235         if data:
 236             return SpecialHostFormset(data, initial=hosts_initial)
 237         return SpecialHostFormset(initial=hosts_initial)
 238
 239     def get_host_list(self, grb=None):
 240         return self.repo_get(self.repo.RESOURCE_TEMPLATE_MODELS).get("resources")
 241
 242     def get_context(self):
 243         context = super(Define_Software, self).get_context()
 244
 245         context["formset"] = self.create_hostformset(self.get_host_list())
 246
 247         return context
 248
 249     def post(self, post_data, user):
 250         hosts = self.get_host_list()
 251
 252         # TODO: fix headnode in form, currently doesn't return a selected one
 253         # models['headnode_index'] = post_data.get("headnode", 1)
 254         formset = self.create_hostformset(hosts, data=post_data)
 255         has_headnode = False
 256         if formset.is_valid():
 257             for i, form in enumerate(formset):
 258                 host = hosts[i]
 259                 image = form.cleaned_data['image']
 260                 hostname = form.cleaned_data['host_name']
 261                 headnode = form.cleaned_data['headnode']
 262                 if headnode:
 263                     has_headnode = True
 264                 host.is_head_node = headnode
 265                 host.name = hostname
 266                 host.image = image
 267                 host.save()
 268
 269             if not has_headnode and len(hosts) > 0:
 270                 self.set_invalid("No headnode. Please set a headnode.")
 271                 return
 272
 273             self.set_valid("Completed")
 274         else:
 275             self.set_invalid("Please complete all fields")
 276
 277
 278 class Define_Nets(WorkflowStep):
 279     template = 'resource/steps/pod_definition.html'
 280     title = "Define Networks"
 281     description = "Use the tool below to draw the network topology of your POD"
 282     short_title = "networking"
 283     form = NetworkDefinitionForm
 284
 285     def get_vlans(self):
 286         vlans = self.repo_get(self.repo.VLANS)
 287         if vlans:
 288             return vlans
 289         # try to grab some vlans from lab
 290         models = self.repo_get(self.repo.RESOURCE_TEMPLATE_MODELS, {})
 291         if "bundle" not in models:
 292             return None
 293         lab = models['bundle'].lab
 294         if lab is None or lab.vlan_manager is None:
 295             return None
 296         try:
 297             vlans = lab.vlan_manager.get_vlans(count=lab.vlan_manager.block_size)
 298             self.repo_put(self.repo.VLANS, vlans)
 299             return vlans
 300         except Exception:
 301             return None
 302
 303     def make_mx_network_dict(self, network):
 304         return {
 305             'id': network.id,
 306             'name': network.name,
 307             'public': network.is_public
 308         }
 309
 310     def make_mx_resource_dict(self, resource_config):
 311         resource_dict = {
 312             'id': resource_config.id,
 313             'interfaces': [],
 314             'value': {
 315                 'name': resource_config.name,
 316                 'id': resource_config.id,
 317                 'description': resource_config.profile.description
 318             }
 319         }
 320
 321         for interface_config in resource_config.interface_configs.all():
 322             connections = []
 323             for connection in interface_config.connections.all():
 324                 connections.append({'tagged': connection.vlan_is_tagged, 'network': connection.network.id})
 325
 326             interface_dict = {
 327                 "id": interface_config.id,
 328                 "name": interface_config.profile.name,
 329                 "description": "speed: " + str(interface_config.profile.speed) + "M\ntype: " + interface_config.profile.nic_type,
 330                 "connections": connections
 331             }
 332
 333             resource_dict['interfaces'].append(interface_dict)
 334
 335         return resource_dict
 336
 337     def make_mx_host_dict(self, generic_host):
 338         host = {
 339             'id': generic_host.profile.name,
 340             'interfaces': [],
 341             'value': {
 342                 "name": generic_host.profile.name,
 343                 "description": generic_host.profile.description
 344             }
 345         }
 346         for iface in generic_host.profile.interfaceprofile.all():
 347             host['interfaces'].append({
 348                 "name": iface.name,
 349                 "description": "speed: " + str(iface.speed) + "M\ntype: " + iface.nic_type
 350             })
 351         return host
 352
 353     # first step guards this one, so can't get here without at least empty
 354     # models being populated by step one
 355     def get_context(self):
 356         context = super(Define_Nets, self).get_context()
 357         context.update({
 358             'form': NetworkDefinitionForm(),
 359             'debug': settings.DEBUG,
 360             'resources': {},
 361             'networks': {},
 362             'vlans': [],
 363             # remove others
 364             'hosts': [],
 365             'added_hosts': [],
 366             'removed_hosts': []
 367         })
 368
 369         models = self.repo_get(self.repo.RESOURCE_TEMPLATE_MODELS)  # infallible, guarded by prior step
 370         for resource in models['resources']:
 371             d = self.make_mx_resource_dict(resource)
 372             context['resources'][d['id']] = d
 373
 374         for network in models['networks'].values():
 375             d = self.make_mx_network_dict(network)
 376             context['networks'][d['id']] = d
 377
 378         return context
 379
 380     def post(self, post_data, user):
 381         try:
 382             xmlData = post_data.get("xml")
 383             self.updateModels(xmlData)
 384             # update model with xml
 385             self.set_valid("Networks applied successfully")
 386         except ResourceAvailabilityException:
 387             self.set_invalid("Public network not availble")
 388         except Exception as e:
 389             traceback.print_exc()
 390             self.set_invalid("An error occurred when applying networks: " + str(e))
 391
 392     def resetNetworks(self, networks: List[Network]):  # potentially just pass template here?
 393         for network in networks:
 394             network.delete()
 395
 396     def updateModels(self, xmlData):
 397         models = self.repo_get(self.repo.RESOURCE_TEMPLATE_MODELS, {})
 398         given_hosts = None
 399         interfaces = None
 400         networks = None
 401         try:
 402             given_hosts, interfaces, networks = self.parseXml(xmlData)
 403         except Exception as e:
 404             print("tried to parse Xml, got exception instead:")
 405             print(e)
 406
 407         existing_rconfig_list = models.get("resources", [])
 408         existing_rconfigs = {}  # maps id to host
 409         for rconfig in existing_rconfig_list:
 410             existing_rconfigs["host_" + str(rconfig.id)] = rconfig
 411
 412         bundle = models.get("template")  # hard fail if not in repo
 413
 414         self.resetNetworks(models['networks'].values())
 415         models['networks'] = {}
 416
 417         for net_id, net in networks.items():
 418             network = Network.objects.create(
 419                 name=net['name'],
 420                 bundle=bundle,
 421                 is_public=net['public'])
 422
 423             models['networks'][net_id] = network
 424             network.save()
 425
 426         for hostid, given_host in given_hosts.items():
 427             for ifaceId in given_host['interfaces']:
 428                 iface = interfaces[ifaceId]
 429
 430                 iface_config = InterfaceConfiguration.objects.get(id=iface['config_id'])
 431                 if iface_config.resource_config.template.id != bundle.id:
 432                     raise ValidationError("User does not own the template they are editing")
 433
 434                 for connection in iface['connections']:
 435                     network_id = connection['network']
 436                     net = models['networks'][network_id]
 437                     connection = NetworkConnection(vlan_is_tagged=connection['tagged'], network=net)
 438                     connection.save()
 439                     iface_config.connections.add(connection)
 440                     iface_config.save()
 441         self.repo_put(self.repo.RESOURCE_TEMPLATE_MODELS, models)
 442
 443     def decomposeXml(self, xmlString):
 444         """
 445         Translate XML into useable data.
 446
 447         This function takes in an xml doc from our front end
 448         and returns dictionaries that map cellIds to the xml
 449         nodes themselves. There is no unpacking of the
 450         xml objects, just grouping and organizing
 451         """
 452         connections = {}
 453         networks = {}
 454         hosts = {}
 455         interfaces = {}
 456         network_ports = {}
 457
 458         xmlDom = minidom.parseString(xmlString)
 459         root = xmlDom.documentElement.firstChild
 460         for cell in root.childNodes:
 461             cellId = cell.getAttribute('id')
 462             group = cellId.split("_")[0]
 463             parentGroup = cell.getAttribute("parent").split("_")[0]
 464             # place cell into correct group
 465
 466             if cell.getAttribute("edge"):
 467                 connections[cellId] = cell
 468
 469             elif "network" in group:
 470                 networks[cellId] = cell
 471
 472             elif "host" in group:
 473                 hosts[cellId] = cell
 474
 475             elif "host" in parentGroup:
 476                 interfaces[cellId] = cell
 477
 478             # make network ports also map to thier network
 479             elif "network" in parentGroup:
 480                 network_ports[cellId] = cell.getAttribute("parent")  # maps port ID to net ID
 481
 482         return connections, networks, hosts, interfaces, network_ports
 483
 484     # serialize and deserialize xml from mxGraph
 485     def parseXml(self, xmlString):
 486         networks = {}  # maps net name to network object
 487         hosts = {}  # cotains id -> hosts, each containing interfaces, referencing networks
 488         interfaces = {}  # maps id -> interface
 489         untagged_ifaces = set()  # used to check vlan config
 490         network_names = set()  # used to check network names
 491         xml_connections, xml_nets, xml_hosts, xml_ifaces, xml_ports = self.decomposeXml(xmlString)
 492
 493         # parse Hosts
 494         for cellId, cell in xml_hosts.items():
 495             cell_json_str = cell.getAttribute("value")
 496             cell_json = json.loads(cell_json_str)
 497             host = {"interfaces": [], "name": cellId, "hostname": cell_json['name']}
 498             hosts[cellId] = host
 499
 500         # parse networks
 501         for cellId, cell in xml_nets.items():
 502             escaped_json_str = cell.getAttribute("value")
 503             json_str = escaped_json_str.replace('&quot;', '"')
 504             net_info = json.loads(json_str)
 505             net_name = net_info['name']
 506             public = net_info['public']
 507             if net_name in network_names:
 508                 raise NetworkExistsException("Non unique network name found")
 509             network = {"name": net_name, "public": public, "id": cellId}
 510             networks[cellId] = network
 511             network_names.add(net_name)
 512
 513         # parse interfaces
 514         for cellId, cell in xml_ifaces.items():
 515             parentId = cell.getAttribute('parent')
 516             cell_json_str = cell.getAttribute("value")
 517             cell_json = json.loads(cell_json_str)
 518             iface = {"graph_id": cellId, "connections": [], "config_id": cell_json['id'], "profile_name": cell_json['name']}
 519             hosts[parentId]['interfaces'].append(cellId)
 520             interfaces[cellId] = iface
 521
 522         # parse connections
 523         for cellId, cell in xml_connections.items():
 524             escaped_json_str = cell.getAttribute("value")
 525             json_str = escaped_json_str.replace('&quot;', '"')
 526             attributes = json.loads(json_str)
 527             tagged = attributes['tagged']
 528             interface = None
 529             network = None
 530             src = cell.getAttribute("source")
 531             tgt = cell.getAttribute("target")
 532             if src in interfaces:
 533                 interface = interfaces[src]
 534                 network = networks[xml_ports[tgt]]
 535             else:
 536                 interface = interfaces[tgt]
 537                 network = networks[xml_ports[src]]
 538
 539             if not tagged:
 540                 if interface['config_id'] in untagged_ifaces:
 541                     raise InvalidVlanConfigurationException("More than one untagged vlan on an interface")
 542                 untagged_ifaces.add(interface['config_id'])
 543
 544             # add connection to interface
 545             interface['connections'].append({"tagged": tagged, "network": network['id']})
 546
 547         return hosts, interfaces, networks
 548
 549
 550 class Resource_Meta_Info(WorkflowStep):
 551     template = 'resource/steps/meta_info.html'
 552     title = "Extra Info"
 553     description = "Please fill out the rest of the information about your resource"
 554     short_title = "pod info"
 555
 556     def update_confirmation(self):
 557         confirm = self.repo_get(self.repo.CONFIRMATION, {})
 558         if "template" not in confirm:
 559             confirm['template'] = {}
 560         models = self.repo_get(self.repo.RESOURCE_TEMPLATE_MODELS, {})
 561         if "template" in models:
 562             confirm['template']['description'] = models['template'].description
 563             confirm['template']['name'] = models['template'].name
 564         self.repo_put(self.repo.CONFIRMATION, confirm)
 565
 566     def get_context(self):
 567         context = super(Resource_Meta_Info, self).get_context()
 568         name = ""
 569         desc = ""
 570         models = self.repo_get(self.repo.RESOURCE_TEMPLATE_MODELS, None)
 571         bundle = models['template']
 572         if bundle:
 573             name = bundle.name
 574             desc = bundle.description
 575         context['form'] = ResourceMetaForm(initial={"bundle_name": name, "bundle_description": desc})
 576         return context
 577
 578     def post(self, post_data, user):
 579         form = ResourceMetaForm(post_data)
 580         if form.is_valid():
 581             models = self.repo_get(self.repo.RESOURCE_TEMPLATE_MODELS, {})
 582             name = form.cleaned_data['bundle_name']
 583             desc = form.cleaned_data['bundle_description']
 584             bundle = models['template']  # infallible
 585             bundle.name = name
 586             bundle.description = desc
 587             bundle.save()
 588             self.repo_put(self.repo.RESOURCE_TEMPLATE_MODELS, models)
 589             confirm = self.repo_get(self.repo.CONFIRMATION)
 590             if "resource" not in confirm:
 591                 confirm['resource'] = {}
 592             confirm_info = confirm['resource']
 593             confirm_info["name"] = name
 594             tmp = desc
 595             if len(tmp) > 60:
 596                 tmp = tmp[:60] + "..."
 597             confirm_info["description"] = tmp
 598             self.repo_put(self.repo.CONFIRMATION, confirm)
 599             self.set_valid("Step Completed")
 600         else:
 601             self.set_invalid("Please correct the fields highlighted in red to continue")