dashboard/src/workflow/resource_bundle_workflow.py

   1 ##############################################################################
   2 # Copyright (c) 2018 Parker Berberian, Sawyer Bergeron, and others.
   3 #
   4 # All rights reserved. This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 # http://www.apache.org/licenses/LICENSE-2.0
   8 ##############################################################################
   9
  10
  11 from django.shortcuts import render
  12 from django.forms import formset_factory
  13 from django.conf import settings
  14
  15 import json
  16 import re
  17 from xml.dom import minidom
  18
  19 from workflow.models import WorkflowStep
  20 from account.models import Lab
  21 from workflow.forms import (
  22     HardwareDefinitionForm,
  23     NetworkDefinitionForm,
  24     ResourceMetaForm,
  25     GenericHostMetaForm
  26 )
  27 from resource_inventory.models import (
  28     GenericResourceBundle,
  29     GenericInterface,
  30     GenericHost,
  31     GenericResource,
  32     HostProfile,
  33     Network,
  34     NetworkConnection
  35 )
  36 from dashboard.exceptions import (
  37     InvalidVlanConfigurationException,
  38     NetworkExistsException,
  39     InvalidHostnameException,
  40     NonUniqueHostnameException,
  41     ResourceAvailabilityException
  42 )
  43
  44 import logging
  45 logger = logging.getLogger(__name__)
  46
  47
  48 class Define_Hardware(WorkflowStep):
  49
  50     template = 'resource/steps/define_hardware.html'
  51     title = "Define Hardware"
  52     description = "Choose the type and amount of machines you want"
  53     short_title = "hosts"
  54
  55     def get_context(self):
  56         context = super(Define_Hardware, self).get_context()
  57         selection_data = {"hosts": {}, "labs": {}}
  58         models = self.repo_get(self.repo.GRESOURCE_BUNDLE_MODELS, {})
  59         hosts = models.get("hosts", [])
  60         for host in hosts:
  61             profile_id = "host_" + str(host.profile.id)
  62             if profile_id not in selection_data['hosts']:
  63                 selection_data['hosts'][profile_id] = []
  64             selection_data['hosts'][profile_id].append({"host_name": host.resource.name, "class": profile_id})
  65
  66         if models.get("bundle", GenericResourceBundle()).lab:
  67             selection_data['labs'] = {"lab_" + str(models.get("bundle").lab.lab_user.id): "true"}
  68
  69         form = HardwareDefinitionForm(
  70             selection_data=selection_data
  71         )
  72         context['form'] = form
  73         return context
  74
  75     def render(self, request):
  76         self.context = self.get_context()
  77         return render(request, self.template, self.context)
  78
  79     def update_models(self, data):
  80         data = json.loads(data['filter_field'])
  81         models = self.repo_get(self.repo.GRESOURCE_BUNDLE_MODELS, {})
  82         models['hosts'] = []  # This will always clear existing data when this step changes
  83         models['interfaces'] = {}
  84         if "bundle" not in models:
  85             models['bundle'] = GenericResourceBundle(owner=self.repo_get(self.repo.SESSION_USER))
  86         host_data = data['hosts']
  87         names = {}
  88         for host_dict in host_data:
  89             id = host_dict['class']
  90             # bit of formatting
  91             id = int(id.split("_")[-1])
  92             profile = HostProfile.objects.get(id=id)
  93             # instantiate genericHost and store in repo
  94             name = host_dict['host_name']
  95             if not re.match(r"(?=^.{1,253}$)(^([A-Za-z0-9-_]{1,62}\.)*[A-Za-z0-9-_]{1,63})", name):
  96                 raise InvalidHostnameException("Hostname must comply to RFC 952 and all extensions to it until this point")
  97             if name in names:
  98                 raise NonUniqueHostnameException("All hosts must have unique names")
  99             names[name] = True
 100             genericResource = GenericResource(bundle=models['bundle'], name=name)
 101             genericHost = GenericHost(profile=profile, resource=genericResource)
 102             models['hosts'].append(genericHost)
 103             for interface_profile in profile.interfaceprofile.all():
 104                 genericInterface = GenericInterface(profile=interface_profile, host=genericHost)
 105                 if genericHost.resource.name not in models['interfaces']:
 106                     models['interfaces'][genericHost.resource.name] = []
 107                 models['interfaces'][genericHost.resource.name].append(genericInterface)
 108
 109         # add selected lab to models
 110         for lab_dict in data['labs']:
 111             if list(lab_dict.values())[0]:  # True for lab the user selected
 112                 lab_user_id = int(list(lab_dict.keys())[0].split("_")[-1])
 113                 models['bundle'].lab = Lab.objects.get(lab_user__id=lab_user_id)
 114                 break  # if somehow we get two 'true' labs, we only use one
 115
 116         # return to repo
 117         self.repo_put(self.repo.GRESOURCE_BUNDLE_MODELS, models)
 118
 119     def update_confirmation(self):
 120         confirm = self.repo_get(self.repo.CONFIRMATION, {})
 121         if "resource" not in confirm:
 122             confirm['resource'] = {}
 123         confirm['resource']['hosts'] = []
 124         models = self.repo_get(self.repo.GRESOURCE_BUNDLE_MODELS, {"hosts": []})
 125         for host in models['hosts']:
 126             host_dict = {"name": host.resource.name, "profile": host.profile.name}
 127             confirm['resource']['hosts'].append(host_dict)
 128         if "lab" in models:
 129             confirm['resource']['lab'] = models['lab'].lab_user.username
 130         self.repo_put(self.repo.CONFIRMATION, confirm)
 131
 132     def post_render(self, request):
 133         try:
 134             self.form = HardwareDefinitionForm(request.POST)
 135             if self.form.is_valid():
 136                 if len(json.loads(self.form.cleaned_data['filter_field'])['labs']) != 1:
 137                     self.set_invalid("Please select one lab")
 138                 else:
 139                     self.update_models(self.form.cleaned_data)
 140                     self.update_confirmation()
 141                     self.set_valid("Step Completed")
 142             else:
 143                 self.set_invalid("Please complete the fields highlighted in red to continue")
 144                 pass
 145         except Exception as e:
 146             self.set_invalid(str(e))
 147         self.context = self.get_context()
 148         return render(request, self.template, self.context)
 149
 150
 151 class Define_Nets(WorkflowStep):
 152     template = 'resource/steps/pod_definition.html'
 153     title = "Define Networks"
 154     description = "Use the tool below to draw the network topology of your POD"
 155     short_title = "networking"
 156     form = NetworkDefinitionForm
 157
 158     def get_vlans(self):
 159         vlans = self.repo_get(self.repo.VLANS)
 160         if vlans:
 161             return vlans
 162         # try to grab some vlans from lab
 163         models = self.repo_get(self.repo.GRESOURCE_BUNDLE_MODELS, {})
 164         if "bundle" not in models:
 165             return None
 166         lab = models['bundle'].lab
 167         if lab is None or lab.vlan_manager is None:
 168             return None
 169         try:
 170             vlans = lab.vlan_manager.get_vlan(count=lab.vlan_manager.block_size)
 171             self.repo_put(self.repo.VLANS, vlans)
 172             return vlans
 173         except Exception:
 174             return None
 175
 176     def get_context(self):
 177         # TODO: render *primarily* on hosts in repo models
 178         context = super(Define_Nets, self).get_context()
 179         context['form'] = NetworkDefinitionForm()
 180         try:
 181             context['hosts'] = []
 182             models = self.repo_get(self.repo.GRESOURCE_BUNDLE_MODELS, {})
 183             vlans = self.get_vlans()
 184             if vlans:
 185                 context['vlans'] = vlans
 186             hosts = models.get("hosts", [])
 187             hostlist = self.repo_get(self.repo.GRB_LAST_HOSTLIST, None)
 188             added_list = []
 189             added_dict = {}
 190             context['debug'] = settings.DEBUG
 191             context['added_hosts'] = []
 192             if hostlist is not None:
 193                 new_hostlist = []
 194                 for host in models['hosts']:
 195                     intcount = host.profile.interfaceprofile.count()
 196                     new_hostlist.append(str(host.resource.name) + "*" + str(host.profile) + "&" + str(intcount))
 197                 context['removed_hosts'] = list(set(hostlist) - set(new_hostlist))
 198                 added_list = list(set(new_hostlist) - set(hostlist))
 199                 for hoststr in added_list:
 200                     key = hoststr.split("*")[0]
 201                     added_dict[key] = hoststr
 202             for generic_host in hosts:
 203                 host_profile = generic_host.profile
 204                 host = {}
 205                 host['id'] = generic_host.resource.name
 206                 host['interfaces'] = []
 207                 for iface in host_profile.interfaceprofile.all():
 208                     host['interfaces'].append(
 209                         {
 210                             "name": iface.name,
 211                             "description": "speed: " + str(iface.speed) + "M\ntype: " + iface.nic_type
 212                         }
 213                     )
 214                 host['value'] = {"name": generic_host.resource.name}
 215                 host['value']['description'] = generic_host.profile.description
 216                 context['hosts'].append(json.dumps(host))
 217                 if host['id'] in added_dict:
 218                     context['added_hosts'].append(json.dumps(host))
 219             bundle = models.get("bundle", False)
 220             if bundle and bundle.xml:
 221                 context['xml'] = bundle.xml
 222             else:
 223                 context['xml'] = False
 224
 225         except Exception:
 226             pass
 227
 228         return context
 229
 230     def post_render(self, request):
 231         models = self.repo_get(self.repo.GRESOURCE_BUNDLE_MODELS, {})
 232         if 'hosts' in models:
 233             hostlist = []
 234             for host in models['hosts']:
 235                 intcount = host.profile.interfaceprofile.count()
 236                 hostlist.append(str(host.resource.name) + "*" + str(host.profile) + "&" + str(intcount))
 237             self.repo_put(self.repo.GRB_LAST_HOSTLIST, hostlist)
 238         try:
 239             xmlData = request.POST.get("xml")
 240             self.updateModels(xmlData)
 241             # update model with xml
 242             self.set_valid("Networks applied successfully")
 243         except ResourceAvailabilityException:
 244             self.set_invalid("Public network not availble")
 245         except Exception as e:
 246             self.set_invalid("An error occurred when applying networks: " + str(e))
 247         return self.render(request)
 248
 249     def updateModels(self, xmlData):
 250         models = self.repo_get(self.repo.GRESOURCE_BUNDLE_MODELS, {})
 251         models["connections"] = {}
 252         models['networks'] = {}
 253         given_hosts, interfaces, networks = self.parseXml(xmlData)
 254         existing_host_list = models.get("hosts", [])
 255         existing_hosts = {}  # maps id to host
 256         for host in existing_host_list:
 257             existing_hosts[host.resource.name] = host
 258
 259         bundle = models.get("bundle", GenericResourceBundle(owner=self.repo_get(self.repo.SESSION_USER)))
 260
 261         for net_id, net in networks.items():
 262             network = Network()
 263             network.name = net['name']
 264             network.bundle = bundle
 265             network.is_public = net['public']
 266             models['networks'][net_id] = network
 267
 268         for hostid, given_host in given_hosts.items():
 269             existing_host = existing_hosts[hostid[5:]]
 270
 271             for ifaceId in given_host['interfaces']:
 272                 iface = interfaces[ifaceId]
 273                 if existing_host.resource.name not in models['connections']:
 274                     models['connections'][existing_host.resource.name] = {}
 275                 models['connections'][existing_host.resource.name][iface['profile_name']] = []
 276                 for connection in iface['connections']:
 277                     network_id = connection['network']
 278                     net = models['networks'][network_id]
 279                     connection = NetworkConnection(vlan_is_tagged=connection['tagged'], network=net)
 280                     models['connections'][existing_host.resource.name][iface['profile_name']].append(connection)
 281         bundle.xml = xmlData
 282         self.repo_put(self.repo.GRESOURCE_BUNDLE_MODELS, models)
 283
 284     def decomposeXml(self, xmlString):
 285         """
 286         This function takes in an xml doc from our front end
 287         and returns dictionaries that map cellIds to the xml
 288         nodes themselves. There is no unpacking of the
 289         xml objects, just grouping and organizing
 290         """
 291
 292         connections = {}
 293         networks = {}
 294         hosts = {}
 295         interfaces = {}
 296         network_ports = {}
 297
 298         xmlDom = minidom.parseString(xmlString)
 299         root = xmlDom.documentElement.firstChild
 300         for cell in root.childNodes:
 301             cellId = cell.getAttribute('id')
 302             group = cellId.split("_")[0]
 303             parentGroup = cell.getAttribute("parent").split("_")[0]
 304             # place cell into correct group
 305
 306             if cell.getAttribute("edge"):
 307                 connections[cellId] = cell
 308
 309             elif "network" in group:
 310                 networks[cellId] = cell
 311
 312             elif "host" in group:
 313                 hosts[cellId] = cell
 314
 315             elif "host" in parentGroup:
 316                 interfaces[cellId] = cell
 317
 318             # make network ports also map to thier network
 319             elif "network" in parentGroup:
 320                 network_ports[cellId] = cell.getAttribute("parent")  # maps port ID to net ID
 321
 322         return connections, networks, hosts, interfaces, network_ports
 323
 324     # serialize and deserialize xml from mxGraph
 325     def parseXml(self, xmlString):
 326         networks = {}  # maps net name to network object
 327         hosts = {}  # cotains id -> hosts, each containing interfaces, referencing networks
 328         interfaces = {}  # maps id -> interface
 329         untagged_ifaces = set()  # used to check vlan config
 330         network_names = set()  # used to check network names
 331         xml_connections, xml_nets, xml_hosts, xml_ifaces, xml_ports = self.decomposeXml(xmlString)
 332
 333         # parse Hosts
 334         for cellId, cell in xml_hosts.items():
 335             cell_json_str = cell.getAttribute("value")
 336             cell_json = json.loads(cell_json_str)
 337             host = {"interfaces": [], "name": cellId, "profile_name": cell_json['name']}
 338             hosts[cellId] = host
 339
 340         # parse networks
 341         for cellId, cell in xml_nets.items():
 342             escaped_json_str = cell.getAttribute("value")
 343             json_str = escaped_json_str.replace('&quot;', '"')
 344             net_info = json.loads(json_str)
 345             net_name = net_info['name']
 346             public = net_info['public']
 347             if net_name in network_names:
 348                 raise NetworkExistsException("Non unique network name found")
 349             network = {"name": net_name, "public": public, "id": cellId}
 350             networks[cellId] = network
 351             network_names.add(net_name)
 352
 353         # parse interfaces
 354         for cellId, cell in xml_ifaces.items():
 355             parentId = cell.getAttribute('parent')
 356             cell_json_str = cell.getAttribute("value")
 357             cell_json = json.loads(cell_json_str)
 358             iface = {"name": cellId, "connections": [], "profile_name": cell_json['name']}
 359             hosts[parentId]['interfaces'].append(cellId)
 360             interfaces[cellId] = iface
 361
 362         # parse connections
 363         for cellId, cell in xml_connections.items():
 364             escaped_json_str = cell.getAttribute("value")
 365             json_str = escaped_json_str.replace('&quot;', '"')
 366             attributes = json.loads(json_str)
 367             tagged = attributes['tagged']
 368             interface = None
 369             network = None
 370             src = cell.getAttribute("source")
 371             tgt = cell.getAttribute("target")
 372             if src in interfaces:
 373                 interface = interfaces[src]
 374                 network = networks[xml_ports[tgt]]
 375             else:
 376                 interface = interfaces[tgt]
 377                 network = networks[xml_ports[src]]
 378
 379             if not tagged:
 380                 if interface['name'] in untagged_ifaces:
 381                     raise InvalidVlanConfigurationException("More than one untagged vlan on an interface")
 382                 untagged_ifaces.add(interface['name'])
 383
 384             # add connection to interface
 385             interface['connections'].append({"tagged": tagged, "network": network['id']})
 386
 387         return hosts, interfaces, networks
 388
 389
 390 class Resource_Meta_Info(WorkflowStep):
 391     template = 'resource/steps/meta_info.html'
 392     title = "Extra Info"
 393     description = "Please fill out the rest of the information about your resource"
 394     short_title = "pod info"
 395
 396     def get_context(self):
 397         context = super(Resource_Meta_Info, self).get_context()
 398         name = ""
 399         desc = ""
 400         bundle = self.repo_get(self.repo.GRESOURCE_BUNDLE_MODELS, {}).get("bundle", False)
 401         if bundle and bundle.name:
 402             name = bundle.name
 403             desc = bundle.description
 404         context['form'] = ResourceMetaForm(initial={"bundle_name": name, "bundle_description": desc})
 405         return context
 406
 407     def post_render(self, request):
 408         form = ResourceMetaForm(request.POST)
 409         if form.is_valid():
 410             models = self.repo_get(self.repo.GRESOURCE_BUNDLE_MODELS, {})
 411             name = form.cleaned_data['bundle_name']
 412             desc = form.cleaned_data['bundle_description']
 413             bundle = models.get("bundle", GenericResourceBundle(owner=self.repo_get(self.repo.SESSION_USER)))
 414             bundle.name = name
 415             bundle.description = desc
 416             models['bundle'] = bundle
 417             self.repo_put(self.repo.GRESOURCE_BUNDLE_MODELS, models)
 418             confirm = self.repo_get(self.repo.CONFIRMATION)
 419             if "resource" not in confirm:
 420                 confirm['resource'] = {}
 421             confirm_info = confirm['resource']
 422             confirm_info["name"] = name
 423             tmp = desc
 424             if len(tmp) > 60:
 425                 tmp = tmp[:60] + "..."
 426             confirm_info["description"] = tmp
 427             self.repo_put(self.repo.CONFIRMATION, confirm)
 428             self.set_valid("Step Completed")
 429
 430         else:
 431             self.set_invalid("Please correct the fields highlighted in red to continue")
 432             pass
 433         return self.render(request)
 434
 435
 436 class Host_Meta_Info(WorkflowStep):
 437     template = "resource/steps/host_info.html"
 438     title = "Host Info"
 439     description = "We need a little bit of information about your chosen machines"
 440     short_title = "host info"
 441
 442     def __init__(self, *args, **kwargs):
 443         super(Host_Meta_Info, self).__init__(*args, **kwargs)
 444         self.formset = formset_factory(GenericHostMetaForm, extra=0)
 445
 446     def get_context(self):
 447         context = super(Host_Meta_Info, self).get_context()
 448         GenericHostFormset = self.formset
 449         models = self.repo_get(self.repo.GRESOURCE_BUNDLE_MODELS, {})
 450         initial_data = []
 451         if "hosts" not in models:
 452             context['error'] = "Please go back and select your hosts"
 453         else:
 454             for host in models['hosts']:
 455                 profile = host.profile.name
 456                 name = host.resource.name
 457                 if not name:
 458                     name = ""
 459                 initial_data.append({"host_profile": profile, "host_name": name})
 460         context['formset'] = GenericHostFormset(initial=initial_data)
 461         return context
 462
 463     def post_render(self, request):
 464         models = self.repo_get(self.repo.GRESOURCE_BUNDLE_MODELS, {})
 465         if 'hosts' not in models:
 466             models['hosts'] = []
 467         hosts = models['hosts']
 468         i = 0
 469         confirm_hosts = []
 470         GenericHostFormset = self.formset
 471         formset = GenericHostFormset(request.POST)
 472         if formset.is_valid():
 473             for form in formset:
 474                 host = hosts[i]
 475                 host.resource.name = form.cleaned_data['host_name']
 476                 i += 1
 477                 confirm_hosts.append({"name": host.resource.name, "profile": host.profile.name})
 478             models['hosts'] = hosts
 479             self.repo_put(self.repo.GRESOURCE_BUNDLE_MODELS, models)
 480             confirm = self.repo_get(self.repo.CONFIRMATION, {})
 481             if "resource" not in confirm:
 482                 confirm['resource'] = {}
 483             confirm['resource']['hosts'] = confirm_hosts
 484             self.repo_put(self.repo.CONFIRMATION, confirm)
 485         else:
 486             pass
 487         return self.render(request)