dashboard/src/workflow/resource_bundle_workflow.py

   1 ##############################################################################
   2 # Copyright (c) 2018 Parker Berberian, Sawyer Bergeron, and others.
   3 #
   4 # All rights reserved. This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 # http://www.apache.org/licenses/LICENSE-2.0
   8 ##############################################################################
   9
  10
  11 from django.shortcuts import render
  12 from django.forms import formset_factory
  13 from django.conf import settings
  14
  15 import json
  16 import re
  17 from xml.dom import minidom
  18
  19 from workflow.models import WorkflowStep
  20 from account.models import Lab
  21 from workflow.forms import (
  22     HardwareDefinitionForm,
  23     NetworkDefinitionForm,
  24     ResourceMetaForm,
  25     GenericHostMetaForm
  26 )
  27 from resource_inventory.models import (
  28     GenericResourceBundle,
  29     GenericInterface,
  30     GenericHost,
  31     GenericResource,
  32     HostProfile,
  33     Network,
  34     NetworkConnection
  35 )
  36 from dashboard.exceptions import (
  37     InvalidVlanConfigurationException,
  38     NetworkExistsException,
  39     InvalidHostnameException,
  40     NonUniqueHostnameException,
  41     ResourceAvailabilityException
  42 )
  43
  44 import logging
  45 logger = logging.getLogger(__name__)
  46
  47
  48 class Define_Hardware(WorkflowStep):
  49
  50     template = 'resource/steps/define_hardware.html'
  51     title = "Define Hardware"
  52     description = "Choose the type and amount of machines you want"
  53     short_title = "hosts"
  54
  55     def __init__(self, *args, **kwargs):
  56         self.form = None
  57         super().__init__(*args, **kwargs)
  58
  59     def get_context(self):
  60         context = super(Define_Hardware, self).get_context()
  61         context['form'] = self.form or HardwareDefinitionForm()
  62         return context
  63
  64     def update_models(self, data):
  65         data = data['filter_field']
  66         models = self.repo_get(self.repo.GRESOURCE_BUNDLE_MODELS, {})
  67         models['hosts'] = []  # This will always clear existing data when this step changes
  68         models['interfaces'] = {}
  69         if "bundle" not in models:
  70             models['bundle'] = GenericResourceBundle(owner=self.repo_get(self.repo.SESSION_USER))
  71         host_data = data['host']
  72         names = {}
  73         for host_profile_dict in host_data.values():
  74             id = host_profile_dict['id']
  75             profile = HostProfile.objects.get(id=id)
  76             # instantiate genericHost and store in repo
  77             for name in host_profile_dict['values'].values():
  78                 if not re.match(r"(?=^.{1,253}$)(^([A-Za-z0-9-_]{1,62}\.)*[A-Za-z0-9-_]{1,63})", name):
  79                     raise InvalidHostnameException("Invalid hostname: '" + name + "'")
  80                 if name in names:
  81                     raise NonUniqueHostnameException("All hosts must have unique names")
  82                 names[name] = True
  83                 genericResource = GenericResource(bundle=models['bundle'], name=name)
  84                 genericHost = GenericHost(profile=profile, resource=genericResource)
  85                 models['hosts'].append(genericHost)
  86                 for interface_profile in profile.interfaceprofile.all():
  87                     genericInterface = GenericInterface(profile=interface_profile, host=genericHost)
  88                     if genericHost.resource.name not in models['interfaces']:
  89                         models['interfaces'][genericHost.resource.name] = []
  90                     models['interfaces'][genericHost.resource.name].append(genericInterface)
  91
  92         # add selected lab to models
  93         for lab_dict in data['lab'].values():
  94             if lab_dict['selected']:
  95                 models['bundle'].lab = Lab.objects.get(lab_user__id=lab_dict['id'])
  96                 break  # if somehow we get two 'true' labs, we only use one
  97
  98         # return to repo
  99         self.repo_put(self.repo.GRESOURCE_BUNDLE_MODELS, models)
 100
 101     def update_confirmation(self):
 102         confirm = self.repo_get(self.repo.CONFIRMATION, {})
 103         if "resource" not in confirm:
 104             confirm['resource'] = {}
 105         confirm['resource']['hosts'] = []
 106         models = self.repo_get(self.repo.GRESOURCE_BUNDLE_MODELS, {"hosts": []})
 107         for host in models['hosts']:
 108             host_dict = {"name": host.resource.name, "profile": host.profile.name}
 109             confirm['resource']['hosts'].append(host_dict)
 110         if "lab" in models:
 111             confirm['resource']['lab'] = models['lab'].lab_user.username
 112         self.repo_put(self.repo.CONFIRMATION, confirm)
 113
 114     def post_render(self, request):
 115         try:
 116             self.form = HardwareDefinitionForm(request.POST)
 117             if self.form.is_valid():
 118                 self.update_models(self.form.cleaned_data)
 119                 self.update_confirmation()
 120                 self.set_valid("Step Completed")
 121             else:
 122                 self.set_invalid("Please complete the fields highlighted in red to continue")
 123         except Exception as e:
 124             self.set_invalid(str(e))
 125         self.context = self.get_context()
 126         return render(request, self.template, self.context)
 127
 128
 129 class Define_Nets(WorkflowStep):
 130     template = 'resource/steps/pod_definition.html'
 131     title = "Define Networks"
 132     description = "Use the tool below to draw the network topology of your POD"
 133     short_title = "networking"
 134     form = NetworkDefinitionForm
 135
 136     def get_vlans(self):
 137         vlans = self.repo_get(self.repo.VLANS)
 138         if vlans:
 139             return vlans
 140         # try to grab some vlans from lab
 141         models = self.repo_get(self.repo.GRESOURCE_BUNDLE_MODELS, {})
 142         if "bundle" not in models:
 143             return None
 144         lab = models['bundle'].lab
 145         if lab is None or lab.vlan_manager is None:
 146             return None
 147         try:
 148             vlans = lab.vlan_manager.get_vlan(count=lab.vlan_manager.block_size)
 149             self.repo_put(self.repo.VLANS, vlans)
 150             return vlans
 151         except Exception:
 152             return None
 153
 154     def get_context(self):
 155         # TODO: render *primarily* on hosts in repo models
 156         context = super(Define_Nets, self).get_context()
 157         context['form'] = NetworkDefinitionForm()
 158         try:
 159             context['hosts'] = []
 160             models = self.repo_get(self.repo.GRESOURCE_BUNDLE_MODELS, {})
 161             vlans = self.get_vlans()
 162             if vlans:
 163                 context['vlans'] = vlans
 164             hosts = models.get("hosts", [])
 165             hostlist = self.repo_get(self.repo.GRB_LAST_HOSTLIST, None)
 166             added_list = []
 167             added_dict = {}
 168             context['debug'] = settings.DEBUG
 169             context['added_hosts'] = []
 170             if hostlist is not None:
 171                 new_hostlist = []
 172                 for host in models['hosts']:
 173                     intcount = host.profile.interfaceprofile.count()
 174                     new_hostlist.append(str(host.resource.name) + "*" + str(host.profile) + "&" + str(intcount))
 175                 context['removed_hosts'] = list(set(hostlist) - set(new_hostlist))
 176                 added_list = list(set(new_hostlist) - set(hostlist))
 177                 for hoststr in added_list:
 178                     key = hoststr.split("*")[0]
 179                     added_dict[key] = hoststr
 180             for generic_host in hosts:
 181                 host_profile = generic_host.profile
 182                 host = {}
 183                 host['id'] = generic_host.resource.name
 184                 host['interfaces'] = []
 185                 for iface in host_profile.interfaceprofile.all():
 186                     host['interfaces'].append(
 187                         {
 188                             "name": iface.name,
 189                             "description": "speed: " + str(iface.speed) + "M\ntype: " + iface.nic_type
 190                         }
 191                     )
 192                 host['value'] = {"name": generic_host.resource.name}
 193                 host['value']['description'] = generic_host.profile.description
 194                 context['hosts'].append(json.dumps(host))
 195                 if host['id'] in added_dict:
 196                     context['added_hosts'].append(json.dumps(host))
 197             bundle = models.get("bundle", False)
 198             if bundle and bundle.xml:
 199                 context['xml'] = bundle.xml
 200             else:
 201                 context['xml'] = False
 202
 203         except Exception:
 204             pass
 205
 206         return context
 207
 208     def post_render(self, request):
 209         models = self.repo_get(self.repo.GRESOURCE_BUNDLE_MODELS, {})
 210         if 'hosts' in models:
 211             hostlist = []
 212             for host in models['hosts']:
 213                 intcount = host.profile.interfaceprofile.count()
 214                 hostlist.append(str(host.resource.name) + "*" + str(host.profile) + "&" + str(intcount))
 215             self.repo_put(self.repo.GRB_LAST_HOSTLIST, hostlist)
 216         try:
 217             xmlData = request.POST.get("xml")
 218             self.updateModels(xmlData)
 219             # update model with xml
 220             self.set_valid("Networks applied successfully")
 221         except ResourceAvailabilityException:
 222             self.set_invalid("Public network not availble")
 223         except Exception as e:
 224             self.set_invalid("An error occurred when applying networks: " + str(e))
 225         return self.render(request)
 226
 227     def updateModels(self, xmlData):
 228         models = self.repo_get(self.repo.GRESOURCE_BUNDLE_MODELS, {})
 229         models["connections"] = {}
 230         models['networks'] = {}
 231         given_hosts, interfaces, networks = self.parseXml(xmlData)
 232         existing_host_list = models.get("hosts", [])
 233         existing_hosts = {}  # maps id to host
 234         for host in existing_host_list:
 235             existing_hosts[host.resource.name] = host
 236
 237         bundle = models.get("bundle", GenericResourceBundle(owner=self.repo_get(self.repo.SESSION_USER)))
 238
 239         for net_id, net in networks.items():
 240             network = Network()
 241             network.name = net['name']
 242             network.bundle = bundle
 243             network.is_public = net['public']
 244             models['networks'][net_id] = network
 245
 246         for hostid, given_host in given_hosts.items():
 247             existing_host = existing_hosts[hostid[5:]]
 248
 249             for ifaceId in given_host['interfaces']:
 250                 iface = interfaces[ifaceId]
 251                 if existing_host.resource.name not in models['connections']:
 252                     models['connections'][existing_host.resource.name] = {}
 253                 models['connections'][existing_host.resource.name][iface['profile_name']] = []
 254                 for connection in iface['connections']:
 255                     network_id = connection['network']
 256                     net = models['networks'][network_id]
 257                     connection = NetworkConnection(vlan_is_tagged=connection['tagged'], network=net)
 258                     models['connections'][existing_host.resource.name][iface['profile_name']].append(connection)
 259         bundle.xml = xmlData
 260         self.repo_put(self.repo.GRESOURCE_BUNDLE_MODELS, models)
 261
 262     def decomposeXml(self, xmlString):
 263         """
 264         This function takes in an xml doc from our front end
 265         and returns dictionaries that map cellIds to the xml
 266         nodes themselves. There is no unpacking of the
 267         xml objects, just grouping and organizing
 268         """
 269
 270         connections = {}
 271         networks = {}
 272         hosts = {}
 273         interfaces = {}
 274         network_ports = {}
 275
 276         xmlDom = minidom.parseString(xmlString)
 277         root = xmlDom.documentElement.firstChild
 278         for cell in root.childNodes:
 279             cellId = cell.getAttribute('id')
 280             group = cellId.split("_")[0]
 281             parentGroup = cell.getAttribute("parent").split("_")[0]
 282             # place cell into correct group
 283
 284             if cell.getAttribute("edge"):
 285                 connections[cellId] = cell
 286
 287             elif "network" in group:
 288                 networks[cellId] = cell
 289
 290             elif "host" in group:
 291                 hosts[cellId] = cell
 292
 293             elif "host" in parentGroup:
 294                 interfaces[cellId] = cell
 295
 296             # make network ports also map to thier network
 297             elif "network" in parentGroup:
 298                 network_ports[cellId] = cell.getAttribute("parent")  # maps port ID to net ID
 299
 300         return connections, networks, hosts, interfaces, network_ports
 301
 302     # serialize and deserialize xml from mxGraph
 303     def parseXml(self, xmlString):
 304         networks = {}  # maps net name to network object
 305         hosts = {}  # cotains id -> hosts, each containing interfaces, referencing networks
 306         interfaces = {}  # maps id -> interface
 307         untagged_ifaces = set()  # used to check vlan config
 308         network_names = set()  # used to check network names
 309         xml_connections, xml_nets, xml_hosts, xml_ifaces, xml_ports = self.decomposeXml(xmlString)
 310
 311         # parse Hosts
 312         for cellId, cell in xml_hosts.items():
 313             cell_json_str = cell.getAttribute("value")
 314             cell_json = json.loads(cell_json_str)
 315             host = {"interfaces": [], "name": cellId, "profile_name": cell_json['name']}
 316             hosts[cellId] = host
 317
 318         # parse networks
 319         for cellId, cell in xml_nets.items():
 320             escaped_json_str = cell.getAttribute("value")
 321             json_str = escaped_json_str.replace('&quot;', '"')
 322             net_info = json.loads(json_str)
 323             net_name = net_info['name']
 324             public = net_info['public']
 325             if net_name in network_names:
 326                 raise NetworkExistsException("Non unique network name found")
 327             network = {"name": net_name, "public": public, "id": cellId}
 328             networks[cellId] = network
 329             network_names.add(net_name)
 330
 331         # parse interfaces
 332         for cellId, cell in xml_ifaces.items():
 333             parentId = cell.getAttribute('parent')
 334             cell_json_str = cell.getAttribute("value")
 335             cell_json = json.loads(cell_json_str)
 336             iface = {"name": cellId, "connections": [], "profile_name": cell_json['name']}
 337             hosts[parentId]['interfaces'].append(cellId)
 338             interfaces[cellId] = iface
 339
 340         # parse connections
 341         for cellId, cell in xml_connections.items():
 342             escaped_json_str = cell.getAttribute("value")
 343             json_str = escaped_json_str.replace('&quot;', '"')
 344             attributes = json.loads(json_str)
 345             tagged = attributes['tagged']
 346             interface = None
 347             network = None
 348             src = cell.getAttribute("source")
 349             tgt = cell.getAttribute("target")
 350             if src in interfaces:
 351                 interface = interfaces[src]
 352                 network = networks[xml_ports[tgt]]
 353             else:
 354                 interface = interfaces[tgt]
 355                 network = networks[xml_ports[src]]
 356
 357             if not tagged:
 358                 if interface['name'] in untagged_ifaces:
 359                     raise InvalidVlanConfigurationException("More than one untagged vlan on an interface")
 360                 untagged_ifaces.add(interface['name'])
 361
 362             # add connection to interface
 363             interface['connections'].append({"tagged": tagged, "network": network['id']})
 364
 365         return hosts, interfaces, networks
 366
 367
 368 class Resource_Meta_Info(WorkflowStep):
 369     template = 'resource/steps/meta_info.html'
 370     title = "Extra Info"
 371     description = "Please fill out the rest of the information about your resource"
 372     short_title = "pod info"
 373
 374     def get_context(self):
 375         context = super(Resource_Meta_Info, self).get_context()
 376         name = ""
 377         desc = ""
 378         bundle = self.repo_get(self.repo.GRESOURCE_BUNDLE_MODELS, {}).get("bundle", False)
 379         if bundle and bundle.name:
 380             name = bundle.name
 381             desc = bundle.description
 382         context['form'] = ResourceMetaForm(initial={"bundle_name": name, "bundle_description": desc})
 383         return context
 384
 385     def post_render(self, request):
 386         form = ResourceMetaForm(request.POST)
 387         if form.is_valid():
 388             models = self.repo_get(self.repo.GRESOURCE_BUNDLE_MODELS, {})
 389             name = form.cleaned_data['bundle_name']
 390             desc = form.cleaned_data['bundle_description']
 391             bundle = models.get("bundle", GenericResourceBundle(owner=self.repo_get(self.repo.SESSION_USER)))
 392             bundle.name = name
 393             bundle.description = desc
 394             models['bundle'] = bundle
 395             self.repo_put(self.repo.GRESOURCE_BUNDLE_MODELS, models)
 396             confirm = self.repo_get(self.repo.CONFIRMATION)
 397             if "resource" not in confirm:
 398                 confirm['resource'] = {}
 399             confirm_info = confirm['resource']
 400             confirm_info["name"] = name
 401             tmp = desc
 402             if len(tmp) > 60:
 403                 tmp = tmp[:60] + "..."
 404             confirm_info["description"] = tmp
 405             self.repo_put(self.repo.CONFIRMATION, confirm)
 406             self.set_valid("Step Completed")
 407
 408         else:
 409             self.set_invalid("Please correct the fields highlighted in red to continue")
 410             pass
 411         return self.render(request)
 412
 413
 414 class Host_Meta_Info(WorkflowStep):
 415     template = "resource/steps/host_info.html"
 416     title = "Host Info"
 417     description = "We need a little bit of information about your chosen machines"
 418     short_title = "host info"
 419
 420     def __init__(self, *args, **kwargs):
 421         super(Host_Meta_Info, self).__init__(*args, **kwargs)
 422         self.formset = formset_factory(GenericHostMetaForm, extra=0)
 423
 424     def get_context(self):
 425         context = super(Host_Meta_Info, self).get_context()
 426         GenericHostFormset = self.formset
 427         models = self.repo_get(self.repo.GRESOURCE_BUNDLE_MODELS, {})
 428         initial_data = []
 429         if "hosts" not in models:
 430             context['error'] = "Please go back and select your hosts"
 431         else:
 432             for host in models['hosts']:
 433                 profile = host.profile.name
 434                 name = host.resource.name
 435                 if not name:
 436                     name = ""
 437                 initial_data.append({"host_profile": profile, "host_name": name})
 438         context['formset'] = GenericHostFormset(initial=initial_data)
 439         return context
 440
 441     def post_render(self, request):
 442         models = self.repo_get(self.repo.GRESOURCE_BUNDLE_MODELS, {})
 443         if 'hosts' not in models:
 444             models['hosts'] = []
 445         hosts = models['hosts']
 446         i = 0
 447         confirm_hosts = []
 448         GenericHostFormset = self.formset
 449         formset = GenericHostFormset(request.POST)
 450         if formset.is_valid():
 451             for form in formset:
 452                 host = hosts[i]
 453                 host.resource.name = form.cleaned_data['host_name']
 454                 i += 1
 455                 confirm_hosts.append({"name": host.resource.name, "profile": host.profile.name})
 456             models['hosts'] = hosts
 457             self.repo_put(self.repo.GRESOURCE_BUNDLE_MODELS, models)
 458             confirm = self.repo_get(self.repo.CONFIRMATION, {})
 459             if "resource" not in confirm:
 460                 confirm['resource'] = {}
 461             confirm['resource']['hosts'] = confirm_hosts
 462             self.repo_put(self.repo.CONFIRMATION, confirm)
 463         else:
 464             pass
 465         return self.render(request)