dashboard/src/workflow/resource_bundle_workflow.py

   1 ##############################################################################
   2 # Copyright (c) 2018 Parker Berberian, Sawyer Bergeron, and others.
   3 #
   4 # All rights reserved. This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 # http://www.apache.org/licenses/LICENSE-2.0
   8 ##############################################################################
   9
  10
  11 from django.shortcuts import render
  12 from django.forms import formset_factory
  13
  14 import json
  15 import re
  16 from xml.dom import minidom
  17
  18 from workflow.models import WorkflowStep
  19 from account.models import Lab
  20 from workflow.forms import (
  21     HardwareDefinitionForm,
  22     NetworkDefinitionForm,
  23     ResourceMetaForm,
  24     GenericHostMetaForm
  25 )
  26 from resource_inventory.models import (
  27     GenericResourceBundle,
  28     Vlan,
  29     GenericInterface,
  30     GenericHost,
  31     GenericResource,
  32     HostProfile
  33 )
  34 from dashboard.exceptions import (
  35     InvalidVlanConfigurationException,
  36     NetworkExistsException,
  37     InvalidHostnameException,
  38     NonUniqueHostnameException,
  39     ResourceAvailabilityException
  40 )
  41
  42 import logging
  43 logger = logging.getLogger(__name__)
  44
  45
  46 class Define_Hardware(WorkflowStep):
  47
  48     template = 'resource/steps/define_hardware.html'
  49     title = "Define Hardware"
  50     description = "Choose the type and amount of machines you want"
  51     short_title = "hosts"
  52
  53     def get_context(self):
  54         context = super(Define_Hardware, self).get_context()
  55         selection_data = {"hosts": {}, "labs": {}}
  56         models = self.repo_get(self.repo.GRESOURCE_BUNDLE_MODELS, {})
  57         hosts = models.get("hosts", [])
  58         for host in hosts:
  59             profile_id = "host_" + str(host.profile.id)
  60             if profile_id not in selection_data['hosts']:
  61                 selection_data['hosts'][profile_id] = []
  62             selection_data['hosts'][profile_id].append({"host_name": host.resource.name, "class": profile_id})
  63
  64         if models.get("bundle", GenericResourceBundle()).lab:
  65             selection_data['labs'] = {"lab_" + str(models.get("bundle").lab.lab_user.id): "true"}
  66
  67         form = HardwareDefinitionForm(
  68             selection_data=selection_data
  69         )
  70         context['form'] = form
  71         return context
  72
  73     def render(self, request):
  74         self.context = self.get_context()
  75         return render(request, self.template, self.context)
  76
  77     def update_models(self, data):
  78         data = json.loads(data['filter_field'])
  79         models = self.repo_get(self.repo.GRESOURCE_BUNDLE_MODELS, {})
  80         models['hosts'] = []  # This will always clear existing data when this step changes
  81         models['interfaces'] = {}
  82         if "bundle" not in models:
  83             models['bundle'] = GenericResourceBundle(owner=self.repo_get(self.repo.SESSION_USER))
  84         host_data = data['hosts']
  85         names = {}
  86         for host_dict in host_data:
  87             id = host_dict['class']
  88             # bit of formatting
  89             id = int(id.split("_")[-1])
  90             profile = HostProfile.objects.get(id=id)
  91             # instantiate genericHost and store in repo
  92             name = host_dict['host_name']
  93             if not re.match(r"(?=^.{1,253}$)(^([A-Za-z0-9-_]{1,62}\.)*[A-Za-z0-9-_]{1,63})", name):
  94                 raise InvalidHostnameException("Hostname must comply to RFC 952 and all extensions to it until this point")
  95             if name in names:
  96                 raise NonUniqueHostnameException("All hosts must have unique names")
  97             names[name] = True
  98             genericResource = GenericResource(bundle=models['bundle'], name=name)
  99             genericHost = GenericHost(profile=profile, resource=genericResource)
 100             models['hosts'].append(genericHost)
 101             for interface_profile in profile.interfaceprofile.all():
 102                 genericInterface = GenericInterface(profile=interface_profile, host=genericHost)
 103                 if genericHost.resource.name not in models['interfaces']:
 104                     models['interfaces'][genericHost.resource.name] = []
 105                 models['interfaces'][genericHost.resource.name].append(genericInterface)
 106
 107         # add selected lab to models
 108         for lab_dict in data['labs']:
 109             if list(lab_dict.values())[0]:  # True for lab the user selected
 110                 lab_user_id = int(list(lab_dict.keys())[0].split("_")[-1])
 111                 models['bundle'].lab = Lab.objects.get(lab_user__id=lab_user_id)
 112                 break  # if somehow we get two 'true' labs, we only use one
 113
 114         # return to repo
 115         self.repo_put(self.repo.GRESOURCE_BUNDLE_MODELS, models)
 116
 117     def update_confirmation(self):
 118         confirm = self.repo_get(self.repo.CONFIRMATION, {})
 119         if "resource" not in confirm:
 120             confirm['resource'] = {}
 121         confirm['resource']['hosts'] = []
 122         models = self.repo_get(self.repo.GRESOURCE_BUNDLE_MODELS, {"hosts": []})
 123         for host in models['hosts']:
 124             host_dict = {"name": host.resource.name, "profile": host.profile.name}
 125             confirm['resource']['hosts'].append(host_dict)
 126         if "lab" in models:
 127             confirm['resource']['lab'] = models['lab'].lab_user.username
 128         self.repo_put(self.repo.CONFIRMATION, confirm)
 129
 130     def post_render(self, request):
 131         try:
 132             self.form = HardwareDefinitionForm(request.POST)
 133             if self.form.is_valid():
 134                 if len(json.loads(self.form.cleaned_data['filter_field'])['labs']) != 1:
 135                     self.metastep.set_invalid("Please select one lab")
 136                 else:
 137                     self.update_models(self.form.cleaned_data)
 138                     self.update_confirmation()
 139                     self.metastep.set_valid("Step Completed")
 140             else:
 141                 self.metastep.set_invalid("Please complete the fields highlighted in red to continue")
 142                 pass
 143         except Exception as e:
 144             self.metastep.set_invalid(str(e))
 145         self.context = self.get_context()
 146         return render(request, self.template, self.context)
 147
 148
 149 class Define_Nets(WorkflowStep):
 150     template = 'resource/steps/pod_definition.html'
 151     title = "Define Networks"
 152     description = "Use the tool below to draw the network topology of your POD"
 153     short_title = "networking"
 154     form = NetworkDefinitionForm
 155
 156     def get_vlans(self):
 157         vlans = self.repo_get(self.repo.VLANS)
 158         if vlans:
 159             return vlans
 160         # try to grab some vlans from lab
 161         models = self.repo_get(self.repo.GRESOURCE_BUNDLE_MODELS, {})
 162         if "bundle" not in models:
 163             return None
 164         lab = models['bundle'].lab
 165         if lab is None or lab.vlan_manager is None:
 166             return None
 167         try:
 168             vlans = lab.vlan_manager.get_vlan(count=lab.vlan_manager.block_size)
 169             self.repo_put(self.repo.VLANS, vlans)
 170             return vlans
 171         except Exception:
 172             return None
 173
 174     def get_context(self):
 175         # TODO: render *primarily* on hosts in repo models
 176         context = super(Define_Nets, self).get_context()
 177         context['form'] = NetworkDefinitionForm()
 178         try:
 179             context['hosts'] = []
 180             models = self.repo_get(self.repo.GRESOURCE_BUNDLE_MODELS, {})
 181             vlans = self.get_vlans()
 182             if vlans:
 183                 context['vlans'] = vlans
 184             hosts = models.get("hosts", [])
 185             hostlist = self.repo_get(self.repo.GRB_LAST_HOSTLIST, None)
 186             added_list = []
 187             added_dict = {}
 188             context['added_hosts'] = []
 189             if hostlist is not None:
 190                 new_hostlist = []
 191                 for host in models['hosts']:
 192                     intcount = host.profile.interfaceprofile.count()
 193                     new_hostlist.append(str(host.resource.name) + "*" + str(host.profile) + "&" + str(intcount))
 194                 context['removed_hosts'] = list(set(hostlist) - set(new_hostlist))
 195                 added_list = list(set(new_hostlist) - set(hostlist))
 196                 for hoststr in added_list:
 197                     key = hoststr.split("*")[0]
 198                     added_dict[key] = hoststr
 199             for generic_host in hosts:
 200                 host_profile = generic_host.profile
 201                 host = {}
 202                 host['id'] = generic_host.resource.name
 203                 host['interfaces'] = []
 204                 for iface in host_profile.interfaceprofile.all():
 205                     host['interfaces'].append(
 206                         {
 207                             "name": iface.name,
 208                             "description": "speed: " + str(iface.speed) + "M\ntype: " + iface.nic_type
 209                         }
 210                     )
 211                 host['value'] = {"name": generic_host.resource.name}
 212                 host['value']['description'] = generic_host.profile.description
 213                 context['hosts'].append(json.dumps(host))
 214                 if host['id'] in added_dict:
 215                     context['added_hosts'].append(json.dumps(host))
 216             bundle = models.get("bundle", False)
 217             if bundle and bundle.xml:
 218                 context['xml'] = bundle.xml
 219             else:
 220                 context['xml'] = False
 221
 222         except Exception:
 223             pass
 224
 225         return context
 226
 227     def post_render(self, request):
 228         models = self.repo_get(self.repo.GRESOURCE_BUNDLE_MODELS, {})
 229         if 'hosts' in models:
 230             hostlist = []
 231             for host in models['hosts']:
 232                 intcount = host.profile.interfaceprofile.count()
 233                 hostlist.append(str(host.resource.name) + "*" + str(host.profile) + "&" + str(intcount))
 234             self.repo_put(self.repo.GRB_LAST_HOSTLIST, hostlist)
 235         try:
 236             xmlData = request.POST.get("xml")
 237             self.updateModels(xmlData)
 238             # update model with xml
 239             self.metastep.set_valid("Networks applied successfully")
 240         except ResourceAvailabilityException:
 241             self.metastep.set_invalid("Public network not availble")
 242         except Exception:
 243             self.metastep.set_invalid("An error occurred when applying networks")
 244         return self.render(request)
 245
 246     def updateModels(self, xmlData):
 247         models = self.repo_get(self.repo.GRESOURCE_BUNDLE_MODELS, {})
 248         models["vlans"] = {}
 249         given_hosts, interfaces = self.parseXml(xmlData)
 250         vlan_manager = models['bundle'].lab.vlan_manager
 251         existing_host_list = models.get("hosts", [])
 252         existing_hosts = {}  # maps id to host
 253         for host in existing_host_list:
 254             existing_hosts[host.resource.name] = host
 255
 256         bundle = models.get("bundle", GenericResourceBundle(owner=self.repo_get(self.repo.SESSION_USER)))
 257
 258         for hostid, given_host in given_hosts.items():
 259             existing_host = existing_hosts[hostid[5:]]
 260
 261             for ifaceId in given_host['interfaces']:
 262                 iface = interfaces[ifaceId]
 263                 if existing_host.resource.name not in models['vlans']:
 264                     models['vlans'][existing_host.resource.name] = {}
 265                 models['vlans'][existing_host.resource.name][iface['profile_name']] = []
 266                 for network in iface['networks']:
 267                     vlan_id = network['network']['vlan']
 268                     is_public = network['network']['public']
 269                     if is_public:
 270                         public_net = vlan_manager.get_public_vlan()
 271                         if public_net is None:
 272                             raise ResourceAvailabilityException("No public networks available")
 273                         vlan_id = vlan_manager.get_public_vlan().vlan
 274                     vlan = Vlan(vlan_id=vlan_id, tagged=network['tagged'], public=is_public)
 275                     models['vlans'][existing_host.resource.name][iface['profile_name']].append(vlan)
 276         bundle.xml = xmlData
 277         self.repo_put(self.repo.GRESOURCE_BUNDLE_MODELS, models)
 278
 279     # serialize and deserialize xml from mxGraph
 280     def parseXml(self, xmlString):
 281         parent_nets = {}  # map network ports to networks
 282         networks = {}  # maps net id to network object
 283         hosts = {}  # cotains id -> hosts, each containing interfaces, referencing networks
 284         interfaces = {}  # maps id -> interface
 285         xmlDom = minidom.parseString(xmlString)
 286         root = xmlDom.documentElement.firstChild
 287         netids = {}
 288         untagged_ints = {}
 289         for cell in root.childNodes:
 290             cellId = cell.getAttribute('id')
 291
 292             if cell.getAttribute("edge"):
 293                 # cell is a network connection
 294                 escaped_json_str = cell.getAttribute("value")
 295                 json_str = escaped_json_str.replace('&quot;', '"')
 296                 attributes = json.loads(json_str)
 297                 tagged = attributes['tagged']
 298                 interface = None
 299                 network = None
 300                 src = cell.getAttribute("source")
 301                 tgt = cell.getAttribute("target")
 302                 if src in parent_nets:
 303                     # src is a network port
 304                     network = networks[parent_nets[src]]
 305                     if tgt in untagged_ints and not tagged:
 306                         raise InvalidVlanConfigurationException("More than one untagged vlan on an interface")
 307                     interface = interfaces[tgt]
 308                     untagged_ints[tgt] = True
 309                 else:
 310                     network = networks[parent_nets[tgt]]
 311                     if src in untagged_ints and not tagged:
 312                         raise InvalidVlanConfigurationException("More than one untagged vlan on an interface")
 313                     interface = interfaces[src]
 314                     untagged_ints[src] = True
 315                 interface['networks'].append({"network": network, "tagged": tagged})
 316
 317             elif "network" in cellId:  # cell is a network
 318                 escaped_json_str = cell.getAttribute("value")
 319                 json_str = escaped_json_str.replace('&quot;', '"')
 320                 net_info = json.loads(json_str)
 321                 nid = net_info['vlan_id']
 322                 public = net_info['public']
 323                 try:
 324                     int_netid = int(nid)
 325                     assert public or int_netid > 1, "Net id is 1 or lower"
 326                     assert int_netid < 4095, "Net id is 4095 or greater"
 327                 except Exception:
 328                     raise InvalidVlanConfigurationException("VLAN ID is not an integer more than 1 and less than 4095")
 329                 if nid in netids:
 330                     raise NetworkExistsException("Non unique network id found")
 331                 else:
 332                     pass
 333                 network = {"name": net_info['name'], "vlan": net_info['vlan_id'], "public": public}
 334                 netids[net_info['vlan_id']] = True
 335                 networks[cellId] = network
 336
 337             elif "host" in cellId:  # cell is a host/machine
 338                 # TODO gather host info
 339                 cell_json_str = cell.getAttribute("value")
 340                 cell_json = json.loads(cell_json_str)
 341                 host = {"interfaces": [], "name": cellId, "profile_name": cell_json['name']}
 342                 hosts[cellId] = host
 343
 344             elif cell.hasAttribute("parent"):
 345                 parentId = cell.getAttribute('parent')
 346                 if "network" in parentId:
 347                     parent_nets[cellId] = parentId
 348                 elif "host" in parentId:
 349                     # TODO gather iface info
 350                     cell_json_str = cell.getAttribute("value")
 351                     cell_json = json.loads(cell_json_str)
 352                     iface = {"name": cellId, "networks": [], "profile_name": cell_json['name']}
 353                     hosts[parentId]['interfaces'].append(cellId)
 354                     interfaces[cellId] = iface
 355         return hosts, interfaces
 356
 357
 358 class Resource_Meta_Info(WorkflowStep):
 359     template = 'resource/steps/meta_info.html'
 360     title = "Extra Info"
 361     description = "Please fill out the rest of the information about your resource"
 362     short_title = "pod info"
 363
 364     def get_context(self):
 365         context = super(Resource_Meta_Info, self).get_context()
 366         name = ""
 367         desc = ""
 368         bundle = self.repo_get(self.repo.GRESOURCE_BUNDLE_MODELS, {}).get("bundle", False)
 369         if bundle and bundle.name:
 370             name = bundle.name
 371             desc = bundle.description
 372         context['form'] = ResourceMetaForm(initial={"bundle_name": name, "bundle_description": desc})
 373         return context
 374
 375     def post_render(self, request):
 376         form = ResourceMetaForm(request.POST)
 377         if form.is_valid():
 378             models = self.repo_get(self.repo.GRESOURCE_BUNDLE_MODELS, {})
 379             name = form.cleaned_data['bundle_name']
 380             desc = form.cleaned_data['bundle_description']
 381             bundle = models.get("bundle", GenericResourceBundle(owner=self.repo_get(self.repo.SESSION_USER)))
 382             bundle.name = name
 383             bundle.description = desc
 384             models['bundle'] = bundle
 385             self.repo_put(self.repo.GRESOURCE_BUNDLE_MODELS, models)
 386             confirm = self.repo_get(self.repo.CONFIRMATION)
 387             if "resource" not in confirm:
 388                 confirm['resource'] = {}
 389             confirm_info = confirm['resource']
 390             confirm_info["name"] = name
 391             tmp = desc
 392             if len(tmp) > 60:
 393                 tmp = tmp[:60] + "..."
 394             confirm_info["description"] = tmp
 395             self.repo_put(self.repo.CONFIRMATION, confirm)
 396             self.metastep.set_valid("Step Completed")
 397
 398         else:
 399             self.metastep.set_invalid("Please correct the fields highlighted in red to continue")
 400             pass
 401         return self.render(request)
 402
 403
 404 class Host_Meta_Info(WorkflowStep):
 405     template = "resource/steps/host_info.html"
 406     title = "Host Info"
 407     description = "We need a little bit of information about your chosen machines"
 408     short_title = "host info"
 409
 410     def __init__(self, *args, **kwargs):
 411         super(Host_Meta_Info, self).__init__(*args, **kwargs)
 412         self.formset = formset_factory(GenericHostMetaForm, extra=0)
 413
 414     def get_context(self):
 415         context = super(Host_Meta_Info, self).get_context()
 416         GenericHostFormset = self.formset
 417         models = self.repo_get(self.repo.GRESOURCE_BUNDLE_MODELS, {})
 418         initial_data = []
 419         if "hosts" not in models:
 420             context['error'] = "Please go back and select your hosts"
 421         else:
 422             for host in models['hosts']:
 423                 profile = host.profile.name
 424                 name = host.resource.name
 425                 if not name:
 426                     name = ""
 427                 initial_data.append({"host_profile": profile, "host_name": name})
 428         context['formset'] = GenericHostFormset(initial=initial_data)
 429         return context
 430
 431     def post_render(self, request):
 432         models = self.repo_get(self.repo.GRESOURCE_BUNDLE_MODELS, {})
 433         if 'hosts' not in models:
 434             models['hosts'] = []
 435         hosts = models['hosts']
 436         i = 0
 437         confirm_hosts = []
 438         GenericHostFormset = self.formset
 439         formset = GenericHostFormset(request.POST)
 440         if formset.is_valid():
 441             for form in formset:
 442                 host = hosts[i]
 443                 host.resource.name = form.cleaned_data['host_name']
 444                 i += 1
 445                 confirm_hosts.append({"name": host.resource.name, "profile": host.profile.name})
 446             models['hosts'] = hosts
 447             self.repo_put(self.repo.GRESOURCE_BUNDLE_MODELS, models)
 448             confirm = self.repo_get(self.repo.CONFIRMATION, {})
 449             if "resource" not in confirm:
 450                 confirm['resource'] = {}
 451             confirm['resource']['hosts'] = confirm_hosts
 452             self.repo_put(self.repo.CONFIRMATION, confirm)
 453         else:
 454             pass
 455         return self.render(request)