src/workflow/resource_bundle_workflow.py

   1 ##############################################################################
   2 # Copyright (c) 2018 Parker Berberian, Sawyer Bergeron, and others.
   3 #
   4 # All rights reserved. This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 # http://www.apache.org/licenses/LICENSE-2.0
   8 ##############################################################################
   9
  10
  11 from django.shortcuts import render
  12 from django.forms import formset_factory
  13
  14 import json
  15 import re
  16 from xml.dom import minidom
  17
  18 from workflow.models import WorkflowStep
  19 from account.models import Lab
  20 from workflow.forms import (
  21     HardwareDefinitionForm,
  22     NetworkDefinitionForm,
  23     ResourceMetaForm,
  24     GenericHostMetaForm
  25 )
  26 from resource_inventory.models import (
  27     GenericResourceBundle,
  28     Vlan,
  29     GenericInterface,
  30     GenericHost,
  31     GenericResource,
  32     HostProfile
  33 )
  34 from dashboard.exceptions import (
  35     InvalidVlanConfigurationException,
  36     NetworkExistsException,
  37     InvalidHostnameException,
  38     NonUniqueHostnameException,
  39     ResourceAvailabilityException
  40 )
  41
  42 import logging
  43 logger = logging.getLogger(__name__)
  44
  45
  46 class Define_Hardware(WorkflowStep):
  47
  48     template = 'resource/steps/define_hardware.html'
  49     title = "Define Hardware"
  50     description = "Choose the type and amount of machines you want"
  51     short_title = "hosts"
  52
  53     def get_context(self):
  54         context = super(Define_Hardware, self).get_context()
  55         selection_data = {"hosts": {}, "labs": {}}
  56         models = self.repo_get(self.repo.GRESOURCE_BUNDLE_MODELS, {})
  57         hosts = models.get("hosts", [])
  58         for host in hosts:
  59             profile_id = "host_" + str(host.profile.id)
  60             if profile_id not in selection_data['hosts']:
  61                 selection_data['hosts'][profile_id] = []
  62             selection_data['hosts'][profile_id].append({"host_name": host.resource.name, "class": profile_id})
  63
  64         if models.get("bundle", GenericResourceBundle()).lab:
  65             selection_data['labs'] = {"lab_" + str(models.get("bundle").lab.lab_user.id): "true"}
  66
  67         form = HardwareDefinitionForm(
  68             selection_data=selection_data
  69         )
  70         context['form'] = form
  71         return context
  72
  73     def render(self, request):
  74         self.context = self.get_context()
  75         return render(request, self.template, self.context)
  76
  77     def update_models(self, data):
  78         data = json.loads(data['filter_field'])
  79         models = self.repo_get(self.repo.GRESOURCE_BUNDLE_MODELS, {})
  80         models['hosts'] = []  # This will always clear existing data when this step changes
  81         models['interfaces'] = {}
  82         if "bundle" not in models:
  83             models['bundle'] = GenericResourceBundle(owner=self.repo_get(self.repo.SESSION_USER))
  84         host_data = data['hosts']
  85         names = {}
  86         for host_dict in host_data:
  87             id = host_dict['class']
  88             # bit of formatting
  89             id = int(id.split("_")[-1])
  90             profile = HostProfile.objects.get(id=id)
  91             # instantiate genericHost and store in repo
  92             name = host_dict['host_name']
  93             if not re.match(r"(?=^.{1,253}$)(^([A-Za-z0-9-_]{1,62}\.)*[A-Za-z0-9-_]{1,63})", name):
  94                 raise InvalidHostnameException("Hostname must comply to RFC 952 and all extensions to it until this point")
  95             if name in names:
  96                 raise NonUniqueHostnameException("All hosts must have unique names")
  97             names[name] = True
  98             genericResource = GenericResource(bundle=models['bundle'], name=name)
  99             genericHost = GenericHost(profile=profile, resource=genericResource)
 100             models['hosts'].append(genericHost)
 101             for interface_profile in profile.interfaceprofile.all():
 102                 genericInterface = GenericInterface(profile=interface_profile, host=genericHost)
 103                 if genericHost.resource.name not in models['interfaces']:
 104                     models['interfaces'][genericHost.resource.name] = []
 105                 models['interfaces'][genericHost.resource.name].append(genericInterface)
 106
 107         # add selected lab to models
 108         for lab_dict in data['labs']:
 109             if list(lab_dict.values())[0]:  # True for lab the user selected
 110                 lab_user_id = int(list(lab_dict.keys())[0].split("_")[-1])
 111                 models['bundle'].lab = Lab.objects.get(lab_user__id=lab_user_id)
 112                 break  # if somehow we get two 'true' labs, we only use one
 113
 114         # return to repo
 115         self.repo_put(self.repo.GRESOURCE_BUNDLE_MODELS, models)
 116
 117     def update_confirmation(self):
 118         confirm = self.repo_get(self.repo.CONFIRMATION, {})
 119         if "resource" not in confirm:
 120             confirm['resource'] = {}
 121         confirm['resource']['hosts'] = []
 122         models = self.repo_get(self.repo.GRESOURCE_BUNDLE_MODELS, {"hosts": []})
 123         for host in models['hosts']:
 124             host_dict = {"name": host.resource.name, "profile": host.profile.name}
 125             confirm['resource']['hosts'].append(host_dict)
 126         if "lab" in models:
 127             confirm['resource']['lab'] = models['lab'].lab_user.username
 128         self.repo_put(self.repo.CONFIRMATION, confirm)
 129
 130     def post_render(self, request):
 131         try:
 132             self.form = HardwareDefinitionForm(request.POST)
 133             if self.form.is_valid():
 134                 self.update_models(self.form.cleaned_data)
 135                 self.update_confirmation()
 136                 self.metastep.set_valid("Step Completed")
 137             else:
 138                 self.metastep.set_invalid("Please complete the fields highlighted in red to continue")
 139                 pass
 140         except Exception as e:
 141             self.metastep.set_invalid(str(e))
 142         self.context = self.get_context()
 143         return render(request, self.template, self.context)
 144
 145
 146 class Define_Nets(WorkflowStep):
 147     template = 'resource/steps/pod_definition.html'
 148     title = "Define Networks"
 149     description = "Use the tool below to draw the network topology of your POD"
 150     short_title = "networking"
 151     form = NetworkDefinitionForm
 152
 153     def get_vlans(self):
 154         vlans = self.repo_get(self.repo.VLANS)
 155         if vlans:
 156             return vlans
 157         # try to grab some vlans from lab
 158         models = self.repo_get(self.repo.GRESOURCE_BUNDLE_MODELS, {})
 159         if "bundle" not in models:
 160             return None
 161         lab = models['bundle'].lab
 162         if lab is None or lab.vlan_manager is None:
 163             return None
 164         try:
 165             vlans = lab.vlan_manager.get_vlan(count=lab.vlan_manager.block_size)
 166             self.repo_put(self.repo.VLANS, vlans)
 167             return vlans
 168         except Exception:
 169             return None
 170
 171     def get_context(self):
 172         # TODO: render *primarily* on hosts in repo models
 173         context = super(Define_Nets, self).get_context()
 174         context['form'] = NetworkDefinitionForm()
 175         try:
 176             context['hosts'] = []
 177             models = self.repo_get(self.repo.GRESOURCE_BUNDLE_MODELS, {})
 178             vlans = self.get_vlans()
 179             if vlans:
 180                 context['vlans'] = vlans
 181             hosts = models.get("hosts", [])
 182             hostlist = self.repo_get(self.repo.GRB_LAST_HOSTLIST, None)
 183             added_list = []
 184             added_dict = {}
 185             context['added_hosts'] = []
 186             if hostlist is not None:
 187                 new_hostlist = []
 188                 for host in models['hosts']:
 189                     intcount = host.profile.interfaceprofile.count()
 190                     new_hostlist.append(str(host.resource.name) + "*" + str(host.profile) + "&" + str(intcount))
 191                 context['removed_hosts'] = list(set(hostlist) - set(new_hostlist))
 192                 added_list = list(set(new_hostlist) - set(hostlist))
 193                 for hoststr in added_list:
 194                     key = hoststr.split("*")[0]
 195                     added_dict[key] = hoststr
 196             for generic_host in hosts:
 197                 host_profile = generic_host.profile
 198                 host = {}
 199                 host['id'] = generic_host.resource.name
 200                 host['interfaces'] = []
 201                 for iface in host_profile.interfaceprofile.all():
 202                     host['interfaces'].append(
 203                         {
 204                             "name": iface.name,
 205                             "description": "speed: " + str(iface.speed) + "M\ntype: " + iface.nic_type
 206                         }
 207                     )
 208                 host['value'] = {"name": generic_host.resource.name}
 209                 host['value']['description'] = generic_host.profile.description
 210                 context['hosts'].append(json.dumps(host))
 211                 if host['id'] in added_dict:
 212                     context['added_hosts'].append(json.dumps(host))
 213             bundle = models.get("bundle", False)
 214             if bundle and bundle.xml:
 215                 context['xml'] = bundle.xml
 216             else:
 217                 context['xml'] = False
 218
 219         except Exception:
 220             pass
 221
 222         return context
 223
 224     def post_render(self, request):
 225         models = self.repo_get(self.repo.GRESOURCE_BUNDLE_MODELS, {})
 226         if 'hosts' in models:
 227             hostlist = []
 228             for host in models['hosts']:
 229                 intcount = host.profile.interfaceprofile.count()
 230                 hostlist.append(str(host.resource.name) + "*" + str(host.profile) + "&" + str(intcount))
 231             self.repo_put(self.repo.GRB_LAST_HOSTLIST, hostlist)
 232         try:
 233             xmlData = request.POST.get("xml")
 234             self.updateModels(xmlData)
 235             # update model with xml
 236             self.metastep.set_valid("Networks applied successfully")
 237         except ResourceAvailabilityException:
 238             self.metastep.set_invalid("Public network not availble")
 239         except Exception:
 240             self.metastep.set_invalid("An error occurred when applying networks")
 241         return self.render(request)
 242
 243     def updateModels(self, xmlData):
 244         models = self.repo_get(self.repo.GRESOURCE_BUNDLE_MODELS, {})
 245         models["vlans"] = {}
 246         given_hosts, interfaces = self.parseXml(xmlData)
 247         vlan_manager = models['bundle'].lab.vlan_manager
 248         existing_host_list = models.get("hosts", [])
 249         existing_hosts = {}  # maps id to host
 250         for host in existing_host_list:
 251             existing_hosts[host.resource.name] = host
 252
 253         bundle = models.get("bundle", GenericResourceBundle(owner=self.repo_get(self.repo.SESSION_USER)))
 254
 255         for hostid, given_host in given_hosts.items():
 256             existing_host = existing_hosts[hostid[5:]]
 257
 258             for ifaceId in given_host['interfaces']:
 259                 iface = interfaces[ifaceId]
 260                 if existing_host.resource.name not in models['vlans']:
 261                     models['vlans'][existing_host.resource.name] = {}
 262                 models['vlans'][existing_host.resource.name][iface['profile_name']] = []
 263                 for network in iface['networks']:
 264                     vlan_id = network['network']['vlan']
 265                     is_public = network['network']['public']
 266                     if is_public:
 267                         public_net = vlan_manager.get_public_vlan()
 268                         if public_net is None:
 269                             raise ResourceAvailabilityException("No public networks available")
 270                         vlan_id = vlan_manager.get_public_vlan().vlan
 271                     vlan = Vlan(vlan_id=vlan_id, tagged=network['tagged'], public=is_public)
 272                     models['vlans'][existing_host.resource.name][iface['profile_name']].append(vlan)
 273         bundle.xml = xmlData
 274         self.repo_put(self.repo.GRESOURCE_BUNDLE_MODELS, models)
 275
 276     # serialize and deserialize xml from mxGraph
 277     def parseXml(self, xmlString):
 278         parent_nets = {}  # map network ports to networks
 279         networks = {}  # maps net id to network object
 280         hosts = {}  # cotains id -> hosts, each containing interfaces, referencing networks
 281         interfaces = {}  # maps id -> interface
 282         xmlDom = minidom.parseString(xmlString)
 283         root = xmlDom.documentElement.firstChild
 284         netids = {}
 285         untagged_ints = {}
 286         for cell in root.childNodes:
 287             cellId = cell.getAttribute('id')
 288
 289             if cell.getAttribute("edge"):
 290                 # cell is a network connection
 291                 escaped_json_str = cell.getAttribute("value")
 292                 json_str = escaped_json_str.replace('&quot;', '"')
 293                 attributes = json.loads(json_str)
 294                 tagged = attributes['tagged']
 295                 interface = None
 296                 network = None
 297                 src = cell.getAttribute("source")
 298                 tgt = cell.getAttribute("target")
 299                 if src in parent_nets:
 300                     # src is a network port
 301                     network = networks[parent_nets[src]]
 302                     if tgt in untagged_ints and not tagged:
 303                         raise InvalidVlanConfigurationException("More than one untagged vlan on an interface")
 304                     interface = interfaces[tgt]
 305                     untagged_ints[tgt] = True
 306                 else:
 307                     network = networks[parent_nets[tgt]]
 308                     if src in untagged_ints and not tagged:
 309                         raise InvalidVlanConfigurationException("More than one untagged vlan on an interface")
 310                     interface = interfaces[src]
 311                     untagged_ints[src] = True
 312                 interface['networks'].append({"network": network, "tagged": tagged})
 313
 314             elif "network" in cellId:  # cell is a network
 315                 escaped_json_str = cell.getAttribute("value")
 316                 json_str = escaped_json_str.replace('&quot;', '"')
 317                 net_info = json.loads(json_str)
 318                 nid = net_info['vlan_id']
 319                 public = net_info['public']
 320                 try:
 321                     int_netid = int(nid)
 322                     assert public or int_netid > 1, "Net id is 1 or lower"
 323                     assert int_netid < 4095, "Net id is 4095 or greater"
 324                 except Exception:
 325                     raise InvalidVlanConfigurationException("VLAN ID is not an integer more than 1 and less than 4095")
 326                 if nid in netids:
 327                     raise NetworkExistsException("Non unique network id found")
 328                 else:
 329                     pass
 330                 network = {"name": net_info['name'], "vlan": net_info['vlan_id'], "public": public}
 331                 netids[net_info['vlan_id']] = True
 332                 networks[cellId] = network
 333
 334             elif "host" in cellId:  # cell is a host/machine
 335                 # TODO gather host info
 336                 cell_json_str = cell.getAttribute("value")
 337                 cell_json = json.loads(cell_json_str)
 338                 host = {"interfaces": [], "name": cellId, "profile_name": cell_json['name']}
 339                 hosts[cellId] = host
 340
 341             elif cell.hasAttribute("parent"):
 342                 parentId = cell.getAttribute('parent')
 343                 if "network" in parentId:
 344                     parent_nets[cellId] = parentId
 345                 elif "host" in parentId:
 346                     # TODO gather iface info
 347                     cell_json_str = cell.getAttribute("value")
 348                     cell_json = json.loads(cell_json_str)
 349                     iface = {"name": cellId, "networks": [], "profile_name": cell_json['name']}
 350                     hosts[parentId]['interfaces'].append(cellId)
 351                     interfaces[cellId] = iface
 352         return hosts, interfaces
 353
 354
 355 class Resource_Meta_Info(WorkflowStep):
 356     template = 'resource/steps/meta_info.html'
 357     title = "Extra Info"
 358     description = "Please fill out the rest of the information about your resource"
 359     short_title = "pod info"
 360
 361     def get_context(self):
 362         context = super(Resource_Meta_Info, self).get_context()
 363         name = ""
 364         desc = ""
 365         bundle = self.repo_get(self.repo.GRESOURCE_BUNDLE_MODELS, {}).get("bundle", False)
 366         if bundle and bundle.name:
 367             name = bundle.name
 368             desc = bundle.description
 369         context['form'] = ResourceMetaForm(initial={"bundle_name": name, "bundle_description": desc})
 370         return context
 371
 372     def post_render(self, request):
 373         form = ResourceMetaForm(request.POST)
 374         if form.is_valid():
 375             models = self.repo_get(self.repo.GRESOURCE_BUNDLE_MODELS, {})
 376             name = form.cleaned_data['bundle_name']
 377             desc = form.cleaned_data['bundle_description']
 378             bundle = models.get("bundle", GenericResourceBundle(owner=self.repo_get(self.repo.SESSION_USER)))
 379             bundle.name = name
 380             bundle.description = desc
 381             models['bundle'] = bundle
 382             self.repo_put(self.repo.GRESOURCE_BUNDLE_MODELS, models)
 383             confirm = self.repo_get(self.repo.CONFIRMATION)
 384             if "resource" not in confirm:
 385                 confirm['resource'] = {}
 386             confirm_info = confirm['resource']
 387             confirm_info["name"] = name
 388             tmp = desc
 389             if len(tmp) > 60:
 390                 tmp = tmp[:60] + "..."
 391             confirm_info["description"] = tmp
 392             self.repo_put(self.repo.CONFIRMATION, confirm)
 393             self.metastep.set_valid("Step Completed")
 394
 395         else:
 396             self.metastep.set_invalid("Please correct the fields highlighted in red to continue")
 397             pass
 398         return self.render(request)
 399
 400
 401 class Host_Meta_Info(WorkflowStep):
 402     template = "resource/steps/host_info.html"
 403     title = "Host Info"
 404     description = "We need a little bit of information about your chosen machines"
 405     short_title = "host info"
 406
 407     def __init__(self, *args, **kwargs):
 408         super(Host_Meta_Info, self).__init__(*args, **kwargs)
 409         self.formset = formset_factory(GenericHostMetaForm, extra=0)
 410
 411     def get_context(self):
 412         context = super(Host_Meta_Info, self).get_context()
 413         GenericHostFormset = self.formset
 414         models = self.repo_get(self.repo.GRESOURCE_BUNDLE_MODELS, {})
 415         initial_data = []
 416         if "hosts" not in models:
 417             context['error'] = "Please go back and select your hosts"
 418         else:
 419             for host in models['hosts']:
 420                 profile = host.profile.name
 421                 name = host.resource.name
 422                 if not name:
 423                     name = ""
 424                 initial_data.append({"host_profile": profile, "host_name": name})
 425         context['formset'] = GenericHostFormset(initial=initial_data)
 426         return context
 427
 428     def post_render(self, request):
 429         models = self.repo_get(self.repo.GRESOURCE_BUNDLE_MODELS, {})
 430         if 'hosts' not in models:
 431             models['hosts'] = []
 432         hosts = models['hosts']
 433         i = 0
 434         confirm_hosts = []
 435         GenericHostFormset = self.formset
 436         formset = GenericHostFormset(request.POST)
 437         if formset.is_valid():
 438             for form in formset:
 439                 host = hosts[i]
 440                 host.resource.name = form.cleaned_data['host_name']
 441                 i += 1
 442                 confirm_hosts.append({"name": host.resource.name, "profile": host.profile.name})
 443             models['hosts'] = hosts
 444             self.repo_put(self.repo.GRESOURCE_BUNDLE_MODELS, models)
 445             confirm = self.repo_get(self.repo.CONFIRMATION, {})
 446             if "resource" not in confirm:
 447                 confirm['resource'] = {}
 448             confirm['resource']['hosts'] = confirm_hosts
 449             self.repo_put(self.repo.CONFIRMATION, confirm)
 450         else:
 451             pass
 452         return self.render(request)