dashboard/src/workflow/models.py

   1 ##############################################################################
   2 # Copyright (c) 2018 Sawyer Bergeron, Parker Berberian, and others.
   3 #
   4 # All rights reserved. This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 # http://www.apache.org/licenses/LICENSE-2.0
   8 ##############################################################################
   9
  10
  11 from django.shortcuts import render
  12 from django.contrib import messages
  13 from django.http import HttpResponse
  14
  15 import yaml
  16 import requests
  17
  18 from workflow.forms import ConfirmationForm
  19 from api.models import JobFactory
  20 from dashboard.exceptions import ResourceAvailabilityException, ModelValidationException
  21 from resource_inventory.models import Image, GenericInterface
  22 from resource_inventory.resource_manager import ResourceManager
  23 from notifier.manager import NotificationHandler
  24 from booking.models import Booking
  25
  26
  27 class BookingAuthManager():
  28     LFN_PROJECTS = ["opnfv"]  # TODO
  29
  30     def parse_github_url(self, url):
  31         project_leads = []
  32         try:
  33             parts = url.split("/")
  34             if "http" in parts[0]:  # the url include http(s)://
  35                 parts = parts[2:]
  36             if parts[-1] != "INFO.yaml":
  37                 return None
  38             if parts[0] not in ["github.com", "raw.githubusercontent.com"]:
  39                 return None
  40             if parts[1] not in self.LFN_PROJECTS:
  41                 return None
  42             # now to download and parse file
  43             if parts[3] == "blob":
  44                 parts[3] = "raw"
  45             url = "https://" + "/".join(parts)
  46             info_file = requests.get(url, timeout=15).text
  47             info_parsed = yaml.load(info_file)
  48             ptl = info_parsed.get('project_lead')
  49             if ptl:
  50                 project_leads.append(ptl)
  51             sub_ptl = info_parsed.get("subproject_lead")
  52             if sub_ptl:
  53                 project_leads.append(sub_ptl)
  54
  55         except Exception:
  56             pass
  57
  58         return project_leads
  59
  60     def parse_gerrit_url(self, url):
  61         project_leads = []
  62         try:
  63             parts = url.split("/")
  64             if "http" in parts[0]:  # the url include http(s)://
  65                 parts = parts[2:]
  66             if "f=INFO.yaml" not in parts[-1].split(";"):
  67                 return None
  68             if "gerrit.opnfv.org" not in parts[0]:
  69                 return None
  70             # now to download and parse file
  71             url = "https://" + "/".join(parts)
  72             info_file = requests.get(url, timeout=15).text
  73             info_parsed = yaml.load(info_file)
  74             ptl = info_parsed.get('project_lead')
  75             if ptl:
  76                 project_leads.append(ptl)
  77             sub_ptl = info_parsed.get("subproject_lead")
  78             if sub_ptl:
  79                 project_leads.append(sub_ptl)
  80
  81         except Exception:
  82             return None
  83
  84         return project_leads
  85
  86     def parse_opnfv_git_url(self, url):
  87         project_leads = []
  88         try:
  89             parts = url.split("/")
  90             if "http" in parts[0]:  # the url include http(s)://
  91                 parts = parts[2:]
  92             if "INFO.yaml" not in parts[-1]:
  93                 return None
  94             if "git.opnfv.org" not in parts[0]:
  95                 return None
  96             if parts[-2] == "tree":
  97                 parts[-2] = "plain"
  98             # now to download and parse file
  99             url = "https://" + "/".join(parts)
 100             info_file = requests.get(url, timeout=15).text
 101             info_parsed = yaml.load(info_file)
 102             ptl = info_parsed.get('project_lead')
 103             if ptl:
 104                 project_leads.append(ptl)
 105             sub_ptl = info_parsed.get("subproject_lead")
 106             if sub_ptl:
 107                 project_leads.append(sub_ptl)
 108
 109         except Exception:
 110             return None
 111
 112         return project_leads
 113
 114     def parse_url(self, info_url):
 115         """
 116         will return the PTL in the INFO file on success, or None
 117         """
 118         if "github" in info_url:
 119             return self.parse_github_url(info_url)
 120
 121         if "gerrit.opnfv.org" in info_url:
 122             return self.parse_gerrit_url(info_url)
 123
 124         if "git.opnfv.org" in info_url:
 125             return self.parse_opnfv_git_url(info_url)
 126
 127     def booking_allowed(self, booking, repo):
 128         """
 129         This is the method that will have to change whenever the booking policy changes in the Infra
 130         group / LFN. This is a nice isolation of that administration crap
 131         currently checks if the booking uses multiple servers. if it does, then the owner must be a PTL,
 132         which is checked using the provided info file
 133         """
 134         if len(booking.resource.template.getHosts()) < 2:
 135             return True  # if they only have one server, we dont care
 136         if booking.owner.userprofile.booking_privledge:
 137             return True  # admin override for this user
 138         if repo.BOOKING_INFO_FILE not in repo.el:
 139             return False  # INFO file not provided
 140         ptl_info = self.parse_url(repo.BOOKING_INFO_FILE)
 141         return ptl_info and ptl_info == booking.owner.userprofile.email_addr
 142
 143
 144 class WorkflowStep(object):
 145     template = 'bad_request.html'
 146     title = "Generic Step"
 147     description = "You were led here by mistake"
 148     short_title = "error"
 149     metastep = None
 150
 151     def __init__(self, id, repo=None):
 152         self.repo = repo
 153         self.id = id
 154
 155     def get_context(self):
 156         context = {}
 157         context['step_number'] = self.repo_get('steps')
 158         context['active_step'] = self.repo_get('active_step')
 159         context['render_correct'] = "true"
 160         context['step_title'] = self.title
 161         context['description'] = self.description
 162         return context
 163
 164     def render(self, request):
 165         self.context = self.get_context()
 166         return render(request, self.template, self.context)
 167
 168     def post_render(self, request):
 169         return self.render(request)
 170
 171     def test_render(self, request):
 172         if request.method == "POST":
 173             return self.post_render(request)
 174         return self.render(request)
 175
 176     def validate(self, request):
 177         pass
 178
 179     def repo_get(self, key, default=None):
 180         return self.repo.get(key, default, self.id)
 181
 182     def repo_put(self, key, value):
 183         return self.repo.put(key, value, self.id)
 184
 185
 186 class Confirmation_Step(WorkflowStep):
 187     template = 'workflow/confirm.html'
 188     title = "Confirm Changes"
 189     description = "Does this all look right?"
 190
 191     def get_vlan_warning(self):
 192         grb = self.repo_get(self.repo.SELECTED_GRESOURCE_BUNDLE, False)
 193         if not grb:
 194             return 0
 195         if self.repo.BOOKING_MODELS not in self.repo.el:
 196             return 0
 197         vlan_manager = grb.lab.vlan_manager
 198         if vlan_manager is None:
 199             return 0
 200         hosts = grb.getHosts()
 201         for host in hosts:
 202             for interface in host.generic_interfaces.all():
 203                 for vlan in interface.vlans.all():
 204                     if vlan.public:
 205                         if not vlan_manager.public_vlan_is_available(vlan.vlan_id):
 206                             return 1
 207                     else:
 208                         if not vlan_manager.is_available(vlan.vlan_id):
 209                             return 1  # There is a problem with these vlans
 210         return 0
 211
 212     def get_context(self):
 213         context = super(Confirmation_Step, self).get_context()
 214         context['form'] = ConfirmationForm()
 215         context['confirmation_info'] = yaml.dump(
 216             self.repo_get(self.repo.CONFIRMATION),
 217             default_flow_style=False
 218         ).strip()
 219         context['vlan_warning'] = self.get_vlan_warning()
 220
 221         return context
 222
 223     def flush_to_db(self):
 224         errors = self.repo.make_models()
 225         if errors:
 226             return errors
 227
 228     def post_render(self, request):
 229         form = ConfirmationForm(request.POST)
 230         if form.is_valid():
 231             data = form.cleaned_data['confirm']
 232             context = self.get_context()
 233             if data == "True":
 234                 context["bypassed"] = "true"
 235                 errors = self.flush_to_db()
 236                 if errors:
 237                     messages.add_message(request, messages.ERROR, "ERROR OCCURRED: " + errors)
 238                 else:
 239                     messages.add_message(request, messages.SUCCESS, "Confirmed")
 240
 241                 return HttpResponse('')
 242             elif data == "False":
 243                 context["bypassed"] = "true"
 244                 messages.add_message(request, messages.SUCCESS, "Canceled")
 245                 return render(request, self.template, context)
 246             else:
 247                 pass
 248
 249         else:
 250             if "vlan_input" in request.POST:
 251                 if request.POST.get("vlan_input") == "True":
 252                     self.translate_vlans()
 253                     return self.render(request)
 254             pass
 255
 256     def translate_vlans(self):
 257         grb = self.repo_get(self.repo.SELECTED_GRESOURCE_BUNDLE, False)
 258         if not grb:
 259             return 0
 260         vlan_manager = grb.lab.vlan_manager
 261         if vlan_manager is None:
 262             return 0
 263         hosts = grb.getHosts()
 264         for host in hosts:
 265             for interface in host.generic_interfaces.all():
 266                 for vlan in interface.vlans.all():
 267                     if not vlan.public:
 268                         if not vlan_manager.is_available(vlan.vlan_id):
 269                             vlan.vlan_id = vlan_manager.get_vlan()
 270                             vlan.save()
 271                     else:
 272                         if not vlan_manager.public_vlan_is_available(vlan.vlan_id):
 273                             pub_vlan = vlan_manager.get_public_vlan()
 274                             vlan.vlan_id = pub_vlan.vlan
 275                             vlan.save()
 276
 277
 278 class Workflow():
 279
 280     steps = []
 281     active_index = 0
 282
 283
 284 class Repository():
 285
 286     EDIT = "editing"
 287     MODELS = "models"
 288     RESOURCE_SELECT = "resource_select"
 289     CONFIRMATION = "confirmation"
 290     SELECTED_GRESOURCE_BUNDLE = "selected generic bundle pk"
 291     SELECTED_CONFIG_BUNDLE = "selected config bundle pk"
 292     GRESOURCE_BUNDLE_MODELS = "generic_resource_bundle_models"
 293     GRESOURCE_BUNDLE_INFO = "generic_resource_bundle_info"
 294     BOOKING = "booking"
 295     LAB = "lab"
 296     GRB_LAST_HOSTLIST = "grb_network_previous_hostlist"
 297     BOOKING_FORMS = "booking_forms"
 298     SWCONF_HOSTS = "swconf_hosts"
 299     BOOKING_MODELS = "booking models"
 300     CONFIG_MODELS = "configuration bundle models"
 301     SESSION_USER = "session owner user account"
 302     VALIDATED_MODEL_GRB = "valid grb config model instance in db"
 303     VALIDATED_MODEL_CONFIG = "valid config model instance in db"
 304     VALIDATED_MODEL_BOOKING = "valid booking model instance in db"
 305     VLANS = "a list of vlans"
 306     SNAPSHOT_MODELS = "the models for snapshotting"
 307     SNAPSHOT_BOOKING_ID = "the booking id for snapshotting"
 308     SNAPSHOT_NAME = "the name of the snapshot"
 309     SNAPSHOT_DESC = "description of the snapshot"
 310     BOOKING_INFO_FILE = "the INFO.yaml file for this user's booking"
 311
 312     #migratory elements of segmented workflow
 313     #each of these is the end result of a different workflow.
 314     HAS_RESULT = "whether or not workflow has a result"
 315     RESULT_KEY = "key for target index that result will be put into in parent"
 316     RESULT = "result object from workflow"
 317
 318     def get_child_defaults(self):
 319         return_tuples = []
 320         for key in [self.SELECTED_GRESOURCE_BUNDLE, self.SESSION_USER]:
 321             return_tuples.append((key, self.el.get(key)))
 322         return return_tuples
 323
 324     def set_defaults(self, defaults):
 325         for key, value in defaults:
 326             self.el[key] = value
 327
 328     def get(self, key, default, id):
 329         self.add_get_history(key, id)
 330         return self.el.get(key, default)
 331
 332     def put(self, key, val, id):
 333         self.add_put_history(key, id)
 334         self.el[key] = val
 335
 336     def add_get_history(self, key, id):
 337         self.add_history(key, id, self.get_history)
 338
 339     def add_put_history(self, key, id):
 340         self.add_history(key, id, self.put_history)
 341
 342     def add_history(self, key, id, history):
 343         if key not in history:
 344             history[key] = [id]
 345         else:
 346             history[key].append(id)
 347
 348     def make_models(self):
 349         if self.SNAPSHOT_MODELS in self.el:
 350             errors = self.make_snapshot()
 351             if errors:
 352                 return errors
 353         # if GRB WF, create it
 354         if self.GRESOURCE_BUNDLE_MODELS in self.el:
 355             errors = self.make_generic_resource_bundle()
 356             if errors:
 357                 return errors
 358             else:
 359                 self.el[self.HAS_RESULT] = True
 360                 self.el[self.RESULT_KEY] = self.SELECTED_GRESOURCE_BUNDLE
 361                 return
 362
 363         if self.CONFIG_MODELS in self.el:
 364             errors = self.make_software_config_bundle()
 365             if errors:
 366                 return errors
 367             else:
 368                 self.el[self.HAS_RESULT] = True
 369                 self.el[self.RESULT_KEY] = self.SELECTED_CONFIG_BUNDLE
 370                 return
 371
 372         if self.BOOKING_MODELS in self.el:
 373             errors = self.make_booking()
 374             if errors:
 375                 return errors
 376             # create notification
 377             booking = self.el[self.BOOKING_MODELS]['booking']
 378             NotificationHandler.notify_new_booking(booking)
 379
 380     def make_snapshot(self):
 381         owner = self.el[self.SESSION_USER]
 382         models = self.el[self.SNAPSHOT_MODELS]
 383         image = models.get('snapshot', Image())
 384         booking_id = self.el.get(self.SNAPSHOT_BOOKING_ID)
 385         if not booking_id:
 386             return "SNAP, No booking ID provided"
 387         booking = Booking.objects.get(pk=booking_id)
 388         name = self.el.get(self.SNAPSHOT_NAME)
 389         if not name:
 390             return "SNAP, no name provided"
 391         host = models.get('host')
 392         if not host:
 393             return "SNAP, no host provided"
 394         description = self.el.get(self.SNAPSHOT_DESC, "")
 395         image.from_lab = booking.lab
 396         image.name = name
 397         image.description = description
 398         image.public = False
 399         image.lab_id = -1
 400         image.owner = owner
 401         image.host_type = host.profile
 402         image.save()
 403
 404     def make_generic_resource_bundle(self):
 405         owner = self.el[self.SESSION_USER]
 406         if self.GRESOURCE_BUNDLE_MODELS in self.el:
 407             models = self.el[self.GRESOURCE_BUNDLE_MODELS]
 408             if 'hosts' in models:
 409                 hosts = models['hosts']
 410             else:
 411                 return "GRB has no hosts. CODE:0x0002"
 412             if 'bundle' in models:
 413                 bundle = models['bundle']
 414             else:
 415                 return "GRB, no bundle in models. CODE:0x0003"
 416
 417             try:
 418                 bundle.owner = owner
 419                 bundle.save()
 420             except Exception as e:
 421                 return "GRB, saving bundle generated exception: " + str(e) + " CODE:0x0004"
 422             try:
 423                 for host in hosts:
 424                     genericresource = host.resource
 425                     genericresource.bundle = bundle
 426                     genericresource.save()
 427                     host.resource = genericresource
 428                     host.save()
 429             except Exception as e:
 430                 return "GRB, saving hosts generated exception: " + str(e) + " CODE:0x0005"
 431
 432             if 'interfaces' in models:
 433                 for interface_set in models['interfaces'].values():
 434                     for interface in interface_set:
 435                         try:
 436                             interface.host = interface.host
 437                             interface.save()
 438                         except Exception as e:
 439                             return "GRB, saving interface " + str(interface) + " failed. CODE:0x0019"
 440             else:
 441                 return "GRB, no interface set provided. CODE:0x001a"
 442
 443             if 'vlans' in models:
 444                 for resource_name, mapping in models['vlans'].items():
 445                     for profile_name, vlan_set in mapping.items():
 446                         interface = GenericInterface.objects.get(
 447                             profile__name=profile_name,
 448                             host__resource__name=resource_name,
 449                             host__resource__bundle=models['bundle']
 450                         )
 451                         for vlan in vlan_set:
 452                             try:
 453                                 vlan.save()
 454                                 interface.vlans.add(vlan)
 455                             except Exception as e:
 456                                 return "GRB, saving vlan " + str(vlan) + " failed. Exception: " + str(e) + ". CODE:0x0017"
 457             else:
 458                 return "GRB, no vlan set provided. CODE:0x0018"
 459
 460         else:
 461             return "GRB no models given. CODE:0x0001"
 462
 463         self.el[self.RESULT] = bundle
 464         return False
 465
 466     def make_software_config_bundle(self):
 467         models = self.el[self.CONFIG_MODELS]
 468         if 'bundle' in models:
 469             bundle = models['bundle']
 470             bundle.bundle = bundle.bundle
 471             try:
 472                 bundle.save()
 473             except Exception as e:
 474                 return "SWC, saving bundle generated exception: " + str(e) + "CODE:0x0007"
 475
 476         else:
 477             return "SWC, no bundle in models. CODE:0x0006"
 478         if 'host_configs' in models:
 479             host_configs = models['host_configs']
 480             for host_config in host_configs:
 481                 host_config.bundle = host_config.bundle
 482                 host_config.host = host_config.host
 483                 try:
 484                     host_config.save()
 485                 except Exception as e:
 486                     return "SWC, saving host configs generated exception: " + str(e) + "CODE:0x0009"
 487         else:
 488             return "SWC, no host configs in models. CODE:0x0008"
 489         if 'opnfv' in models:
 490             opnfvconfig = models['opnfv']
 491             opnfvconfig.bundle = opnfvconfig.bundle
 492             if opnfvconfig.scenario not in opnfvconfig.installer.sup_scenarios.all():
 493                 return "SWC, scenario not supported by installer. CODE:0x000d"
 494             try:
 495                 opnfvconfig.save()
 496             except Exception as e:
 497                 return "SWC, saving opnfv config generated exception: " + str(e) + "CODE:0x000b"
 498         else:
 499             pass
 500
 501         self.el[self.RESULT] = bundle
 502         return False
 503
 504     def make_booking(self):
 505         models = self.el[self.BOOKING_MODELS]
 506         owner = self.el[self.SESSION_USER]
 507
 508         if self.SELECTED_GRESOURCE_BUNDLE in self.el:
 509             selected_grb = self.el[self.SELECTED_GRESOURCE_BUNDLE]
 510         else:
 511             return "BOOK, no selected resource. CODE:0x000e"
 512
 513         if not self.reserve_vlans(selected_grb):
 514             return "BOOK, vlans not available"
 515
 516         if 'booking' in models:
 517             booking = models['booking']
 518         else:
 519             return "BOOK, no booking model exists. CODE:0x000f"
 520
 521         if not booking.start:
 522             return "BOOK, booking has no start. CODE:0x0010"
 523         if not booking.end:
 524             return "BOOK, booking has no end. CODE:0x0011"
 525         if booking.end <= booking.start:
 526             return "BOOK, end before/same time as start. CODE:0x0012"
 527
 528         if 'collaborators' in models:
 529             collaborators = models['collaborators']
 530         else:
 531             return "BOOK, collaborators not defined. CODE:0x0013"
 532         try:
 533             resource_bundle = ResourceManager.getInstance().convertResourceBundle(selected_grb, config=booking.config_bundle)
 534         except ResourceAvailabilityException as e:
 535             return "BOOK, requested resources are not available. Exception: " + str(e) + " CODE:0x0014"
 536         except ModelValidationException as e:
 537             return "Error encountered when saving bundle. " + str(e) + " CODE: 0x001b"
 538
 539         booking.resource = resource_bundle
 540         booking.owner = owner
 541         booking.config_bundle = booking.config_bundle
 542         booking.lab = selected_grb.lab
 543
 544         is_allowed = BookingAuthManager().booking_allowed(booking, self)
 545         if not is_allowed:
 546             return "BOOK, you are not allowed to book the requested resources"
 547
 548         try:
 549             booking.save()
 550         except Exception as e:
 551             return "BOOK, saving booking generated exception: " + str(e) + " CODE:0x0015"
 552
 553         for collaborator in collaborators:
 554             booking.collaborators.add(collaborator)
 555
 556         try:
 557             booking.pdf = ResourceManager().makePDF(booking.resource)
 558             booking.save()
 559         except Exception as e:
 560             return "BOOK, failed to create Pod Desriptor File: " + str(e)
 561
 562         try:
 563             JobFactory.makeCompleteJob(booking)
 564         except Exception as e:
 565             return "BOOK, serializing for api generated exception: " + str(e) + " CODE:0xFFFF"
 566
 567         try:
 568             booking.save()
 569         except Exception as e:
 570             return "BOOK, saving booking generated exception: " + str(e) + " CODE:0x0016"
 571
 572     def reserve_vlans(self, grb):
 573         """
 574         True is success
 575         """
 576         vlans = []
 577         public_vlan = None
 578         vlan_manager = grb.lab.vlan_manager
 579         if vlan_manager is None:
 580             return True
 581         for host in grb.getHosts():
 582             for interface in host.generic_interfaces.all():
 583                 for vlan in interface.vlans.all():
 584                     if vlan.public:
 585                         public_vlan = vlan
 586                     else:
 587                         vlans.append(vlan.vlan_id)
 588
 589         try:
 590             vlan_manager.reserve_vlans(vlans)
 591             vlan_manager.reserve_public_vlan(public_vlan.vlan_id)
 592             return True
 593         except Exception:
 594             return False
 595
 596     def __init__(self):
 597         self.el = {}
 598         self.el[self.CONFIRMATION] = {}
 599         self.el["active_step"] = 0
 600         self.get_history = {}
 601         self.put_history = {}