dashboard/src/workflow/models.py

   1 ##############################################################################
   2 # Copyright (c) 2018 Sawyer Bergeron, Parker Berberian, and others.
   3 #
   4 # All rights reserved. This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 # http://www.apache.org/licenses/LICENSE-2.0
   8 ##############################################################################
   9
  10
  11 from django.shortcuts import render
  12 from django.contrib import messages
  13 from django.http import HttpResponse
  14 from django.utils import timezone
  15
  16 import yaml
  17 import requests
  18
  19 from workflow.forms import ConfirmationForm
  20 from api.models import JobFactory
  21 from dashboard.exceptions import ResourceAvailabilityException, ModelValidationException
  22 from resource_inventory.models import Image, GenericInterface
  23 from resource_inventory.resource_manager import ResourceManager
  24 from resource_inventory.pdf_templater import PDFTemplater
  25 from notifier.manager import NotificationHandler
  26 from booking.models import Booking
  27
  28
  29 class BookingAuthManager():
  30     LFN_PROJECTS = ["opnfv"]  # TODO
  31
  32     def parse_github_url(self, url):
  33         project_leads = []
  34         try:
  35             parts = url.split("/")
  36             if "http" in parts[0]:  # the url include http(s)://
  37                 parts = parts[2:]
  38             if parts[-1] != "INFO.yaml":
  39                 return None
  40             if parts[0] not in ["github.com", "raw.githubusercontent.com"]:
  41                 return None
  42             if parts[1] not in self.LFN_PROJECTS:
  43                 return None
  44             # now to download and parse file
  45             if parts[3] == "blob":
  46                 parts[3] = "raw"
  47             url = "https://" + "/".join(parts)
  48             info_file = requests.get(url, timeout=15).text
  49             info_parsed = yaml.load(info_file)
  50             ptl = info_parsed.get('project_lead')
  51             if ptl:
  52                 project_leads.append(ptl)
  53             sub_ptl = info_parsed.get("subproject_lead")
  54             if sub_ptl:
  55                 project_leads.append(sub_ptl)
  56
  57         except Exception:
  58             pass
  59
  60         return project_leads
  61
  62     def parse_gerrit_url(self, url):
  63         project_leads = []
  64         try:
  65             halfs = url.split("?")
  66             parts = halfs[0].split("/")
  67             args = halfs[1].split(";")
  68             if "http" in parts[0]:  # the url include http(s)://
  69                 parts = parts[2:]
  70             if "f=INFO.yaml" not in args:
  71                 return None
  72             if "gerrit.opnfv.org" not in parts[0]:
  73                 return None
  74             try:
  75                 i = args.index("a=blob")
  76                 args[i] = "a=blob_plain"
  77             except ValueError:
  78                 pass
  79             # recreate url
  80             halfs[1] = ";".join(args)
  81             halfs[0] = "/".join(parts)
  82             # now to download and parse file
  83             url = "https://" + "?".join(halfs)
  84             info_file = requests.get(url, timeout=15).text
  85             info_parsed = yaml.load(info_file)
  86             ptl = info_parsed.get('project_lead')
  87             if ptl:
  88                 project_leads.append(ptl)
  89             sub_ptl = info_parsed.get("subproject_lead")
  90             if sub_ptl:
  91                 project_leads.append(sub_ptl)
  92
  93         except Exception:
  94             return None
  95
  96         return project_leads
  97
  98     def parse_opnfv_git_url(self, url):
  99         project_leads = []
 100         try:
 101             parts = url.split("/")
 102             if "http" in parts[0]:  # the url include http(s)://
 103                 parts = parts[2:]
 104             if "INFO.yaml" not in parts[-1]:
 105                 return None
 106             if "git.opnfv.org" not in parts[0]:
 107                 return None
 108             if parts[-2] == "tree":
 109                 parts[-2] = "plain"
 110             # now to download and parse file
 111             url = "https://" + "/".join(parts)
 112             info_file = requests.get(url, timeout=15).text
 113             info_parsed = yaml.load(info_file)
 114             ptl = info_parsed.get('project_lead')
 115             if ptl:
 116                 project_leads.append(ptl)
 117             sub_ptl = info_parsed.get("subproject_lead")
 118             if sub_ptl:
 119                 project_leads.append(sub_ptl)
 120
 121         except Exception:
 122             return None
 123
 124         return project_leads
 125
 126     def parse_url(self, info_url):
 127         """
 128         will return the PTL in the INFO file on success, or None
 129         """
 130         if "github" in info_url:
 131             return self.parse_github_url(info_url)
 132
 133         if "gerrit.opnfv.org" in info_url:
 134             return self.parse_gerrit_url(info_url)
 135
 136         if "git.opnfv.org" in info_url:
 137             return self.parse_opnfv_git_url(info_url)
 138
 139     def booking_allowed(self, booking, repo):
 140         """
 141         This is the method that will have to change whenever the booking policy changes in the Infra
 142         group / LFN. This is a nice isolation of that administration crap
 143         currently checks if the booking uses multiple servers. if it does, then the owner must be a PTL,
 144         which is checked using the provided info file
 145         """
 146         if len(booking.resource.template.getHosts()) < 2:
 147             return True  # if they only have one server, we dont care
 148         if booking.owner.userprofile.booking_privledge:
 149             return True  # admin override for this user
 150         if repo.BOOKING_INFO_FILE not in repo.el:
 151             return False  # INFO file not provided
 152         ptl_info = self.parse_url(repo.el.get(repo.BOOKING_INFO_FILE))
 153         for ptl in ptl_info:
 154             if ptl['email'] == booking.owner.userprofile.email_addr:
 155                 return True
 156         return False
 157
 158
 159 class WorkflowStep(object):
 160     template = 'bad_request.html'
 161     title = "Generic Step"
 162     description = "You were led here by mistake"
 163     short_title = "error"
 164     metastep = None
 165
 166     def __init__(self, id, repo=None):
 167         self.repo = repo
 168         self.id = id
 169
 170     def get_context(self):
 171         context = {}
 172         context['step_number'] = self.repo_get('steps')
 173         context['active_step'] = self.repo_get('active_step')
 174         context['render_correct'] = "true"
 175         context['step_title'] = self.title
 176         context['description'] = self.description
 177         return context
 178
 179     def render(self, request):
 180         self.context = self.get_context()
 181         return render(request, self.template, self.context)
 182
 183     def post_render(self, request):
 184         return self.render(request)
 185
 186     def test_render(self, request):
 187         if request.method == "POST":
 188             return self.post_render(request)
 189         return self.render(request)
 190
 191     def validate(self, request):
 192         pass
 193
 194     def repo_get(self, key, default=None):
 195         return self.repo.get(key, default, self.id)
 196
 197     def repo_put(self, key, value):
 198         return self.repo.put(key, value, self.id)
 199
 200
 201 class Confirmation_Step(WorkflowStep):
 202     template = 'workflow/confirm.html'
 203     title = "Confirm Changes"
 204     description = "Does this all look right?"
 205
 206     def get_vlan_warning(self):
 207         grb = self.repo_get(self.repo.SELECTED_GRESOURCE_BUNDLE, False)
 208         if not grb:
 209             return 0
 210         if self.repo.BOOKING_MODELS not in self.repo.el:
 211             return 0
 212         vlan_manager = grb.lab.vlan_manager
 213         if vlan_manager is None:
 214             return 0
 215         hosts = grb.getHosts()
 216         for host in hosts:
 217             for interface in host.generic_interfaces.all():
 218                 for vlan in interface.vlans.all():
 219                     if vlan.public:
 220                         if not vlan_manager.public_vlan_is_available(vlan.vlan_id):
 221                             return 1
 222                     else:
 223                         if not vlan_manager.is_available(vlan.vlan_id):
 224                             return 1  # There is a problem with these vlans
 225         return 0
 226
 227     def get_context(self):
 228         context = super(Confirmation_Step, self).get_context()
 229         context['form'] = ConfirmationForm()
 230         context['confirmation_info'] = yaml.dump(
 231             self.repo_get(self.repo.CONFIRMATION),
 232             default_flow_style=False
 233         ).strip()
 234         context['vlan_warning'] = self.get_vlan_warning()
 235
 236         return context
 237
 238     def flush_to_db(self):
 239         errors = self.repo.make_models()
 240         if errors:
 241             return errors
 242
 243     def post_render(self, request):
 244         form = ConfirmationForm(request.POST)
 245         if form.is_valid():
 246             data = form.cleaned_data['confirm']
 247             context = self.get_context()
 248             if data == "True":
 249                 context["bypassed"] = "true"
 250                 errors = self.flush_to_db()
 251                 if errors:
 252                     messages.add_message(request, messages.ERROR, "ERROR OCCURRED: " + errors)
 253                 else:
 254                     messages.add_message(request, messages.SUCCESS, "Confirmed")
 255
 256                 return HttpResponse('')
 257             elif data == "False":
 258                 context["bypassed"] = "true"
 259                 messages.add_message(request, messages.SUCCESS, "Canceled")
 260                 return render(request, self.template, context)
 261             else:
 262                 pass
 263
 264         else:
 265             if "vlan_input" in request.POST:
 266                 if request.POST.get("vlan_input") == "True":
 267                     self.translate_vlans()
 268                     return self.render(request)
 269             pass
 270
 271     def translate_vlans(self):
 272         grb = self.repo_get(self.repo.SELECTED_GRESOURCE_BUNDLE, False)
 273         if not grb:
 274             return 0
 275         vlan_manager = grb.lab.vlan_manager
 276         if vlan_manager is None:
 277             return 0
 278         hosts = grb.getHosts()
 279         for host in hosts:
 280             for interface in host.generic_interfaces.all():
 281                 for vlan in interface.vlans.all():
 282                     if not vlan.public:
 283                         if not vlan_manager.is_available(vlan.vlan_id):
 284                             vlan.vlan_id = vlan_manager.get_vlan()
 285                             vlan.save()
 286                     else:
 287                         if not vlan_manager.public_vlan_is_available(vlan.vlan_id):
 288                             pub_vlan = vlan_manager.get_public_vlan()
 289                             vlan.vlan_id = pub_vlan.vlan
 290                             vlan.save()
 291
 292
 293 class Workflow():
 294
 295     steps = []
 296     active_index = 0
 297
 298
 299 class Repository():
 300
 301     EDIT = "editing"
 302     MODELS = "models"
 303     RESOURCE_SELECT = "resource_select"
 304     CONFIRMATION = "confirmation"
 305     SELECTED_GRESOURCE_BUNDLE = "selected generic bundle pk"
 306     SELECTED_CONFIG_BUNDLE = "selected config bundle pk"
 307     GRESOURCE_BUNDLE_MODELS = "generic_resource_bundle_models"
 308     GRESOURCE_BUNDLE_INFO = "generic_resource_bundle_info"
 309     BOOKING = "booking"
 310     LAB = "lab"
 311     GRB_LAST_HOSTLIST = "grb_network_previous_hostlist"
 312     BOOKING_FORMS = "booking_forms"
 313     SWCONF_HOSTS = "swconf_hosts"
 314     BOOKING_MODELS = "booking models"
 315     CONFIG_MODELS = "configuration bundle models"
 316     SESSION_USER = "session owner user account"
 317     VALIDATED_MODEL_GRB = "valid grb config model instance in db"
 318     VALIDATED_MODEL_CONFIG = "valid config model instance in db"
 319     VALIDATED_MODEL_BOOKING = "valid booking model instance in db"
 320     VLANS = "a list of vlans"
 321     SNAPSHOT_MODELS = "the models for snapshotting"
 322     SNAPSHOT_BOOKING_ID = "the booking id for snapshotting"
 323     SNAPSHOT_NAME = "the name of the snapshot"
 324     SNAPSHOT_DESC = "description of the snapshot"
 325     BOOKING_INFO_FILE = "the INFO.yaml file for this user's booking"
 326
 327     # migratory elements of segmented workflow
 328     # each of these is the end result of a different workflow.
 329     HAS_RESULT = "whether or not workflow has a result"
 330     RESULT_KEY = "key for target index that result will be put into in parent"
 331     RESULT = "result object from workflow"
 332
 333     def get_child_defaults(self):
 334         return_tuples = []
 335         for key in [self.SELECTED_GRESOURCE_BUNDLE, self.SESSION_USER]:
 336             return_tuples.append((key, self.el.get(key)))
 337         return return_tuples
 338
 339     def set_defaults(self, defaults):
 340         for key, value in defaults:
 341             self.el[key] = value
 342
 343     def get(self, key, default, id):
 344         self.add_get_history(key, id)
 345         return self.el.get(key, default)
 346
 347     def put(self, key, val, id):
 348         self.add_put_history(key, id)
 349         self.el[key] = val
 350
 351     def add_get_history(self, key, id):
 352         self.add_history(key, id, self.get_history)
 353
 354     def add_put_history(self, key, id):
 355         self.add_history(key, id, self.put_history)
 356
 357     def add_history(self, key, id, history):
 358         if key not in history:
 359             history[key] = [id]
 360         else:
 361             history[key].append(id)
 362
 363     def make_models(self):
 364         if self.SNAPSHOT_MODELS in self.el:
 365             errors = self.make_snapshot()
 366             if errors:
 367                 return errors
 368         # if GRB WF, create it
 369         if self.GRESOURCE_BUNDLE_MODELS in self.el:
 370             errors = self.make_generic_resource_bundle()
 371             if errors:
 372                 return errors
 373             else:
 374                 self.el[self.HAS_RESULT] = True
 375                 self.el[self.RESULT_KEY] = self.SELECTED_GRESOURCE_BUNDLE
 376                 return
 377
 378         if self.CONFIG_MODELS in self.el:
 379             errors = self.make_software_config_bundle()
 380             if errors:
 381                 return errors
 382             else:
 383                 self.el[self.HAS_RESULT] = True
 384                 self.el[self.RESULT_KEY] = self.SELECTED_CONFIG_BUNDLE
 385                 return
 386
 387         if self.BOOKING_MODELS in self.el:
 388             errors = self.make_booking()
 389             if errors:
 390                 return errors
 391             # create notification
 392             booking = self.el[self.BOOKING_MODELS]['booking']
 393             NotificationHandler.notify_new_booking(booking)
 394
 395     def make_snapshot(self):
 396         owner = self.el[self.SESSION_USER]
 397         models = self.el[self.SNAPSHOT_MODELS]
 398         image = models.get('snapshot', Image())
 399         booking_id = self.el.get(self.SNAPSHOT_BOOKING_ID)
 400         if not booking_id:
 401             return "SNAP, No booking ID provided"
 402         booking = Booking.objects.get(pk=booking_id)
 403         if booking.start > timezone.now() or booking.end < timezone.now():
 404             return "Booking is not active"
 405         name = self.el.get(self.SNAPSHOT_NAME)
 406         if not name:
 407             return "SNAP, no name provided"
 408         host = models.get('host')
 409         if not host:
 410             return "SNAP, no host provided"
 411         description = self.el.get(self.SNAPSHOT_DESC, "")
 412         image.from_lab = booking.lab
 413         image.name = name
 414         image.description = description
 415         image.public = False
 416         image.lab_id = -1
 417         image.owner = owner
 418         image.host_type = host.profile
 419         image.save()
 420         try:
 421             current_image = host.config.image
 422             image.os = current_image.os
 423             image.save()
 424         except Exception:
 425             pass
 426         JobFactory.makeSnapshotTask(image, booking, host)
 427
 428     def make_generic_resource_bundle(self):
 429         owner = self.el[self.SESSION_USER]
 430         if self.GRESOURCE_BUNDLE_MODELS in self.el:
 431             models = self.el[self.GRESOURCE_BUNDLE_MODELS]
 432             if 'hosts' in models:
 433                 hosts = models['hosts']
 434             else:
 435                 return "GRB has no hosts. CODE:0x0002"
 436             if 'bundle' in models:
 437                 bundle = models['bundle']
 438             else:
 439                 return "GRB, no bundle in models. CODE:0x0003"
 440
 441             try:
 442                 bundle.owner = owner
 443                 bundle.save()
 444             except Exception as e:
 445                 return "GRB, saving bundle generated exception: " + str(e) + " CODE:0x0004"
 446             try:
 447                 for host in hosts:
 448                     genericresource = host.resource
 449                     genericresource.bundle = bundle
 450                     genericresource.save()
 451                     host.resource = genericresource
 452                     host.save()
 453             except Exception as e:
 454                 return "GRB, saving hosts generated exception: " + str(e) + " CODE:0x0005"
 455
 456             if 'interfaces' in models:
 457                 for interface_set in models['interfaces'].values():
 458                     for interface in interface_set:
 459                         try:
 460                             interface.host = interface.host
 461                             interface.save()
 462                         except Exception:
 463                             return "GRB, saving interface " + str(interface) + " failed. CODE:0x0019"
 464             else:
 465                 return "GRB, no interface set provided. CODE:0x001a"
 466
 467             if 'vlans' in models:
 468                 for resource_name, mapping in models['vlans'].items():
 469                     for profile_name, vlan_set in mapping.items():
 470                         interface = GenericInterface.objects.get(
 471                             profile__name=profile_name,
 472                             host__resource__name=resource_name,
 473                             host__resource__bundle=models['bundle']
 474                         )
 475                         for vlan in vlan_set:
 476                             try:
 477                                 vlan.save()
 478                                 interface.vlans.add(vlan)
 479                             except Exception as e:
 480                                 return "GRB, saving vlan " + str(vlan) + " failed. Exception: " + str(e) + ". CODE:0x0017"
 481             else:
 482                 return "GRB, no vlan set provided. CODE:0x0018"
 483
 484         else:
 485             return "GRB no models given. CODE:0x0001"
 486
 487         self.el[self.RESULT] = bundle
 488         return False
 489
 490     def make_software_config_bundle(self):
 491         models = self.el[self.CONFIG_MODELS]
 492         if 'bundle' in models:
 493             bundle = models['bundle']
 494             bundle.bundle = bundle.bundle
 495             try:
 496                 bundle.save()
 497             except Exception as e:
 498                 return "SWC, saving bundle generated exception: " + str(e) + "CODE:0x0007"
 499
 500         else:
 501             return "SWC, no bundle in models. CODE:0x0006"
 502         if 'host_configs' in models:
 503             host_configs = models['host_configs']
 504             for host_config in host_configs:
 505                 host_config.bundle = host_config.bundle
 506                 host_config.host = host_config.host
 507                 try:
 508                     host_config.save()
 509                 except Exception as e:
 510                     return "SWC, saving host configs generated exception: " + str(e) + "CODE:0x0009"
 511         else:
 512             return "SWC, no host configs in models. CODE:0x0008"
 513         if 'opnfv' in models:
 514             opnfvconfig = models['opnfv']
 515             opnfvconfig.bundle = opnfvconfig.bundle
 516             if opnfvconfig.scenario not in opnfvconfig.installer.sup_scenarios.all():
 517                 return "SWC, scenario not supported by installer. CODE:0x000d"
 518             try:
 519                 opnfvconfig.save()
 520             except Exception as e:
 521                 return "SWC, saving opnfv config generated exception: " + str(e) + "CODE:0x000b"
 522         else:
 523             pass
 524
 525         self.el[self.RESULT] = bundle
 526         return False
 527
 528     def make_booking(self):
 529         models = self.el[self.BOOKING_MODELS]
 530         owner = self.el[self.SESSION_USER]
 531
 532         if self.SELECTED_GRESOURCE_BUNDLE in self.el:
 533             selected_grb = self.el[self.SELECTED_GRESOURCE_BUNDLE]
 534         else:
 535             return "BOOK, no selected resource. CODE:0x000e"
 536
 537         if not self.reserve_vlans(selected_grb):
 538             return "BOOK, vlans not available"
 539
 540         if 'booking' in models:
 541             booking = models['booking']
 542         else:
 543             return "BOOK, no booking model exists. CODE:0x000f"
 544
 545         if not booking.start:
 546             return "BOOK, booking has no start. CODE:0x0010"
 547         if not booking.end:
 548             return "BOOK, booking has no end. CODE:0x0011"
 549         if booking.end <= booking.start:
 550             return "BOOK, end before/same time as start. CODE:0x0012"
 551
 552         if 'collaborators' in models:
 553             collaborators = models['collaborators']
 554         else:
 555             return "BOOK, collaborators not defined. CODE:0x0013"
 556         try:
 557             resource_bundle = ResourceManager.getInstance().convertResourceBundle(selected_grb, config=booking.config_bundle)
 558         except ResourceAvailabilityException as e:
 559             return "BOOK, requested resources are not available. Exception: " + str(e) + " CODE:0x0014"
 560         except ModelValidationException as e:
 561             return "Error encountered when saving bundle. " + str(e) + " CODE: 0x001b"
 562
 563         booking.resource = resource_bundle
 564         booking.owner = owner
 565         booking.config_bundle = booking.config_bundle
 566         booking.lab = selected_grb.lab
 567
 568         is_allowed = BookingAuthManager().booking_allowed(booking, self)
 569         if not is_allowed:
 570             return "BOOK, you are not allowed to book the requested resources"
 571
 572         try:
 573             booking.save()
 574         except Exception as e:
 575             return "BOOK, saving booking generated exception: " + str(e) + " CODE:0x0015"
 576
 577         for collaborator in collaborators:
 578             booking.collaborators.add(collaborator)
 579
 580         try:
 581             booking.pdf = PDFTemplater.makePDF(booking.resource)
 582             booking.save()
 583         except Exception as e:
 584             return "BOOK, failed to create Pod Desriptor File: " + str(e)
 585
 586         try:
 587             JobFactory.makeCompleteJob(booking)
 588         except Exception as e:
 589             return "BOOK, serializing for api generated exception: " + str(e) + " CODE:0xFFFF"
 590
 591         try:
 592             booking.save()
 593         except Exception as e:
 594             return "BOOK, saving booking generated exception: " + str(e) + " CODE:0x0016"
 595
 596     def reserve_vlans(self, grb):
 597         """
 598         True is success
 599         """
 600         vlans = []
 601         public_vlan = None
 602         vlan_manager = grb.lab.vlan_manager
 603         if vlan_manager is None:
 604             return True
 605         for host in grb.getHosts():
 606             for interface in host.generic_interfaces.all():
 607                 for vlan in interface.vlans.all():
 608                     if vlan.public:
 609                         public_vlan = vlan
 610                     else:
 611                         vlans.append(vlan.vlan_id)
 612
 613         try:
 614             vlan_manager.reserve_vlans(vlans)
 615             vlan_manager.reserve_public_vlan(public_vlan.vlan_id)
 616             return True
 617         except Exception:
 618             return False
 619
 620     def __init__(self):
 621         self.el = {}
 622         self.el[self.CONFIRMATION] = {}
 623         self.el["active_step"] = 0
 624         self.get_history = {}
 625         self.put_history = {}