src/workflow/models.py

   1 ##############################################################################
   2 # Copyright (c) 2018 Sawyer Bergeron, Parker Berberian, and others.
   3 #
   4 # All rights reserved. This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 # http://www.apache.org/licenses/LICENSE-2.0
   8 ##############################################################################
   9
  10
  11 from django.template.loader import get_template
  12 from django.http import HttpResponse
  13 from django.utils import timezone
  14 from django.db import transaction
  15
  16 import yaml
  17 import requests
  18
  19 from workflow.forms import ConfirmationForm
  20 from api.models import JobFactory
  21 from dashboard.exceptions import ResourceAvailabilityException, ModelValidationException
  22 from resource_inventory.models import Image, OPNFVConfig, ResourceOPNFVConfig, NetworkRole
  23 from resource_inventory.resource_manager import ResourceManager
  24 from resource_inventory.pdf_templater import PDFTemplater
  25 from notifier.manager import NotificationHandler
  26 from booking.models import Booking
  27
  28
  29 class BookingAuthManager():
  30     """
  31     Verifies Booking Authorization.
  32
  33     Class to verify that the user is allowed to book the requested resource
  34     The user must input a url to the INFO.yaml file to prove that they are the ptl of
  35     an approved project if they are booking a multi-node pod.
  36     This class parses the url and checks the logged in user against the info file.
  37     """
  38
  39     LFN_PROJECTS = ["opnfv"]  # TODO
  40
  41     def parse_github_url(self, url):
  42         project_leads = []
  43         try:
  44             parts = url.split("/")
  45             if "http" in parts[0]:  # the url include http(s)://
  46                 parts = parts[2:]
  47             if parts[-1] != "INFO.yaml":
  48                 return None
  49             if parts[0] not in ["github.com", "raw.githubusercontent.com"]:
  50                 return None
  51             if parts[1] not in self.LFN_PROJECTS:
  52                 return None
  53             # now to download and parse file
  54             if parts[3] == "blob":
  55                 parts[3] = "raw"
  56             url = "https://" + "/".join(parts)
  57             info_file = requests.get(url, timeout=15).text
  58             info_parsed = yaml.load(info_file)
  59             ptl = info_parsed.get('project_lead')
  60             if ptl:
  61                 project_leads.append(ptl)
  62             sub_ptl = info_parsed.get("subproject_lead")
  63             if sub_ptl:
  64                 project_leads.append(sub_ptl)
  65
  66         except Exception:
  67             pass
  68
  69         return project_leads
  70
  71     def parse_gerrit_url(self, url):
  72         project_leads = []
  73         try:
  74             halfs = url.split("?")
  75             parts = halfs[0].split("/")
  76             args = halfs[1].split(";")
  77             if "http" in parts[0]:  # the url include http(s)://
  78                 parts = parts[2:]
  79             if "f=INFO.yaml" not in args:
  80                 return None
  81             if "gerrit.opnfv.org" not in parts[0]:
  82                 return None
  83             try:
  84                 i = args.index("a=blob")
  85                 args[i] = "a=blob_plain"
  86             except ValueError:
  87                 pass
  88             # recreate url
  89             halfs[1] = ";".join(args)
  90             halfs[0] = "/".join(parts)
  91             # now to download and parse file
  92             url = "https://" + "?".join(halfs)
  93             info_file = requests.get(url, timeout=15).text
  94             info_parsed = yaml.load(info_file)
  95             ptl = info_parsed.get('project_lead')
  96             if ptl:
  97                 project_leads.append(ptl)
  98             sub_ptl = info_parsed.get("subproject_lead")
  99             if sub_ptl:
 100                 project_leads.append(sub_ptl)
 101
 102         except Exception:
 103             return None
 104
 105         return project_leads
 106
 107     def parse_opnfv_git_url(self, url):
 108         project_leads = []
 109         try:
 110             parts = url.split("/")
 111             if "http" in parts[0]:  # the url include http(s)://
 112                 parts = parts[2:]
 113             if "INFO.yaml" not in parts[-1]:
 114                 return None
 115             if "git.opnfv.org" not in parts[0]:
 116                 return None
 117             if parts[-2] == "tree":
 118                 parts[-2] = "plain"
 119             # now to download and parse file
 120             url = "https://" + "/".join(parts)
 121             info_file = requests.get(url, timeout=15).text
 122             info_parsed = yaml.load(info_file)
 123             ptl = info_parsed.get('project_lead')
 124             if ptl:
 125                 project_leads.append(ptl)
 126             sub_ptl = info_parsed.get("subproject_lead")
 127             if sub_ptl:
 128                 project_leads.append(sub_ptl)
 129
 130         except Exception:
 131             return None
 132
 133         return project_leads
 134
 135     def parse_url(self, info_url):
 136         """
 137         Parse the project URL.
 138
 139         Gets the INFO.yaml file from the project and returns the PTL info.
 140         """
 141         if "github" in info_url:
 142             return self.parse_github_url(info_url)
 143
 144         if "gerrit.opnfv.org" in info_url:
 145             return self.parse_gerrit_url(info_url)
 146
 147         if "git.opnfv.org" in info_url:
 148             return self.parse_opnfv_git_url(info_url)
 149
 150     def booking_allowed(self, booking, repo):
 151         """
 152         Assert the current Booking Policy.
 153
 154         This is the method that will have to change whenever the booking policy changes in the Infra
 155         group / LFN. This is a nice isolation of that administration crap
 156         currently checks if the booking uses multiple servers. if it does, then the owner must be a PTL,
 157         which is checked using the provided info file
 158         """
 159         if booking.owner.userprofile.booking_privledge:
 160             return True  # admin override for this user
 161         if Booking.objects.filter(owner=booking.owner, end__gt=timezone.now()).count() >= 3:
 162             return False
 163         if len(booking.resource.template.get_required_resources()) < 2:
 164             return True  # if they only have one server, we dont care
 165         if repo.BOOKING_INFO_FILE not in repo.el:
 166             return False  # INFO file not provided
 167         ptl_info = self.parse_url(repo.el.get(repo.BOOKING_INFO_FILE))
 168         for ptl in ptl_info:
 169             if ptl['email'] == booking.owner.userprofile.email_addr:
 170                 return True
 171         return False
 172
 173
 174 class WorkflowStepStatus(object):
 175     """
 176     Poor man's enum for the status of a workflow step.
 177
 178     The steps in a workflow are not completed (UNTOUCHED)
 179     or they have been completed correctly (VALID) or they were filled out
 180     incorrectly (INVALID)
 181     """
 182
 183     UNTOUCHED = 0
 184     INVALID = 100
 185     VALID = 200
 186
 187
 188 class WorkflowStep(object):
 189     template = 'bad_request.html'
 190     title = "Generic Step"
 191     description = "You were led here by mistake"
 192     short_title = "error"
 193     metastep = None
 194     # phasing out metastep:
 195
 196     valid = WorkflowStepStatus.UNTOUCHED
 197     message = ""
 198
 199     enabled = True
 200
 201     def cleanup(self):
 202         raise Exception("WorkflowStep subclass of type " + str(type(self)) + " has no concrete implemented cleanup() method")
 203
 204     def enable(self):
 205         if not self.enabled:
 206             self.enabled = True
 207
 208     def disable(self):
 209         if self.enabled:
 210             self.cleanup()
 211             self.enabled = False
 212
 213     def set_invalid(self, message, code=WorkflowStepStatus.INVALID):
 214         self.valid = code
 215         self.message = message
 216
 217     def set_valid(self, message, code=WorkflowStepStatus.VALID):
 218         self.valid = code
 219         self.message = message
 220
 221     def to_json(self):
 222         return {
 223             'title': self.short_title,
 224             'enabled': self.enabled,
 225             'valid': self.valid,
 226             'message': self.message,
 227         }
 228
 229     def __init__(self, id, repo=None):
 230         self.repo = repo
 231         self.id = id
 232
 233     def get_context(self):
 234         context = {}
 235         context['step_number'] = self.repo_get('steps')
 236         context['active_step'] = self.repo_get('active_step')
 237         context['render_correct'] = "true"
 238         context['step_title'] = self.title
 239         context['description'] = self.description
 240         return context
 241
 242     def render(self, request):
 243         return HttpResponse(self.render_to_string(request))
 244
 245     def render_to_string(self, request):
 246         template = get_template(self.template)
 247         return template.render(self.get_context(), request)
 248
 249     def post(self, post_content, user):
 250         raise Exception("WorkflowStep subclass of type " + str(type(self)) + " has no concrete post() method")
 251
 252     def validate(self, request):
 253         pass
 254
 255     def repo_get(self, key, default=None):
 256         return self.repo.get(key, default, self.id)
 257
 258     def repo_put(self, key, value):
 259         return self.repo.put(key, value, self.id)
 260
 261
 262 """
 263 subclassing notes:
 264     subclasses have to define the following class attributes:
 265         self.select_repo_key: where the selected "object" or "bundle" is to be placed in the repo
 266         self.form: the form to be used
 267         alert_bundle_missing(): what message to display if a user does not select/selects an invalid object
 268         get_form_queryset(): generate a queryset to be used to filter available items for the field
 269         get_page_context(): return simple context such as page header and other info
 270 """
 271
 272
 273 class AbstractSelectOrCreate(WorkflowStep):
 274     template = 'dashboard/genericselect.html'
 275     title = "Select a Bundle"
 276     short_title = "select"
 277     description = "Generic bundle selector step"
 278
 279     select_repo_key = None
 280     form = None  # subclasses are expected to use a form that is a subclass of SearchableSelectGenericForm
 281
 282     def alert_bundle_missing(self):  # override in subclasses to change message if field isn't filled out
 283         self.set_invalid("Please select a valid bundle")
 284
 285     def post(self, post_data, user):
 286         form = self.form(post_data, queryset=self.get_form_queryset())
 287         if form.is_valid():
 288             bundle = form.get_validated_bundle()
 289             if not bundle:
 290                 self.alert_bundle_missing()
 291                 return
 292             self.repo_put(self.select_repo_key, bundle)
 293             self.put_confirm_info(bundle)
 294             self.set_valid("Step Completed")
 295         else:
 296             self.alert_bundle_missing()
 297
 298     def get_context(self):
 299         default = []
 300
 301         bundle = self.repo_get(self.select_repo_key, False)
 302         if bundle:
 303             default.append(bundle)
 304
 305         form = self.form(queryset=self.get_form_queryset(), initial=default)
 306
 307         context = {'form': form, **self.get_page_context()}
 308         context.update(super().get_context())
 309
 310         return context
 311
 312     def get_page_context():
 313         return {
 314             'select_type': 'generic',
 315             'select_type_title': 'Generic Bundle'
 316         }
 317
 318
 319 class Confirmation_Step(WorkflowStep):
 320     template = 'workflow/confirm.html'
 321     title = "Confirm Changes"
 322     description = "Does this all look right?"
 323
 324     short_title = "confirm"
 325
 326     def get_context(self):
 327         context = super(Confirmation_Step, self).get_context()
 328         context['form'] = ConfirmationForm()
 329         context['confirmation_info'] = yaml.dump(
 330             self.repo_get(self.repo.CONFIRMATION),
 331             default_flow_style=False
 332         ).strip()
 333
 334         if self.valid == WorkflowStepStatus.VALID:
 335             context["confirm_succeeded"] = "true"
 336
 337         return context
 338
 339     def flush_to_db(self):
 340         errors = self.repo.make_models()
 341         if errors:
 342             return errors
 343
 344     def post(self, post_data, user):
 345         form = ConfirmationForm(post_data)
 346         if form.is_valid():
 347             data = form.cleaned_data['confirm']
 348             if data == "True":
 349                 errors = self.flush_to_db()
 350                 if errors:
 351                     self.set_invalid("ERROR OCCURRED: " + errors)
 352                 else:
 353                     self.set_valid("Confirmed")
 354
 355             elif data == "False":
 356                 self.repo.cancel()
 357                 self.set_valid("Canceled")
 358             else:
 359                 self.set_invalid("Bad Form Contents")
 360
 361         else:
 362             self.set_invalid("Bad Form Contents")
 363
 364
 365 class Repository():
 366
 367     EDIT = "editing"
 368     MODELS = "models"
 369     RESOURCE_SELECT = "resource_select"
 370     CONFIRMATION = "confirmation"
 371     SELECTED_RESOURCE_TEMPLATE = "selected resource template pk"
 372     SELECTED_OPNFV_CONFIG = "selected opnfv deployment config"
 373     RESOURCE_TEMPLATE_MODELS = "generic_resource_template_models"
 374     RESOURCE_TEMPLATE_INFO = "generic_resource_template_info"
 375     BOOKING = "booking"
 376     LAB = "lab"
 377     RCONFIG_LAST_HOSTLIST = "resource_configuration_network_previous_hostlist"
 378     BOOKING_FORMS = "booking_forms"
 379     SWCONF_HOSTS = "swconf_hosts"
 380     BOOKING_MODELS = "booking models"
 381     CONFIG_MODELS = "configuration bundle models"
 382     OPNFV_MODELS = "opnfv configuration models"
 383     SESSION_USER = "session owner user account"
 384     SESSION_MANAGER = "session manager for current session"
 385     VALIDATED_MODEL_GRB = "valid grb config model instance in db"
 386     VALIDATED_MODEL_CONFIG = "valid config model instance in db"
 387     VALIDATED_MODEL_BOOKING = "valid booking model instance in db"
 388     VLANS = "a list of vlans"
 389     SNAPSHOT_MODELS = "the models for snapshotting"
 390     SNAPSHOT_BOOKING_ID = "the booking id for snapshotting"
 391     SNAPSHOT_NAME = "the name of the snapshot"
 392     SNAPSHOT_DESC = "description of the snapshot"
 393     BOOKING_INFO_FILE = "the INFO.yaml file for this user's booking"
 394
 395     # new keys for migration to using ResourceTemplates:
 396     RESOURCE_TEMPLATE_MODELS = "current working model of resource template"
 397
 398     # migratory elements of segmented workflow
 399     # each of these is the end result of a different workflow.
 400     HAS_RESULT = "whether or not workflow has a result"
 401     RESULT_KEY = "key for target index that result will be put into in parent"
 402     RESULT = "result object from workflow"
 403
 404     def get_child_defaults(self):
 405         return_tuples = []
 406         for key in [self.SELECTED_RESOURCE_TEMPLATE, self.SESSION_USER]:
 407             return_tuples.append((key, self.el.get(key)))
 408         return return_tuples
 409
 410     def set_defaults(self, defaults):
 411         for key, value in defaults:
 412             self.el[key] = value
 413
 414     def get(self, key, default, id):
 415
 416         self.add_get_history(key, id)
 417         return self.el.get(key, default)
 418
 419     def put(self, key, val, id):
 420         self.add_put_history(key, id)
 421         self.el[key] = val
 422
 423     def add_get_history(self, key, id):
 424         self.add_history(key, id, self.get_history)
 425
 426     def add_put_history(self, key, id):
 427         self.add_history(key, id, self.put_history)
 428
 429     def add_history(self, key, id, history):
 430         if key not in history:
 431             history[key] = [id]
 432         else:
 433             history[key].append(id)
 434
 435     def cancel(self):
 436         if self.RESOURCE_TEMPLATE_MODELS in self.el:
 437             models = self.el[self.RESOURCE_TEMPLATE_MODELS]
 438             if models['template'].temporary:
 439                 models['template'].delete()
 440                 # deleting current template should cascade delete all
 441                 # necessary related models
 442
 443     def make_models(self):
 444         if self.SNAPSHOT_MODELS in self.el:
 445             errors = self.make_snapshot()
 446             if errors:
 447                 return errors
 448
 449         # if GRB WF, create it
 450         if self.RESOURCE_TEMPLATE_MODELS in self.el:
 451             errors = self.make_generic_resource_bundle()
 452             if errors:
 453                 return errors
 454             else:
 455                 self.el[self.HAS_RESULT] = True
 456                 self.el[self.RESULT_KEY] = self.SELECTED_RESOURCE_TEMPLATE
 457                 return
 458
 459         if self.OPNFV_MODELS in self.el:
 460             errors = self.make_opnfv_config()
 461             if errors:
 462                 return errors
 463             else:
 464                 self.el[self.HAS_RESULT] = True
 465                 self.el[self.RESULT_KEY] = self.SELECTED_OPNFV_CONFIG
 466
 467         if self.BOOKING_MODELS in self.el:
 468             errors = self.make_booking()
 469             if errors:
 470                 return errors
 471             # create notification
 472             booking = self.el[self.BOOKING_MODELS]['booking']
 473             NotificationHandler.notify_new_booking(booking)
 474
 475     def make_snapshot(self):
 476         owner = self.el[self.SESSION_USER]
 477         models = self.el[self.SNAPSHOT_MODELS]
 478         image = models.get('snapshot', Image())
 479         booking_id = self.el.get(self.SNAPSHOT_BOOKING_ID)
 480         if not booking_id:
 481             return "SNAP, No booking ID provided"
 482         booking = Booking.objects.get(pk=booking_id)
 483         if booking.start > timezone.now() or booking.end < timezone.now():
 484             return "Booking is not active"
 485         name = self.el.get(self.SNAPSHOT_NAME)
 486         if not name:
 487             return "SNAP, no name provided"
 488         host = models.get('host')
 489         if not host:
 490             return "SNAP, no host provided"
 491         description = self.el.get(self.SNAPSHOT_DESC, "")
 492         image.from_lab = booking.lab
 493         image.name = name
 494         image.description = description
 495         image.public = False
 496         image.lab_id = -1
 497         image.owner = owner
 498         image.host_type = host.profile
 499         image.save()
 500         try:
 501             current_image = host.config.image
 502             image.os = current_image.os
 503             image.save()
 504         except Exception:
 505             pass
 506         JobFactory.makeSnapshotTask(image, booking, host)
 507
 508         self.el[self.RESULT] = image
 509         self.el[self.HAS_RESULT] = True
 510
 511     def make_generic_resource_bundle(self):
 512         owner = self.el[self.SESSION_USER]
 513         if self.RESOURCE_TEMPLATE_MODELS in self.el:
 514             models = self.el[self.RESOURCE_TEMPLATE_MODELS]
 515             models['template'].owner = owner
 516             models['template'].temporary = False
 517             models['template'].save()
 518             self.el[self.RESULT] = models['template']
 519             self.el[self.HAS_RESULT] = True
 520             return False
 521
 522         else:
 523             return "GRB no models given. CODE:0x0001"
 524
 525     def make_software_config_bundle(self):
 526         models = self.el[self.CONFIG_MODELS]
 527         if 'bundle' in models:
 528             bundle = models['bundle']
 529             bundle.bundle = self.el[self.SELECTED_RESOURCE_TEMPLATE]
 530             try:
 531                 bundle.save()
 532             except Exception as e:
 533                 return "SWC, saving bundle generated exception: " + str(e) + "CODE:0x0007"
 534
 535         else:
 536             return "SWC, no bundle in models. CODE:0x0006"
 537         if 'host_configs' in models:
 538             host_configs = models['host_configs']
 539             for host_config in host_configs:
 540                 host_config.template = host_config.template
 541                 host_config.profile = host_config.profile
 542                 try:
 543                     host_config.save()
 544                 except Exception as e:
 545                     return "SWC, saving host configs generated exception: " + str(e) + "CODE:0x0009"
 546         else:
 547             return "SWC, no host configs in models. CODE:0x0008"
 548         if 'opnfv' in models:
 549             opnfvconfig = models['opnfv']
 550             opnfvconfig.bundle = opnfvconfig.bundle
 551             if opnfvconfig.scenario not in opnfvconfig.installer.sup_scenarios.all():
 552                 return "SWC, scenario not supported by installer. CODE:0x000d"
 553             try:
 554                 opnfvconfig.save()
 555             except Exception as e:
 556                 return "SWC, saving opnfv config generated exception: " + str(e) + "CODE:0x000b"
 557         else:
 558             pass
 559
 560         self.el[self.RESULT] = bundle
 561         return False
 562
 563     @transaction.atomic  # TODO: Rewrite transactions with savepoints at user level for all workflows
 564     def make_booking(self):
 565         models = self.el[self.BOOKING_MODELS]
 566         owner = self.el[self.SESSION_USER]
 567
 568         if 'booking' in models:
 569             booking = models['booking']
 570         else:
 571             return "BOOK, no booking model exists. CODE:0x000f"
 572
 573         selected_grb = None
 574
 575         if self.SELECTED_RESOURCE_TEMPLATE in self.el:
 576             selected_grb = self.el[self.SELECTED_RESOURCE_TEMPLATE]
 577         else:
 578             return "BOOK, no selected resource. CODE:0x000e"
 579
 580         if not booking.start:
 581             return "BOOK, booking has no start. CODE:0x0010"
 582         if not booking.end:
 583             return "BOOK, booking has no end. CODE:0x0011"
 584         if booking.end <= booking.start:
 585             return "BOOK, end before/same time as start. CODE:0x0012"
 586
 587         if 'collaborators' in models:
 588             collaborators = models['collaborators']
 589         else:
 590             return "BOOK, collaborators not defined. CODE:0x0013"
 591         try:
 592             res_manager = ResourceManager.getInstance()
 593             resource_bundle = res_manager.instantiateTemplate(selected_grb)
 594         except ResourceAvailabilityException as e:
 595             return "BOOK, requested resources are not available. Exception: " + str(e) + " CODE:0x0014"
 596         except ModelValidationException as e:
 597             return "Error encountered when saving bundle. " + str(e) + " CODE: 0x001b"
 598
 599         booking.resource = resource_bundle
 600         booking.owner = owner
 601         booking.lab = selected_grb.lab
 602
 603         is_allowed = BookingAuthManager().booking_allowed(booking, self)
 604         if not is_allowed:
 605             return "BOOK, you are not allowed to book the requested resources"
 606
 607         try:
 608             booking.save()
 609         except Exception as e:
 610             return "BOOK, saving booking generated exception: " + str(e) + " CODE:0x0015"
 611
 612         for collaborator in collaborators:
 613             booking.collaborators.add(collaborator)
 614
 615         try:
 616             booking.pdf = PDFTemplater.makePDF(booking)
 617             booking.save()
 618         except Exception as e:
 619             return "BOOK, failed to create Pod Desriptor File: " + str(e)
 620
 621         try:
 622             JobFactory.makeCompleteJob(booking)
 623         except Exception as e:
 624             return "BOOK, serializing for api generated exception: " + str(e) + " CODE:0xFFFF"
 625
 626         try:
 627             booking.save()
 628         except Exception as e:
 629             return "BOOK, saving booking generated exception: " + str(e) + " CODE:0x0016"
 630
 631         self.el[self.RESULT] = booking
 632         self.el[self.HAS_RESULT] = True
 633
 634     def make_opnfv_config(self):
 635         opnfv_models = self.el[self.OPNFV_MODELS]
 636         config_bundle = self.el[self.SELECTED_CONFIG_BUNDLE]
 637         if not config_bundle:
 638             return "No Configuration bundle selected"
 639         info = opnfv_models.get("meta", {})
 640         name = info.get("name", False)
 641         desc = info.get("description", False)
 642         if not (name and desc):
 643             return "No name or description given"
 644         installer = opnfv_models['installer_chosen']
 645         if not installer:
 646             return "No OPNFV Installer chosen"
 647         scenario = opnfv_models['scenario_chosen']
 648         if not scenario:
 649             return "No OPNFV Scenario chosen"
 650
 651         opnfv_config = OPNFVConfig.objects.create(
 652             bundle=config_bundle,
 653             name=name,
 654             description=desc,
 655             installer=installer,
 656             scenario=scenario
 657         )
 658
 659         network_roles = opnfv_models['network_roles']
 660         for net_role in network_roles:
 661             opnfv_config.networks.add(
 662                 NetworkRole.objects.create(
 663                     name=net_role['role'],
 664                     network=net_role['network']
 665                 )
 666             )
 667
 668         host_roles = opnfv_models['host_roles']
 669         for host_role in host_roles:
 670             config = config_bundle.hostConfigurations.get(
 671                 host__resource__name=host_role['host_name']
 672             )
 673             ResourceOPNFVConfig.objects.create(
 674                 role=host_role['role'],
 675                 host_config=config,
 676                 opnfv_config=opnfv_config
 677             )
 678
 679         self.el[self.RESULT] = opnfv_config
 680         self.el[self.HAS_RESULT] = True
 681
 682     def __init__(self):
 683         self.el = {}
 684         self.el[self.CONFIRMATION] = {}
 685         self.el["active_step"] = 0
 686         self.el[self.HAS_RESULT] = False
 687         self.el[self.RESULT] = None
 688         self.get_history = {}
 689         self.put_history = {}