dashboard/src/workflow/models.py

   1 ##############################################################################
   2 # Copyright (c) 2018 Sawyer Bergeron, Parker Berberian, and others.
   3 #
   4 # All rights reserved. This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 # http://www.apache.org/licenses/LICENSE-2.0
   8 ##############################################################################
   9
  10
  11 from django.shortcuts import render
  12 from django.contrib import messages
  13 from django.http import HttpResponse
  14 from django.utils import timezone
  15
  16 import yaml
  17 import requests
  18
  19 from workflow.forms import ConfirmationForm
  20 from api.models import JobFactory
  21 from dashboard.exceptions import ResourceAvailabilityException, ModelValidationException
  22 from resource_inventory.models import Image, GenericInterface, OPNFVConfig, HostOPNFVConfig, NetworkRole
  23 from resource_inventory.resource_manager import ResourceManager
  24 from resource_inventory.pdf_templater import PDFTemplater
  25 from notifier.manager import NotificationHandler
  26 from booking.models import Booking
  27
  28
  29 class BookingAuthManager():
  30     LFN_PROJECTS = ["opnfv"]  # TODO
  31
  32     def parse_github_url(self, url):
  33         project_leads = []
  34         try:
  35             parts = url.split("/")
  36             if "http" in parts[0]:  # the url include http(s)://
  37                 parts = parts[2:]
  38             if parts[-1] != "INFO.yaml":
  39                 return None
  40             if parts[0] not in ["github.com", "raw.githubusercontent.com"]:
  41                 return None
  42             if parts[1] not in self.LFN_PROJECTS:
  43                 return None
  44             # now to download and parse file
  45             if parts[3] == "blob":
  46                 parts[3] = "raw"
  47             url = "https://" + "/".join(parts)
  48             info_file = requests.get(url, timeout=15).text
  49             info_parsed = yaml.load(info_file)
  50             ptl = info_parsed.get('project_lead')
  51             if ptl:
  52                 project_leads.append(ptl)
  53             sub_ptl = info_parsed.get("subproject_lead")
  54             if sub_ptl:
  55                 project_leads.append(sub_ptl)
  56
  57         except Exception:
  58             pass
  59
  60         return project_leads
  61
  62     def parse_gerrit_url(self, url):
  63         project_leads = []
  64         try:
  65             halfs = url.split("?")
  66             parts = halfs[0].split("/")
  67             args = halfs[1].split(";")
  68             if "http" in parts[0]:  # the url include http(s)://
  69                 parts = parts[2:]
  70             if "f=INFO.yaml" not in args:
  71                 return None
  72             if "gerrit.opnfv.org" not in parts[0]:
  73                 return None
  74             try:
  75                 i = args.index("a=blob")
  76                 args[i] = "a=blob_plain"
  77             except ValueError:
  78                 pass
  79             # recreate url
  80             halfs[1] = ";".join(args)
  81             halfs[0] = "/".join(parts)
  82             # now to download and parse file
  83             url = "https://" + "?".join(halfs)
  84             info_file = requests.get(url, timeout=15).text
  85             info_parsed = yaml.load(info_file)
  86             ptl = info_parsed.get('project_lead')
  87             if ptl:
  88                 project_leads.append(ptl)
  89             sub_ptl = info_parsed.get("subproject_lead")
  90             if sub_ptl:
  91                 project_leads.append(sub_ptl)
  92
  93         except Exception:
  94             return None
  95
  96         return project_leads
  97
  98     def parse_opnfv_git_url(self, url):
  99         project_leads = []
 100         try:
 101             parts = url.split("/")
 102             if "http" in parts[0]:  # the url include http(s)://
 103                 parts = parts[2:]
 104             if "INFO.yaml" not in parts[-1]:
 105                 return None
 106             if "git.opnfv.org" not in parts[0]:
 107                 return None
 108             if parts[-2] == "tree":
 109                 parts[-2] = "plain"
 110             # now to download and parse file
 111             url = "https://" + "/".join(parts)
 112             info_file = requests.get(url, timeout=15).text
 113             info_parsed = yaml.load(info_file)
 114             ptl = info_parsed.get('project_lead')
 115             if ptl:
 116                 project_leads.append(ptl)
 117             sub_ptl = info_parsed.get("subproject_lead")
 118             if sub_ptl:
 119                 project_leads.append(sub_ptl)
 120
 121         except Exception:
 122             return None
 123
 124         return project_leads
 125
 126     def parse_url(self, info_url):
 127         """
 128         will return the PTL in the INFO file on success, or None
 129         """
 130         if "github" in info_url:
 131             return self.parse_github_url(info_url)
 132
 133         if "gerrit.opnfv.org" in info_url:
 134             return self.parse_gerrit_url(info_url)
 135
 136         if "git.opnfv.org" in info_url:
 137             return self.parse_opnfv_git_url(info_url)
 138
 139     def booking_allowed(self, booking, repo):
 140         """
 141         This is the method that will have to change whenever the booking policy changes in the Infra
 142         group / LFN. This is a nice isolation of that administration crap
 143         currently checks if the booking uses multiple servers. if it does, then the owner must be a PTL,
 144         which is checked using the provided info file
 145         """
 146         if booking.owner.userprofile.booking_privledge:
 147             return True  # admin override for this user
 148         if Booking.objects.filter(owner=booking.owner, end__gt=timezone.now()).count() >= 3:
 149             return False
 150         if len(booking.resource.template.getHosts()) < 2:
 151             return True  # if they only have one server, we dont care
 152         if repo.BOOKING_INFO_FILE not in repo.el:
 153             return False  # INFO file not provided
 154         ptl_info = self.parse_url(repo.el.get(repo.BOOKING_INFO_FILE))
 155         for ptl in ptl_info:
 156             if ptl['email'] == booking.owner.userprofile.email_addr:
 157                 return True
 158         return False
 159
 160
 161 class WorkflowStep(object):
 162     template = 'bad_request.html'
 163     title = "Generic Step"
 164     description = "You were led here by mistake"
 165     short_title = "error"
 166     metastep = None
 167
 168     def __init__(self, id, repo=None):
 169         self.repo = repo
 170         self.id = id
 171
 172     def get_context(self):
 173         context = {}
 174         context['step_number'] = self.repo_get('steps')
 175         context['active_step'] = self.repo_get('active_step')
 176         context['render_correct'] = "true"
 177         context['step_title'] = self.title
 178         context['description'] = self.description
 179         return context
 180
 181     def render(self, request):
 182         self.context = self.get_context()
 183         return render(request, self.template, self.context)
 184
 185     def post_render(self, request):
 186         return self.render(request)
 187
 188     def test_render(self, request):
 189         if request.method == "POST":
 190             return self.post_render(request)
 191         return self.render(request)
 192
 193     def validate(self, request):
 194         pass
 195
 196     def repo_get(self, key, default=None):
 197         return self.repo.get(key, default, self.id)
 198
 199     def repo_put(self, key, value):
 200         return self.repo.put(key, value, self.id)
 201
 202
 203 class Confirmation_Step(WorkflowStep):
 204     template = 'workflow/confirm.html'
 205     title = "Confirm Changes"
 206     description = "Does this all look right?"
 207
 208     def get_context(self):
 209         context = super(Confirmation_Step, self).get_context()
 210         context['form'] = ConfirmationForm()
 211         context['confirmation_info'] = yaml.dump(
 212             self.repo_get(self.repo.CONFIRMATION),
 213             default_flow_style=False
 214         ).strip()
 215
 216         return context
 217
 218     def flush_to_db(self):
 219         errors = self.repo.make_models()
 220         if errors:
 221             return errors
 222
 223     def post_render(self, request):
 224         form = ConfirmationForm(request.POST)
 225         if form.is_valid():
 226             data = form.cleaned_data['confirm']
 227             context = self.get_context()
 228             if data == "True":
 229                 context["bypassed"] = "true"
 230                 errors = self.flush_to_db()
 231                 if errors:
 232                     messages.add_message(request, messages.ERROR, "ERROR OCCURRED: " + errors)
 233                 else:
 234                     messages.add_message(request, messages.SUCCESS, "Confirmed")
 235
 236                 return HttpResponse('')
 237             elif data == "False":
 238                 context["bypassed"] = "true"
 239                 messages.add_message(request, messages.SUCCESS, "Canceled")
 240                 return render(request, self.template, context)
 241             else:
 242                 pass
 243
 244         else:
 245             pass
 246
 247
 248 class Workflow():
 249
 250     steps = []
 251     active_index = 0
 252
 253
 254 class Repository():
 255
 256     EDIT = "editing"
 257     MODELS = "models"
 258     RESOURCE_SELECT = "resource_select"
 259     CONFIRMATION = "confirmation"
 260     SELECTED_GRESOURCE_BUNDLE = "selected generic bundle pk"
 261     SELECTED_CONFIG_BUNDLE = "selected config bundle pk"
 262     SELECTED_OPNFV_CONFIG = "selected opnfv deployment config"
 263     GRESOURCE_BUNDLE_MODELS = "generic_resource_bundle_models"
 264     GRESOURCE_BUNDLE_INFO = "generic_resource_bundle_info"
 265     BOOKING = "booking"
 266     LAB = "lab"
 267     GRB_LAST_HOSTLIST = "grb_network_previous_hostlist"
 268     BOOKING_FORMS = "booking_forms"
 269     SWCONF_HOSTS = "swconf_hosts"
 270     BOOKING_MODELS = "booking models"
 271     CONFIG_MODELS = "configuration bundle models"
 272     OPNFV_MODELS = "opnfv configuration models"
 273     SESSION_USER = "session owner user account"
 274     VALIDATED_MODEL_GRB = "valid grb config model instance in db"
 275     VALIDATED_MODEL_CONFIG = "valid config model instance in db"
 276     VALIDATED_MODEL_BOOKING = "valid booking model instance in db"
 277     VLANS = "a list of vlans"
 278     SNAPSHOT_MODELS = "the models for snapshotting"
 279     SNAPSHOT_BOOKING_ID = "the booking id for snapshotting"
 280     SNAPSHOT_NAME = "the name of the snapshot"
 281     SNAPSHOT_DESC = "description of the snapshot"
 282     BOOKING_INFO_FILE = "the INFO.yaml file for this user's booking"
 283
 284     # migratory elements of segmented workflow
 285     # each of these is the end result of a different workflow.
 286     HAS_RESULT = "whether or not workflow has a result"
 287     RESULT_KEY = "key for target index that result will be put into in parent"
 288     RESULT = "result object from workflow"
 289
 290     def get_child_defaults(self):
 291         return_tuples = []
 292         for key in [self.SELECTED_GRESOURCE_BUNDLE, self.SESSION_USER]:
 293             return_tuples.append((key, self.el.get(key)))
 294         return return_tuples
 295
 296     def set_defaults(self, defaults):
 297         for key, value in defaults:
 298             self.el[key] = value
 299
 300     def get(self, key, default, id):
 301         self.add_get_history(key, id)
 302         return self.el.get(key, default)
 303
 304     def put(self, key, val, id):
 305         self.add_put_history(key, id)
 306         self.el[key] = val
 307
 308     def add_get_history(self, key, id):
 309         self.add_history(key, id, self.get_history)
 310
 311     def add_put_history(self, key, id):
 312         self.add_history(key, id, self.put_history)
 313
 314     def add_history(self, key, id, history):
 315         if key not in history:
 316             history[key] = [id]
 317         else:
 318             history[key].append(id)
 319
 320     def make_models(self):
 321         if self.SNAPSHOT_MODELS in self.el:
 322             errors = self.make_snapshot()
 323             if errors:
 324                 return errors
 325         # if GRB WF, create it
 326         if self.GRESOURCE_BUNDLE_MODELS in self.el:
 327             errors = self.make_generic_resource_bundle()
 328             if errors:
 329                 return errors
 330             else:
 331                 self.el[self.HAS_RESULT] = True
 332                 self.el[self.RESULT_KEY] = self.SELECTED_GRESOURCE_BUNDLE
 333                 return
 334
 335         if self.CONFIG_MODELS in self.el:
 336             errors = self.make_software_config_bundle()
 337             if errors:
 338                 return errors
 339             else:
 340                 self.el[self.HAS_RESULT] = True
 341                 self.el[self.RESULT_KEY] = self.SELECTED_CONFIG_BUNDLE
 342                 return
 343
 344         if self.OPNFV_MODELS in self.el:
 345             errors = self.make_opnfv_config()
 346             if errors:
 347                 return errors
 348             else:
 349                 self.el[self.HAS_RESULT] = True
 350                 self.el[self.RESULT_KEY] = self.SELECTED_OPNFV_CONFIG
 351
 352         if self.BOOKING_MODELS in self.el:
 353             errors = self.make_booking()
 354             if errors:
 355                 return errors
 356             # create notification
 357             booking = self.el[self.BOOKING_MODELS]['booking']
 358             NotificationHandler.notify_new_booking(booking)
 359
 360     def make_snapshot(self):
 361         owner = self.el[self.SESSION_USER]
 362         models = self.el[self.SNAPSHOT_MODELS]
 363         image = models.get('snapshot', Image())
 364         booking_id = self.el.get(self.SNAPSHOT_BOOKING_ID)
 365         if not booking_id:
 366             return "SNAP, No booking ID provided"
 367         booking = Booking.objects.get(pk=booking_id)
 368         if booking.start > timezone.now() or booking.end < timezone.now():
 369             return "Booking is not active"
 370         name = self.el.get(self.SNAPSHOT_NAME)
 371         if not name:
 372             return "SNAP, no name provided"
 373         host = models.get('host')
 374         if not host:
 375             return "SNAP, no host provided"
 376         description = self.el.get(self.SNAPSHOT_DESC, "")
 377         image.from_lab = booking.lab
 378         image.name = name
 379         image.description = description
 380         image.public = False
 381         image.lab_id = -1
 382         image.owner = owner
 383         image.host_type = host.profile
 384         image.save()
 385         try:
 386             current_image = host.config.image
 387             image.os = current_image.os
 388             image.save()
 389         except Exception:
 390             pass
 391         JobFactory.makeSnapshotTask(image, booking, host)
 392
 393     def make_generic_resource_bundle(self):
 394         owner = self.el[self.SESSION_USER]
 395         if self.GRESOURCE_BUNDLE_MODELS in self.el:
 396             models = self.el[self.GRESOURCE_BUNDLE_MODELS]
 397             if 'hosts' in models:
 398                 hosts = models['hosts']
 399             else:
 400                 return "GRB has no hosts. CODE:0x0002"
 401             if 'bundle' in models:
 402                 bundle = models['bundle']
 403             else:
 404                 return "GRB, no bundle in models. CODE:0x0003"
 405
 406             try:
 407                 bundle.owner = owner
 408                 bundle.save()
 409             except Exception as e:
 410                 return "GRB, saving bundle generated exception: " + str(e) + " CODE:0x0004"
 411             try:
 412                 for host in hosts:
 413                     genericresource = host.resource
 414                     genericresource.bundle = bundle
 415                     genericresource.save()
 416                     host.resource = genericresource
 417                     host.save()
 418             except Exception as e:
 419                 return "GRB, saving hosts generated exception: " + str(e) + " CODE:0x0005"
 420
 421             if 'networks' in models:
 422                 for net in models['networks'].values():
 423                     net.bundle = bundle
 424                     net.save()
 425
 426             if 'interfaces' in models:
 427                 for interface_set in models['interfaces'].values():
 428                     for interface in interface_set:
 429                         try:
 430                             interface.host = interface.host
 431                             interface.save()
 432                         except Exception:
 433                             return "GRB, saving interface " + str(interface) + " failed. CODE:0x0019"
 434             else:
 435                 return "GRB, no interface set provided. CODE:0x001a"
 436
 437             if 'connections' in models:
 438                 for resource_name, mapping in models['connections'].items():
 439                     for profile_name, connection_set in mapping.items():
 440                         interface = GenericInterface.objects.get(
 441                             profile__name=profile_name,
 442                             host__resource__name=resource_name,
 443                             host__resource__bundle=models['bundle']
 444                         )
 445                         for connection in connection_set:
 446                             try:
 447                                 connection.network = connection.network
 448                                 connection.save()
 449                                 interface.connections.add(connection)
 450                             except Exception as e:
 451                                 return "GRB, saving vlan " + str(connection) + " failed. Exception: " + str(e) + ". CODE:0x0017"
 452             else:
 453                 return "GRB, no vlan set provided. CODE:0x0018"
 454
 455         else:
 456             return "GRB no models given. CODE:0x0001"
 457
 458         self.el[self.RESULT] = bundle
 459         return False
 460
 461     def make_software_config_bundle(self):
 462         models = self.el[self.CONFIG_MODELS]
 463         if 'bundle' in models:
 464             bundle = models['bundle']
 465             bundle.bundle = bundle.bundle
 466             try:
 467                 bundle.save()
 468             except Exception as e:
 469                 return "SWC, saving bundle generated exception: " + str(e) + "CODE:0x0007"
 470
 471         else:
 472             return "SWC, no bundle in models. CODE:0x0006"
 473         if 'host_configs' in models:
 474             host_configs = models['host_configs']
 475             for host_config in host_configs:
 476                 host_config.bundle = host_config.bundle
 477                 host_config.host = host_config.host
 478                 try:
 479                     host_config.save()
 480                 except Exception as e:
 481                     return "SWC, saving host configs generated exception: " + str(e) + "CODE:0x0009"
 482         else:
 483             return "SWC, no host configs in models. CODE:0x0008"
 484         if 'opnfv' in models:
 485             opnfvconfig = models['opnfv']
 486             opnfvconfig.bundle = opnfvconfig.bundle
 487             if opnfvconfig.scenario not in opnfvconfig.installer.sup_scenarios.all():
 488                 return "SWC, scenario not supported by installer. CODE:0x000d"
 489             try:
 490                 opnfvconfig.save()
 491             except Exception as e:
 492                 return "SWC, saving opnfv config generated exception: " + str(e) + "CODE:0x000b"
 493         else:
 494             pass
 495
 496         self.el[self.RESULT] = bundle
 497         return False
 498
 499     def make_booking(self):
 500         models = self.el[self.BOOKING_MODELS]
 501         owner = self.el[self.SESSION_USER]
 502
 503         if self.SELECTED_GRESOURCE_BUNDLE in self.el:
 504             selected_grb = self.el[self.SELECTED_GRESOURCE_BUNDLE]
 505         else:
 506             return "BOOK, no selected resource. CODE:0x000e"
 507
 508         if 'booking' in models:
 509             booking = models['booking']
 510         else:
 511             return "BOOK, no booking model exists. CODE:0x000f"
 512
 513         if not booking.start:
 514             return "BOOK, booking has no start. CODE:0x0010"
 515         if not booking.end:
 516             return "BOOK, booking has no end. CODE:0x0011"
 517         if booking.end <= booking.start:
 518             return "BOOK, end before/same time as start. CODE:0x0012"
 519
 520         if 'collaborators' in models:
 521             collaborators = models['collaborators']
 522         else:
 523             return "BOOK, collaborators not defined. CODE:0x0013"
 524         try:
 525             resource_bundle = ResourceManager.getInstance().convertResourceBundle(selected_grb, config=booking.config_bundle)
 526         except ResourceAvailabilityException as e:
 527             return "BOOK, requested resources are not available. Exception: " + str(e) + " CODE:0x0014"
 528         except ModelValidationException as e:
 529             return "Error encountered when saving bundle. " + str(e) + " CODE: 0x001b"
 530
 531         booking.resource = resource_bundle
 532         booking.owner = owner
 533         booking.config_bundle = booking.config_bundle
 534         booking.lab = selected_grb.lab
 535
 536         is_allowed = BookingAuthManager().booking_allowed(booking, self)
 537         if not is_allowed:
 538             return "BOOK, you are not allowed to book the requested resources"
 539
 540         try:
 541             booking.save()
 542         except Exception as e:
 543             return "BOOK, saving booking generated exception: " + str(e) + " CODE:0x0015"
 544
 545         for collaborator in collaborators:
 546             booking.collaborators.add(collaborator)
 547
 548         try:
 549             booking.pdf = PDFTemplater.makePDF(booking)
 550             booking.save()
 551         except Exception as e:
 552             return "BOOK, failed to create Pod Desriptor File: " + str(e)
 553
 554         try:
 555             JobFactory.makeCompleteJob(booking)
 556         except Exception as e:
 557             return "BOOK, serializing for api generated exception: " + str(e) + " CODE:0xFFFF"
 558
 559         try:
 560             booking.save()
 561         except Exception as e:
 562             return "BOOK, saving booking generated exception: " + str(e) + " CODE:0x0016"
 563
 564     def make_opnfv_config(self):
 565         opnfv_models = self.el[self.OPNFV_MODELS]
 566         config_bundle = opnfv_models['configbundle']
 567         if not config_bundle:
 568             return "No Configuration bundle selected"
 569         info = opnfv_models.get("meta", {})
 570         name = info.get("name", False)
 571         desc = info.get("description", False)
 572         if not (name and desc):
 573             return "No name or description given"
 574         installer = opnfv_models['installer_chosen']
 575         if not installer:
 576             return "No OPNFV Installer chosen"
 577         scenario = opnfv_models['scenario_chosen']
 578         if not scenario:
 579             return "No OPNFV Scenario chosen"
 580
 581         opnfv_config = OPNFVConfig.objects.create(
 582             bundle=config_bundle,
 583             name=name,
 584             description=desc,
 585             installer=installer,
 586             scenario=scenario
 587         )
 588
 589         network_roles = opnfv_models['network_roles']
 590         for net_role in network_roles:
 591             opnfv_config.networks.add(
 592                 NetworkRole.objects.create(
 593                     name=net_role['role'],
 594                     network=net_role['network']
 595                 )
 596             )
 597
 598         host_roles = opnfv_models['host_roles']
 599         for host_role in host_roles:
 600             config = config_bundle.hostConfigurations.get(
 601                 host__resource__name=host_role['host_name']
 602             )
 603             HostOPNFVConfig.objects.create(
 604                 role=host_role['role'],
 605                 host_config=config,
 606                 opnfv_config=opnfv_config
 607             )
 608
 609         self.el[self.RESULT] = opnfv_config
 610
 611     def __init__(self):
 612         self.el = {}
 613         self.el[self.CONFIRMATION] = {}
 614         self.el["active_step"] = 0
 615         self.get_history = {}
 616         self.put_history = {}