src/workflow/models.py

   1 ##############################################################################
   2 # Copyright (c) 2018 Sawyer Bergeron, Parker Berberian, and others.
   3 #
   4 # All rights reserved. This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 # http://www.apache.org/licenses/LICENSE-2.0
   8 ##############################################################################
   9
  10
  11 from django.shortcuts import render
  12 from django.contrib import messages
  13 from django.http import HttpResponse
  14 from django.utils import timezone
  15
  16 import yaml
  17 import requests
  18
  19 from workflow.forms import ConfirmationForm
  20 from api.models import JobFactory
  21 from dashboard.exceptions import ResourceAvailabilityException, ModelValidationException
  22 from resource_inventory.models import Image, GenericInterface, OPNFVConfig, HostOPNFVConfig, NetworkRole
  23 from resource_inventory.resource_manager import ResourceManager
  24 from resource_inventory.pdf_templater import PDFTemplater
  25 from notifier.manager import NotificationHandler
  26 from booking.models import Booking
  27
  28
  29 class BookingAuthManager():
  30     LFN_PROJECTS = ["opnfv"]  # TODO
  31
  32     def parse_github_url(self, url):
  33         project_leads = []
  34         try:
  35             parts = url.split("/")
  36             if "http" in parts[0]:  # the url include http(s)://
  37                 parts = parts[2:]
  38             if parts[-1] != "INFO.yaml":
  39                 return None
  40             if parts[0] not in ["github.com", "raw.githubusercontent.com"]:
  41                 return None
  42             if parts[1] not in self.LFN_PROJECTS:
  43                 return None
  44             # now to download and parse file
  45             if parts[3] == "blob":
  46                 parts[3] = "raw"
  47             url = "https://" + "/".join(parts)
  48             info_file = requests.get(url, timeout=15).text
  49             info_parsed = yaml.load(info_file)
  50             ptl = info_parsed.get('project_lead')
  51             if ptl:
  52                 project_leads.append(ptl)
  53             sub_ptl = info_parsed.get("subproject_lead")
  54             if sub_ptl:
  55                 project_leads.append(sub_ptl)
  56
  57         except Exception:
  58             pass
  59
  60         return project_leads
  61
  62     def parse_gerrit_url(self, url):
  63         project_leads = []
  64         try:
  65             halfs = url.split("?")
  66             parts = halfs[0].split("/")
  67             args = halfs[1].split(";")
  68             if "http" in parts[0]:  # the url include http(s)://
  69                 parts = parts[2:]
  70             if "f=INFO.yaml" not in args:
  71                 return None
  72             if "gerrit.opnfv.org" not in parts[0]:
  73                 return None
  74             try:
  75                 i = args.index("a=blob")
  76                 args[i] = "a=blob_plain"
  77             except ValueError:
  78                 pass
  79             # recreate url
  80             halfs[1] = ";".join(args)
  81             halfs[0] = "/".join(parts)
  82             # now to download and parse file
  83             url = "https://" + "?".join(halfs)
  84             info_file = requests.get(url, timeout=15).text
  85             info_parsed = yaml.load(info_file)
  86             ptl = info_parsed.get('project_lead')
  87             if ptl:
  88                 project_leads.append(ptl)
  89             sub_ptl = info_parsed.get("subproject_lead")
  90             if sub_ptl:
  91                 project_leads.append(sub_ptl)
  92
  93         except Exception:
  94             return None
  95
  96         return project_leads
  97
  98     def parse_opnfv_git_url(self, url):
  99         project_leads = []
 100         try:
 101             parts = url.split("/")
 102             if "http" in parts[0]:  # the url include http(s)://
 103                 parts = parts[2:]
 104             if "INFO.yaml" not in parts[-1]:
 105                 return None
 106             if "git.opnfv.org" not in parts[0]:
 107                 return None
 108             if parts[-2] == "tree":
 109                 parts[-2] = "plain"
 110             # now to download and parse file
 111             url = "https://" + "/".join(parts)
 112             info_file = requests.get(url, timeout=15).text
 113             info_parsed = yaml.load(info_file)
 114             ptl = info_parsed.get('project_lead')
 115             if ptl:
 116                 project_leads.append(ptl)
 117             sub_ptl = info_parsed.get("subproject_lead")
 118             if sub_ptl:
 119                 project_leads.append(sub_ptl)
 120
 121         except Exception:
 122             return None
 123
 124         return project_leads
 125
 126     def parse_url(self, info_url):
 127         """
 128         will return the PTL in the INFO file on success, or None
 129         """
 130         if "github" in info_url:
 131             return self.parse_github_url(info_url)
 132
 133         if "gerrit.opnfv.org" in info_url:
 134             return self.parse_gerrit_url(info_url)
 135
 136         if "git.opnfv.org" in info_url:
 137             return self.parse_opnfv_git_url(info_url)
 138
 139     def booking_allowed(self, booking, repo):
 140         """
 141         This is the method that will have to change whenever the booking policy changes in the Infra
 142         group / LFN. This is a nice isolation of that administration crap
 143         currently checks if the booking uses multiple servers. if it does, then the owner must be a PTL,
 144         which is checked using the provided info file
 145         """
 146         if booking.owner.userprofile.booking_privledge:
 147             return True  # admin override for this user
 148         if Booking.objects.filter(owner=booking.owner, end__gt=timezone.now()).count() >= 3:
 149             return False
 150         if len(booking.resource.template.getHosts()) < 2:
 151             return True  # if they only have one server, we dont care
 152         if repo.BOOKING_INFO_FILE not in repo.el:
 153             return False  # INFO file not provided
 154         ptl_info = self.parse_url(repo.el.get(repo.BOOKING_INFO_FILE))
 155         for ptl in ptl_info:
 156             if ptl['email'] == booking.owner.userprofile.email_addr:
 157                 return True
 158         return False
 159
 160
 161 class WorkflowStepStatus(object):
 162     UNTOUCHED = 0
 163     INVALID = 100
 164     VALID = 200
 165
 166
 167 class WorkflowStep(object):
 168     template = 'bad_request.html'
 169     title = "Generic Step"
 170     description = "You were led here by mistake"
 171     short_title = "error"
 172     metastep = None
 173     # phasing out metastep:
 174
 175     valid = WorkflowStepStatus.UNTOUCHED
 176     message = ""
 177
 178     enabled = True
 179
 180     def cleanup(self):
 181         raise Exception("WorkflowStep subclass of type " + str(type(self)) + " has no concrete implemented cleanup() method")
 182
 183     def enable(self):
 184         if not self.enabled:
 185             self.enabled = True
 186
 187     def disable(self):
 188         if self.enabled:
 189             self.cleanup()
 190             self.enabled = False
 191
 192     def set_invalid(self, message, code=WorkflowStepStatus.INVALID):
 193         self.valid = code
 194         self.message = message
 195
 196     def set_valid(self, message, code=WorkflowStepStatus.VALID):
 197         self.valid = code
 198         self.message = message
 199
 200     def to_json(self):
 201         return {
 202             'title': self.short_title,
 203             'enabled': self.enabled,
 204             'valid': self.valid,
 205             'message': self.message,
 206         }
 207
 208     def __init__(self, id, repo=None):
 209         self.repo = repo
 210         self.id = id
 211
 212     def get_context(self):
 213         context = {}
 214         context['step_number'] = self.repo_get('steps')
 215         context['active_step'] = self.repo_get('active_step')
 216         context['render_correct'] = "true"
 217         context['step_title'] = self.title
 218         context['description'] = self.description
 219         return context
 220
 221     def render(self, request):
 222         self.context = self.get_context()
 223         return render(request, self.template, self.context)
 224
 225     def post_render(self, request):
 226         return self.render(request)
 227
 228     def test_render(self, request):
 229         if request.method == "POST":
 230             return self.post_render(request)
 231         return self.render(request)
 232
 233     def validate(self, request):
 234         pass
 235
 236     def repo_get(self, key, default=None):
 237         return self.repo.get(key, default, self.id)
 238
 239     def repo_put(self, key, value):
 240         return self.repo.put(key, value, self.id)
 241
 242
 243 class Confirmation_Step(WorkflowStep):
 244     template = 'workflow/confirm.html'
 245     title = "Confirm Changes"
 246     description = "Does this all look right?"
 247
 248     short_title = "confirm"
 249
 250     def get_context(self):
 251         context = super(Confirmation_Step, self).get_context()
 252         context['form'] = ConfirmationForm()
 253         context['confirmation_info'] = yaml.dump(
 254             self.repo_get(self.repo.CONFIRMATION),
 255             default_flow_style=False
 256         ).strip()
 257
 258         return context
 259
 260     def flush_to_db(self):
 261         errors = self.repo.make_models()
 262         if errors:
 263             return errors
 264
 265     def post_render(self, request):
 266         form = ConfirmationForm(request.POST)
 267         if form.is_valid():
 268             data = form.cleaned_data['confirm']
 269             context = self.get_context()
 270             if data == "True":
 271                 context["bypassed"] = "true"
 272                 errors = self.flush_to_db()
 273                 if errors:
 274                     messages.add_message(request, messages.ERROR, "ERROR OCCURRED: " + errors)
 275                 else:
 276                     messages.add_message(request, messages.SUCCESS, "Confirmed")
 277
 278                 return HttpResponse('')
 279             elif data == "False":
 280                 context["bypassed"] = "true"
 281                 messages.add_message(request, messages.SUCCESS, "Canceled")
 282                 return render(request, self.template, context)
 283             else:
 284                 pass
 285
 286         else:
 287             pass
 288
 289
 290 class Repository():
 291
 292     EDIT = "editing"
 293     MODELS = "models"
 294     RESOURCE_SELECT = "resource_select"
 295     CONFIRMATION = "confirmation"
 296     SELECTED_GRESOURCE_BUNDLE = "selected generic bundle pk"
 297     SELECTED_CONFIG_BUNDLE = "selected config bundle pk"
 298     SELECTED_OPNFV_CONFIG = "selected opnfv deployment config"
 299     GRESOURCE_BUNDLE_MODELS = "generic_resource_bundle_models"
 300     GRESOURCE_BUNDLE_INFO = "generic_resource_bundle_info"
 301     BOOKING = "booking"
 302     LAB = "lab"
 303     GRB_LAST_HOSTLIST = "grb_network_previous_hostlist"
 304     BOOKING_FORMS = "booking_forms"
 305     SWCONF_HOSTS = "swconf_hosts"
 306     BOOKING_MODELS = "booking models"
 307     CONFIG_MODELS = "configuration bundle models"
 308     OPNFV_MODELS = "opnfv configuration models"
 309     SESSION_USER = "session owner user account"
 310     SESSION_MANAGER = "session manager for current session"
 311     VALIDATED_MODEL_GRB = "valid grb config model instance in db"
 312     VALIDATED_MODEL_CONFIG = "valid config model instance in db"
 313     VALIDATED_MODEL_BOOKING = "valid booking model instance in db"
 314     VLANS = "a list of vlans"
 315     SNAPSHOT_MODELS = "the models for snapshotting"
 316     SNAPSHOT_BOOKING_ID = "the booking id for snapshotting"
 317     SNAPSHOT_NAME = "the name of the snapshot"
 318     SNAPSHOT_DESC = "description of the snapshot"
 319     BOOKING_INFO_FILE = "the INFO.yaml file for this user's booking"
 320
 321     # migratory elements of segmented workflow
 322     # each of these is the end result of a different workflow.
 323     HAS_RESULT = "whether or not workflow has a result"
 324     RESULT_KEY = "key for target index that result will be put into in parent"
 325     RESULT = "result object from workflow"
 326
 327     def get_child_defaults(self):
 328         return_tuples = []
 329         for key in [self.SELECTED_GRESOURCE_BUNDLE, self.SESSION_USER]:
 330             return_tuples.append((key, self.el.get(key)))
 331         return return_tuples
 332
 333     def set_defaults(self, defaults):
 334         for key, value in defaults:
 335             self.el[key] = value
 336
 337     def get(self, key, default, id):
 338         self.add_get_history(key, id)
 339         return self.el.get(key, default)
 340
 341     def put(self, key, val, id):
 342         self.add_put_history(key, id)
 343         self.el[key] = val
 344
 345     def add_get_history(self, key, id):
 346         self.add_history(key, id, self.get_history)
 347
 348     def add_put_history(self, key, id):
 349         self.add_history(key, id, self.put_history)
 350
 351     def add_history(self, key, id, history):
 352         if key not in history:
 353             history[key] = [id]
 354         else:
 355             history[key].append(id)
 356
 357     def make_models(self):
 358         if self.SNAPSHOT_MODELS in self.el:
 359             errors = self.make_snapshot()
 360             if errors:
 361                 return errors
 362         # if GRB WF, create it
 363         if self.GRESOURCE_BUNDLE_MODELS in self.el:
 364             errors = self.make_generic_resource_bundle()
 365             if errors:
 366                 return errors
 367             else:
 368                 self.el[self.HAS_RESULT] = True
 369                 self.el[self.RESULT_KEY] = self.SELECTED_GRESOURCE_BUNDLE
 370                 return
 371
 372         if self.CONFIG_MODELS in self.el:
 373             errors = self.make_software_config_bundle()
 374             if errors:
 375                 return errors
 376             else:
 377                 self.el[self.HAS_RESULT] = True
 378                 self.el[self.RESULT_KEY] = self.SELECTED_CONFIG_BUNDLE
 379                 return
 380
 381         if self.OPNFV_MODELS in self.el:
 382             errors = self.make_opnfv_config()
 383             if errors:
 384                 return errors
 385             else:
 386                 self.el[self.HAS_RESULT] = True
 387                 self.el[self.RESULT_KEY] = self.SELECTED_OPNFV_CONFIG
 388
 389         if self.BOOKING_MODELS in self.el:
 390             errors = self.make_booking()
 391             if errors:
 392                 return errors
 393             # create notification
 394             booking = self.el[self.BOOKING_MODELS]['booking']
 395             NotificationHandler.notify_new_booking(booking)
 396
 397     def make_snapshot(self):
 398         owner = self.el[self.SESSION_USER]
 399         models = self.el[self.SNAPSHOT_MODELS]
 400         image = models.get('snapshot', Image())
 401         booking_id = self.el.get(self.SNAPSHOT_BOOKING_ID)
 402         if not booking_id:
 403             return "SNAP, No booking ID provided"
 404         booking = Booking.objects.get(pk=booking_id)
 405         if booking.start > timezone.now() or booking.end < timezone.now():
 406             return "Booking is not active"
 407         name = self.el.get(self.SNAPSHOT_NAME)
 408         if not name:
 409             return "SNAP, no name provided"
 410         host = models.get('host')
 411         if not host:
 412             return "SNAP, no host provided"
 413         description = self.el.get(self.SNAPSHOT_DESC, "")
 414         image.from_lab = booking.lab
 415         image.name = name
 416         image.description = description
 417         image.public = False
 418         image.lab_id = -1
 419         image.owner = owner
 420         image.host_type = host.profile
 421         image.save()
 422         try:
 423             current_image = host.config.image
 424             image.os = current_image.os
 425             image.save()
 426         except Exception:
 427             pass
 428         JobFactory.makeSnapshotTask(image, booking, host)
 429
 430     def make_generic_resource_bundle(self):
 431         owner = self.el[self.SESSION_USER]
 432         if self.GRESOURCE_BUNDLE_MODELS in self.el:
 433             models = self.el[self.GRESOURCE_BUNDLE_MODELS]
 434             if 'hosts' in models:
 435                 hosts = models['hosts']
 436             else:
 437                 return "GRB has no hosts. CODE:0x0002"
 438             if 'bundle' in models:
 439                 bundle = models['bundle']
 440             else:
 441                 return "GRB, no bundle in models. CODE:0x0003"
 442
 443             try:
 444                 bundle.owner = owner
 445                 bundle.save()
 446             except Exception as e:
 447                 return "GRB, saving bundle generated exception: " + str(e) + " CODE:0x0004"
 448             try:
 449                 for host in hosts:
 450                     genericresource = host.resource
 451                     genericresource.bundle = bundle
 452                     genericresource.save()
 453                     host.resource = genericresource
 454                     host.save()
 455             except Exception as e:
 456                 return "GRB, saving hosts generated exception: " + str(e) + " CODE:0x0005"
 457
 458             if 'networks' in models:
 459                 for net in models['networks'].values():
 460                     net.bundle = bundle
 461                     net.save()
 462
 463             if 'interfaces' in models:
 464                 for interface_set in models['interfaces'].values():
 465                     for interface in interface_set:
 466                         try:
 467                             interface.host = interface.host
 468                             interface.save()
 469                         except Exception:
 470                             return "GRB, saving interface " + str(interface) + " failed. CODE:0x0019"
 471             else:
 472                 return "GRB, no interface set provided. CODE:0x001a"
 473
 474             if 'connections' in models:
 475                 for resource_name, mapping in models['connections'].items():
 476                     for profile_name, connection_set in mapping.items():
 477                         interface = GenericInterface.objects.get(
 478                             profile__name=profile_name,
 479                             host__resource__name=resource_name,
 480                             host__resource__bundle=models['bundle']
 481                         )
 482                         for connection in connection_set:
 483                             try:
 484                                 connection.network = connection.network
 485                                 connection.save()
 486                                 interface.connections.add(connection)
 487                             except Exception as e:
 488                                 return "GRB, saving vlan " + str(connection) + " failed. Exception: " + str(e) + ". CODE:0x0017"
 489             else:
 490                 return "GRB, no vlan set provided. CODE:0x0018"
 491
 492         else:
 493             return "GRB no models given. CODE:0x0001"
 494
 495         self.el[self.RESULT] = bundle
 496         return False
 497
 498     def make_software_config_bundle(self):
 499         models = self.el[self.CONFIG_MODELS]
 500         if 'bundle' in models:
 501             bundle = models['bundle']
 502             bundle.bundle = bundle.bundle
 503             try:
 504                 bundle.save()
 505             except Exception as e:
 506                 return "SWC, saving bundle generated exception: " + str(e) + "CODE:0x0007"
 507
 508         else:
 509             return "SWC, no bundle in models. CODE:0x0006"
 510         if 'host_configs' in models:
 511             host_configs = models['host_configs']
 512             for host_config in host_configs:
 513                 host_config.bundle = host_config.bundle
 514                 host_config.host = host_config.host
 515                 try:
 516                     host_config.save()
 517                 except Exception as e:
 518                     return "SWC, saving host configs generated exception: " + str(e) + "CODE:0x0009"
 519         else:
 520             return "SWC, no host configs in models. CODE:0x0008"
 521         if 'opnfv' in models:
 522             opnfvconfig = models['opnfv']
 523             opnfvconfig.bundle = opnfvconfig.bundle
 524             if opnfvconfig.scenario not in opnfvconfig.installer.sup_scenarios.all():
 525                 return "SWC, scenario not supported by installer. CODE:0x000d"
 526             try:
 527                 opnfvconfig.save()
 528             except Exception as e:
 529                 return "SWC, saving opnfv config generated exception: " + str(e) + "CODE:0x000b"
 530         else:
 531             pass
 532
 533         self.el[self.RESULT] = bundle
 534         return False
 535
 536     def make_booking(self):
 537         models = self.el[self.BOOKING_MODELS]
 538         owner = self.el[self.SESSION_USER]
 539
 540         if self.SELECTED_GRESOURCE_BUNDLE in self.el:
 541             selected_grb = self.el[self.SELECTED_GRESOURCE_BUNDLE]
 542         else:
 543             return "BOOK, no selected resource. CODE:0x000e"
 544
 545         if 'booking' in models:
 546             booking = models['booking']
 547         else:
 548             return "BOOK, no booking model exists. CODE:0x000f"
 549
 550         if not booking.start:
 551             return "BOOK, booking has no start. CODE:0x0010"
 552         if not booking.end:
 553             return "BOOK, booking has no end. CODE:0x0011"
 554         if booking.end <= booking.start:
 555             return "BOOK, end before/same time as start. CODE:0x0012"
 556
 557         if 'collaborators' in models:
 558             collaborators = models['collaborators']
 559         else:
 560             return "BOOK, collaborators not defined. CODE:0x0013"
 561         try:
 562             resource_bundle = ResourceManager.getInstance().convertResourceBundle(selected_grb, config=booking.config_bundle)
 563         except ResourceAvailabilityException as e:
 564             return "BOOK, requested resources are not available. Exception: " + str(e) + " CODE:0x0014"
 565         except ModelValidationException as e:
 566             return "Error encountered when saving bundle. " + str(e) + " CODE: 0x001b"
 567
 568         booking.resource = resource_bundle
 569         booking.owner = owner
 570         booking.config_bundle = booking.config_bundle
 571         booking.lab = selected_grb.lab
 572
 573         is_allowed = BookingAuthManager().booking_allowed(booking, self)
 574         if not is_allowed:
 575             return "BOOK, you are not allowed to book the requested resources"
 576
 577         try:
 578             booking.save()
 579         except Exception as e:
 580             return "BOOK, saving booking generated exception: " + str(e) + " CODE:0x0015"
 581
 582         for collaborator in collaborators:
 583             booking.collaborators.add(collaborator)
 584
 585         try:
 586             booking.pdf = PDFTemplater.makePDF(booking)
 587             booking.save()
 588         except Exception as e:
 589             return "BOOK, failed to create Pod Desriptor File: " + str(e)
 590
 591         try:
 592             JobFactory.makeCompleteJob(booking)
 593         except Exception as e:
 594             return "BOOK, serializing for api generated exception: " + str(e) + " CODE:0xFFFF"
 595
 596         try:
 597             booking.save()
 598         except Exception as e:
 599             return "BOOK, saving booking generated exception: " + str(e) + " CODE:0x0016"
 600
 601     def make_opnfv_config(self):
 602         opnfv_models = self.el[self.OPNFV_MODELS]
 603         config_bundle = opnfv_models['configbundle']
 604         if not config_bundle:
 605             return "No Configuration bundle selected"
 606         info = opnfv_models.get("meta", {})
 607         name = info.get("name", False)
 608         desc = info.get("description", False)
 609         if not (name and desc):
 610             return "No name or description given"
 611         installer = opnfv_models['installer_chosen']
 612         if not installer:
 613             return "No OPNFV Installer chosen"
 614         scenario = opnfv_models['scenario_chosen']
 615         if not scenario:
 616             return "No OPNFV Scenario chosen"
 617
 618         opnfv_config = OPNFVConfig.objects.create(
 619             bundle=config_bundle,
 620             name=name,
 621             description=desc,
 622             installer=installer,
 623             scenario=scenario
 624         )
 625
 626         network_roles = opnfv_models['network_roles']
 627         for net_role in network_roles:
 628             opnfv_config.networks.add(
 629                 NetworkRole.objects.create(
 630                     name=net_role['role'],
 631                     network=net_role['network']
 632                 )
 633             )
 634
 635         host_roles = opnfv_models['host_roles']
 636         for host_role in host_roles:
 637             config = config_bundle.hostConfigurations.get(
 638                 host__resource__name=host_role['host_name']
 639             )
 640             HostOPNFVConfig.objects.create(
 641                 role=host_role['role'],
 642                 host_config=config,
 643                 opnfv_config=opnfv_config
 644             )
 645
 646         self.el[self.RESULT] = opnfv_config
 647
 648     def __init__(self):
 649         self.el = {}
 650         self.el[self.CONFIRMATION] = {}
 651         self.el["active_step"] = 0
 652         self.get_history = {}
 653         self.put_history = {}