dashboard/src/workflow/models.py

   1 ##############################################################################
   2 # Copyright (c) 2018 Sawyer Bergeron, Parker Berberian, and others.
   3 #
   4 # All rights reserved. This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 # http://www.apache.org/licenses/LICENSE-2.0
   8 ##############################################################################
   9
  10
  11 from django.shortcuts import render
  12 from django.contrib import messages
  13 from django.http import HttpResponse
  14 from django.utils import timezone
  15
  16 import yaml
  17 import requests
  18
  19 from workflow.forms import ConfirmationForm
  20 from api.models import JobFactory
  21 from dashboard.exceptions import ResourceAvailabilityException, ModelValidationException
  22 from resource_inventory.models import Image, GenericInterface
  23 from resource_inventory.resource_manager import ResourceManager
  24 from notifier.manager import NotificationHandler
  25 from booking.models import Booking
  26
  27
  28 class BookingAuthManager():
  29     LFN_PROJECTS = ["opnfv"]  # TODO
  30
  31     def parse_github_url(self, url):
  32         project_leads = []
  33         try:
  34             parts = url.split("/")
  35             if "http" in parts[0]:  # the url include http(s)://
  36                 parts = parts[2:]
  37             if parts[-1] != "INFO.yaml":
  38                 return None
  39             if parts[0] not in ["github.com", "raw.githubusercontent.com"]:
  40                 return None
  41             if parts[1] not in self.LFN_PROJECTS:
  42                 return None
  43             # now to download and parse file
  44             if parts[3] == "blob":
  45                 parts[3] = "raw"
  46             url = "https://" + "/".join(parts)
  47             info_file = requests.get(url, timeout=15).text
  48             info_parsed = yaml.load(info_file)
  49             ptl = info_parsed.get('project_lead')
  50             if ptl:
  51                 project_leads.append(ptl)
  52             sub_ptl = info_parsed.get("subproject_lead")
  53             if sub_ptl:
  54                 project_leads.append(sub_ptl)
  55
  56         except Exception:
  57             pass
  58
  59         return project_leads
  60
  61     def parse_gerrit_url(self, url):
  62         project_leads = []
  63         try:
  64             parts = url.split("/")
  65             if "http" in parts[0]:  # the url include http(s)://
  66                 parts = parts[2:]
  67             if "f=INFO.yaml" not in parts[-1].split(";"):
  68                 return None
  69             if "gerrit.opnfv.org" not in parts[0]:
  70                 return None
  71             # now to download and parse file
  72             url = "https://" + "/".join(parts)
  73             info_file = requests.get(url, timeout=15).text
  74             info_parsed = yaml.load(info_file)
  75             ptl = info_parsed.get('project_lead')
  76             if ptl:
  77                 project_leads.append(ptl)
  78             sub_ptl = info_parsed.get("subproject_lead")
  79             if sub_ptl:
  80                 project_leads.append(sub_ptl)
  81
  82         except Exception:
  83             return None
  84
  85         return project_leads
  86
  87     def parse_opnfv_git_url(self, url):
  88         project_leads = []
  89         try:
  90             parts = url.split("/")
  91             if "http" in parts[0]:  # the url include http(s)://
  92                 parts = parts[2:]
  93             if "INFO.yaml" not in parts[-1]:
  94                 return None
  95             if "git.opnfv.org" not in parts[0]:
  96                 return None
  97             if parts[-2] == "tree":
  98                 parts[-2] = "plain"
  99             # now to download and parse file
 100             url = "https://" + "/".join(parts)
 101             info_file = requests.get(url, timeout=15).text
 102             info_parsed = yaml.load(info_file)
 103             ptl = info_parsed.get('project_lead')
 104             if ptl:
 105                 project_leads.append(ptl)
 106             sub_ptl = info_parsed.get("subproject_lead")
 107             if sub_ptl:
 108                 project_leads.append(sub_ptl)
 109
 110         except Exception:
 111             return None
 112
 113         return project_leads
 114
 115     def parse_url(self, info_url):
 116         """
 117         will return the PTL in the INFO file on success, or None
 118         """
 119         if "github" in info_url:
 120             return self.parse_github_url(info_url)
 121
 122         if "gerrit.opnfv.org" in info_url:
 123             return self.parse_gerrit_url(info_url)
 124
 125         if "git.opnfv.org" in info_url:
 126             return self.parse_opnfv_git_url(info_url)
 127
 128     def booking_allowed(self, booking, repo):
 129         """
 130         This is the method that will have to change whenever the booking policy changes in the Infra
 131         group / LFN. This is a nice isolation of that administration crap
 132         currently checks if the booking uses multiple servers. if it does, then the owner must be a PTL,
 133         which is checked using the provided info file
 134         """
 135         if len(booking.resource.template.getHosts()) < 2:
 136             return True  # if they only have one server, we dont care
 137         if booking.owner.userprofile.booking_privledge:
 138             return True  # admin override for this user
 139         if repo.BOOKING_INFO_FILE not in repo.el:
 140             return False  # INFO file not provided
 141         ptl_info = self.parse_url(repo.BOOKING_INFO_FILE)
 142         return ptl_info and ptl_info == booking.owner.userprofile.email_addr
 143
 144
 145 class WorkflowStep(object):
 146     template = 'bad_request.html'
 147     title = "Generic Step"
 148     description = "You were led here by mistake"
 149     short_title = "error"
 150     metastep = None
 151
 152     def __init__(self, id, repo=None):
 153         self.repo = repo
 154         self.id = id
 155
 156     def get_context(self):
 157         context = {}
 158         context['step_number'] = self.repo_get('steps')
 159         context['active_step'] = self.repo_get('active_step')
 160         context['render_correct'] = "true"
 161         context['step_title'] = self.title
 162         context['description'] = self.description
 163         return context
 164
 165     def render(self, request):
 166         self.context = self.get_context()
 167         return render(request, self.template, self.context)
 168
 169     def post_render(self, request):
 170         return self.render(request)
 171
 172     def test_render(self, request):
 173         if request.method == "POST":
 174             return self.post_render(request)
 175         return self.render(request)
 176
 177     def validate(self, request):
 178         pass
 179
 180     def repo_get(self, key, default=None):
 181         return self.repo.get(key, default, self.id)
 182
 183     def repo_put(self, key, value):
 184         return self.repo.put(key, value, self.id)
 185
 186
 187 class Confirmation_Step(WorkflowStep):
 188     template = 'workflow/confirm.html'
 189     title = "Confirm Changes"
 190     description = "Does this all look right?"
 191
 192     def get_vlan_warning(self):
 193         grb = self.repo_get(self.repo.SELECTED_GRESOURCE_BUNDLE, False)
 194         if not grb:
 195             return 0
 196         if self.repo.BOOKING_MODELS not in self.repo.el:
 197             return 0
 198         vlan_manager = grb.lab.vlan_manager
 199         if vlan_manager is None:
 200             return 0
 201         hosts = grb.getHosts()
 202         for host in hosts:
 203             for interface in host.generic_interfaces.all():
 204                 for vlan in interface.vlans.all():
 205                     if vlan.public:
 206                         if not vlan_manager.public_vlan_is_available(vlan.vlan_id):
 207                             return 1
 208                     else:
 209                         if not vlan_manager.is_available(vlan.vlan_id):
 210                             return 1  # There is a problem with these vlans
 211         return 0
 212
 213     def get_context(self):
 214         context = super(Confirmation_Step, self).get_context()
 215         context['form'] = ConfirmationForm()
 216         context['confirmation_info'] = yaml.dump(
 217             self.repo_get(self.repo.CONFIRMATION),
 218             default_flow_style=False
 219         ).strip()
 220         context['vlan_warning'] = self.get_vlan_warning()
 221
 222         return context
 223
 224     def flush_to_db(self):
 225         errors = self.repo.make_models()
 226         if errors:
 227             return errors
 228
 229     def post_render(self, request):
 230         form = ConfirmationForm(request.POST)
 231         if form.is_valid():
 232             data = form.cleaned_data['confirm']
 233             context = self.get_context()
 234             if data == "True":
 235                 context["bypassed"] = "true"
 236                 errors = self.flush_to_db()
 237                 if errors:
 238                     messages.add_message(request, messages.ERROR, "ERROR OCCURRED: " + errors)
 239                 else:
 240                     messages.add_message(request, messages.SUCCESS, "Confirmed")
 241
 242                 return HttpResponse('')
 243             elif data == "False":
 244                 context["bypassed"] = "true"
 245                 messages.add_message(request, messages.SUCCESS, "Canceled")
 246                 return render(request, self.template, context)
 247             else:
 248                 pass
 249
 250         else:
 251             if "vlan_input" in request.POST:
 252                 if request.POST.get("vlan_input") == "True":
 253                     self.translate_vlans()
 254                     return self.render(request)
 255             pass
 256
 257     def translate_vlans(self):
 258         grb = self.repo_get(self.repo.SELECTED_GRESOURCE_BUNDLE, False)
 259         if not grb:
 260             return 0
 261         vlan_manager = grb.lab.vlan_manager
 262         if vlan_manager is None:
 263             return 0
 264         hosts = grb.getHosts()
 265         for host in hosts:
 266             for interface in host.generic_interfaces.all():
 267                 for vlan in interface.vlans.all():
 268                     if not vlan.public:
 269                         if not vlan_manager.is_available(vlan.vlan_id):
 270                             vlan.vlan_id = vlan_manager.get_vlan()
 271                             vlan.save()
 272                     else:
 273                         if not vlan_manager.public_vlan_is_available(vlan.vlan_id):
 274                             pub_vlan = vlan_manager.get_public_vlan()
 275                             vlan.vlan_id = pub_vlan.vlan
 276                             vlan.save()
 277
 278
 279 class Workflow():
 280
 281     steps = []
 282     active_index = 0
 283
 284
 285 class Repository():
 286
 287     EDIT = "editing"
 288     MODELS = "models"
 289     RESOURCE_SELECT = "resource_select"
 290     CONFIRMATION = "confirmation"
 291     SELECTED_GRESOURCE_BUNDLE = "selected generic bundle pk"
 292     SELECTED_CONFIG_BUNDLE = "selected config bundle pk"
 293     GRESOURCE_BUNDLE_MODELS = "generic_resource_bundle_models"
 294     GRESOURCE_BUNDLE_INFO = "generic_resource_bundle_info"
 295     BOOKING = "booking"
 296     LAB = "lab"
 297     GRB_LAST_HOSTLIST = "grb_network_previous_hostlist"
 298     BOOKING_FORMS = "booking_forms"
 299     SWCONF_HOSTS = "swconf_hosts"
 300     BOOKING_MODELS = "booking models"
 301     CONFIG_MODELS = "configuration bundle models"
 302     SESSION_USER = "session owner user account"
 303     VALIDATED_MODEL_GRB = "valid grb config model instance in db"
 304     VALIDATED_MODEL_CONFIG = "valid config model instance in db"
 305     VALIDATED_MODEL_BOOKING = "valid booking model instance in db"
 306     VLANS = "a list of vlans"
 307     SNAPSHOT_MODELS = "the models for snapshotting"
 308     SNAPSHOT_BOOKING_ID = "the booking id for snapshotting"
 309     SNAPSHOT_NAME = "the name of the snapshot"
 310     SNAPSHOT_DESC = "description of the snapshot"
 311     BOOKING_INFO_FILE = "the INFO.yaml file for this user's booking"
 312
 313     #migratory elements of segmented workflow
 314     #each of these is the end result of a different workflow.
 315     HAS_RESULT = "whether or not workflow has a result"
 316     RESULT_KEY = "key for target index that result will be put into in parent"
 317     RESULT = "result object from workflow"
 318
 319     def get_child_defaults(self):
 320         return_tuples = []
 321         for key in [self.SELECTED_GRESOURCE_BUNDLE, self.SESSION_USER]:
 322             return_tuples.append((key, self.el.get(key)))
 323         return return_tuples
 324
 325     def set_defaults(self, defaults):
 326         for key, value in defaults:
 327             self.el[key] = value
 328
 329     def get(self, key, default, id):
 330         self.add_get_history(key, id)
 331         return self.el.get(key, default)
 332
 333     def put(self, key, val, id):
 334         self.add_put_history(key, id)
 335         self.el[key] = val
 336
 337     def add_get_history(self, key, id):
 338         self.add_history(key, id, self.get_history)
 339
 340     def add_put_history(self, key, id):
 341         self.add_history(key, id, self.put_history)
 342
 343     def add_history(self, key, id, history):
 344         if key not in history:
 345             history[key] = [id]
 346         else:
 347             history[key].append(id)
 348
 349     def make_models(self):
 350         if self.SNAPSHOT_MODELS in self.el:
 351             errors = self.make_snapshot()
 352             if errors:
 353                 return errors
 354         # if GRB WF, create it
 355         if self.GRESOURCE_BUNDLE_MODELS in self.el:
 356             errors = self.make_generic_resource_bundle()
 357             if errors:
 358                 return errors
 359             else:
 360                 self.el[self.HAS_RESULT] = True
 361                 self.el[self.RESULT_KEY] = self.SELECTED_GRESOURCE_BUNDLE
 362                 return
 363
 364         if self.CONFIG_MODELS in self.el:
 365             errors = self.make_software_config_bundle()
 366             if errors:
 367                 return errors
 368             else:
 369                 self.el[self.HAS_RESULT] = True
 370                 self.el[self.RESULT_KEY] = self.SELECTED_CONFIG_BUNDLE
 371                 return
 372
 373         if self.BOOKING_MODELS in self.el:
 374             errors = self.make_booking()
 375             if errors:
 376                 return errors
 377             # create notification
 378             booking = self.el[self.BOOKING_MODELS]['booking']
 379             NotificationHandler.notify_new_booking(booking)
 380
 381     def make_snapshot(self):
 382         owner = self.el[self.SESSION_USER]
 383         models = self.el[self.SNAPSHOT_MODELS]
 384         image = models.get('snapshot', Image())
 385         booking_id = self.el.get(self.SNAPSHOT_BOOKING_ID)
 386         if not booking_id:
 387             return "SNAP, No booking ID provided"
 388         booking = Booking.objects.get(pk=booking_id)
 389         if booking.start > timezone.now() or booking.end < timezone.now():
 390             return "Booking is not active"
 391         name = self.el.get(self.SNAPSHOT_NAME)
 392         if not name:
 393             return "SNAP, no name provided"
 394         host = models.get('host')
 395         if not host:
 396             return "SNAP, no host provided"
 397         description = self.el.get(self.SNAPSHOT_DESC, "")
 398         image.from_lab = booking.lab
 399         image.name = name
 400         image.description = description
 401         image.public = False
 402         image.lab_id = -1
 403         image.owner = owner
 404         image.host_type = host.profile
 405         image.save()
 406         try:
 407             current_image = host.config.image
 408             image.os = current_image.os
 409             image.save()
 410         except Exception:
 411             pass
 412         JobFactory.makeSnapshotTask(image, booking, host)
 413
 414     def make_generic_resource_bundle(self):
 415         owner = self.el[self.SESSION_USER]
 416         if self.GRESOURCE_BUNDLE_MODELS in self.el:
 417             models = self.el[self.GRESOURCE_BUNDLE_MODELS]
 418             if 'hosts' in models:
 419                 hosts = models['hosts']
 420             else:
 421                 return "GRB has no hosts. CODE:0x0002"
 422             if 'bundle' in models:
 423                 bundle = models['bundle']
 424             else:
 425                 return "GRB, no bundle in models. CODE:0x0003"
 426
 427             try:
 428                 bundle.owner = owner
 429                 bundle.save()
 430             except Exception as e:
 431                 return "GRB, saving bundle generated exception: " + str(e) + " CODE:0x0004"
 432             try:
 433                 for host in hosts:
 434                     genericresource = host.resource
 435                     genericresource.bundle = bundle
 436                     genericresource.save()
 437                     host.resource = genericresource
 438                     host.save()
 439             except Exception as e:
 440                 return "GRB, saving hosts generated exception: " + str(e) + " CODE:0x0005"
 441
 442             if 'interfaces' in models:
 443                 for interface_set in models['interfaces'].values():
 444                     for interface in interface_set:
 445                         try:
 446                             interface.host = interface.host
 447                             interface.save()
 448                         except Exception as e:
 449                             return "GRB, saving interface " + str(interface) + " failed. CODE:0x0019"
 450             else:
 451                 return "GRB, no interface set provided. CODE:0x001a"
 452
 453             if 'vlans' in models:
 454                 for resource_name, mapping in models['vlans'].items():
 455                     for profile_name, vlan_set in mapping.items():
 456                         interface = GenericInterface.objects.get(
 457                             profile__name=profile_name,
 458                             host__resource__name=resource_name,
 459                             host__resource__bundle=models['bundle']
 460                         )
 461                         for vlan in vlan_set:
 462                             try:
 463                                 vlan.save()
 464                                 interface.vlans.add(vlan)
 465                             except Exception as e:
 466                                 return "GRB, saving vlan " + str(vlan) + " failed. Exception: " + str(e) + ". CODE:0x0017"
 467             else:
 468                 return "GRB, no vlan set provided. CODE:0x0018"
 469
 470         else:
 471             return "GRB no models given. CODE:0x0001"
 472
 473         self.el[self.RESULT] = bundle
 474         return False
 475
 476     def make_software_config_bundle(self):
 477         models = self.el[self.CONFIG_MODELS]
 478         if 'bundle' in models:
 479             bundle = models['bundle']
 480             bundle.bundle = bundle.bundle
 481             try:
 482                 bundle.save()
 483             except Exception as e:
 484                 return "SWC, saving bundle generated exception: " + str(e) + "CODE:0x0007"
 485
 486         else:
 487             return "SWC, no bundle in models. CODE:0x0006"
 488         if 'host_configs' in models:
 489             host_configs = models['host_configs']
 490             for host_config in host_configs:
 491                 host_config.bundle = host_config.bundle
 492                 host_config.host = host_config.host
 493                 try:
 494                     host_config.save()
 495                 except Exception as e:
 496                     return "SWC, saving host configs generated exception: " + str(e) + "CODE:0x0009"
 497         else:
 498             return "SWC, no host configs in models. CODE:0x0008"
 499         if 'opnfv' in models:
 500             opnfvconfig = models['opnfv']
 501             opnfvconfig.bundle = opnfvconfig.bundle
 502             if opnfvconfig.scenario not in opnfvconfig.installer.sup_scenarios.all():
 503                 return "SWC, scenario not supported by installer. CODE:0x000d"
 504             try:
 505                 opnfvconfig.save()
 506             except Exception as e:
 507                 return "SWC, saving opnfv config generated exception: " + str(e) + "CODE:0x000b"
 508         else:
 509             pass
 510
 511         self.el[self.RESULT] = bundle
 512         return False
 513
 514     def make_booking(self):
 515         models = self.el[self.BOOKING_MODELS]
 516         owner = self.el[self.SESSION_USER]
 517
 518         if self.SELECTED_GRESOURCE_BUNDLE in self.el:
 519             selected_grb = self.el[self.SELECTED_GRESOURCE_BUNDLE]
 520         else:
 521             return "BOOK, no selected resource. CODE:0x000e"
 522
 523         if not self.reserve_vlans(selected_grb):
 524             return "BOOK, vlans not available"
 525
 526         if 'booking' in models:
 527             booking = models['booking']
 528         else:
 529             return "BOOK, no booking model exists. CODE:0x000f"
 530
 531         if not booking.start:
 532             return "BOOK, booking has no start. CODE:0x0010"
 533         if not booking.end:
 534             return "BOOK, booking has no end. CODE:0x0011"
 535         if booking.end <= booking.start:
 536             return "BOOK, end before/same time as start. CODE:0x0012"
 537
 538         if 'collaborators' in models:
 539             collaborators = models['collaborators']
 540         else:
 541             return "BOOK, collaborators not defined. CODE:0x0013"
 542         try:
 543             resource_bundle = ResourceManager.getInstance().convertResourceBundle(selected_grb, config=booking.config_bundle)
 544         except ResourceAvailabilityException as e:
 545             return "BOOK, requested resources are not available. Exception: " + str(e) + " CODE:0x0014"
 546         except ModelValidationException as e:
 547             return "Error encountered when saving bundle. " + str(e) + " CODE: 0x001b"
 548
 549         booking.resource = resource_bundle
 550         booking.owner = owner
 551         booking.config_bundle = booking.config_bundle
 552         booking.lab = selected_grb.lab
 553
 554         is_allowed = BookingAuthManager().booking_allowed(booking, self)
 555         if not is_allowed:
 556             return "BOOK, you are not allowed to book the requested resources"
 557
 558         try:
 559             booking.save()
 560         except Exception as e:
 561             return "BOOK, saving booking generated exception: " + str(e) + " CODE:0x0015"
 562
 563         for collaborator in collaborators:
 564             booking.collaborators.add(collaborator)
 565
 566         try:
 567             booking.pdf = ResourceManager().makePDF(booking.resource)
 568             booking.save()
 569         except Exception as e:
 570             return "BOOK, failed to create Pod Desriptor File: " + str(e)
 571
 572         try:
 573             JobFactory.makeCompleteJob(booking)
 574         except Exception as e:
 575             return "BOOK, serializing for api generated exception: " + str(e) + " CODE:0xFFFF"
 576
 577         try:
 578             booking.save()
 579         except Exception as e:
 580             return "BOOK, saving booking generated exception: " + str(e) + " CODE:0x0016"
 581
 582     def reserve_vlans(self, grb):
 583         """
 584         True is success
 585         """
 586         vlans = []
 587         public_vlan = None
 588         vlan_manager = grb.lab.vlan_manager
 589         if vlan_manager is None:
 590             return True
 591         for host in grb.getHosts():
 592             for interface in host.generic_interfaces.all():
 593                 for vlan in interface.vlans.all():
 594                     if vlan.public:
 595                         public_vlan = vlan
 596                     else:
 597                         vlans.append(vlan.vlan_id)
 598
 599         try:
 600             vlan_manager.reserve_vlans(vlans)
 601             vlan_manager.reserve_public_vlan(public_vlan.vlan_id)
 602             return True
 603         except Exception:
 604             return False
 605
 606     def __init__(self):
 607         self.el = {}
 608         self.el[self.CONFIRMATION] = {}
 609         self.el["active_step"] = 0
 610         self.get_history = {}
 611         self.put_history = {}