src/workflow/models.py

   1 ##############################################################################
   2 # Copyright (c) 2018 Sawyer Bergeron, Parker Berberian, and others.
   3 #
   4 # All rights reserved. This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 # http://www.apache.org/licenses/LICENSE-2.0
   8 ##############################################################################
   9
  10
  11 from django.shortcuts import render
  12 from django.contrib import messages
  13 from django.http import HttpResponse
  14 from django.utils import timezone
  15
  16 import yaml
  17 import requests
  18
  19 from workflow.forms import ConfirmationForm
  20 from api.models import JobFactory
  21 from dashboard.exceptions import ResourceAvailabilityException, ModelValidationException
  22 from resource_inventory.models import Image, GenericInterface
  23 from resource_inventory.resource_manager import ResourceManager
  24 from notifier.manager import NotificationHandler
  25 from booking.models import Booking
  26
  27
  28 class BookingAuthManager():
  29     LFN_PROJECTS = ["opnfv"]  # TODO
  30
  31     def parse_github_url(self, url):
  32         project_leads = []
  33         try:
  34             parts = url.split("/")
  35             if "http" in parts[0]:  # the url include http(s)://
  36                 parts = parts[2:]
  37             if parts[-1] != "INFO.yaml":
  38                 return None
  39             if parts[0] not in ["github.com", "raw.githubusercontent.com"]:
  40                 return None
  41             if parts[1] not in self.LFN_PROJECTS:
  42                 return None
  43             # now to download and parse file
  44             if parts[3] == "blob":
  45                 parts[3] = "raw"
  46             url = "https://" + "/".join(parts)
  47             info_file = requests.get(url, timeout=15).text
  48             info_parsed = yaml.load(info_file)
  49             ptl = info_parsed.get('project_lead')
  50             if ptl:
  51                 project_leads.append(ptl)
  52             sub_ptl = info_parsed.get("subproject_lead")
  53             if sub_ptl:
  54                 project_leads.append(sub_ptl)
  55
  56         except Exception:
  57             pass
  58
  59         return project_leads
  60
  61     def parse_gerrit_url(self, url):
  62         project_leads = []
  63         try:
  64             halfs = url.split("?")
  65             parts = halfs[0].split("/")
  66             args = halfs[1].split(";")
  67             if "http" in parts[0]:  # the url include http(s)://
  68                 parts = parts[2:]
  69             if "f=INFO.yaml" not in args:
  70                 return None
  71             if "gerrit.opnfv.org" not in parts[0]:
  72                 return None
  73             try:
  74                 i = args.index("a=blob")
  75                 args[i] = "a=blob_plain"
  76             except ValueError:
  77                 pass
  78             # recreate url
  79             halfs[1] = ";".join(args)
  80             halfs[0] = "/".join(parts)
  81             # now to download and parse file
  82             url = "https://" + "?".join(halfs)
  83             info_file = requests.get(url, timeout=15).text
  84             info_parsed = yaml.load(info_file)
  85             ptl = info_parsed.get('project_lead')
  86             if ptl:
  87                 project_leads.append(ptl)
  88             sub_ptl = info_parsed.get("subproject_lead")
  89             if sub_ptl:
  90                 project_leads.append(sub_ptl)
  91
  92         except Exception:
  93             return None
  94
  95         return project_leads
  96
  97     def parse_opnfv_git_url(self, url):
  98         project_leads = []
  99         try:
 100             parts = url.split("/")
 101             if "http" in parts[0]:  # the url include http(s)://
 102                 parts = parts[2:]
 103             if "INFO.yaml" not in parts[-1]:
 104                 return None
 105             if "git.opnfv.org" not in parts[0]:
 106                 return None
 107             if parts[-2] == "tree":
 108                 parts[-2] = "plain"
 109             # now to download and parse file
 110             url = "https://" + "/".join(parts)
 111             info_file = requests.get(url, timeout=15).text
 112             info_parsed = yaml.load(info_file)
 113             ptl = info_parsed.get('project_lead')
 114             if ptl:
 115                 project_leads.append(ptl)
 116             sub_ptl = info_parsed.get("subproject_lead")
 117             if sub_ptl:
 118                 project_leads.append(sub_ptl)
 119
 120         except Exception:
 121             return None
 122
 123         return project_leads
 124
 125     def parse_url(self, info_url):
 126         """
 127         will return the PTL in the INFO file on success, or None
 128         """
 129         if "github" in info_url:
 130             return self.parse_github_url(info_url)
 131
 132         if "gerrit.opnfv.org" in info_url:
 133             return self.parse_gerrit_url(info_url)
 134
 135         if "git.opnfv.org" in info_url:
 136             return self.parse_opnfv_git_url(info_url)
 137
 138     def booking_allowed(self, booking, repo):
 139         """
 140         This is the method that will have to change whenever the booking policy changes in the Infra
 141         group / LFN. This is a nice isolation of that administration crap
 142         currently checks if the booking uses multiple servers. if it does, then the owner must be a PTL,
 143         which is checked using the provided info file
 144         """
 145         if len(booking.resource.template.getHosts()) < 2:
 146             return True  # if they only have one server, we dont care
 147         if booking.owner.userprofile.booking_privledge:
 148             return True  # admin override for this user
 149         if repo.BOOKING_INFO_FILE not in repo.el:
 150             return False  # INFO file not provided
 151         ptl_info = self.parse_url(repo.el.get(repo.BOOKING_INFO_FILE))
 152         for ptl in ptl_info:
 153             if ptl['email'] == booking.owner.userprofile.email_addr:
 154                 return True
 155         return False
 156
 157
 158 class WorkflowStep(object):
 159     template = 'bad_request.html'
 160     title = "Generic Step"
 161     description = "You were led here by mistake"
 162     short_title = "error"
 163     metastep = None
 164
 165     def __init__(self, id, repo=None):
 166         self.repo = repo
 167         self.id = id
 168
 169     def get_context(self):
 170         context = {}
 171         context['step_number'] = self.repo_get('steps')
 172         context['active_step'] = self.repo_get('active_step')
 173         context['render_correct'] = "true"
 174         context['step_title'] = self.title
 175         context['description'] = self.description
 176         return context
 177
 178     def render(self, request):
 179         self.context = self.get_context()
 180         return render(request, self.template, self.context)
 181
 182     def post_render(self, request):
 183         return self.render(request)
 184
 185     def test_render(self, request):
 186         if request.method == "POST":
 187             return self.post_render(request)
 188         return self.render(request)
 189
 190     def validate(self, request):
 191         pass
 192
 193     def repo_get(self, key, default=None):
 194         return self.repo.get(key, default, self.id)
 195
 196     def repo_put(self, key, value):
 197         return self.repo.put(key, value, self.id)
 198
 199
 200 class Confirmation_Step(WorkflowStep):
 201     template = 'workflow/confirm.html'
 202     title = "Confirm Changes"
 203     description = "Does this all look right?"
 204
 205     def get_vlan_warning(self):
 206         grb = self.repo_get(self.repo.SELECTED_GRESOURCE_BUNDLE, False)
 207         if not grb:
 208             return 0
 209         if self.repo.BOOKING_MODELS not in self.repo.el:
 210             return 0
 211         vlan_manager = grb.lab.vlan_manager
 212         if vlan_manager is None:
 213             return 0
 214         hosts = grb.getHosts()
 215         for host in hosts:
 216             for interface in host.generic_interfaces.all():
 217                 for vlan in interface.vlans.all():
 218                     if vlan.public:
 219                         if not vlan_manager.public_vlan_is_available(vlan.vlan_id):
 220                             return 1
 221                     else:
 222                         if not vlan_manager.is_available(vlan.vlan_id):
 223                             return 1  # There is a problem with these vlans
 224         return 0
 225
 226     def get_context(self):
 227         context = super(Confirmation_Step, self).get_context()
 228         context['form'] = ConfirmationForm()
 229         context['confirmation_info'] = yaml.dump(
 230             self.repo_get(self.repo.CONFIRMATION),
 231             default_flow_style=False
 232         ).strip()
 233         context['vlan_warning'] = self.get_vlan_warning()
 234
 235         return context
 236
 237     def flush_to_db(self):
 238         errors = self.repo.make_models()
 239         if errors:
 240             return errors
 241
 242     def post_render(self, request):
 243         form = ConfirmationForm(request.POST)
 244         if form.is_valid():
 245             data = form.cleaned_data['confirm']
 246             context = self.get_context()
 247             if data == "True":
 248                 context["bypassed"] = "true"
 249                 errors = self.flush_to_db()
 250                 if errors:
 251                     messages.add_message(request, messages.ERROR, "ERROR OCCURRED: " + errors)
 252                 else:
 253                     messages.add_message(request, messages.SUCCESS, "Confirmed")
 254
 255                 return HttpResponse('')
 256             elif data == "False":
 257                 context["bypassed"] = "true"
 258                 messages.add_message(request, messages.SUCCESS, "Canceled")
 259                 return render(request, self.template, context)
 260             else:
 261                 pass
 262
 263         else:
 264             if "vlan_input" in request.POST:
 265                 if request.POST.get("vlan_input") == "True":
 266                     self.translate_vlans()
 267                     return self.render(request)
 268             pass
 269
 270     def translate_vlans(self):
 271         grb = self.repo_get(self.repo.SELECTED_GRESOURCE_BUNDLE, False)
 272         if not grb:
 273             return 0
 274         vlan_manager = grb.lab.vlan_manager
 275         if vlan_manager is None:
 276             return 0
 277         hosts = grb.getHosts()
 278         for host in hosts:
 279             for interface in host.generic_interfaces.all():
 280                 for vlan in interface.vlans.all():
 281                     if not vlan.public:
 282                         if not vlan_manager.is_available(vlan.vlan_id):
 283                             vlan.vlan_id = vlan_manager.get_vlan()
 284                             vlan.save()
 285                     else:
 286                         if not vlan_manager.public_vlan_is_available(vlan.vlan_id):
 287                             pub_vlan = vlan_manager.get_public_vlan()
 288                             vlan.vlan_id = pub_vlan.vlan
 289                             vlan.save()
 290
 291
 292 class Workflow():
 293
 294     steps = []
 295     active_index = 0
 296
 297
 298 class Repository():
 299
 300     EDIT = "editing"
 301     MODELS = "models"
 302     RESOURCE_SELECT = "resource_select"
 303     CONFIRMATION = "confirmation"
 304     SELECTED_GRESOURCE_BUNDLE = "selected generic bundle pk"
 305     SELECTED_CONFIG_BUNDLE = "selected config bundle pk"
 306     GRESOURCE_BUNDLE_MODELS = "generic_resource_bundle_models"
 307     GRESOURCE_BUNDLE_INFO = "generic_resource_bundle_info"
 308     BOOKING = "booking"
 309     LAB = "lab"
 310     GRB_LAST_HOSTLIST = "grb_network_previous_hostlist"
 311     BOOKING_FORMS = "booking_forms"
 312     SWCONF_HOSTS = "swconf_hosts"
 313     BOOKING_MODELS = "booking models"
 314     CONFIG_MODELS = "configuration bundle models"
 315     SESSION_USER = "session owner user account"
 316     VALIDATED_MODEL_GRB = "valid grb config model instance in db"
 317     VALIDATED_MODEL_CONFIG = "valid config model instance in db"
 318     VALIDATED_MODEL_BOOKING = "valid booking model instance in db"
 319     VLANS = "a list of vlans"
 320     SNAPSHOT_MODELS = "the models for snapshotting"
 321     SNAPSHOT_BOOKING_ID = "the booking id for snapshotting"
 322     SNAPSHOT_NAME = "the name of the snapshot"
 323     SNAPSHOT_DESC = "description of the snapshot"
 324     BOOKING_INFO_FILE = "the INFO.yaml file for this user's booking"
 325
 326     # migratory elements of segmented workflow
 327     # each of these is the end result of a different workflow.
 328     HAS_RESULT = "whether or not workflow has a result"
 329     RESULT_KEY = "key for target index that result will be put into in parent"
 330     RESULT = "result object from workflow"
 331
 332     def get_child_defaults(self):
 333         return_tuples = []
 334         for key in [self.SELECTED_GRESOURCE_BUNDLE, self.SESSION_USER]:
 335             return_tuples.append((key, self.el.get(key)))
 336         return return_tuples
 337
 338     def set_defaults(self, defaults):
 339         for key, value in defaults:
 340             self.el[key] = value
 341
 342     def get(self, key, default, id):
 343         self.add_get_history(key, id)
 344         return self.el.get(key, default)
 345
 346     def put(self, key, val, id):
 347         self.add_put_history(key, id)
 348         self.el[key] = val
 349
 350     def add_get_history(self, key, id):
 351         self.add_history(key, id, self.get_history)
 352
 353     def add_put_history(self, key, id):
 354         self.add_history(key, id, self.put_history)
 355
 356     def add_history(self, key, id, history):
 357         if key not in history:
 358             history[key] = [id]
 359         else:
 360             history[key].append(id)
 361
 362     def make_models(self):
 363         if self.SNAPSHOT_MODELS in self.el:
 364             errors = self.make_snapshot()
 365             if errors:
 366                 return errors
 367         # if GRB WF, create it
 368         if self.GRESOURCE_BUNDLE_MODELS in self.el:
 369             errors = self.make_generic_resource_bundle()
 370             if errors:
 371                 return errors
 372             else:
 373                 self.el[self.HAS_RESULT] = True
 374                 self.el[self.RESULT_KEY] = self.SELECTED_GRESOURCE_BUNDLE
 375                 return
 376
 377         if self.CONFIG_MODELS in self.el:
 378             errors = self.make_software_config_bundle()
 379             if errors:
 380                 return errors
 381             else:
 382                 self.el[self.HAS_RESULT] = True
 383                 self.el[self.RESULT_KEY] = self.SELECTED_CONFIG_BUNDLE
 384                 return
 385
 386         if self.BOOKING_MODELS in self.el:
 387             errors = self.make_booking()
 388             if errors:
 389                 return errors
 390             # create notification
 391             booking = self.el[self.BOOKING_MODELS]['booking']
 392             NotificationHandler.notify_new_booking(booking)
 393
 394     def make_snapshot(self):
 395         owner = self.el[self.SESSION_USER]
 396         models = self.el[self.SNAPSHOT_MODELS]
 397         image = models.get('snapshot', Image())
 398         booking_id = self.el.get(self.SNAPSHOT_BOOKING_ID)
 399         if not booking_id:
 400             return "SNAP, No booking ID provided"
 401         booking = Booking.objects.get(pk=booking_id)
 402         if booking.start > timezone.now() or booking.end < timezone.now():
 403             return "Booking is not active"
 404         name = self.el.get(self.SNAPSHOT_NAME)
 405         if not name:
 406             return "SNAP, no name provided"
 407         host = models.get('host')
 408         if not host:
 409             return "SNAP, no host provided"
 410         description = self.el.get(self.SNAPSHOT_DESC, "")
 411         image.from_lab = booking.lab
 412         image.name = name
 413         image.description = description
 414         image.public = False
 415         image.lab_id = -1
 416         image.owner = owner
 417         image.host_type = host.profile
 418         image.save()
 419         try:
 420             current_image = host.config.image
 421             image.os = current_image.os
 422             image.save()
 423         except Exception:
 424             pass
 425         JobFactory.makeSnapshotTask(image, booking, host)
 426
 427     def make_generic_resource_bundle(self):
 428         owner = self.el[self.SESSION_USER]
 429         if self.GRESOURCE_BUNDLE_MODELS in self.el:
 430             models = self.el[self.GRESOURCE_BUNDLE_MODELS]
 431             if 'hosts' in models:
 432                 hosts = models['hosts']
 433             else:
 434                 return "GRB has no hosts. CODE:0x0002"
 435             if 'bundle' in models:
 436                 bundle = models['bundle']
 437             else:
 438                 return "GRB, no bundle in models. CODE:0x0003"
 439
 440             try:
 441                 bundle.owner = owner
 442                 bundle.save()
 443             except Exception as e:
 444                 return "GRB, saving bundle generated exception: " + str(e) + " CODE:0x0004"
 445             try:
 446                 for host in hosts:
 447                     genericresource = host.resource
 448                     genericresource.bundle = bundle
 449                     genericresource.save()
 450                     host.resource = genericresource
 451                     host.save()
 452             except Exception as e:
 453                 return "GRB, saving hosts generated exception: " + str(e) + " CODE:0x0005"
 454
 455             if 'interfaces' in models:
 456                 for interface_set in models['interfaces'].values():
 457                     for interface in interface_set:
 458                         try:
 459                             interface.host = interface.host
 460                             interface.save()
 461                         except Exception:
 462                             return "GRB, saving interface " + str(interface) + " failed. CODE:0x0019"
 463             else:
 464                 return "GRB, no interface set provided. CODE:0x001a"
 465
 466             if 'vlans' in models:
 467                 for resource_name, mapping in models['vlans'].items():
 468                     for profile_name, vlan_set in mapping.items():
 469                         interface = GenericInterface.objects.get(
 470                             profile__name=profile_name,
 471                             host__resource__name=resource_name,
 472                             host__resource__bundle=models['bundle']
 473                         )
 474                         for vlan in vlan_set:
 475                             try:
 476                                 vlan.save()
 477                                 interface.vlans.add(vlan)
 478                             except Exception as e:
 479                                 return "GRB, saving vlan " + str(vlan) + " failed. Exception: " + str(e) + ". CODE:0x0017"
 480             else:
 481                 return "GRB, no vlan set provided. CODE:0x0018"
 482
 483         else:
 484             return "GRB no models given. CODE:0x0001"
 485
 486         self.el[self.RESULT] = bundle
 487         return False
 488
 489     def make_software_config_bundle(self):
 490         models = self.el[self.CONFIG_MODELS]
 491         if 'bundle' in models:
 492             bundle = models['bundle']
 493             bundle.bundle = bundle.bundle
 494             try:
 495                 bundle.save()
 496             except Exception as e:
 497                 return "SWC, saving bundle generated exception: " + str(e) + "CODE:0x0007"
 498
 499         else:
 500             return "SWC, no bundle in models. CODE:0x0006"
 501         if 'host_configs' in models:
 502             host_configs = models['host_configs']
 503             for host_config in host_configs:
 504                 host_config.bundle = host_config.bundle
 505                 host_config.host = host_config.host
 506                 try:
 507                     host_config.save()
 508                 except Exception as e:
 509                     return "SWC, saving host configs generated exception: " + str(e) + "CODE:0x0009"
 510         else:
 511             return "SWC, no host configs in models. CODE:0x0008"
 512         if 'opnfv' in models:
 513             opnfvconfig = models['opnfv']
 514             opnfvconfig.bundle = opnfvconfig.bundle
 515             if opnfvconfig.scenario not in opnfvconfig.installer.sup_scenarios.all():
 516                 return "SWC, scenario not supported by installer. CODE:0x000d"
 517             try:
 518                 opnfvconfig.save()
 519             except Exception as e:
 520                 return "SWC, saving opnfv config generated exception: " + str(e) + "CODE:0x000b"
 521         else:
 522             pass
 523
 524         self.el[self.RESULT] = bundle
 525         return False
 526
 527     def make_booking(self):
 528         models = self.el[self.BOOKING_MODELS]
 529         owner = self.el[self.SESSION_USER]
 530
 531         if self.SELECTED_GRESOURCE_BUNDLE in self.el:
 532             selected_grb = self.el[self.SELECTED_GRESOURCE_BUNDLE]
 533         else:
 534             return "BOOK, no selected resource. CODE:0x000e"
 535
 536         if not self.reserve_vlans(selected_grb):
 537             return "BOOK, vlans not available"
 538
 539         if 'booking' in models:
 540             booking = models['booking']
 541         else:
 542             return "BOOK, no booking model exists. CODE:0x000f"
 543
 544         if not booking.start:
 545             return "BOOK, booking has no start. CODE:0x0010"
 546         if not booking.end:
 547             return "BOOK, booking has no end. CODE:0x0011"
 548         if booking.end <= booking.start:
 549             return "BOOK, end before/same time as start. CODE:0x0012"
 550
 551         if 'collaborators' in models:
 552             collaborators = models['collaborators']
 553         else:
 554             return "BOOK, collaborators not defined. CODE:0x0013"
 555         try:
 556             resource_bundle = ResourceManager.getInstance().convertResourceBundle(selected_grb, config=booking.config_bundle)
 557         except ResourceAvailabilityException as e:
 558             return "BOOK, requested resources are not available. Exception: " + str(e) + " CODE:0x0014"
 559         except ModelValidationException as e:
 560             return "Error encountered when saving bundle. " + str(e) + " CODE: 0x001b"
 561
 562         booking.resource = resource_bundle
 563         booking.owner = owner
 564         booking.config_bundle = booking.config_bundle
 565         booking.lab = selected_grb.lab
 566
 567         is_allowed = BookingAuthManager().booking_allowed(booking, self)
 568         if not is_allowed:
 569             return "BOOK, you are not allowed to book the requested resources"
 570
 571         try:
 572             booking.save()
 573         except Exception as e:
 574             return "BOOK, saving booking generated exception: " + str(e) + " CODE:0x0015"
 575
 576         for collaborator in collaborators:
 577             booking.collaborators.add(collaborator)
 578
 579         try:
 580             booking.pdf = ResourceManager().makePDF(booking.resource)
 581             booking.save()
 582         except Exception as e:
 583             return "BOOK, failed to create Pod Desriptor File: " + str(e)
 584
 585         try:
 586             JobFactory.makeCompleteJob(booking)
 587         except Exception as e:
 588             return "BOOK, serializing for api generated exception: " + str(e) + " CODE:0xFFFF"
 589
 590         try:
 591             booking.save()
 592         except Exception as e:
 593             return "BOOK, saving booking generated exception: " + str(e) + " CODE:0x0016"
 594
 595     def reserve_vlans(self, grb):
 596         """
 597         True is success
 598         """
 599         vlans = []
 600         public_vlan = None
 601         vlan_manager = grb.lab.vlan_manager
 602         if vlan_manager is None:
 603             return True
 604         for host in grb.getHosts():
 605             for interface in host.generic_interfaces.all():
 606                 for vlan in interface.vlans.all():
 607                     if vlan.public:
 608                         public_vlan = vlan
 609                     else:
 610                         vlans.append(vlan.vlan_id)
 611
 612         try:
 613             vlan_manager.reserve_vlans(vlans)
 614             vlan_manager.reserve_public_vlan(public_vlan.vlan_id)
 615             return True
 616         except Exception:
 617             return False
 618
 619     def __init__(self):
 620         self.el = {}
 621         self.el[self.CONFIRMATION] = {}
 622         self.el["active_step"] = 0
 623         self.get_history = {}
 624         self.put_history = {}