e065202a6afec74e59d4df9dad676f48098f92d5
[laas.git] / src / workflow / models.py
1 ##############################################################################
2 # Copyright (c) 2018 Sawyer Bergeron, Parker Berberian, and others.
3 #
4 # All rights reserved. This program and the accompanying materials
5 # are made available under the terms of the Apache License, Version 2.0
6 # which accompanies this distribution, and is available at
7 # http://www.apache.org/licenses/LICENSE-2.0
8 ##############################################################################
9
10
11 from django.template.loader import get_template
12 from django.http import HttpResponse
13 from django.utils import timezone
14 from django.db import transaction
15
16 import yaml
17 import requests
18
19 from workflow.forms import ConfirmationForm
20 from api.models import JobFactory
21 from dashboard.exceptions import ResourceAvailabilityException, ModelValidationException
22 from resource_inventory.models import Image, OPNFVConfig, ResourceOPNFVConfig, NetworkRole
23 from resource_inventory.resource_manager import ResourceManager
24 from resource_inventory.pdf_templater import PDFTemplater
25 from notifier.manager import NotificationHandler
26 from booking.models import Booking
27
28
29 class BookingAuthManager():
30     """
31     Verifies Booking Authorization.
32
33     Class to verify that the user is allowed to book the requested resource
34     The user must input a url to the INFO.yaml file to prove that they are the ptl of
35     an approved project if they are booking a multi-node pod.
36     This class parses the url and checks the logged in user against the info file.
37     """
38
39     LFN_PROJECTS = ["opnfv"]  # TODO
40
41     def parse_github_url(self, url):
42         project_leads = []
43         try:
44             parts = url.split("/")
45             if "http" in parts[0]:  # the url include http(s)://
46                 parts = parts[2:]
47             if parts[-1] != "INFO.yaml":
48                 return None
49             if parts[0] not in ["github.com", "raw.githubusercontent.com"]:
50                 return None
51             if parts[1] not in self.LFN_PROJECTS:
52                 return None
53             # now to download and parse file
54             if parts[3] == "blob":
55                 parts[3] = "raw"
56             url = "https://" + "/".join(parts)
57             info_file = requests.get(url, timeout=15).text
58             info_parsed = yaml.load(info_file)
59             ptl = info_parsed.get('project_lead')
60             if ptl:
61                 project_leads.append(ptl)
62             sub_ptl = info_parsed.get("subproject_lead")
63             if sub_ptl:
64                 project_leads.append(sub_ptl)
65
66         except Exception:
67             pass
68
69         return project_leads
70
71     def parse_gerrit_url(self, url):
72         project_leads = []
73         try:
74             halfs = url.split("?")
75             parts = halfs[0].split("/")
76             args = halfs[1].split(";")
77             if "http" in parts[0]:  # the url include http(s)://
78                 parts = parts[2:]
79             if "f=INFO.yaml" not in args:
80                 return None
81             if "gerrit.opnfv.org" not in parts[0]:
82                 return None
83             try:
84                 i = args.index("a=blob")
85                 args[i] = "a=blob_plain"
86             except ValueError:
87                 pass
88             # recreate url
89             halfs[1] = ";".join(args)
90             halfs[0] = "/".join(parts)
91             # now to download and parse file
92             url = "https://" + "?".join(halfs)
93             info_file = requests.get(url, timeout=15).text
94             info_parsed = yaml.load(info_file)
95             ptl = info_parsed.get('project_lead')
96             if ptl:
97                 project_leads.append(ptl)
98             sub_ptl = info_parsed.get("subproject_lead")
99             if sub_ptl:
100                 project_leads.append(sub_ptl)
101
102         except Exception:
103             return None
104
105         return project_leads
106
107     def parse_opnfv_git_url(self, url):
108         project_leads = []
109         try:
110             parts = url.split("/")
111             if "http" in parts[0]:  # the url include http(s)://
112                 parts = parts[2:]
113             if "INFO.yaml" not in parts[-1]:
114                 return None
115             if "git.opnfv.org" not in parts[0]:
116                 return None
117             if parts[-2] == "tree":
118                 parts[-2] = "plain"
119             # now to download and parse file
120             url = "https://" + "/".join(parts)
121             info_file = requests.get(url, timeout=15).text
122             info_parsed = yaml.load(info_file)
123             ptl = info_parsed.get('project_lead')
124             if ptl:
125                 project_leads.append(ptl)
126             sub_ptl = info_parsed.get("subproject_lead")
127             if sub_ptl:
128                 project_leads.append(sub_ptl)
129
130         except Exception:
131             return None
132
133         return project_leads
134
135     def parse_url(self, info_url):
136         """
137         Parse the project URL.
138
139         Gets the INFO.yaml file from the project and returns the PTL info.
140         """
141         if "github" in info_url:
142             return self.parse_github_url(info_url)
143
144         if "gerrit.opnfv.org" in info_url:
145             return self.parse_gerrit_url(info_url)
146
147         if "git.opnfv.org" in info_url:
148             return self.parse_opnfv_git_url(info_url)
149
150     def booking_allowed(self, booking, repo):
151         """
152         Assert the current Booking Policy.
153
154         This is the method that will have to change whenever the booking policy changes in the Infra
155         group / LFN. This is a nice isolation of that administration crap
156         currently checks if the booking uses multiple servers. if it does, then the owner must be a PTL,
157         which is checked using the provided info file
158         """
159         if booking.owner.userprofile.booking_privledge:
160             return True  # admin override for this user
161         if Booking.objects.filter(owner=booking.owner, end__gt=timezone.now()).count() >= 3:
162             return False
163         if len(booking.resource.template.get_required_resources()) < 2:
164             return True  # if they only have one server, we dont care
165         if repo.BOOKING_INFO_FILE not in repo.el:
166             return False  # INFO file not provided
167         ptl_info = self.parse_url(repo.el.get(repo.BOOKING_INFO_FILE))
168         for ptl in ptl_info:
169             if ptl['email'] == booking.owner.userprofile.email_addr:
170                 return True
171         return False
172
173
174 class WorkflowStepStatus(object):
175     """
176     Poor man's enum for the status of a workflow step.
177
178     The steps in a workflow are not completed (UNTOUCHED)
179     or they have been completed correctly (VALID) or they were filled out
180     incorrectly (INVALID)
181     """
182
183     UNTOUCHED = 0
184     INVALID = 100
185     VALID = 200
186
187
188 class WorkflowStep(object):
189     template = 'bad_request.html'
190     title = "Generic Step"
191     description = "You were led here by mistake"
192     short_title = "error"
193     metastep = None
194     # phasing out metastep:
195
196     valid = WorkflowStepStatus.UNTOUCHED
197     message = ""
198
199     enabled = True
200
201     def cleanup(self):
202         raise Exception("WorkflowStep subclass of type " + str(type(self)) + " has no concrete implemented cleanup() method")
203
204     def enable(self):
205         if not self.enabled:
206             self.enabled = True
207
208     def disable(self):
209         if self.enabled:
210             self.cleanup()
211             self.enabled = False
212
213     def set_invalid(self, message, code=WorkflowStepStatus.INVALID):
214         self.valid = code
215         self.message = message
216
217     def set_valid(self, message, code=WorkflowStepStatus.VALID):
218         self.valid = code
219         self.message = message
220
221     def to_json(self):
222         return {
223             'title': self.short_title,
224             'enabled': self.enabled,
225             'valid': self.valid,
226             'message': self.message,
227         }
228
229     def __init__(self, id, repo=None):
230         self.repo = repo
231         self.id = id
232
233     def get_context(self):
234         context = {}
235         context['step_number'] = self.repo_get('steps')
236         context['active_step'] = self.repo_get('active_step')
237         context['render_correct'] = "true"
238         context['step_title'] = self.title
239         context['description'] = self.description
240         return context
241
242     def render(self, request):
243         return HttpResponse(self.render_to_string(request))
244
245     def render_to_string(self, request):
246         template = get_template(self.template)
247         return template.render(self.get_context(), request)
248
249     def post(self, post_content, user):
250         raise Exception("WorkflowStep subclass of type " + str(type(self)) + " has no concrete post() method")
251
252     def validate(self, request):
253         pass
254
255     def repo_get(self, key, default=None):
256         return self.repo.get(key, default, self.id)
257
258     def repo_put(self, key, value):
259         return self.repo.put(key, value, self.id)
260
261
262 """
263 subclassing notes:
264     subclasses have to define the following class attributes:
265         self.select_repo_key: where the selected "object" or "bundle" is to be placed in the repo
266         self.form: the form to be used
267         alert_bundle_missing(): what message to display if a user does not select/selects an invalid object
268         get_form_queryset(): generate a queryset to be used to filter available items for the field
269         get_page_context(): return simple context such as page header and other info
270 """
271
272
273 class AbstractSelectOrCreate(WorkflowStep):
274     template = 'dashboard/genericselect.html'
275     title = "Select a Bundle"
276     short_title = "select"
277     description = "Generic bundle selector step"
278
279     select_repo_key = None
280     form = None  # subclasses are expected to use a form that is a subclass of SearchableSelectGenericForm
281
282     def alert_bundle_missing(self):  # override in subclasses to change message if field isn't filled out
283         self.set_invalid("Please select a valid bundle")
284
285     def post(self, post_data, user):
286         form = self.form(post_data, queryset=self.get_form_queryset())
287         if form.is_valid():
288             bundle = form.get_validated_bundle()
289             if not bundle:
290                 self.alert_bundle_missing()
291                 return
292             self.repo_put(self.select_repo_key, bundle)
293             self.put_confirm_info(bundle)
294             self.set_valid("Step Completed")
295         else:
296             self.alert_bundle_missing()
297
298     def get_context(self):
299         default = []
300
301         bundle = self.repo_get(self.select_repo_key, False)
302         if bundle:
303             default.append(bundle)
304
305         form = self.form(queryset=self.get_form_queryset(), initial=default)
306
307         context = {'form': form, **self.get_page_context()}
308         context.update(super().get_context())
309
310         return context
311
312     def get_page_context():
313         return {
314             'select_type': 'generic',
315             'select_type_title': 'Generic Bundle'
316         }
317
318
319 class Confirmation_Step(WorkflowStep):
320     template = 'workflow/confirm.html'
321     title = "Confirm Changes"
322     description = "Does this all look right?"
323
324     short_title = "confirm"
325
326     def get_context(self):
327         context = super(Confirmation_Step, self).get_context()
328         context['form'] = ConfirmationForm()
329         # Summary of submitted form data shown on the 'confirm' step of the workflow
330         confirm_details = "\nPod:\n  Name: '{name}'\n  Description: '{desc}'\nLab: '{lab}'".format(
331             name=self.repo_get(self.repo.CONFIRMATION)['resource']['name'],
332             desc=self.repo_get(self.repo.CONFIRMATION)['resource']['description'],
333             lab=self.repo_get(self.repo.CONFIRMATION)['template']['lab'])
334         confirm_details += "\nResources:"
335         for i, device in enumerate(self.repo_get(self.repo.RESOURCE_TEMPLATE_MODELS)['resources']):
336             confirm_details += "\n  " + str(device) + ": " + str(self.repo_get(self.repo.CONFIRMATION)['template']['resources'][i]['profile'])
337         context['confirmation_info'] = confirm_details
338         if self.valid == WorkflowStepStatus.VALID:
339             context["confirm_succeeded"] = "true"
340
341         return context
342
343     def flush_to_db(self):
344         errors = self.repo.make_models()
345         if errors:
346             return errors
347
348     def post(self, post_data, user):
349         form = ConfirmationForm(post_data)
350         if form.is_valid():
351             data = form.cleaned_data['confirm']
352             if data == "True":
353                 errors = self.flush_to_db()
354                 if errors:
355                     self.set_invalid("ERROR OCCURRED: " + errors)
356                 else:
357                     self.set_valid("Confirmed")
358
359             elif data == "False":
360                 self.repo.cancel()
361                 self.set_valid("Canceled")
362             else:
363                 self.set_invalid("Bad Form Contents")
364
365         else:
366             self.set_invalid("Bad Form Contents")
367
368
369 class Repository():
370
371     EDIT = "editing"
372     MODELS = "models"
373     RESOURCE_SELECT = "resource_select"
374     CONFIRMATION = "confirmation"
375     SELECTED_RESOURCE_TEMPLATE = "selected resource template pk"
376     SELECTED_OPNFV_CONFIG = "selected opnfv deployment config"
377     RESOURCE_TEMPLATE_MODELS = "generic_resource_template_models"
378     RESOURCE_TEMPLATE_INFO = "generic_resource_template_info"
379     BOOKING = "booking"
380     LAB = "lab"
381     RCONFIG_LAST_HOSTLIST = "resource_configuration_network_previous_hostlist"
382     BOOKING_FORMS = "booking_forms"
383     SWCONF_HOSTS = "swconf_hosts"
384     BOOKING_MODELS = "booking models"
385     CONFIG_MODELS = "configuration bundle models"
386     OPNFV_MODELS = "opnfv configuration models"
387     SESSION_USER = "session owner user account"
388     SESSION_MANAGER = "session manager for current session"
389     VALIDATED_MODEL_GRB = "valid grb config model instance in db"
390     VALIDATED_MODEL_CONFIG = "valid config model instance in db"
391     VALIDATED_MODEL_BOOKING = "valid booking model instance in db"
392     VLANS = "a list of vlans"
393     SNAPSHOT_MODELS = "the models for snapshotting"
394     SNAPSHOT_BOOKING_ID = "the booking id for snapshotting"
395     SNAPSHOT_NAME = "the name of the snapshot"
396     SNAPSHOT_DESC = "description of the snapshot"
397     BOOKING_INFO_FILE = "the INFO.yaml file for this user's booking"
398
399     # new keys for migration to using ResourceTemplates:
400     RESOURCE_TEMPLATE_MODELS = "current working model of resource template"
401
402     # migratory elements of segmented workflow
403     # each of these is the end result of a different workflow.
404     HAS_RESULT = "whether or not workflow has a result"
405     RESULT_KEY = "key for target index that result will be put into in parent"
406     RESULT = "result object from workflow"
407
408     def get_child_defaults(self):
409         return_tuples = []
410         for key in [self.SELECTED_RESOURCE_TEMPLATE, self.SESSION_USER]:
411             return_tuples.append((key, self.el.get(key)))
412         return return_tuples
413
414     def set_defaults(self, defaults):
415         for key, value in defaults:
416             self.el[key] = value
417
418     def get(self, key, default, id):
419
420         self.add_get_history(key, id)
421         return self.el.get(key, default)
422
423     def put(self, key, val, id):
424         self.add_put_history(key, id)
425         self.el[key] = val
426
427     def add_get_history(self, key, id):
428         self.add_history(key, id, self.get_history)
429
430     def add_put_history(self, key, id):
431         self.add_history(key, id, self.put_history)
432
433     def add_history(self, key, id, history):
434         if key not in history:
435             history[key] = [id]
436         else:
437             history[key].append(id)
438
439     def cancel(self):
440         if self.RESOURCE_TEMPLATE_MODELS in self.el:
441             models = self.el[self.RESOURCE_TEMPLATE_MODELS]
442             if models['template'].temporary:
443                 models['template'].delete()
444                 # deleting current template should cascade delete all
445                 # necessary related models
446
447     def make_models(self):
448         if self.SNAPSHOT_MODELS in self.el:
449             errors = self.make_snapshot()
450             if errors:
451                 return errors
452
453         # if GRB WF, create it
454         if self.RESOURCE_TEMPLATE_MODELS in self.el:
455             errors = self.make_generic_resource_bundle()
456             if errors:
457                 return errors
458             else:
459                 self.el[self.HAS_RESULT] = True
460                 self.el[self.RESULT_KEY] = self.SELECTED_RESOURCE_TEMPLATE
461                 return
462
463         if self.OPNFV_MODELS in self.el:
464             errors = self.make_opnfv_config()
465             if errors:
466                 return errors
467             else:
468                 self.el[self.HAS_RESULT] = True
469                 self.el[self.RESULT_KEY] = self.SELECTED_OPNFV_CONFIG
470
471         if self.BOOKING_MODELS in self.el:
472             errors = self.make_booking()
473             if errors:
474                 return errors
475             # create notification
476             booking = self.el[self.BOOKING_MODELS]['booking']
477             NotificationHandler.notify_new_booking(booking)
478
479     def make_snapshot(self):
480         owner = self.el[self.SESSION_USER]
481         models = self.el[self.SNAPSHOT_MODELS]
482         image = models.get('snapshot', Image())
483         booking_id = self.el.get(self.SNAPSHOT_BOOKING_ID)
484         if not booking_id:
485             return "SNAP, No booking ID provided"
486         booking = Booking.objects.get(pk=booking_id)
487         if booking.start > timezone.now() or booking.end < timezone.now():
488             return "Booking is not active"
489         name = self.el.get(self.SNAPSHOT_NAME)
490         if not name:
491             return "SNAP, no name provided"
492         host = models.get('host')
493         if not host:
494             return "SNAP, no host provided"
495         description = self.el.get(self.SNAPSHOT_DESC, "")
496         image.from_lab = booking.lab
497         image.name = name
498         image.description = description
499         image.public = False
500         image.lab_id = -1
501         image.owner = owner
502         image.host_type = host.profile
503         image.save()
504         try:
505             current_image = host.config.image
506             image.os = current_image.os
507             image.save()
508         except Exception:
509             pass
510         JobFactory.makeSnapshotTask(image, booking, host)
511
512         self.el[self.RESULT] = image
513         self.el[self.HAS_RESULT] = True
514
515     def make_generic_resource_bundle(self):
516         owner = self.el[self.SESSION_USER]
517         if self.RESOURCE_TEMPLATE_MODELS in self.el:
518             models = self.el[self.RESOURCE_TEMPLATE_MODELS]
519             models['template'].owner = owner
520             models['template'].temporary = False
521             models['template'].save()
522             self.el[self.RESULT] = models['template']
523             self.el[self.HAS_RESULT] = True
524             return False
525
526         else:
527             return "GRB no models given. CODE:0x0001"
528
529     def make_software_config_bundle(self):
530         models = self.el[self.CONFIG_MODELS]
531         if 'bundle' in models:
532             bundle = models['bundle']
533             bundle.bundle = self.el[self.SELECTED_RESOURCE_TEMPLATE]
534             try:
535                 bundle.save()
536             except Exception as e:
537                 return "SWC, saving bundle generated exception: " + str(e) + "CODE:0x0007"
538
539         else:
540             return "SWC, no bundle in models. CODE:0x0006"
541         if 'host_configs' in models:
542             host_configs = models['host_configs']
543             for host_config in host_configs:
544                 host_config.template = host_config.template
545                 host_config.profile = host_config.profile
546                 try:
547                     host_config.save()
548                 except Exception as e:
549                     return "SWC, saving host configs generated exception: " + str(e) + "CODE:0x0009"
550         else:
551             return "SWC, no host configs in models. CODE:0x0008"
552         if 'opnfv' in models:
553             opnfvconfig = models['opnfv']
554             opnfvconfig.bundle = opnfvconfig.bundle
555             if opnfvconfig.scenario not in opnfvconfig.installer.sup_scenarios.all():
556                 return "SWC, scenario not supported by installer. CODE:0x000d"
557             try:
558                 opnfvconfig.save()
559             except Exception as e:
560                 return "SWC, saving opnfv config generated exception: " + str(e) + "CODE:0x000b"
561         else:
562             pass
563
564         self.el[self.RESULT] = bundle
565         return False
566
567     @transaction.atomic  # TODO: Rewrite transactions with savepoints at user level for all workflows
568     def make_booking(self):
569         models = self.el[self.BOOKING_MODELS]
570         owner = self.el[self.SESSION_USER]
571
572         if 'booking' in models:
573             booking = models['booking']
574         else:
575             return "BOOK, no booking model exists. CODE:0x000f"
576
577         selected_grb = None
578
579         if self.SELECTED_RESOURCE_TEMPLATE in self.el:
580             selected_grb = self.el[self.SELECTED_RESOURCE_TEMPLATE]
581         else:
582             return "BOOK, no selected resource. CODE:0x000e"
583
584         if not booking.start:
585             return "BOOK, booking has no start. CODE:0x0010"
586         if not booking.end:
587             return "BOOK, booking has no end. CODE:0x0011"
588         if booking.end <= booking.start:
589             return "BOOK, end before/same time as start. CODE:0x0012"
590
591         if 'collaborators' in models:
592             collaborators = models['collaborators']
593         else:
594             return "BOOK, collaborators not defined. CODE:0x0013"
595         try:
596             res_manager = ResourceManager.getInstance()
597             resource_bundle = res_manager.instantiateTemplate(selected_grb)
598         except ResourceAvailabilityException as e:
599             return "BOOK, requested resources are not available. Exception: " + str(e) + " CODE:0x0014"
600         except ModelValidationException as e:
601             return "Error encountered when saving bundle. " + str(e) + " CODE: 0x001b"
602
603         booking.resource = resource_bundle
604         booking.owner = owner
605         booking.lab = selected_grb.lab
606
607         is_allowed = BookingAuthManager().booking_allowed(booking, self)
608         if not is_allowed:
609             return "BOOK, you are not allowed to book the requested resources"
610
611         try:
612             booking.save()
613         except Exception as e:
614             return "BOOK, saving booking generated exception: " + str(e) + " CODE:0x0015"
615
616         for collaborator in collaborators:
617             booking.collaborators.add(collaborator)
618
619         try:
620             booking.pdf = PDFTemplater.makePDF(booking)
621             booking.save()
622         except Exception as e:
623             return "BOOK, failed to create Pod Desriptor File: " + str(e)
624
625         try:
626             JobFactory.makeCompleteJob(booking)
627         except Exception as e:
628             return "BOOK, serializing for api generated exception: " + str(e) + " CODE:0xFFFF"
629
630         try:
631             booking.save()
632         except Exception as e:
633             return "BOOK, saving booking generated exception: " + str(e) + " CODE:0x0016"
634
635         self.el[self.RESULT] = booking
636         self.el[self.HAS_RESULT] = True
637
638     def make_opnfv_config(self):
639         opnfv_models = self.el[self.OPNFV_MODELS]
640         config_bundle = self.el[self.SELECTED_CONFIG_BUNDLE]
641         if not config_bundle:
642             return "No Configuration bundle selected"
643         info = opnfv_models.get("meta", {})
644         name = info.get("name", False)
645         desc = info.get("description", False)
646         if not (name and desc):
647             return "No name or description given"
648         installer = opnfv_models['installer_chosen']
649         if not installer:
650             return "No OPNFV Installer chosen"
651         scenario = opnfv_models['scenario_chosen']
652         if not scenario:
653             return "No OPNFV Scenario chosen"
654
655         opnfv_config = OPNFVConfig.objects.create(
656             bundle=config_bundle,
657             name=name,
658             description=desc,
659             installer=installer,
660             scenario=scenario
661         )
662
663         network_roles = opnfv_models['network_roles']
664         for net_role in network_roles:
665             opnfv_config.networks.add(
666                 NetworkRole.objects.create(
667                     name=net_role['role'],
668                     network=net_role['network']
669                 )
670             )
671
672         host_roles = opnfv_models['host_roles']
673         for host_role in host_roles:
674             config = config_bundle.hostConfigurations.get(
675                 host__resource__name=host_role['host_name']
676             )
677             ResourceOPNFVConfig.objects.create(
678                 role=host_role['role'],
679                 host_config=config,
680                 opnfv_config=opnfv_config
681             )
682
683         self.el[self.RESULT] = opnfv_config
684         self.el[self.HAS_RESULT] = True
685
686     def __init__(self):
687         self.el = {}
688         self.el[self.CONFIRMATION] = {}
689         self.el["active_step"] = 0
690         self.el[self.HAS_RESULT] = False
691         self.el[self.RESULT] = None
692         self.get_history = {}
693         self.put_history = {}