Refactor selector step logic
[pharos-tools.git] / dashboard / src / workflow / models.py
1 ##############################################################################
2 # Copyright (c) 2018 Sawyer Bergeron, Parker Berberian, and others.
3 #
4 # All rights reserved. This program and the accompanying materials
5 # are made available under the terms of the Apache License, Version 2.0
6 # which accompanies this distribution, and is available at
7 # http://www.apache.org/licenses/LICENSE-2.0
8 ##############################################################################
9
10
11 from django.shortcuts import render
12 from django.contrib import messages
13 from django.http import HttpResponse
14 from django.utils import timezone
15
16 import yaml
17 import requests
18
19 from workflow.forms import ConfirmationForm
20 from api.models import JobFactory
21 from dashboard.exceptions import ResourceAvailabilityException, ModelValidationException
22 from resource_inventory.models import Image, GenericInterface, OPNFVConfig, HostOPNFVConfig, NetworkRole
23 from resource_inventory.resource_manager import ResourceManager
24 from resource_inventory.pdf_templater import PDFTemplater
25 from notifier.manager import NotificationHandler
26 from booking.models import Booking
27
28
29 class BookingAuthManager():
30     LFN_PROJECTS = ["opnfv"]  # TODO
31
32     def parse_github_url(self, url):
33         project_leads = []
34         try:
35             parts = url.split("/")
36             if "http" in parts[0]:  # the url include http(s)://
37                 parts = parts[2:]
38             if parts[-1] != "INFO.yaml":
39                 return None
40             if parts[0] not in ["github.com", "raw.githubusercontent.com"]:
41                 return None
42             if parts[1] not in self.LFN_PROJECTS:
43                 return None
44             # now to download and parse file
45             if parts[3] == "blob":
46                 parts[3] = "raw"
47             url = "https://" + "/".join(parts)
48             info_file = requests.get(url, timeout=15).text
49             info_parsed = yaml.load(info_file)
50             ptl = info_parsed.get('project_lead')
51             if ptl:
52                 project_leads.append(ptl)
53             sub_ptl = info_parsed.get("subproject_lead")
54             if sub_ptl:
55                 project_leads.append(sub_ptl)
56
57         except Exception:
58             pass
59
60         return project_leads
61
62     def parse_gerrit_url(self, url):
63         project_leads = []
64         try:
65             halfs = url.split("?")
66             parts = halfs[0].split("/")
67             args = halfs[1].split(";")
68             if "http" in parts[0]:  # the url include http(s)://
69                 parts = parts[2:]
70             if "f=INFO.yaml" not in args:
71                 return None
72             if "gerrit.opnfv.org" not in parts[0]:
73                 return None
74             try:
75                 i = args.index("a=blob")
76                 args[i] = "a=blob_plain"
77             except ValueError:
78                 pass
79             # recreate url
80             halfs[1] = ";".join(args)
81             halfs[0] = "/".join(parts)
82             # now to download and parse file
83             url = "https://" + "?".join(halfs)
84             info_file = requests.get(url, timeout=15).text
85             info_parsed = yaml.load(info_file)
86             ptl = info_parsed.get('project_lead')
87             if ptl:
88                 project_leads.append(ptl)
89             sub_ptl = info_parsed.get("subproject_lead")
90             if sub_ptl:
91                 project_leads.append(sub_ptl)
92
93         except Exception:
94             return None
95
96         return project_leads
97
98     def parse_opnfv_git_url(self, url):
99         project_leads = []
100         try:
101             parts = url.split("/")
102             if "http" in parts[0]:  # the url include http(s)://
103                 parts = parts[2:]
104             if "INFO.yaml" not in parts[-1]:
105                 return None
106             if "git.opnfv.org" not in parts[0]:
107                 return None
108             if parts[-2] == "tree":
109                 parts[-2] = "plain"
110             # now to download and parse file
111             url = "https://" + "/".join(parts)
112             info_file = requests.get(url, timeout=15).text
113             info_parsed = yaml.load(info_file)
114             ptl = info_parsed.get('project_lead')
115             if ptl:
116                 project_leads.append(ptl)
117             sub_ptl = info_parsed.get("subproject_lead")
118             if sub_ptl:
119                 project_leads.append(sub_ptl)
120
121         except Exception:
122             return None
123
124         return project_leads
125
126     def parse_url(self, info_url):
127         """
128         will return the PTL in the INFO file on success, or None
129         """
130         if "github" in info_url:
131             return self.parse_github_url(info_url)
132
133         if "gerrit.opnfv.org" in info_url:
134             return self.parse_gerrit_url(info_url)
135
136         if "git.opnfv.org" in info_url:
137             return self.parse_opnfv_git_url(info_url)
138
139     def booking_allowed(self, booking, repo):
140         """
141         This is the method that will have to change whenever the booking policy changes in the Infra
142         group / LFN. This is a nice isolation of that administration crap
143         currently checks if the booking uses multiple servers. if it does, then the owner must be a PTL,
144         which is checked using the provided info file
145         """
146         if booking.owner.userprofile.booking_privledge:
147             return True  # admin override for this user
148         if Booking.objects.filter(owner=booking.owner, end__gt=timezone.now()).count() >= 3:
149             return False
150         if len(booking.resource.template.getHosts()) < 2:
151             return True  # if they only have one server, we dont care
152         if repo.BOOKING_INFO_FILE not in repo.el:
153             return False  # INFO file not provided
154         ptl_info = self.parse_url(repo.el.get(repo.BOOKING_INFO_FILE))
155         for ptl in ptl_info:
156             if ptl['email'] == booking.owner.userprofile.email_addr:
157                 return True
158         return False
159
160
161 class WorkflowStepStatus(object):
162     UNTOUCHED = 0
163     INVALID = 100
164     VALID = 200
165
166
167 class WorkflowStep(object):
168     template = 'bad_request.html'
169     title = "Generic Step"
170     description = "You were led here by mistake"
171     short_title = "error"
172     metastep = None
173     # phasing out metastep:
174
175     valid = WorkflowStepStatus.UNTOUCHED
176     message = ""
177
178     enabled = True
179
180     def cleanup(self):
181         raise Exception("WorkflowStep subclass of type " + str(type(self)) + " has no concrete implemented cleanup() method")
182
183     def enable(self):
184         if not self.enabled:
185             self.enabled = True
186
187     def disable(self):
188         if self.enabled:
189             self.cleanup()
190             self.enabled = False
191
192     def set_invalid(self, message, code=WorkflowStepStatus.INVALID):
193         self.valid = code
194         self.message = message
195
196     def set_valid(self, message, code=WorkflowStepStatus.VALID):
197         self.valid = code
198         self.message = message
199
200     def to_json(self):
201         return {
202             'title': self.short_title,
203             'enabled': self.enabled,
204             'valid': self.valid,
205             'message': self.message,
206         }
207
208     def __init__(self, id, repo=None):
209         self.repo = repo
210         self.id = id
211
212     def get_context(self):
213         context = {}
214         context['step_number'] = self.repo_get('steps')
215         context['active_step'] = self.repo_get('active_step')
216         context['render_correct'] = "true"
217         context['step_title'] = self.title
218         context['description'] = self.description
219         return context
220
221     def render(self, request):
222         self.context = self.get_context()
223         return render(request, self.template, self.context)
224
225     def post_render(self, request):
226         return self.render(request)
227
228     def test_render(self, request):
229         if request.method == "POST":
230             return self.post_render(request)
231         return self.render(request)
232
233     def validate(self, request):
234         pass
235
236     def repo_get(self, key, default=None):
237         return self.repo.get(key, default, self.id)
238
239     def repo_put(self, key, value):
240         return self.repo.put(key, value, self.id)
241
242
243 """
244 subclassing notes:
245     subclasses have to define the following class attributes:
246         self.select_repo_key: where the selected "object" or "bundle" is to be placed in the repo
247         self.form: the form to be used
248         alert_bundle_missing(): what message to display if a user does not select/selects an invalid object
249         get_form_queryset(): generate a queryset to be used to filter available items for the field
250         get_page_context(): return simple context such as page header and other info
251 """
252
253
254 class AbstractSelectOrCreate(WorkflowStep):
255     template = 'dashboard/genericselect.html'
256     title = "Select a Bundle"
257     short_title = "select"
258     description = "Generic bundle selector step"
259
260     select_repo_key = None
261     form = None  # subclasses are expected to use a form that is a subclass of SearchableSelectGenericForm
262
263     def alert_bundle_missing(self):  # override in subclasses to change message if field isn't filled out
264         self.set_invalid("Please select a valid bundle")
265
266     def post_render(self, request):
267         context = self.get_context()
268         form = self.form(request.POST, queryset=self.get_form_queryset())
269         if form.is_valid():
270             bundle = form.get_validated_bundle()
271             if not bundle:
272                 self.alert_bundle_missing()
273                 return render(request, self.template, context)
274             self.repo_put(self.select_repo_key, bundle)
275             self.put_confirm_info(bundle)
276             self.set_valid("Step Completed")
277         else:
278             self.alert_bundle_missing()
279             messages.add_message(request, messages.ERROR, "Form Didn't Validate", fail_silently=True)
280
281         return self.render(request)
282
283     def get_context(self):
284         default = []
285
286         bundle = self.repo_get(self.select_repo_key, False)
287         if bundle:
288             default.append(bundle)
289
290         form = self.form(queryset=self.get_form_queryset(), initial=default)
291
292         context = {'form': form, **self.get_page_context()}
293         context.update(super().get_context())
294
295         return context
296
297     def get_page_context():
298         return {
299             'select_type': 'generic',
300             'select_type_title': 'Generic Bundle'
301         }
302
303
304 class Confirmation_Step(WorkflowStep):
305     template = 'workflow/confirm.html'
306     title = "Confirm Changes"
307     description = "Does this all look right?"
308
309     short_title = "confirm"
310
311     def get_context(self):
312         context = super(Confirmation_Step, self).get_context()
313         context['form'] = ConfirmationForm()
314         context['confirmation_info'] = yaml.dump(
315             self.repo_get(self.repo.CONFIRMATION),
316             default_flow_style=False
317         ).strip()
318
319         return context
320
321     def flush_to_db(self):
322         errors = self.repo.make_models()
323         if errors:
324             return errors
325
326     def post_render(self, request):
327         form = ConfirmationForm(request.POST)
328         if form.is_valid():
329             data = form.cleaned_data['confirm']
330             context = self.get_context()
331             if data == "True":
332                 context["bypassed"] = "true"
333                 errors = self.flush_to_db()
334                 if errors:
335                     messages.add_message(request, messages.ERROR, "ERROR OCCURRED: " + errors)
336                 else:
337                     messages.add_message(request, messages.SUCCESS, "Confirmed")
338
339                 return HttpResponse('')
340             elif data == "False":
341                 context["bypassed"] = "true"
342                 messages.add_message(request, messages.SUCCESS, "Canceled")
343                 return render(request, self.template, context)
344             else:
345                 pass
346
347         else:
348             pass
349
350
351 class Repository():
352
353     EDIT = "editing"
354     MODELS = "models"
355     RESOURCE_SELECT = "resource_select"
356     CONFIRMATION = "confirmation"
357     SELECTED_GRESOURCE_BUNDLE = "selected generic bundle pk"
358     SELECTED_CONFIG_BUNDLE = "selected config bundle pk"
359     SELECTED_OPNFV_CONFIG = "selected opnfv deployment config"
360     GRESOURCE_BUNDLE_MODELS = "generic_resource_bundle_models"
361     GRESOURCE_BUNDLE_INFO = "generic_resource_bundle_info"
362     BOOKING = "booking"
363     LAB = "lab"
364     GRB_LAST_HOSTLIST = "grb_network_previous_hostlist"
365     BOOKING_FORMS = "booking_forms"
366     SWCONF_HOSTS = "swconf_hosts"
367     BOOKING_MODELS = "booking models"
368     CONFIG_MODELS = "configuration bundle models"
369     OPNFV_MODELS = "opnfv configuration models"
370     SESSION_USER = "session owner user account"
371     SESSION_MANAGER = "session manager for current session"
372     VALIDATED_MODEL_GRB = "valid grb config model instance in db"
373     VALIDATED_MODEL_CONFIG = "valid config model instance in db"
374     VALIDATED_MODEL_BOOKING = "valid booking model instance in db"
375     VLANS = "a list of vlans"
376     SNAPSHOT_MODELS = "the models for snapshotting"
377     SNAPSHOT_BOOKING_ID = "the booking id for snapshotting"
378     SNAPSHOT_NAME = "the name of the snapshot"
379     SNAPSHOT_DESC = "description of the snapshot"
380     BOOKING_INFO_FILE = "the INFO.yaml file for this user's booking"
381
382     # migratory elements of segmented workflow
383     # each of these is the end result of a different workflow.
384     HAS_RESULT = "whether or not workflow has a result"
385     RESULT_KEY = "key for target index that result will be put into in parent"
386     RESULT = "result object from workflow"
387
388     def get_child_defaults(self):
389         return_tuples = []
390         for key in [self.SELECTED_GRESOURCE_BUNDLE, self.SESSION_USER]:
391             return_tuples.append((key, self.el.get(key)))
392         return return_tuples
393
394     def set_defaults(self, defaults):
395         for key, value in defaults:
396             self.el[key] = value
397
398     def get(self, key, default, id):
399
400         self.add_get_history(key, id)
401         return self.el.get(key, default)
402
403     def put(self, key, val, id):
404         self.add_put_history(key, id)
405         self.el[key] = val
406
407     def add_get_history(self, key, id):
408         self.add_history(key, id, self.get_history)
409
410     def add_put_history(self, key, id):
411         self.add_history(key, id, self.put_history)
412
413     def add_history(self, key, id, history):
414         if key not in history:
415             history[key] = [id]
416         else:
417             history[key].append(id)
418
419     def make_models(self):
420         if self.SNAPSHOT_MODELS in self.el:
421             errors = self.make_snapshot()
422             if errors:
423                 return errors
424
425         # if GRB WF, create it
426         if self.GRESOURCE_BUNDLE_MODELS in self.el:
427             errors = self.make_generic_resource_bundle()
428             if errors:
429                 return errors
430             else:
431                 self.el[self.HAS_RESULT] = True
432                 self.el[self.RESULT_KEY] = self.SELECTED_GRESOURCE_BUNDLE
433                 return
434
435         if self.CONFIG_MODELS in self.el:
436             errors = self.make_software_config_bundle()
437             if errors:
438                 return errors
439             else:
440                 self.el[self.HAS_RESULT] = True
441                 self.el[self.RESULT_KEY] = self.SELECTED_CONFIG_BUNDLE
442                 return
443
444         if self.OPNFV_MODELS in self.el:
445             errors = self.make_opnfv_config()
446             if errors:
447                 return errors
448             else:
449                 self.el[self.HAS_RESULT] = True
450                 self.el[self.RESULT_KEY] = self.SELECTED_OPNFV_CONFIG
451
452         if self.BOOKING_MODELS in self.el:
453             errors = self.make_booking()
454             if errors:
455                 return errors
456             # create notification
457             booking = self.el[self.BOOKING_MODELS]['booking']
458             NotificationHandler.notify_new_booking(booking)
459
460     def make_snapshot(self):
461         owner = self.el[self.SESSION_USER]
462         models = self.el[self.SNAPSHOT_MODELS]
463         image = models.get('snapshot', Image())
464         booking_id = self.el.get(self.SNAPSHOT_BOOKING_ID)
465         if not booking_id:
466             return "SNAP, No booking ID provided"
467         booking = Booking.objects.get(pk=booking_id)
468         if booking.start > timezone.now() or booking.end < timezone.now():
469             return "Booking is not active"
470         name = self.el.get(self.SNAPSHOT_NAME)
471         if not name:
472             return "SNAP, no name provided"
473         host = models.get('host')
474         if not host:
475             return "SNAP, no host provided"
476         description = self.el.get(self.SNAPSHOT_DESC, "")
477         image.from_lab = booking.lab
478         image.name = name
479         image.description = description
480         image.public = False
481         image.lab_id = -1
482         image.owner = owner
483         image.host_type = host.profile
484         image.save()
485         try:
486             current_image = host.config.image
487             image.os = current_image.os
488             image.save()
489         except Exception:
490             pass
491         JobFactory.makeSnapshotTask(image, booking, host)
492
493     def make_generic_resource_bundle(self):
494         owner = self.el[self.SESSION_USER]
495         if self.GRESOURCE_BUNDLE_MODELS in self.el:
496             models = self.el[self.GRESOURCE_BUNDLE_MODELS]
497             if 'hosts' in models:
498                 hosts = models['hosts']
499             else:
500                 return "GRB has no hosts. CODE:0x0002"
501             if 'bundle' in models:
502                 bundle = models['bundle']
503             else:
504                 return "GRB, no bundle in models. CODE:0x0003"
505
506             try:
507                 bundle.owner = owner
508                 bundle.save()
509             except Exception as e:
510                 return "GRB, saving bundle generated exception: " + str(e) + " CODE:0x0004"
511             try:
512                 for host in hosts:
513                     genericresource = host.resource
514                     genericresource.bundle = bundle
515                     genericresource.save()
516                     host.resource = genericresource
517                     host.save()
518             except Exception as e:
519                 return "GRB, saving hosts generated exception: " + str(e) + " CODE:0x0005"
520
521             if 'networks' in models:
522                 for net in models['networks'].values():
523                     net.bundle = bundle
524                     net.save()
525
526             if 'interfaces' in models:
527                 for interface_set in models['interfaces'].values():
528                     for interface in interface_set:
529                         try:
530                             interface.host = interface.host
531                             interface.save()
532                         except Exception:
533                             return "GRB, saving interface " + str(interface) + " failed. CODE:0x0019"
534             else:
535                 return "GRB, no interface set provided. CODE:0x001a"
536
537             if 'connections' in models:
538                 for resource_name, mapping in models['connections'].items():
539                     for profile_name, connection_set in mapping.items():
540                         interface = GenericInterface.objects.get(
541                             profile__name=profile_name,
542                             host__resource__name=resource_name,
543                             host__resource__bundle=models['bundle']
544                         )
545                         for connection in connection_set:
546                             try:
547                                 connection.network = connection.network
548                                 connection.save()
549                                 interface.connections.add(connection)
550                             except Exception as e:
551                                 return "GRB, saving vlan " + str(connection) + " failed. Exception: " + str(e) + ". CODE:0x0017"
552             else:
553                 return "GRB, no vlan set provided. CODE:0x0018"
554
555         else:
556             return "GRB no models given. CODE:0x0001"
557
558         self.el[self.RESULT] = bundle
559         return False
560
561     def make_software_config_bundle(self):
562         models = self.el[self.CONFIG_MODELS]
563         if 'bundle' in models:
564             bundle = models['bundle']
565             bundle.bundle = self.el[self.SELECTED_GRESOURCE_BUNDLE]
566             try:
567                 bundle.save()
568             except Exception as e:
569                 return "SWC, saving bundle generated exception: " + str(e) + "CODE:0x0007"
570
571         else:
572             return "SWC, no bundle in models. CODE:0x0006"
573         if 'host_configs' in models:
574             host_configs = models['host_configs']
575             for host_config in host_configs:
576                 host_config.bundle = host_config.bundle
577                 host_config.host = host_config.host
578                 try:
579                     host_config.save()
580                 except Exception as e:
581                     return "SWC, saving host configs generated exception: " + str(e) + "CODE:0x0009"
582         else:
583             return "SWC, no host configs in models. CODE:0x0008"
584         if 'opnfv' in models:
585             opnfvconfig = models['opnfv']
586             opnfvconfig.bundle = opnfvconfig.bundle
587             if opnfvconfig.scenario not in opnfvconfig.installer.sup_scenarios.all():
588                 return "SWC, scenario not supported by installer. CODE:0x000d"
589             try:
590                 opnfvconfig.save()
591             except Exception as e:
592                 return "SWC, saving opnfv config generated exception: " + str(e) + "CODE:0x000b"
593         else:
594             pass
595
596         self.el[self.RESULT] = bundle
597         return False
598
599     def make_booking(self):
600         models = self.el[self.BOOKING_MODELS]
601         owner = self.el[self.SESSION_USER]
602
603         if 'booking' in models:
604             booking = models['booking']
605         else:
606             return "BOOK, no booking model exists. CODE:0x000f"
607
608         selected_grb = None
609
610         if self.SELECTED_GRESOURCE_BUNDLE in self.el:
611             selected_grb = self.el[self.SELECTED_GRESOURCE_BUNDLE]
612         else:
613             return "BOOK, no selected resource. CODE:0x000e"
614
615         if self.SELECTED_CONFIG_BUNDLE not in self.el:
616             return "BOOK, no selected config bundle. CODE:0x001f"
617
618         booking.config_bundle = self.el[self.SELECTED_CONFIG_BUNDLE]
619
620         if not booking.start:
621             return "BOOK, booking has no start. CODE:0x0010"
622         if not booking.end:
623             return "BOOK, booking has no end. CODE:0x0011"
624         if booking.end <= booking.start:
625             return "BOOK, end before/same time as start. CODE:0x0012"
626
627         if 'collaborators' in models:
628             collaborators = models['collaborators']
629         else:
630             return "BOOK, collaborators not defined. CODE:0x0013"
631         try:
632             resource_bundle = ResourceManager.getInstance().convertResourceBundle(selected_grb, config=booking.config_bundle)
633         except ResourceAvailabilityException as e:
634             return "BOOK, requested resources are not available. Exception: " + str(e) + " CODE:0x0014"
635         except ModelValidationException as e:
636             return "Error encountered when saving bundle. " + str(e) + " CODE: 0x001b"
637
638         booking.resource = resource_bundle
639         booking.owner = owner
640         booking.lab = selected_grb.lab
641
642         is_allowed = BookingAuthManager().booking_allowed(booking, self)
643         if not is_allowed:
644             return "BOOK, you are not allowed to book the requested resources"
645
646         try:
647             booking.save()
648         except Exception as e:
649             return "BOOK, saving booking generated exception: " + str(e) + " CODE:0x0015"
650
651         for collaborator in collaborators:
652             booking.collaborators.add(collaborator)
653
654         try:
655             booking.pdf = PDFTemplater.makePDF(booking)
656             booking.save()
657         except Exception as e:
658             return "BOOK, failed to create Pod Desriptor File: " + str(e)
659
660         try:
661             JobFactory.makeCompleteJob(booking)
662         except Exception as e:
663             return "BOOK, serializing for api generated exception: " + str(e) + " CODE:0xFFFF"
664
665         try:
666             booking.save()
667         except Exception as e:
668             return "BOOK, saving booking generated exception: " + str(e) + " CODE:0x0016"
669
670     def make_opnfv_config(self):
671         opnfv_models = self.el[self.OPNFV_MODELS]
672         config_bundle = opnfv_models['configbundle']
673         if not config_bundle:
674             return "No Configuration bundle selected"
675         info = opnfv_models.get("meta", {})
676         name = info.get("name", False)
677         desc = info.get("description", False)
678         if not (name and desc):
679             return "No name or description given"
680         installer = opnfv_models['installer_chosen']
681         if not installer:
682             return "No OPNFV Installer chosen"
683         scenario = opnfv_models['scenario_chosen']
684         if not scenario:
685             return "No OPNFV Scenario chosen"
686
687         opnfv_config = OPNFVConfig.objects.create(
688             bundle=config_bundle,
689             name=name,
690             description=desc,
691             installer=installer,
692             scenario=scenario
693         )
694
695         network_roles = opnfv_models['network_roles']
696         for net_role in network_roles:
697             opnfv_config.networks.add(
698                 NetworkRole.objects.create(
699                     name=net_role['role'],
700                     network=net_role['network']
701                 )
702             )
703
704         host_roles = opnfv_models['host_roles']
705         for host_role in host_roles:
706             config = config_bundle.hostConfigurations.get(
707                 host__resource__name=host_role['host_name']
708             )
709             HostOPNFVConfig.objects.create(
710                 role=host_role['role'],
711                 host_config=config,
712                 opnfv_config=opnfv_config
713             )
714
715         self.el[self.RESULT] = opnfv_config
716
717     def __init__(self):
718         self.el = {}
719         self.el[self.CONFIRMATION] = {}
720         self.el["active_step"] = 0
721         self.get_history = {}
722         self.put_history = {}