src/api/models.py

   1 ##############################################################################
   2 # Copyright (c) 2018 Sawyer Bergeron, Parker Berberian, and others.
   3 #
   4 # All rights reserved. This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 # http://www.apache.org/licenses/LICENSE-2.0
   8 ##############################################################################
   9
  10
  11 from django.contrib.auth.models import User
  12 from django.db import models
  13 from django.core.exceptions import PermissionDenied, ValidationError
  14 from django.shortcuts import get_object_or_404
  15 from django.contrib.postgres.fields import JSONField
  16 from django.http import HttpResponseNotFound
  17 from django.urls import reverse
  18 from django.utils import timezone
  19
  20 import json
  21 import uuid
  22 import yaml
  23 import re
  24
  25 from booking.models import Booking
  26 from resource_inventory.models import (
  27     Lab,
  28     ResourceProfile,
  29     Image,
  30     Opsys,
  31     Interface,
  32     ResourceOPNFVConfig,
  33     RemoteInfo,
  34     OPNFVConfig,
  35     ConfigState,
  36     ResourceQuery,
  37     ResourceConfiguration,
  38     CloudInitFile
  39 )
  40 from resource_inventory.idf_templater import IDFTemplater
  41 from resource_inventory.pdf_templater import PDFTemplater
  42 from account.models import Downtime, UserProfile
  43 from dashboard.utils import AbstractModelQuery
  44
  45
  46 class JobStatus:
  47     """
  48     A poor man's enum for a job's status.
  49
  50     A job is NEW if it has not been started or recognized by the Lab
  51     A job is CURRENT if it has been started by the lab but it is not yet completed
  52     a job is DONE if all the tasks are complete and the booking is ready to use
  53     """
  54
  55     NEW = 0
  56     CURRENT = 100
  57     DONE = 200
  58     ERROR = 300
  59
  60
  61 class LabManagerTracker:
  62
  63     @classmethod
  64     def get(cls, lab_name, token):
  65         """
  66         Get a LabManager.
  67
  68         Takes in a lab name (from a url path)
  69         returns a lab manager instance for that lab, if it exists
  70         Also checks that the given API token is correct
  71         """
  72         try:
  73             lab = Lab.objects.get(name=lab_name)
  74         except Exception:
  75             raise PermissionDenied("Lab not found")
  76         if lab.api_token == token:
  77             return LabManager(lab)
  78         raise PermissionDenied("Lab not authorized")
  79
  80
  81 class LabManager:
  82     """
  83     Handles all lab REST calls.
  84
  85     handles jobs, inventory, status, etc
  86     may need to create helper classes
  87     """
  88
  89     def __init__(self, lab):
  90         self.lab = lab
  91
  92     def get_opsyss(self):
  93         return Opsys.objects.filter(from_lab=self.lab)
  94
  95     def get_images(self):
  96         return Image.objects.filter(from_lab=self.lab)
  97
  98     def get_image(self, image_id):
  99         return Image.objects.filter(from_lab=self.lab, lab_id=image_id)
 100
 101     def get_opsys(self, opsys_id):
 102         return Opsys.objects.filter(from_lab=self.lab, lab_id=opsys_id)
 103
 104     def get_downtime(self):
 105         return Downtime.objects.filter(start__lt=timezone.now(), end__gt=timezone.now(), lab=self.lab)
 106
 107     def get_downtime_json(self):
 108         downtime = self.get_downtime().first()  # should only be one item in queryset
 109         if downtime:
 110             return {
 111                 "is_down": True,
 112                 "start": downtime.start,
 113                 "end": downtime.end,
 114                 "description": downtime.description
 115             }
 116         return {"is_down": False}
 117
 118     def create_downtime(self, form):
 119         """
 120         Create a downtime event.
 121
 122         Takes in a dictionary that describes the model.
 123         {
 124           "start": utc timestamp
 125           "end": utc timestamp
 126           "description": human text (optional)
 127         }
 128         For timestamp structure, https://docs.djangoproject.com/en/2.2/ref/forms/fields/#datetimefield
 129         """
 130         Downtime.objects.create(
 131             start=form.cleaned_data['start'],
 132             end=form.cleaned_data['end'],
 133             description=form.cleaned_data['description'],
 134             lab=self.lab
 135         )
 136         return self.get_downtime_json()
 137
 138     def update_host_remote_info(self, data, res_id):
 139         resource = ResourceQuery.filter(labid=res_id, lab=self.lab)
 140         if len(resource) != 1:
 141             return HttpResponseNotFound("Could not find single host with id " + str(res_id))
 142         resource = resource[0]
 143         info = {}
 144         try:
 145             info['address'] = data['address']
 146             info['mac_address'] = data['mac_address']
 147             info['password'] = data['password']
 148             info['user'] = data['user']
 149             info['type'] = data['type']
 150             info['versions'] = json.dumps(data['versions'])
 151         except Exception as e:
 152             return {"error": "invalid arguement: " + str(e)}
 153         remote_info = resource.remote_management
 154         if "default" in remote_info.mac_address:
 155             remote_info = RemoteInfo()
 156         remote_info.address = info['address']
 157         remote_info.mac_address = info['mac_address']
 158         remote_info.password = info['password']
 159         remote_info.user = info['user']
 160         remote_info.type = info['type']
 161         remote_info.versions = info['versions']
 162         remote_info.save()
 163         resource.remote_management = remote_info
 164         resource.save()
 165         booking = Booking.objects.get(resource=resource.bundle)
 166         self.update_xdf(booking)
 167         return {"status": "success"}
 168
 169     def update_xdf(self, booking):
 170         booking.pdf = PDFTemplater.makePDF(booking)
 171         booking.idf = IDFTemplater().makeIDF(booking)
 172         booking.save()
 173
 174     def get_pdf(self, booking_id):
 175         booking = get_object_or_404(Booking, pk=booking_id, lab=self.lab)
 176         return booking.pdf
 177
 178     def get_idf(self, booking_id):
 179         booking = get_object_or_404(Booking, pk=booking_id, lab=self.lab)
 180         return booking.idf
 181
 182     def get_profile(self):
 183         prof = {}
 184         prof['name'] = self.lab.name
 185         prof['contact'] = {
 186             "phone": self.lab.contact_phone,
 187             "email": self.lab.contact_email
 188         }
 189         prof['host_count'] = [{
 190             "type": profile.name,
 191             "count": len(profile.get_resources(lab=self.lab))}
 192             for profile in ResourceProfile.objects.filter(labs=self.lab)]
 193         return prof
 194
 195     def format_user(self, userprofile):
 196         return {
 197             "id": userprofile.user.id,
 198             "username": userprofile.user.username,
 199             "email": userprofile.email_addr,
 200             "first_name": userprofile.user.first_name,
 201             "last_name": userprofile.user.last_name,
 202             "company": userprofile.company
 203         }
 204
 205     def get_users(self):
 206         userlist = [self.format_user(profile) for profile in UserProfile.objects.select_related("user").all()]
 207
 208         return json.dumps({"users": userlist})
 209
 210     def get_user(self, user_id):
 211         user = User.objects.get(pk=user_id)
 212
 213         profile = get_object_or_404(UserProfile, user=user)
 214
 215         return json.dumps(self.format_user(profile))
 216
 217     def get_inventory(self):
 218         inventory = {}
 219         resources = ResourceQuery.filter(lab=self.lab)
 220         images = Image.objects.filter(from_lab=self.lab)
 221         profiles = ResourceProfile.objects.filter(labs=self.lab)
 222         inventory['resources'] = self.serialize_resources(resources)
 223         inventory['images'] = self.serialize_images(images)
 224         inventory['host_types'] = self.serialize_host_profiles(profiles)
 225         return inventory
 226
 227     def get_host(self, hostname):
 228         resource = ResourceQuery.filter(labid=hostname, lab=self.lab)
 229         if len(resource) != 1:
 230             return HttpResponseNotFound("Could not find single host with id " + str(hostname))
 231         resource = resource[0]
 232         return {
 233             "booked": resource.booked,
 234             "working": resource.working,
 235             "type": resource.profile.name
 236         }
 237
 238     def update_host(self, hostname, data):
 239         resource = ResourceQuery.filter(labid=hostname, lab=self.lab)
 240         if len(resource) != 1:
 241             return HttpResponseNotFound("Could not find single host with id " + str(hostname))
 242         resource = resource[0]
 243         if "working" in data:
 244             working = data['working'] == "true"
 245             resource.working = working
 246         resource.save()
 247         return self.get_host(hostname)
 248
 249     def get_status(self):
 250         return {"status": self.lab.status}
 251
 252     def set_status(self, payload):
 253         {}
 254
 255     def get_current_jobs(self):
 256         jobs = Job.objects.filter(booking__lab=self.lab)
 257
 258         return self.serialize_jobs(jobs, status=JobStatus.CURRENT)
 259
 260     def get_new_jobs(self):
 261         jobs = Job.objects.filter(booking__lab=self.lab)
 262
 263         return self.serialize_jobs(jobs, status=JobStatus.NEW)
 264
 265     def get_done_jobs(self):
 266         jobs = Job.objects.filter(booking__lab=self.lab)
 267
 268         return self.serialize_jobs(jobs, status=JobStatus.DONE)
 269
 270     def get_analytics_job(self):
 271         """ Get analytics job with status new """
 272         jobs = Job.objects.filter(
 273             booking__lab=self.lab,
 274             job_type='DATA'
 275         )
 276
 277         return self.serialize_jobs(jobs, status=JobStatus.NEW)
 278
 279     def get_job(self, jobid):
 280         return Job.objects.get(pk=jobid).to_dict()
 281
 282     def update_job(self, jobid, data):
 283         {}
 284
 285     def serialize_jobs(self, jobs, status=JobStatus.NEW):
 286         job_ser = []
 287         for job in jobs:
 288             jsonized_job = job.get_delta(status)
 289             if len(jsonized_job['payload']) < 1:
 290                 continue
 291             job_ser.append(jsonized_job)
 292
 293         return job_ser
 294
 295     def serialize_resources(self, resources):
 296         # TODO: rewrite for Resource model
 297         host_ser = []
 298         for res in resources:
 299             r = {
 300                 'interfaces': [],
 301                 'hostname': res.name,
 302                 'host_type': res.profile.name
 303             }
 304             for iface in res.get_interfaces():
 305                 r['interfaces'].append({
 306                     'mac': iface.mac_address,
 307                     'busaddr': iface.bus_address,
 308                     'name': iface.name,
 309                     'switchport': {"switch_name": iface.switch_name, "port_name": iface.port_name}
 310                 })
 311         return host_ser
 312
 313     def serialize_images(self, images):
 314         images_ser = []
 315         for image in images:
 316             images_ser.append(
 317                 {
 318                     "name": image.name,
 319                     "lab_id": image.lab_id,
 320                     "dashboard_id": image.id
 321                 }
 322             )
 323         return images_ser
 324
 325     def serialize_resource_profiles(self, profiles):
 326         profile_ser = []
 327         for profile in profiles:
 328             p = {}
 329             p['cpu'] = {
 330                 "cores": profile.cpuprofile.first().cores,
 331                 "arch": profile.cpuprofile.first().architecture,
 332                 "cpus": profile.cpuprofile.first().cpus,
 333             }
 334             p['disks'] = []
 335             for disk in profile.storageprofile.all():
 336                 d = {
 337                     "size": disk.size,
 338                     "type": disk.media_type,
 339                     "name": disk.name
 340                 }
 341                 p['disks'].append(d)
 342             p['description'] = profile.description
 343             p['interfaces'] = []
 344             for iface in profile.interfaceprofile.all():
 345                 p['interfaces'].append(
 346                     {
 347                         "speed": iface.speed,
 348                         "name": iface.name
 349                     }
 350                 )
 351
 352             p['ram'] = {"amount": profile.ramprofile.first().amount}
 353             p['name'] = profile.name
 354             profile_ser.append(p)
 355         return profile_ser
 356
 357
 358 class GeneratedCloudConfig(models.Model):
 359     resource_id = models.CharField(max_length=200)
 360     booking = models.ForeignKey(Booking, on_delete=models.CASCADE)
 361     rconfig = models.ForeignKey(ResourceConfiguration, on_delete=models.CASCADE)
 362     text = models.TextField(null=True, blank=True)
 363
 364     def _normalize_username(self, username: str) -> str:
 365         # TODO: make usernames posix compliant
 366         s = re.sub(r'\W+', '', username)
 367         return s
 368
 369     def _get_ssh_string(self, username: str) -> str:
 370         user = User.objects.get(username=username)
 371         uprofile = user.userprofile
 372
 373         ssh_file = uprofile.ssh_public_key
 374
 375         escaped_file = ssh_file.open().read().decode(encoding="UTF-8").replace("\n", " ")
 376
 377         return escaped_file
 378
 379     def _serialize_users(self):
 380         """
 381         returns the dictionary to be placed behind the `users` field of the toplevel c-i dict
 382         """
 383         # conserves distro default user
 384         user_array = ["default"]
 385
 386         users = list(self.booking.collaborators.all())
 387         users.append(self.booking.owner)
 388         for collaborator in users:
 389             userdict = {}
 390
 391             # TODO: validate if usernames are valid as linux usernames (and provide an override potentially)
 392             userdict['name'] = self._normalize_username(collaborator.username)
 393
 394             userdict['groups'] = "sudo"
 395             userdict['sudo'] = "ALL=(ALL) NOPASSWD:ALL"
 396
 397             userdict['ssh_authorized_keys'] = [self._get_ssh_string(collaborator.username)]
 398
 399             user_array.append(userdict)
 400
 401         # user_array.append({
 402         #    "name": "opnfv",
 403         #    "passwd": "$6$k54L.vim1cLaEc4$5AyUIrufGlbtVBzuCWOlA1yV6QdD7Gr2MzwIs/WhuYR9ebSfh3Qlb7djkqzjwjxpnSAonK1YOabPP6NxUDccu.",
 404         #    "ssh_redirect_user": True,
 405         #    "sudo": "ALL=(ALL) NOPASSWD:ALL",
 406         #    "groups": "sudo",
 407         #    })
 408
 409         return user_array
 410
 411     # TODO: make this configurable
 412     def _serialize_sysinfo(self):
 413         defuser = {}
 414         defuser['name'] = 'opnfv'
 415         defuser['plain_text_passwd'] = 'OPNFV_HOST'
 416         defuser['home'] = '/home/opnfv'
 417         defuser['shell'] = '/bin/bash'
 418         defuser['lock_passwd'] = True
 419         defuser['gecos'] = 'Lab Manager User'
 420         defuser['groups'] = 'sudo'
 421
 422         return {'default_user': defuser}
 423
 424     # TODO: make this configurable
 425     def _serialize_runcmds(self):
 426         cmdlist = []
 427
 428         # have hosts run dhcp on boot
 429         cmdlist.append(['sudo', 'dhclient', '-r'])
 430         cmdlist.append(['sudo', 'dhclient'])
 431
 432         return cmdlist
 433
 434     def _serialize_netconf_v1(self):
 435         # interfaces = {}  # map from iface_name => dhcp_config
 436         # vlans = {}  # map from vlan_id => dhcp_config
 437
 438         config_arr = []
 439
 440         for interface in self._resource().interfaces.all():
 441             interface_name = interface.profile.name
 442             interface_mac = interface.mac_address
 443
 444             iface_dict_entry = {
 445                 "type": "physical",
 446                 "name": interface_name,
 447                 "mac_address": interface_mac,
 448             }
 449
 450             for vlan in interface.config.all():
 451                 if vlan.tagged:
 452                     vlan_dict_entry = {'type': 'vlan'}
 453                     vlan_dict_entry['name'] = str(interface_name) + "." + str(vlan.vlan_id)
 454                     vlan_dict_entry['vlan_link'] = str(interface_name)
 455                     vlan_dict_entry['vlan_id'] = int(vlan.vlan_id)
 456                     vlan_dict_entry['mac_address'] = str(interface_mac)
 457                     if vlan.public:
 458                         vlan_dict_entry["subnets"] = [{"type": "dhcp"}]
 459                     config_arr.append(vlan_dict_entry)
 460                 if (not vlan.tagged) and vlan.public:
 461                     iface_dict_entry["subnets"] = [{"type": "dhcp"}]
 462
 463                 # vlan_dict_entry['mtu'] = # TODO, determine override MTU if needed
 464
 465             config_arr.append(iface_dict_entry)
 466
 467         ns_dict = {
 468             'type': 'nameserver',
 469             'address': ['10.64.0.1', '8.8.8.8']
 470         }
 471
 472         config_arr.append(ns_dict)
 473
 474         full_dict = {'version': 1, 'config': config_arr}
 475
 476         return full_dict
 477
 478     @classmethod
 479     def get(cls, booking_id: int, resource_lab_id: str, file_id: int):
 480         return GeneratedCloudConfig.objects.get(resource_id=resource_lab_id, booking__id=booking_id, file_id=file_id)
 481
 482     def _resource(self):
 483         return ResourceQuery.get(labid=self.resource_id, lab=self.booking.lab)
 484
 485     # def _get_facts(self):
 486         # resource = self._resource()
 487
 488         # hostname = self.rconfig.name
 489         # iface_configs = for_config.interface_configs.all()
 490
 491     def _to_dict(self):
 492         main_dict = {}
 493
 494         main_dict['users'] = self._serialize_users()
 495         main_dict['network'] = self._serialize_netconf_v1()
 496         main_dict['hostname'] = self.rconfig.name
 497
 498         # add first startup commands
 499         main_dict['runcmd'] = self._serialize_runcmds()
 500
 501         # configure distro default user
 502         main_dict['system_info'] = self._serialize_sysinfo()
 503
 504         return main_dict
 505
 506     def serialize(self) -> str:
 507         return yaml.dump(self._to_dict(), width=float("inf"))
 508
 509
 510 class APILog(models.Model):
 511     user = models.ForeignKey(User, on_delete=models.PROTECT)
 512     call_time = models.DateTimeField(auto_now=True)
 513     method = models.CharField(null=True, max_length=6)
 514     endpoint = models.CharField(null=True, max_length=300)
 515     ip_addr = models.GenericIPAddressField(protocol="both", null=True, unpack_ipv4=False)
 516     body = JSONField(null=True)
 517
 518     def __str__(self):
 519         return "Call to {} at {} by {}".format(
 520             self.endpoint,
 521             self.call_time,
 522             self.user.username
 523         )
 524
 525
 526 class AutomationAPIManager:
 527     @staticmethod
 528     def serialize_booking(booking):
 529         sbook = {}
 530         sbook['id'] = booking.pk
 531         sbook['owner'] = booking.owner.username
 532         sbook['collaborators'] = [user.username for user in booking.collaborators.all()]
 533         sbook['start'] = booking.start
 534         sbook['end'] = booking.end
 535         sbook['lab'] = AutomationAPIManager.serialize_lab(booking.lab)
 536         sbook['purpose'] = booking.purpose
 537         sbook['resourceBundle'] = AutomationAPIManager.serialize_bundle(booking.resource)
 538         return sbook
 539
 540     @staticmethod
 541     def serialize_lab(lab):
 542         slab = {}
 543         slab['id'] = lab.pk
 544         slab['name'] = lab.name
 545         return slab
 546
 547     @staticmethod
 548     def serialize_bundle(bundle):
 549         sbundle = {}
 550         sbundle['id'] = bundle.pk
 551         sbundle['resources'] = [
 552             AutomationAPIManager.serialize_server(server)
 553             for server in bundle.get_resources()]
 554         return sbundle
 555
 556     @staticmethod
 557     def serialize_server(server):
 558         sserver = {}
 559         sserver['id'] = server.pk
 560         sserver['name'] = server.name
 561         return sserver
 562
 563     @staticmethod
 564     def serialize_resource_profile(profile):
 565         sprofile = {}
 566         sprofile['id'] = profile.pk
 567         sprofile['name'] = profile.name
 568         return sprofile
 569
 570     @staticmethod
 571     def serialize_template(rec_temp_and_count):
 572         template = rec_temp_and_count[0]
 573         count = rec_temp_and_count[1]
 574
 575         stemplate = {}
 576         stemplate['id'] = template.pk
 577         stemplate['name'] = template.name
 578         stemplate['count_available'] = count
 579         stemplate['resourceProfiles'] = [
 580             AutomationAPIManager.serialize_resource_profile(config.profile)
 581             for config in template.getConfigs()
 582         ]
 583         return stemplate
 584
 585     @staticmethod
 586     def serialize_image(image):
 587         simage = {}
 588         simage['id'] = image.pk
 589         simage['name'] = image.name
 590         return simage
 591
 592     @staticmethod
 593     def serialize_userprofile(up):
 594         sup = {}
 595         sup['id'] = up.pk
 596         sup['username'] = up.user.username
 597         return sup
 598
 599
 600 class Job(models.Model):
 601     """
 602     A Job to be performed by the Lab.
 603
 604     The API uses Jobs and Tasks to communicate actions that need to be taken to the Lab
 605     that is hosting a booking. A booking from a user has an associated Job which tells
 606     the lab how to configure the hardware, networking, etc to fulfill the booking
 607     for the user.
 608     This is the class that is serialized and put into the api
 609     """
 610
 611     JOB_TYPES = (
 612         ('BOOK', 'Booking'),
 613         ('DATA', 'Analytics')
 614     )
 615
 616     booking = models.OneToOneField(Booking, on_delete=models.CASCADE, null=True)
 617     status = models.IntegerField(default=JobStatus.NEW)
 618     complete = models.BooleanField(default=False)
 619     job_type = models.CharField(
 620         max_length=4,
 621         choices=JOB_TYPES,
 622         default='BOOK'
 623     )
 624
 625     def to_dict(self):
 626         d = {}
 627         for relation in self.get_tasklist():
 628             if relation.job_key not in d:
 629                 d[relation.job_key] = {}
 630             d[relation.job_key][relation.task_id] = relation.config.to_dict()
 631
 632         return {"id": self.id, "payload": d}
 633
 634     def get_tasklist(self, status="all"):
 635         if status != "all":
 636             return JobTaskQuery.filter(job=self, status=status)
 637         return JobTaskQuery.filter(job=self)
 638
 639     def is_fulfilled(self):
 640         """
 641         If a job has been completed by the lab.
 642
 643         This method should return true if all of the job's tasks are done,
 644         and false otherwise
 645         """
 646         my_tasks = self.get_tasklist()
 647         for task in my_tasks:
 648             if task.status != JobStatus.DONE:
 649                 return False
 650         return True
 651
 652     def get_delta(self, status):
 653         d = {}
 654         for relation in self.get_tasklist(status=status):
 655             if relation.job_key not in d:
 656                 d[relation.job_key] = {}
 657             d[relation.job_key][relation.task_id] = relation.config.get_delta()
 658
 659         return {"id": self.id, "payload": d}
 660
 661     def to_json(self):
 662         return json.dumps(self.to_dict())
 663
 664
 665 class TaskConfig(models.Model):
 666     state = models.IntegerField(default=ConfigState.NEW)
 667
 668     keys = set()  # TODO: This needs to be an instance variable, not a class variable
 669     delta_keys_list = models.CharField(max_length=200, default="[]")
 670
 671     @property
 672     def delta_keys(self):
 673         return list(set(json.loads(self.delta_keys_list)))
 674
 675     @delta_keys.setter
 676     def delta_keys(self, keylist):
 677         self.delta_keys_list = json.dumps(keylist)
 678
 679     def to_dict(self):
 680         raise NotImplementedError
 681
 682     def get_delta(self):
 683         raise NotImplementedError
 684
 685     def format_delta(self, config, token):
 686         delta = {k: config[k] for k in self.delta_keys}
 687         delta['lab_token'] = token
 688         return delta
 689
 690     def to_json(self):
 691         return json.dumps(self.to_dict())
 692
 693     def clear_delta(self):
 694         self.delta_keys = []
 695
 696     def set(self, *args):
 697         dkeys = self.delta_keys
 698         for arg in args:
 699             if arg in self.keys:
 700                 dkeys.append(arg)
 701         self.delta_keys = dkeys
 702
 703
 704 class BridgeConfig(models.Model):
 705     """Displays mapping between jumphost interfaces and bridges."""
 706
 707     interfaces = models.ManyToManyField(Interface)
 708     opnfv_config = models.ForeignKey(OPNFVConfig, on_delete=models.CASCADE)
 709
 710     def to_dict(self):
 711         d = {}
 712         hid = ResourceQuery.get(interface__pk=self.interfaces.first().pk).labid
 713         d[hid] = {}
 714         for interface in self.interfaces.all():
 715             d[hid][interface.mac_address] = []
 716             for vlan in interface.config.all():
 717                 network_role = self.opnfv_model.networks().filter(network=vlan.network)
 718                 bridge = IDFTemplater.bridge_names[network_role.name]
 719                 br_config = {
 720                     "vlan_id": vlan.vlan_id,
 721                     "tagged": vlan.tagged,
 722                     "bridge": bridge
 723                 }
 724                 d[hid][interface.mac_address].append(br_config)
 725         return d
 726
 727     def to_json(self):
 728         return json.dumps(self.to_dict())
 729
 730
 731 class ActiveUsersConfig(models.Model):
 732     """
 733     Task for getting active VPN users
 734
 735     StackStorm needs no information to run this job
 736     so this task is very bare, but neccessary to fit
 737     job creation convention.
 738     """
 739
 740     def clear_delta(self):
 741         self.delta = '{}'
 742
 743     def get_delta(self):
 744         return json.loads(self.to_json())
 745
 746     def to_json(self):
 747         return json.dumps(self.to_dict())
 748
 749     def to_dict(self):
 750         return {}
 751
 752
 753 class OpnfvApiConfig(models.Model):
 754
 755     installer = models.CharField(max_length=200)
 756     scenario = models.CharField(max_length=300)
 757     roles = models.ManyToManyField(ResourceOPNFVConfig)
 758     # pdf and idf are url endpoints, not the actual file
 759     pdf = models.CharField(max_length=100)
 760     idf = models.CharField(max_length=100)
 761     bridge_config = models.OneToOneField(BridgeConfig, on_delete=models.CASCADE, null=True)
 762     delta = models.TextField()
 763     opnfv_config = models.ForeignKey(OPNFVConfig, null=True, on_delete=models.SET_NULL)
 764
 765     def to_dict(self):
 766         d = {}
 767         if not self.opnfv_config:
 768             return d
 769         if self.installer:
 770             d['installer'] = self.installer
 771         if self.scenario:
 772             d['scenario'] = self.scenario
 773         if self.pdf:
 774             d['pdf'] = self.pdf
 775         if self.idf:
 776             d['idf'] = self.idf
 777         if self.bridge_config:
 778             d['bridged_interfaces'] = self.bridge_config.to_dict()
 779
 780         hosts = self.roles.all()
 781         if hosts.exists():
 782             d['roles'] = []
 783             for host in hosts:
 784                 d['roles'].append({
 785                     host.labid: self.opnfv_config.host_opnfv_config.get(
 786                         host_config__pk=host.config.pk
 787                     ).role.name
 788                 })
 789
 790         return d
 791
 792     def to_json(self):
 793         return json.dumps(self.to_dict())
 794
 795     def set_installer(self, installer):
 796         self.installer = installer
 797         d = json.loads(self.delta)
 798         d['installer'] = installer
 799         self.delta = json.dumps(d)
 800
 801     def set_scenario(self, scenario):
 802         self.scenario = scenario
 803         d = json.loads(self.delta)
 804         d['scenario'] = scenario
 805         self.delta = json.dumps(d)
 806
 807     def set_xdf(self, booking, update_delta=True):
 808         kwargs = {'lab_name': booking.lab.name, 'booking_id': booking.id}
 809         self.pdf = reverse('get-pdf', kwargs=kwargs)
 810         self.idf = reverse('get-idf', kwargs=kwargs)
 811         if update_delta:
 812             d = json.loads(self.delta)
 813             d['pdf'] = self.pdf
 814             d['idf'] = self.idf
 815             self.delta = json.dumps(d)
 816
 817     def add_role(self, host):
 818         self.roles.add(host)
 819         d = json.loads(self.delta)
 820         if 'role' not in d:
 821             d['role'] = []
 822         d['roles'].append({host.labid: host.config.opnfvRole.name})
 823         self.delta = json.dumps(d)
 824
 825     def clear_delta(self):
 826         self.delta = '{}'
 827
 828     def get_delta(self):
 829         return json.loads(self.to_json())
 830
 831
 832 class AccessConfig(TaskConfig):
 833     access_type = models.CharField(max_length=50)
 834     user = models.ForeignKey(User, on_delete=models.CASCADE)
 835     revoke = models.BooleanField(default=False)
 836     context = models.TextField(default="")
 837     delta = models.TextField(default="{}")
 838
 839     def to_dict(self):
 840         d = {}
 841         d['access_type'] = self.access_type
 842         d['user'] = self.user.id
 843         d['revoke'] = self.revoke
 844         try:
 845             d['context'] = json.loads(self.context)
 846         except Exception:
 847             pass
 848         return d
 849
 850     def get_delta(self):
 851         d = json.loads(self.to_json())
 852         d["lab_token"] = self.accessrelation.lab_token
 853
 854         return d
 855
 856     def to_json(self):
 857         return json.dumps(self.to_dict())
 858
 859     def clear_delta(self):
 860         d = {}
 861         d["lab_token"] = self.accessrelation.lab_token
 862         self.delta = json.dumps(d)
 863
 864     def set_access_type(self, access_type):
 865         self.access_type = access_type
 866         d = json.loads(self.delta)
 867         d['access_type'] = access_type
 868         self.delta = json.dumps(d)
 869
 870     def set_user(self, user):
 871         self.user = user
 872         d = json.loads(self.delta)
 873         d['user'] = self.user.id
 874         self.delta = json.dumps(d)
 875
 876     def set_revoke(self, revoke):
 877         self.revoke = revoke
 878         d = json.loads(self.delta)
 879         d['revoke'] = revoke
 880         self.delta = json.dumps(d)
 881
 882     def set_context(self, context):
 883         self.context = json.dumps(context)
 884         d = json.loads(self.delta)
 885         d['context'] = context
 886         self.delta = json.dumps(d)
 887
 888
 889 class SoftwareConfig(TaskConfig):
 890     """Handles software installations, such as OPNFV or ONAP."""
 891
 892     opnfv = models.ForeignKey(OpnfvApiConfig, on_delete=models.CASCADE)
 893
 894     def to_dict(self):
 895         d = {}
 896         if self.opnfv:
 897             d['opnfv'] = self.opnfv.to_dict()
 898
 899         d["lab_token"] = self.softwarerelation.lab_token
 900         self.delta = json.dumps(d)
 901
 902         return d
 903
 904     def get_delta(self):
 905         d = {}
 906         d['opnfv'] = self.opnfv.get_delta()
 907         d['lab_token'] = self.softwarerelation.lab_token
 908
 909         return d
 910
 911     def clear_delta(self):
 912         self.opnfv.clear_delta()
 913
 914     def to_json(self):
 915         return json.dumps(self.to_dict())
 916
 917
 918 class HardwareConfig(TaskConfig):
 919     """Describes the desired configuration of the hardware."""
 920
 921     image = models.CharField(max_length=100, default="defimage")
 922     power = models.CharField(max_length=100, default="off")
 923     hostname = models.CharField(max_length=100, default="hostname")
 924     ipmi_create = models.BooleanField(default=False)
 925     delta = models.TextField()
 926
 927     keys = set(["id", "image", "power", "hostname", "ipmi_create"])
 928
 929     def to_dict(self):
 930         return self.get_delta()
 931
 932     def get_delta(self):
 933         # TODO: grab the GeneratedCloudConfig urls from self.hosthardwarerelation.get_resource()
 934         return self.format_delta(
 935             self.hosthardwarerelation.get_resource().get_configuration(self.state),
 936             self.hosthardwarerelation.lab_token)
 937
 938
 939 class NetworkConfig(TaskConfig):
 940     """Handles network configuration."""
 941
 942     interfaces = models.ManyToManyField(Interface)
 943     delta = models.TextField()
 944
 945     def to_dict(self):
 946         d = {}
 947         hid = self.hostnetworkrelation.resource_id
 948         d[hid] = {}
 949         for interface in self.interfaces.all():
 950             d[hid][interface.mac_address] = []
 951             if self.state != ConfigState.CLEAN:
 952                 for vlan in interface.config.all():
 953                     # TODO: should this come from the interface?
 954                     # e.g. will different interfaces for different resources need different configs?
 955                     d[hid][interface.mac_address].append({"vlan_id": vlan.vlan_id, "tagged": vlan.tagged})
 956
 957         return d
 958
 959     def to_json(self):
 960         return json.dumps(self.to_dict())
 961
 962     def get_delta(self):
 963         d = json.loads(self.to_json())
 964         d['lab_token'] = self.hostnetworkrelation.lab_token
 965         return d
 966
 967     def clear_delta(self):
 968         self.delta = json.dumps(self.to_dict())
 969         self.save()
 970
 971     def add_interface(self, interface):
 972         self.interfaces.add(interface)
 973         d = json.loads(self.delta)
 974         hid = self.hostnetworkrelation.resource_id
 975         if hid not in d:
 976             d[hid] = {}
 977         d[hid][interface.mac_address] = []
 978         for vlan in interface.config.all():
 979             d[hid][interface.mac_address].append({"vlan_id": vlan.vlan_id, "tagged": vlan.tagged})
 980         self.delta = json.dumps(d)
 981
 982
 983 class SnapshotConfig(TaskConfig):
 984
 985     resource_id = models.CharField(max_length=200, default="default_id")
 986     image = models.CharField(max_length=200, null=True)  # cobbler ID
 987     dashboard_id = models.IntegerField()
 988     delta = models.TextField(default="{}")
 989
 990     def to_dict(self):
 991         d = {}
 992         if self.host:
 993             d['host'] = self.host.labid
 994         if self.image:
 995             d['image'] = self.image
 996         d['dashboard_id'] = self.dashboard_id
 997         return d
 998
 999     def to_json(self):
1000         return json.dumps(self.to_dict())
1001
1002     def get_delta(self):
1003         d = json.loads(self.to_json())
1004         return d
1005
1006     def clear_delta(self):
1007         self.delta = json.dumps(self.to_dict())
1008         self.save()
1009
1010     def set_host(self, host):
1011         self.host = host
1012         d = json.loads(self.delta)
1013         d['host'] = host.labid
1014         self.delta = json.dumps(d)
1015
1016     def set_image(self, image):
1017         self.image = image
1018         d = json.loads(self.delta)
1019         d['image'] = self.image
1020         self.delta = json.dumps(d)
1021
1022     def clear_image(self):
1023         self.image = None
1024         d = json.loads(self.delta)
1025         d.pop("image", None)
1026         self.delta = json.dumps(d)
1027
1028     def set_dashboard_id(self, dash):
1029         self.dashboard_id = dash
1030         d = json.loads(self.delta)
1031         d['dashboard_id'] = self.dashboard_id
1032         self.delta = json.dumps(d)
1033
1034     def save(self, *args, **kwargs):
1035         if len(ResourceQuery.filter(labid=self.resource_id)) != 1:
1036             raise ValidationError("resource_id " + str(self.resource_id) + " does not refer to a single resource")
1037         super().save(*args, **kwargs)
1038
1039
1040 def get_task(task_id):
1041     for taskclass in [AccessRelation, SoftwareRelation, HostHardwareRelation, HostNetworkRelation, SnapshotRelation]:
1042         try:
1043             ret = taskclass.objects.get(task_id=task_id)
1044             return ret
1045         except taskclass.DoesNotExist:
1046             pass
1047     from django.core.exceptions import ObjectDoesNotExist
1048     raise ObjectDoesNotExist("Could not find matching TaskRelation instance")
1049
1050
1051 def get_task_uuid():
1052     return str(uuid.uuid4())
1053
1054
1055 class TaskRelation(models.Model):
1056     """
1057     Relates a Job to a TaskConfig.
1058
1059     superclass that relates a Job to tasks anc maintains information
1060     like status and messages from the lab
1061     """
1062
1063     status = models.IntegerField(default=JobStatus.NEW)
1064     job = models.ForeignKey(Job, on_delete=models.CASCADE)
1065     config = models.OneToOneField(TaskConfig, on_delete=models.CASCADE)
1066     task_id = models.CharField(default=get_task_uuid, max_length=37)
1067     lab_token = models.CharField(default="null", max_length=50)
1068     message = models.TextField(default="")
1069
1070     job_key = None
1071
1072     def delete(self, *args, **kwargs):
1073         self.config.delete()
1074         return super(self.__class__, self).delete(*args, **kwargs)
1075
1076     def type_str(self):
1077         return "Generic Task"
1078
1079     class Meta:
1080         abstract = True
1081
1082
1083 class AccessRelation(TaskRelation):
1084     config = models.OneToOneField(AccessConfig, on_delete=models.CASCADE)
1085     job_key = "access"
1086
1087     def type_str(self):
1088         return "Access Task"
1089
1090     def delete(self, *args, **kwargs):
1091         self.config.delete()
1092         return super(self.__class__, self).delete(*args, **kwargs)
1093
1094
1095 class SoftwareRelation(TaskRelation):
1096     config = models.OneToOneField(SoftwareConfig, on_delete=models.CASCADE)
1097     job_key = "software"
1098
1099     def type_str(self):
1100         return "Software Configuration Task"
1101
1102     def delete(self, *args, **kwargs):
1103         self.config.delete()
1104         return super(self.__class__, self).delete(*args, **kwargs)
1105
1106
1107 class HostHardwareRelation(TaskRelation):
1108     resource_id = models.CharField(max_length=200, default="default_id")
1109     config = models.OneToOneField(HardwareConfig, on_delete=models.CASCADE)
1110     job_key = "hardware"
1111
1112     def type_str(self):
1113         return "Hardware Configuration Task"
1114
1115     def get_delta(self):
1116         return self.config.to_dict()
1117
1118     def delete(self, *args, **kwargs):
1119         self.config.delete()
1120         return super(self.__class__, self).delete(*args, **kwargs)
1121
1122     def save(self, *args, **kwargs):
1123         if len(ResourceQuery.filter(labid=self.resource_id)) != 1:
1124             raise ValidationError("resource_id " + str(self.resource_id) + " does not refer to a single resource")
1125         super().save(*args, **kwargs)
1126
1127     def get_resource(self):
1128         return ResourceQuery.get(labid=self.resource_id)
1129
1130
1131 class HostNetworkRelation(TaskRelation):
1132     resource_id = models.CharField(max_length=200, default="default_id")
1133     config = models.OneToOneField(NetworkConfig, on_delete=models.CASCADE)
1134     job_key = "network"
1135
1136     def type_str(self):
1137         return "Network Configuration Task"
1138
1139     def delete(self, *args, **kwargs):
1140         self.config.delete()
1141         return super(self.__class__, self).delete(*args, **kwargs)
1142
1143     def save(self, *args, **kwargs):
1144         if len(ResourceQuery.filter(labid=self.resource_id)) != 1:
1145             raise ValidationError("resource_id " + str(self.resource_id) + " does not refer to a single resource")
1146         super().save(*args, **kwargs)
1147
1148     def get_resource(self):
1149         return ResourceQuery.get(labid=self.resource_id)
1150
1151
1152 class SnapshotRelation(TaskRelation):
1153     snapshot = models.ForeignKey(Image, on_delete=models.CASCADE)
1154     config = models.OneToOneField(SnapshotConfig, on_delete=models.CASCADE)
1155     job_key = "snapshot"
1156
1157     def type_str(self):
1158         return "Snapshot Task"
1159
1160     def get_delta(self):
1161         return self.config.to_dict()
1162
1163     def delete(self, *args, **kwargs):
1164         self.config.delete()
1165         return super(self.__class__, self).delete(*args, **kwargs)
1166
1167
1168 class ActiveUsersRelation(TaskRelation):
1169     config = models.OneToOneField(ActiveUsersConfig, on_delete=models.CASCADE)
1170     job_key = "active users task"
1171
1172     def type_str(self):
1173         return "Active Users Task"
1174
1175
1176 class JobFactory(object):
1177     """This class creates all the API models (jobs, tasks, etc) needed to fulfill a booking."""
1178
1179     @classmethod
1180     def reimageHost(cls, new_image, booking, host):
1181         """Modify an existing job to reimage the given host."""
1182         job = Job.objects.get(booking=booking)
1183         # make hardware task new
1184         hardware_relation = HostHardwareRelation.objects.get(resource_id=host, job=job)
1185         hardware_relation.config.image = new_image.lab_id
1186         hardware_relation.config.save()
1187         hardware_relation.status = JobStatus.NEW
1188
1189         # re-apply networking after host is reset
1190         net_relation = HostNetworkRelation.objects.get(resource_id=host, job=job)
1191         net_relation.status = JobStatus.NEW
1192
1193         # re-apply ssh access after host is reset
1194         for relation in AccessRelation.objects.filter(job=job, config__access_type="ssh"):
1195             relation.status = JobStatus.NEW
1196             relation.save()
1197
1198         hardware_relation.save()
1199         net_relation.save()
1200
1201     @classmethod
1202     def makeSnapshotTask(cls, image, booking, host):
1203         relation = SnapshotRelation()
1204         job = Job.objects.get(booking=booking)
1205         config = SnapshotConfig.objects.create(dashboard_id=image.id)
1206
1207         relation.job = job
1208         relation.config = config
1209         relation.config.save()
1210         relation.config = relation.config
1211         relation.snapshot = image
1212         relation.save()
1213
1214         config.clear_delta()
1215         config.set_host(host)
1216         config.save()
1217
1218     @classmethod
1219     def makeActiveUsersTask(cls):
1220         """ Append active users task to analytics job """
1221         config = ActiveUsersConfig()
1222         relation = ActiveUsersRelation()
1223         job = Job.objects.get(job_type='DATA')
1224
1225         job.status = JobStatus.NEW
1226
1227         relation.job = job
1228         relation.config = config
1229         relation.config.save()
1230         relation.config = relation.config
1231         relation.save()
1232         config.save()
1233
1234     @classmethod
1235     def makeAnalyticsJob(cls, booking):
1236         """
1237         Create the analytics job
1238
1239         This will only run once since there will only be one analytics job.
1240         All analytics tasks get appended to analytics job.
1241         """
1242
1243         if len(Job.objects.filter(job_type='DATA')) > 0:
1244             raise Exception("Cannot have more than one analytics job")
1245
1246         if booking.resource:
1247             raise Exception("Booking is not marker for analytics job, has resoure")
1248
1249         job = Job()
1250         job.booking = booking
1251         job.job_type = 'DATA'
1252         job.save()
1253
1254         cls.makeActiveUsersTask()
1255
1256     @classmethod
1257     def makeCompleteJob(cls, booking):
1258         """Create everything that is needed to fulfill the given booking."""
1259         resources = booking.resource.get_resources()
1260         job = None
1261         try:
1262             job = Job.objects.get(booking=booking)
1263         except Exception:
1264             job = Job.objects.create(status=JobStatus.NEW, booking=booking)
1265         cls.makeHardwareConfigs(
1266             resources=resources,
1267             job=job
1268         )
1269         cls.makeNetworkConfigs(
1270             resources=resources,
1271             job=job
1272         )
1273         cls.makeSoftware(
1274             booking=booking,
1275             job=job
1276         )
1277         cls.makeGeneratedCloudConfigs(
1278             resources=resources,
1279             job=job
1280         )
1281         all_users = list(booking.collaborators.all())
1282         all_users.append(booking.owner)
1283         cls.makeAccessConfig(
1284             users=all_users,
1285             access_type="vpn",
1286             revoke=False,
1287             job=job
1288         )
1289         for user in all_users:
1290             try:
1291                 cls.makeAccessConfig(
1292                     users=[user],
1293                     access_type="ssh",
1294                     revoke=False,
1295                     job=job,
1296                     context={
1297                         "key": user.userprofile.ssh_public_key.open().read().decode(encoding="UTF-8"),
1298                         "hosts": [r.labid for r in resources]
1299                     }
1300                 )
1301             except Exception:
1302                 continue
1303
1304     @classmethod
1305     def makeGeneratedCloudConfigs(cls, resources=[], job=Job()):
1306         for res in resources:
1307             cif = GeneratedCloudConfig.objects.create(resource_id=res.labid, booking=job.booking, rconfig=res.config)
1308             cif.save()
1309
1310             cif = CloudInitFile.create(priority=0, text=cif.serialize())
1311             cif.save()
1312
1313             res.config.cloud_init_files.add(cif)
1314             res.config.save()
1315
1316     @classmethod
1317     def makeHardwareConfigs(cls, resources=[], job=Job()):
1318         """
1319         Create and save HardwareConfig.
1320
1321         Helper function to create the tasks related to
1322         configuring the hardware
1323         """
1324         for res in resources:
1325             hardware_config = None
1326             try:
1327                 hardware_config = HardwareConfig.objects.get(relation__resource_id=res.labid)
1328             except Exception:
1329                 hardware_config = HardwareConfig()
1330
1331             relation = HostHardwareRelation()
1332             relation.resource_id = res.labid
1333             relation.job = job
1334             relation.config = hardware_config
1335             relation.config.save()
1336             relation.config = relation.config
1337             relation.save()
1338
1339             hardware_config.set("id", "image", "hostname", "power", "ipmi_create")
1340             hardware_config.save()
1341
1342     @classmethod
1343     def makeAccessConfig(cls, users, access_type, revoke=False, job=Job(), context=False):
1344         """
1345         Create and save AccessConfig.
1346
1347         Helper function to create the tasks related to
1348         configuring the VPN, SSH, etc access for users
1349         """
1350         for user in users:
1351             relation = AccessRelation()
1352             relation.job = job
1353             config = AccessConfig()
1354             config.access_type = access_type
1355             config.user = user
1356             config.save()
1357             relation.config = config
1358             relation.save()
1359             config.clear_delta()
1360             if context:
1361                 config.set_context(context)
1362             config.set_access_type(access_type)
1363             config.set_revoke(revoke)
1364             config.set_user(user)
1365             config.save()
1366
1367     @classmethod
1368     def makeNetworkConfigs(cls, resources=[], job=Job()):
1369         """
1370         Create and save NetworkConfig.
1371
1372         Helper function to create the tasks related to
1373         configuring the networking
1374         """
1375         for res in resources:
1376             network_config = None
1377             try:
1378                 network_config = NetworkConfig.objects.get(relation__host=res)
1379             except Exception:
1380                 network_config = NetworkConfig.objects.create()
1381
1382             relation = HostNetworkRelation()
1383             relation.resource_id = res.labid
1384             relation.job = job
1385             network_config.save()
1386             relation.config = network_config
1387             relation.save()
1388             network_config.clear_delta()
1389
1390             # TODO: use get_interfaces() on resource
1391             for interface in res.interfaces.all():
1392                 network_config.add_interface(interface)
1393             network_config.save()
1394
1395     @classmethod
1396     def make_bridge_config(cls, booking):
1397         if len(booking.resource.get_resources()) < 2:
1398             return None
1399         try:
1400             jumphost_config = ResourceOPNFVConfig.objects.filter(
1401                 role__name__iexact="jumphost"
1402             )
1403             jumphost = ResourceQuery.filter(
1404                 bundle=booking.resource,
1405                 config=jumphost_config.resource_config
1406             )[0]
1407         except Exception:
1408             return None
1409         br_config = BridgeConfig.objects.create(opnfv_config=booking.opnfv_config)
1410         for iface in jumphost.interfaces.all():
1411             br_config.interfaces.add(iface)
1412         return br_config
1413
1414     @classmethod
1415     def makeSoftware(cls, booking=None, job=Job()):
1416         """
1417         Create and save SoftwareConfig.
1418
1419         Helper function to create the tasks related to
1420         configuring the desired software, e.g. an OPNFV deployment
1421         """
1422         if not booking.opnfv_config:
1423             return None
1424
1425         opnfv_api_config = OpnfvApiConfig.objects.create(
1426             opnfv_config=booking.opnfv_config,
1427             installer=booking.opnfv_config.installer.name,
1428             scenario=booking.opnfv_config.scenario.name,
1429             bridge_config=cls.make_bridge_config(booking)
1430         )
1431
1432         opnfv_api_config.set_xdf(booking, False)
1433         opnfv_api_config.save()
1434
1435         for host in booking.resource.get_resources():
1436             opnfv_api_config.roles.add(host)
1437         software_config = SoftwareConfig.objects.create(opnfv=opnfv_api_config)
1438         software_relation = SoftwareRelation.objects.create(job=job, config=software_config)
1439         return software_relation
1440
1441
1442 JOB_TASK_CLASSLIST = [
1443     HostHardwareRelation,
1444     AccessRelation,
1445     HostNetworkRelation,
1446     SoftwareRelation,
1447     SnapshotRelation,
1448     ActiveUsersRelation
1449 ]
1450
1451
1452 class JobTaskQuery(AbstractModelQuery):
1453     model_list = JOB_TASK_CLASSLIST