src/api/models.py

   1 ##############################################################################
   2 # Copyright (c) 2018 Sawyer Bergeron, Parker Berberian, and others.
   3 #
   4 # All rights reserved. This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 # http://www.apache.org/licenses/LICENSE-2.0
   8 ##############################################################################
   9
  10
  11 from django.contrib.auth.models import User
  12 from django.db import models
  13 from django.core.exceptions import PermissionDenied, ValidationError
  14 from django.shortcuts import get_object_or_404
  15 from django.contrib.postgres.fields import JSONField
  16 from django.http import HttpResponseNotFound
  17 from django.urls import reverse
  18 from django.utils import timezone
  19
  20 import json
  21 import uuid
  22 import yaml
  23
  24 from booking.models import Booking
  25 from resource_inventory.models import (
  26     Lab,
  27     ResourceProfile,
  28     Image,
  29     Opsys,
  30     Interface,
  31     ResourceOPNFVConfig,
  32     RemoteInfo,
  33     OPNFVConfig,
  34     ConfigState,
  35     ResourceQuery,
  36     ResourceConfiguration,
  37     CloudInitFile
  38 )
  39 from resource_inventory.idf_templater import IDFTemplater
  40 from resource_inventory.pdf_templater import PDFTemplater
  41 from account.models import Downtime, UserProfile
  42 from dashboard.utils import AbstractModelQuery
  43
  44
  45 class JobStatus:
  46     """
  47     A poor man's enum for a job's status.
  48
  49     A job is NEW if it has not been started or recognized by the Lab
  50     A job is CURRENT if it has been started by the lab but it is not yet completed
  51     a job is DONE if all the tasks are complete and the booking is ready to use
  52     """
  53
  54     NEW = 0
  55     CURRENT = 100
  56     DONE = 200
  57     ERROR = 300
  58
  59
  60 class LabManagerTracker:
  61
  62     @classmethod
  63     def get(cls, lab_name, token):
  64         """
  65         Get a LabManager.
  66
  67         Takes in a lab name (from a url path)
  68         returns a lab manager instance for that lab, if it exists
  69         Also checks that the given API token is correct
  70         """
  71         try:
  72             lab = Lab.objects.get(name=lab_name)
  73         except Exception:
  74             raise PermissionDenied("Lab not found")
  75         if lab.api_token == token:
  76             return LabManager(lab)
  77         raise PermissionDenied("Lab not authorized")
  78
  79
  80 class LabManager:
  81     """
  82     Handles all lab REST calls.
  83
  84     handles jobs, inventory, status, etc
  85     may need to create helper classes
  86     """
  87
  88     def __init__(self, lab):
  89         self.lab = lab
  90
  91     def get_opsyss(self):
  92         return Opsys.objects.filter(from_lab=self.lab)
  93
  94     def get_images(self):
  95         return Image.objects.filter(from_lab=self.lab)
  96
  97     def get_image(self, image_id):
  98         return Image.objects.filter(from_lab=self.lab, lab_id=image_id)
  99
 100     def get_opsys(self, opsys_id):
 101         return Opsys.objects.filter(from_lab=self.lab, lab_id=opsys_id)
 102
 103     def get_downtime(self):
 104         return Downtime.objects.filter(start__lt=timezone.now(), end__gt=timezone.now(), lab=self.lab)
 105
 106     def get_downtime_json(self):
 107         downtime = self.get_downtime().first()  # should only be one item in queryset
 108         if downtime:
 109             return {
 110                 "is_down": True,
 111                 "start": downtime.start,
 112                 "end": downtime.end,
 113                 "description": downtime.description
 114             }
 115         return {"is_down": False}
 116
 117     def create_downtime(self, form):
 118         """
 119         Create a downtime event.
 120
 121         Takes in a dictionary that describes the model.
 122         {
 123           "start": utc timestamp
 124           "end": utc timestamp
 125           "description": human text (optional)
 126         }
 127         For timestamp structure, https://docs.djangoproject.com/en/2.2/ref/forms/fields/#datetimefield
 128         """
 129         Downtime.objects.create(
 130             start=form.cleaned_data['start'],
 131             end=form.cleaned_data['end'],
 132             description=form.cleaned_data['description'],
 133             lab=self.lab
 134         )
 135         return self.get_downtime_json()
 136
 137     def update_host_remote_info(self, data, res_id):
 138         resource = ResourceQuery.filter(labid=res_id, lab=self.lab)
 139         if len(resource) != 1:
 140             return HttpResponseNotFound("Could not find single host with id " + str(res_id))
 141         resource = resource[0]
 142         info = {}
 143         try:
 144             info['address'] = data['address']
 145             info['mac_address'] = data['mac_address']
 146             info['password'] = data['password']
 147             info['user'] = data['user']
 148             info['type'] = data['type']
 149             info['versions'] = json.dumps(data['versions'])
 150         except Exception as e:
 151             return {"error": "invalid arguement: " + str(e)}
 152         remote_info = resource.remote_management
 153         if "default" in remote_info.mac_address:
 154             remote_info = RemoteInfo()
 155         remote_info.address = info['address']
 156         remote_info.mac_address = info['mac_address']
 157         remote_info.password = info['password']
 158         remote_info.user = info['user']
 159         remote_info.type = info['type']
 160         remote_info.versions = info['versions']
 161         remote_info.save()
 162         resource.remote_management = remote_info
 163         resource.save()
 164         booking = Booking.objects.get(resource=resource.bundle)
 165         self.update_xdf(booking)
 166         return {"status": "success"}
 167
 168     def update_xdf(self, booking):
 169         booking.pdf = PDFTemplater.makePDF(booking)
 170         booking.idf = IDFTemplater().makeIDF(booking)
 171         booking.save()
 172
 173     def get_pdf(self, booking_id):
 174         booking = get_object_or_404(Booking, pk=booking_id, lab=self.lab)
 175         return booking.pdf
 176
 177     def get_idf(self, booking_id):
 178         booking = get_object_or_404(Booking, pk=booking_id, lab=self.lab)
 179         return booking.idf
 180
 181     def get_profile(self):
 182         prof = {}
 183         prof['name'] = self.lab.name
 184         prof['contact'] = {
 185             "phone": self.lab.contact_phone,
 186             "email": self.lab.contact_email
 187         }
 188         prof['host_count'] = [{
 189             "type": profile.name,
 190             "count": len(profile.get_resources(lab=self.lab))}
 191             for profile in ResourceProfile.objects.filter(labs=self.lab)]
 192         return prof
 193
 194     def format_user(self, userprofile):
 195         return {
 196             "id": userprofile.user.id,
 197             "username": userprofile.user.username,
 198             "email": userprofile.email_addr,
 199             "first_name": userprofile.user.first_name,
 200             "last_name": userprofile.user.last_name,
 201             "company": userprofile.company
 202         }
 203
 204     def get_users(self):
 205         userlist = [self.format_user(profile) for profile in UserProfile.objects.select_related("user").all()]
 206
 207         return json.dumps({"users": userlist})
 208
 209     def get_user(self, user_id):
 210         user = User.objects.get(pk=user_id)
 211
 212         profile = get_object_or_404(UserProfile, user=user)
 213
 214         return json.dumps(self.format_user(profile))
 215
 216     def get_inventory(self):
 217         inventory = {}
 218         resources = ResourceQuery.filter(lab=self.lab)
 219         images = Image.objects.filter(from_lab=self.lab)
 220         profiles = ResourceProfile.objects.filter(labs=self.lab)
 221         inventory['resources'] = self.serialize_resources(resources)
 222         inventory['images'] = self.serialize_images(images)
 223         inventory['host_types'] = self.serialize_host_profiles(profiles)
 224         return inventory
 225
 226     def get_host(self, hostname):
 227         resource = ResourceQuery.filter(labid=hostname, lab=self.lab)
 228         if len(resource) != 1:
 229             return HttpResponseNotFound("Could not find single host with id " + str(hostname))
 230         resource = resource[0]
 231         return {
 232             "booked": resource.booked,
 233             "working": resource.working,
 234             "type": resource.profile.name
 235         }
 236
 237     def update_host(self, hostname, data):
 238         resource = ResourceQuery.filter(labid=hostname, lab=self.lab)
 239         if len(resource) != 1:
 240             return HttpResponseNotFound("Could not find single host with id " + str(hostname))
 241         resource = resource[0]
 242         if "working" in data:
 243             working = data['working'] == "true"
 244             resource.working = working
 245         resource.save()
 246         return self.get_host(hostname)
 247
 248     def get_status(self):
 249         return {"status": self.lab.status}
 250
 251     def set_status(self, payload):
 252         {}
 253
 254     def get_current_jobs(self):
 255         jobs = Job.objects.filter(booking__lab=self.lab)
 256
 257         return self.serialize_jobs(jobs, status=JobStatus.CURRENT)
 258
 259     def get_new_jobs(self):
 260         jobs = Job.objects.filter(booking__lab=self.lab)
 261
 262         return self.serialize_jobs(jobs, status=JobStatus.NEW)
 263
 264     def get_done_jobs(self):
 265         jobs = Job.objects.filter(booking__lab=self.lab)
 266
 267         return self.serialize_jobs(jobs, status=JobStatus.DONE)
 268
 269     def get_analytics_job(self):
 270         """ Get analytics job with status new """
 271         jobs = Job.objects.filter(
 272             booking__lab=self.lab,
 273             job_type='DATA'
 274         )
 275
 276         return self.serialize_jobs(jobs, status=JobStatus.NEW)
 277
 278     def get_job(self, jobid):
 279         return Job.objects.get(pk=jobid).to_dict()
 280
 281     def update_job(self, jobid, data):
 282         {}
 283
 284     def serialize_jobs(self, jobs, status=JobStatus.NEW):
 285         job_ser = []
 286         for job in jobs:
 287             jsonized_job = job.get_delta(status)
 288             if len(jsonized_job['payload']) < 1:
 289                 continue
 290             job_ser.append(jsonized_job)
 291
 292         return job_ser
 293
 294     def serialize_resources(self, resources):
 295         # TODO: rewrite for Resource model
 296         host_ser = []
 297         for res in resources:
 298             r = {
 299                 'interfaces': [],
 300                 'hostname': res.name,
 301                 'host_type': res.profile.name
 302             }
 303             for iface in res.get_interfaces():
 304                 r['interfaces'].append({
 305                     'mac': iface.mac_address,
 306                     'busaddr': iface.bus_address,
 307                     'name': iface.name,
 308                     'switchport': {"switch_name": iface.switch_name, "port_name": iface.port_name}
 309                 })
 310         return host_ser
 311
 312     def serialize_images(self, images):
 313         images_ser = []
 314         for image in images:
 315             images_ser.append(
 316                 {
 317                     "name": image.name,
 318                     "lab_id": image.lab_id,
 319                     "dashboard_id": image.id
 320                 }
 321             )
 322         return images_ser
 323
 324     def serialize_resource_profiles(self, profiles):
 325         profile_ser = []
 326         for profile in profiles:
 327             p = {}
 328             p['cpu'] = {
 329                 "cores": profile.cpuprofile.first().cores,
 330                 "arch": profile.cpuprofile.first().architecture,
 331                 "cpus": profile.cpuprofile.first().cpus,
 332             }
 333             p['disks'] = []
 334             for disk in profile.storageprofile.all():
 335                 d = {
 336                     "size": disk.size,
 337                     "type": disk.media_type,
 338                     "name": disk.name
 339                 }
 340                 p['disks'].append(d)
 341             p['description'] = profile.description
 342             p['interfaces'] = []
 343             for iface in profile.interfaceprofile.all():
 344                 p['interfaces'].append(
 345                     {
 346                         "speed": iface.speed,
 347                         "name": iface.name
 348                     }
 349                 )
 350
 351             p['ram'] = {"amount": profile.ramprofile.first().amount}
 352             p['name'] = profile.name
 353             profile_ser.append(p)
 354         return profile_ser
 355
 356
 357 class GeneratedCloudConfig(models.Model):
 358     resource_id = models.CharField(max_length=200)
 359     booking = models.ForeignKey(Booking, on_delete=models.CASCADE)
 360     rconfig = models.ForeignKey(ResourceConfiguration, on_delete=models.CASCADE)
 361     text = models.TextField(null=True, blank=True)
 362
 363     def _normalize_username(self, username: str) -> str:
 364         # TODO: make usernames posix compliant
 365         return username
 366
 367     def _get_ssh_string(self, username: str) -> str:
 368         user = User.objects.get(username=username)
 369         uprofile = user.userprofile
 370
 371         ssh_file = uprofile.ssh_public_key
 372
 373         escaped_file = ssh_file.open().read().decode(encoding="UTF-8").replace("\n", " ")
 374
 375         return escaped_file
 376
 377     def _serialize_users(self):
 378         """
 379         returns the dictionary to be placed behind the `users` field of the toplevel c-i dict
 380         """
 381         # conserves distro default user
 382         user_array = ["default"]
 383
 384         users = list(self.booking.collaborators.all())
 385         users.append(self.booking.owner)
 386         for collaborator in users:
 387             userdict = {}
 388
 389             # TODO: validate if usernames are valid as linux usernames (and provide an override potentially)
 390             userdict['name'] = self._normalize_username(collaborator.username)
 391
 392             userdict['groups'] = "sudo"
 393             userdict['sudo'] = "ALL=(ALL) NOPASSWD:ALL"
 394
 395             userdict['ssh_authorized_keys'] = [self._get_ssh_string(collaborator.username)]
 396
 397             user_array.append(userdict)
 398
 399         # user_array.append({
 400         #    "name": "opnfv",
 401         #    "passwd": "$6$k54L.vim1cLaEc4$5AyUIrufGlbtVBzuCWOlA1yV6QdD7Gr2MzwIs/WhuYR9ebSfh3Qlb7djkqzjwjxpnSAonK1YOabPP6NxUDccu.",
 402         #    "ssh_redirect_user": True,
 403         #    "sudo": "ALL=(ALL) NOPASSWD:ALL",
 404         #    "groups": "sudo",
 405         #    })
 406
 407         return user_array
 408
 409     # TODO: make this configurable
 410     def _serialize_sysinfo(self):
 411         defuser = {}
 412         defuser['name'] = 'opnfv'
 413         defuser['plain_text_passwd'] = 'OPNFV_HOST'
 414         defuser['home'] = '/home/opnfv'
 415         defuser['shell'] = '/bin/bash'
 416         defuser['lock_passwd'] = True
 417         defuser['gecos'] = 'Lab Manager User'
 418         defuser['groups'] = 'sudo'
 419
 420         return {'default_user': defuser}
 421
 422     # TODO: make this configurable
 423     def _serialize_runcmds(self):
 424         cmdlist = []
 425
 426         # have hosts run dhcp on boot
 427         cmdlist.append(['sudo', 'dhclient', '-r'])
 428         cmdlist.append(['sudo', 'dhclient'])
 429
 430         return cmdlist
 431
 432     def _serialize_netconf_v1(self):
 433         # interfaces = {}  # map from iface_name => dhcp_config
 434         # vlans = {}  # map from vlan_id => dhcp_config
 435
 436         config_arr = []
 437
 438         for interface in self._resource().interfaces.all():
 439             interface_name = interface.profile.name
 440             interface_mac = interface.mac_address
 441
 442             iface_dict_entry = {
 443                 "type": "physical",
 444                 "name": interface_name,
 445                 "mac_address": interface_mac,
 446             }
 447
 448             for vlan in interface.config.all():
 449                 if vlan.tagged:
 450                     vlan_dict_entry = {'type': 'vlan'}
 451                     vlan_dict_entry['name'] = str(interface_name) + "." + str(vlan.vlan_id)
 452                     vlan_dict_entry['vlan_link'] = str(interface_name)
 453                     vlan_dict_entry['vlan_id'] = int(vlan.vlan_id)
 454                     vlan_dict_entry['mac_address'] = str(interface_mac)
 455                     if vlan.public:
 456                         vlan_dict_entry["subnets"] = [{"type": "dhcp"}]
 457                     config_arr.append(vlan_dict_entry)
 458                 if (not vlan.tagged) and vlan.public:
 459                     iface_dict_entry["subnets"] = [{"type": "dhcp"}]
 460
 461                 # vlan_dict_entry['mtu'] = # TODO, determine override MTU if needed
 462
 463             config_arr.append(iface_dict_entry)
 464
 465         ns_dict = {
 466             'type': 'nameserver',
 467             'address': ['10.64.0.1', '8.8.8.8']
 468         }
 469
 470         config_arr.append(ns_dict)
 471
 472         full_dict = {'version': 1, 'config': config_arr}
 473
 474         return full_dict
 475
 476     @classmethod
 477     def get(cls, booking_id: int, resource_lab_id: str, file_id: int):
 478         return GeneratedCloudConfig.objects.get(resource_id=resource_lab_id, booking__id=booking_id, file_id=file_id)
 479
 480     def _resource(self):
 481         return ResourceQuery.get(labid=self.resource_id, lab=self.booking.lab)
 482
 483     # def _get_facts(self):
 484         # resource = self._resource()
 485
 486         # hostname = self.rconfig.name
 487         # iface_configs = for_config.interface_configs.all()
 488
 489     def _to_dict(self):
 490         main_dict = {}
 491
 492         main_dict['users'] = self._serialize_users()
 493         main_dict['network'] = self._serialize_netconf_v1()
 494         main_dict['hostname'] = self.rconfig.name
 495
 496         # add first startup commands
 497         main_dict['runcmd'] = self._serialize_runcmds()
 498
 499         # configure distro default user
 500         main_dict['system_info'] = self._serialize_sysinfo()
 501
 502         return main_dict
 503
 504     def serialize(self) -> str:
 505         return yaml.dump(self._to_dict(), width=float("inf"))
 506
 507
 508 class APILog(models.Model):
 509     user = models.ForeignKey(User, on_delete=models.PROTECT)
 510     call_time = models.DateTimeField(auto_now=True)
 511     method = models.CharField(null=True, max_length=6)
 512     endpoint = models.CharField(null=True, max_length=300)
 513     ip_addr = models.GenericIPAddressField(protocol="both", null=True, unpack_ipv4=False)
 514     body = JSONField(null=True)
 515
 516     def __str__(self):
 517         return "Call to {} at {} by {}".format(
 518             self.endpoint,
 519             self.call_time,
 520             self.user.username
 521         )
 522
 523
 524 class AutomationAPIManager:
 525     @staticmethod
 526     def serialize_booking(booking):
 527         sbook = {}
 528         sbook['id'] = booking.pk
 529         sbook['owner'] = booking.owner.username
 530         sbook['collaborators'] = [user.username for user in booking.collaborators.all()]
 531         sbook['start'] = booking.start
 532         sbook['end'] = booking.end
 533         sbook['lab'] = AutomationAPIManager.serialize_lab(booking.lab)
 534         sbook['purpose'] = booking.purpose
 535         sbook['resourceBundle'] = AutomationAPIManager.serialize_bundle(booking.resource)
 536         return sbook
 537
 538     @staticmethod
 539     def serialize_lab(lab):
 540         slab = {}
 541         slab['id'] = lab.pk
 542         slab['name'] = lab.name
 543         return slab
 544
 545     @staticmethod
 546     def serialize_bundle(bundle):
 547         sbundle = {}
 548         sbundle['id'] = bundle.pk
 549         sbundle['resources'] = [
 550             AutomationAPIManager.serialize_server(server)
 551             for server in bundle.get_resources()]
 552         return sbundle
 553
 554     @staticmethod
 555     def serialize_server(server):
 556         sserver = {}
 557         sserver['id'] = server.pk
 558         sserver['name'] = server.name
 559         return sserver
 560
 561     @staticmethod
 562     def serialize_resource_profile(profile):
 563         sprofile = {}
 564         sprofile['id'] = profile.pk
 565         sprofile['name'] = profile.name
 566         return sprofile
 567
 568     @staticmethod
 569     def serialize_template(rec_temp_and_count):
 570         template = rec_temp_and_count[0]
 571         count = rec_temp_and_count[1]
 572
 573         stemplate = {}
 574         stemplate['id'] = template.pk
 575         stemplate['name'] = template.name
 576         stemplate['count_available'] = count
 577         stemplate['resourceProfiles'] = [
 578             AutomationAPIManager.serialize_resource_profile(config.profile)
 579             for config in template.getConfigs()
 580         ]
 581         return stemplate
 582
 583     @staticmethod
 584     def serialize_image(image):
 585         simage = {}
 586         simage['id'] = image.pk
 587         simage['name'] = image.name
 588         return simage
 589
 590     @staticmethod
 591     def serialize_userprofile(up):
 592         sup = {}
 593         sup['id'] = up.pk
 594         sup['username'] = up.user.username
 595         return sup
 596
 597
 598 class Job(models.Model):
 599     """
 600     A Job to be performed by the Lab.
 601
 602     The API uses Jobs and Tasks to communicate actions that need to be taken to the Lab
 603     that is hosting a booking. A booking from a user has an associated Job which tells
 604     the lab how to configure the hardware, networking, etc to fulfill the booking
 605     for the user.
 606     This is the class that is serialized and put into the api
 607     """
 608
 609     JOB_TYPES = (
 610         ('BOOK', 'Booking'),
 611         ('DATA', 'Analytics')
 612     )
 613
 614     booking = models.OneToOneField(Booking, on_delete=models.CASCADE, null=True)
 615     status = models.IntegerField(default=JobStatus.NEW)
 616     complete = models.BooleanField(default=False)
 617     job_type = models.CharField(
 618         max_length=4,
 619         choices=JOB_TYPES,
 620         default='BOOK'
 621     )
 622
 623     def to_dict(self):
 624         d = {}
 625         for relation in self.get_tasklist():
 626             if relation.job_key not in d:
 627                 d[relation.job_key] = {}
 628             d[relation.job_key][relation.task_id] = relation.config.to_dict()
 629
 630         return {"id": self.id, "payload": d}
 631
 632     def get_tasklist(self, status="all"):
 633         if status != "all":
 634             return JobTaskQuery.filter(job=self, status=status)
 635         return JobTaskQuery.filter(job=self)
 636
 637     def is_fulfilled(self):
 638         """
 639         If a job has been completed by the lab.
 640
 641         This method should return true if all of the job's tasks are done,
 642         and false otherwise
 643         """
 644         my_tasks = self.get_tasklist()
 645         for task in my_tasks:
 646             if task.status != JobStatus.DONE:
 647                 return False
 648         return True
 649
 650     def get_delta(self, status):
 651         d = {}
 652         for relation in self.get_tasklist(status=status):
 653             if relation.job_key not in d:
 654                 d[relation.job_key] = {}
 655             d[relation.job_key][relation.task_id] = relation.config.get_delta()
 656
 657         return {"id": self.id, "payload": d}
 658
 659     def to_json(self):
 660         return json.dumps(self.to_dict())
 661
 662
 663 class TaskConfig(models.Model):
 664     state = models.IntegerField(default=ConfigState.NEW)
 665
 666     keys = set()  # TODO: This needs to be an instance variable, not a class variable
 667     delta_keys_list = models.CharField(max_length=200, default="[]")
 668
 669     @property
 670     def delta_keys(self):
 671         return list(set(json.loads(self.delta_keys_list)))
 672
 673     @delta_keys.setter
 674     def delta_keys(self, keylist):
 675         self.delta_keys_list = json.dumps(keylist)
 676
 677     def to_dict(self):
 678         raise NotImplementedError
 679
 680     def get_delta(self):
 681         raise NotImplementedError
 682
 683     def format_delta(self, config, token):
 684         delta = {k: config[k] for k in self.delta_keys}
 685         delta['lab_token'] = token
 686         return delta
 687
 688     def to_json(self):
 689         return json.dumps(self.to_dict())
 690
 691     def clear_delta(self):
 692         self.delta_keys = []
 693
 694     def set(self, *args):
 695         dkeys = self.delta_keys
 696         for arg in args:
 697             if arg in self.keys:
 698                 dkeys.append(arg)
 699         self.delta_keys = dkeys
 700
 701
 702 class BridgeConfig(models.Model):
 703     """Displays mapping between jumphost interfaces and bridges."""
 704
 705     interfaces = models.ManyToManyField(Interface)
 706     opnfv_config = models.ForeignKey(OPNFVConfig, on_delete=models.CASCADE)
 707
 708     def to_dict(self):
 709         d = {}
 710         hid = ResourceQuery.get(interface__pk=self.interfaces.first().pk).labid
 711         d[hid] = {}
 712         for interface in self.interfaces.all():
 713             d[hid][interface.mac_address] = []
 714             for vlan in interface.config.all():
 715                 network_role = self.opnfv_model.networks().filter(network=vlan.network)
 716                 bridge = IDFTemplater.bridge_names[network_role.name]
 717                 br_config = {
 718                     "vlan_id": vlan.vlan_id,
 719                     "tagged": vlan.tagged,
 720                     "bridge": bridge
 721                 }
 722                 d[hid][interface.mac_address].append(br_config)
 723         return d
 724
 725     def to_json(self):
 726         return json.dumps(self.to_dict())
 727
 728
 729 class ActiveUsersConfig(models.Model):
 730     """
 731     Task for getting active VPN users
 732
 733     StackStorm needs no information to run this job
 734     so this task is very bare, but neccessary to fit
 735     job creation convention.
 736     """
 737
 738     def clear_delta(self):
 739         self.delta = '{}'
 740
 741     def get_delta(self):
 742         return json.loads(self.to_json())
 743
 744     def to_json(self):
 745         return json.dumps(self.to_dict())
 746
 747     def to_dict(self):
 748         return {}
 749
 750
 751 class OpnfvApiConfig(models.Model):
 752
 753     installer = models.CharField(max_length=200)
 754     scenario = models.CharField(max_length=300)
 755     roles = models.ManyToManyField(ResourceOPNFVConfig)
 756     # pdf and idf are url endpoints, not the actual file
 757     pdf = models.CharField(max_length=100)
 758     idf = models.CharField(max_length=100)
 759     bridge_config = models.OneToOneField(BridgeConfig, on_delete=models.CASCADE, null=True)
 760     delta = models.TextField()
 761     opnfv_config = models.ForeignKey(OPNFVConfig, null=True, on_delete=models.SET_NULL)
 762
 763     def to_dict(self):
 764         d = {}
 765         if not self.opnfv_config:
 766             return d
 767         if self.installer:
 768             d['installer'] = self.installer
 769         if self.scenario:
 770             d['scenario'] = self.scenario
 771         if self.pdf:
 772             d['pdf'] = self.pdf
 773         if self.idf:
 774             d['idf'] = self.idf
 775         if self.bridge_config:
 776             d['bridged_interfaces'] = self.bridge_config.to_dict()
 777
 778         hosts = self.roles.all()
 779         if hosts.exists():
 780             d['roles'] = []
 781             for host in hosts:
 782                 d['roles'].append({
 783                     host.labid: self.opnfv_config.host_opnfv_config.get(
 784                         host_config__pk=host.config.pk
 785                     ).role.name
 786                 })
 787
 788         return d
 789
 790     def to_json(self):
 791         return json.dumps(self.to_dict())
 792
 793     def set_installer(self, installer):
 794         self.installer = installer
 795         d = json.loads(self.delta)
 796         d['installer'] = installer
 797         self.delta = json.dumps(d)
 798
 799     def set_scenario(self, scenario):
 800         self.scenario = scenario
 801         d = json.loads(self.delta)
 802         d['scenario'] = scenario
 803         self.delta = json.dumps(d)
 804
 805     def set_xdf(self, booking, update_delta=True):
 806         kwargs = {'lab_name': booking.lab.name, 'booking_id': booking.id}
 807         self.pdf = reverse('get-pdf', kwargs=kwargs)
 808         self.idf = reverse('get-idf', kwargs=kwargs)
 809         if update_delta:
 810             d = json.loads(self.delta)
 811             d['pdf'] = self.pdf
 812             d['idf'] = self.idf
 813             self.delta = json.dumps(d)
 814
 815     def add_role(self, host):
 816         self.roles.add(host)
 817         d = json.loads(self.delta)
 818         if 'role' not in d:
 819             d['role'] = []
 820         d['roles'].append({host.labid: host.config.opnfvRole.name})
 821         self.delta = json.dumps(d)
 822
 823     def clear_delta(self):
 824         self.delta = '{}'
 825
 826     def get_delta(self):
 827         return json.loads(self.to_json())
 828
 829
 830 class AccessConfig(TaskConfig):
 831     access_type = models.CharField(max_length=50)
 832     user = models.ForeignKey(User, on_delete=models.CASCADE)
 833     revoke = models.BooleanField(default=False)
 834     context = models.TextField(default="")
 835     delta = models.TextField(default="{}")
 836
 837     def to_dict(self):
 838         d = {}
 839         d['access_type'] = self.access_type
 840         d['user'] = self.user.id
 841         d['revoke'] = self.revoke
 842         try:
 843             d['context'] = json.loads(self.context)
 844         except Exception:
 845             pass
 846         return d
 847
 848     def get_delta(self):
 849         d = json.loads(self.to_json())
 850         d["lab_token"] = self.accessrelation.lab_token
 851
 852         return d
 853
 854     def to_json(self):
 855         return json.dumps(self.to_dict())
 856
 857     def clear_delta(self):
 858         d = {}
 859         d["lab_token"] = self.accessrelation.lab_token
 860         self.delta = json.dumps(d)
 861
 862     def set_access_type(self, access_type):
 863         self.access_type = access_type
 864         d = json.loads(self.delta)
 865         d['access_type'] = access_type
 866         self.delta = json.dumps(d)
 867
 868     def set_user(self, user):
 869         self.user = user
 870         d = json.loads(self.delta)
 871         d['user'] = self.user.id
 872         self.delta = json.dumps(d)
 873
 874     def set_revoke(self, revoke):
 875         self.revoke = revoke
 876         d = json.loads(self.delta)
 877         d['revoke'] = revoke
 878         self.delta = json.dumps(d)
 879
 880     def set_context(self, context):
 881         self.context = json.dumps(context)
 882         d = json.loads(self.delta)
 883         d['context'] = context
 884         self.delta = json.dumps(d)
 885
 886
 887 class SoftwareConfig(TaskConfig):
 888     """Handles software installations, such as OPNFV or ONAP."""
 889
 890     opnfv = models.ForeignKey(OpnfvApiConfig, on_delete=models.CASCADE)
 891
 892     def to_dict(self):
 893         d = {}
 894         if self.opnfv:
 895             d['opnfv'] = self.opnfv.to_dict()
 896
 897         d["lab_token"] = self.softwarerelation.lab_token
 898         self.delta = json.dumps(d)
 899
 900         return d
 901
 902     def get_delta(self):
 903         d = {}
 904         d['opnfv'] = self.opnfv.get_delta()
 905         d['lab_token'] = self.softwarerelation.lab_token
 906
 907         return d
 908
 909     def clear_delta(self):
 910         self.opnfv.clear_delta()
 911
 912     def to_json(self):
 913         return json.dumps(self.to_dict())
 914
 915
 916 class HardwareConfig(TaskConfig):
 917     """Describes the desired configuration of the hardware."""
 918
 919     image = models.CharField(max_length=100, default="defimage")
 920     power = models.CharField(max_length=100, default="off")
 921     hostname = models.CharField(max_length=100, default="hostname")
 922     ipmi_create = models.BooleanField(default=False)
 923     delta = models.TextField()
 924
 925     keys = set(["id", "image", "power", "hostname", "ipmi_create"])
 926
 927     def to_dict(self):
 928         return self.get_delta()
 929
 930     def get_delta(self):
 931         # TODO: grab the GeneratedCloudConfig urls from self.hosthardwarerelation.get_resource()
 932         return self.format_delta(
 933             self.hosthardwarerelation.get_resource().get_configuration(self.state),
 934             self.hosthardwarerelation.lab_token)
 935
 936
 937 class NetworkConfig(TaskConfig):
 938     """Handles network configuration."""
 939
 940     interfaces = models.ManyToManyField(Interface)
 941     delta = models.TextField()
 942
 943     def to_dict(self):
 944         d = {}
 945         hid = self.hostnetworkrelation.resource_id
 946         d[hid] = {}
 947         for interface in self.interfaces.all():
 948             d[hid][interface.mac_address] = []
 949             if self.state != ConfigState.CLEAN:
 950                 for vlan in interface.config.all():
 951                     # TODO: should this come from the interface?
 952                     # e.g. will different interfaces for different resources need different configs?
 953                     d[hid][interface.mac_address].append({"vlan_id": vlan.vlan_id, "tagged": vlan.tagged})
 954
 955         return d
 956
 957     def to_json(self):
 958         return json.dumps(self.to_dict())
 959
 960     def get_delta(self):
 961         d = json.loads(self.to_json())
 962         d['lab_token'] = self.hostnetworkrelation.lab_token
 963         return d
 964
 965     def clear_delta(self):
 966         self.delta = json.dumps(self.to_dict())
 967         self.save()
 968
 969     def add_interface(self, interface):
 970         self.interfaces.add(interface)
 971         d = json.loads(self.delta)
 972         hid = self.hostnetworkrelation.resource_id
 973         if hid not in d:
 974             d[hid] = {}
 975         d[hid][interface.mac_address] = []
 976         for vlan in interface.config.all():
 977             d[hid][interface.mac_address].append({"vlan_id": vlan.vlan_id, "tagged": vlan.tagged})
 978         self.delta = json.dumps(d)
 979
 980
 981 class SnapshotConfig(TaskConfig):
 982
 983     resource_id = models.CharField(max_length=200, default="default_id")
 984     image = models.CharField(max_length=200, null=True)  # cobbler ID
 985     dashboard_id = models.IntegerField()
 986     delta = models.TextField(default="{}")
 987
 988     def to_dict(self):
 989         d = {}
 990         if self.host:
 991             d['host'] = self.host.labid
 992         if self.image:
 993             d['image'] = self.image
 994         d['dashboard_id'] = self.dashboard_id
 995         return d
 996
 997     def to_json(self):
 998         return json.dumps(self.to_dict())
 999
1000     def get_delta(self):
1001         d = json.loads(self.to_json())
1002         return d
1003
1004     def clear_delta(self):
1005         self.delta = json.dumps(self.to_dict())
1006         self.save()
1007
1008     def set_host(self, host):
1009         self.host = host
1010         d = json.loads(self.delta)
1011         d['host'] = host.labid
1012         self.delta = json.dumps(d)
1013
1014     def set_image(self, image):
1015         self.image = image
1016         d = json.loads(self.delta)
1017         d['image'] = self.image
1018         self.delta = json.dumps(d)
1019
1020     def clear_image(self):
1021         self.image = None
1022         d = json.loads(self.delta)
1023         d.pop("image", None)
1024         self.delta = json.dumps(d)
1025
1026     def set_dashboard_id(self, dash):
1027         self.dashboard_id = dash
1028         d = json.loads(self.delta)
1029         d['dashboard_id'] = self.dashboard_id
1030         self.delta = json.dumps(d)
1031
1032     def save(self, *args, **kwargs):
1033         if len(ResourceQuery.filter(labid=self.resource_id)) != 1:
1034             raise ValidationError("resource_id " + str(self.resource_id) + " does not refer to a single resource")
1035         super().save(*args, **kwargs)
1036
1037
1038 def get_task(task_id):
1039     for taskclass in [AccessRelation, SoftwareRelation, HostHardwareRelation, HostNetworkRelation, SnapshotRelation]:
1040         try:
1041             ret = taskclass.objects.get(task_id=task_id)
1042             return ret
1043         except taskclass.DoesNotExist:
1044             pass
1045     from django.core.exceptions import ObjectDoesNotExist
1046     raise ObjectDoesNotExist("Could not find matching TaskRelation instance")
1047
1048
1049 def get_task_uuid():
1050     return str(uuid.uuid4())
1051
1052
1053 class TaskRelation(models.Model):
1054     """
1055     Relates a Job to a TaskConfig.
1056
1057     superclass that relates a Job to tasks anc maintains information
1058     like status and messages from the lab
1059     """
1060
1061     status = models.IntegerField(default=JobStatus.NEW)
1062     job = models.ForeignKey(Job, on_delete=models.CASCADE)
1063     config = models.OneToOneField(TaskConfig, on_delete=models.CASCADE)
1064     task_id = models.CharField(default=get_task_uuid, max_length=37)
1065     lab_token = models.CharField(default="null", max_length=50)
1066     message = models.TextField(default="")
1067
1068     job_key = None
1069
1070     def delete(self, *args, **kwargs):
1071         self.config.delete()
1072         return super(self.__class__, self).delete(*args, **kwargs)
1073
1074     def type_str(self):
1075         return "Generic Task"
1076
1077     class Meta:
1078         abstract = True
1079
1080
1081 class AccessRelation(TaskRelation):
1082     config = models.OneToOneField(AccessConfig, on_delete=models.CASCADE)
1083     job_key = "access"
1084
1085     def type_str(self):
1086         return "Access Task"
1087
1088     def delete(self, *args, **kwargs):
1089         self.config.delete()
1090         return super(self.__class__, self).delete(*args, **kwargs)
1091
1092
1093 class SoftwareRelation(TaskRelation):
1094     config = models.OneToOneField(SoftwareConfig, on_delete=models.CASCADE)
1095     job_key = "software"
1096
1097     def type_str(self):
1098         return "Software Configuration Task"
1099
1100     def delete(self, *args, **kwargs):
1101         self.config.delete()
1102         return super(self.__class__, self).delete(*args, **kwargs)
1103
1104
1105 class HostHardwareRelation(TaskRelation):
1106     resource_id = models.CharField(max_length=200, default="default_id")
1107     config = models.OneToOneField(HardwareConfig, on_delete=models.CASCADE)
1108     job_key = "hardware"
1109
1110     def type_str(self):
1111         return "Hardware Configuration Task"
1112
1113     def get_delta(self):
1114         return self.config.to_dict()
1115
1116     def delete(self, *args, **kwargs):
1117         self.config.delete()
1118         return super(self.__class__, self).delete(*args, **kwargs)
1119
1120     def save(self, *args, **kwargs):
1121         if len(ResourceQuery.filter(labid=self.resource_id)) != 1:
1122             raise ValidationError("resource_id " + str(self.resource_id) + " does not refer to a single resource")
1123         super().save(*args, **kwargs)
1124
1125     def get_resource(self):
1126         return ResourceQuery.get(labid=self.resource_id)
1127
1128
1129 class HostNetworkRelation(TaskRelation):
1130     resource_id = models.CharField(max_length=200, default="default_id")
1131     config = models.OneToOneField(NetworkConfig, on_delete=models.CASCADE)
1132     job_key = "network"
1133
1134     def type_str(self):
1135         return "Network Configuration Task"
1136
1137     def delete(self, *args, **kwargs):
1138         self.config.delete()
1139         return super(self.__class__, self).delete(*args, **kwargs)
1140
1141     def save(self, *args, **kwargs):
1142         if len(ResourceQuery.filter(labid=self.resource_id)) != 1:
1143             raise ValidationError("resource_id " + str(self.resource_id) + " does not refer to a single resource")
1144         super().save(*args, **kwargs)
1145
1146     def get_resource(self):
1147         return ResourceQuery.get(labid=self.resource_id)
1148
1149
1150 class SnapshotRelation(TaskRelation):
1151     snapshot = models.ForeignKey(Image, on_delete=models.CASCADE)
1152     config = models.OneToOneField(SnapshotConfig, on_delete=models.CASCADE)
1153     job_key = "snapshot"
1154
1155     def type_str(self):
1156         return "Snapshot Task"
1157
1158     def get_delta(self):
1159         return self.config.to_dict()
1160
1161     def delete(self, *args, **kwargs):
1162         self.config.delete()
1163         return super(self.__class__, self).delete(*args, **kwargs)
1164
1165
1166 class ActiveUsersRelation(TaskRelation):
1167     config = models.OneToOneField(ActiveUsersConfig, on_delete=models.CASCADE)
1168     job_key = "active users task"
1169
1170     def type_str(self):
1171         return "Active Users Task"
1172
1173
1174 class JobFactory(object):
1175     """This class creates all the API models (jobs, tasks, etc) needed to fulfill a booking."""
1176
1177     @classmethod
1178     def reimageHost(cls, new_image, booking, host):
1179         """Modify an existing job to reimage the given host."""
1180         job = Job.objects.get(booking=booking)
1181         # make hardware task new
1182         hardware_relation = HostHardwareRelation.objects.get(resource_id=host, job=job)
1183         hardware_relation.config.image = new_image.lab_id
1184         hardware_relation.config.save()
1185         hardware_relation.status = JobStatus.NEW
1186
1187         # re-apply networking after host is reset
1188         net_relation = HostNetworkRelation.objects.get(resource_id=host, job=job)
1189         net_relation.status = JobStatus.NEW
1190
1191         # re-apply ssh access after host is reset
1192         for relation in AccessRelation.objects.filter(job=job, config__access_type="ssh"):
1193             relation.status = JobStatus.NEW
1194             relation.save()
1195
1196         hardware_relation.save()
1197         net_relation.save()
1198
1199     @classmethod
1200     def makeSnapshotTask(cls, image, booking, host):
1201         relation = SnapshotRelation()
1202         job = Job.objects.get(booking=booking)
1203         config = SnapshotConfig.objects.create(dashboard_id=image.id)
1204
1205         relation.job = job
1206         relation.config = config
1207         relation.config.save()
1208         relation.config = relation.config
1209         relation.snapshot = image
1210         relation.save()
1211
1212         config.clear_delta()
1213         config.set_host(host)
1214         config.save()
1215
1216     @classmethod
1217     def makeActiveUsersTask(cls):
1218         """ Append active users task to analytics job """
1219         config = ActiveUsersConfig()
1220         relation = ActiveUsersRelation()
1221         job = Job.objects.get(job_type='DATA')
1222
1223         job.status = JobStatus.NEW
1224
1225         relation.job = job
1226         relation.config = config
1227         relation.config.save()
1228         relation.config = relation.config
1229         relation.save()
1230         config.save()
1231
1232     @classmethod
1233     def makeAnalyticsJob(cls, booking):
1234         """
1235         Create the analytics job
1236
1237         This will only run once since there will only be one analytics job.
1238         All analytics tasks get appended to analytics job.
1239         """
1240
1241         if len(Job.objects.filter(job_type='DATA')) > 0:
1242             raise Exception("Cannot have more than one analytics job")
1243
1244         if booking.resource:
1245             raise Exception("Booking is not marker for analytics job, has resoure")
1246
1247         job = Job()
1248         job.booking = booking
1249         job.job_type = 'DATA'
1250         job.save()
1251
1252         cls.makeActiveUsersTask()
1253
1254     @classmethod
1255     def makeCompleteJob(cls, booking):
1256         """Create everything that is needed to fulfill the given booking."""
1257         resources = booking.resource.get_resources()
1258         job = None
1259         try:
1260             job = Job.objects.get(booking=booking)
1261         except Exception:
1262             job = Job.objects.create(status=JobStatus.NEW, booking=booking)
1263         cls.makeHardwareConfigs(
1264             resources=resources,
1265             job=job
1266         )
1267         cls.makeNetworkConfigs(
1268             resources=resources,
1269             job=job
1270         )
1271         cls.makeSoftware(
1272             booking=booking,
1273             job=job
1274         )
1275         cls.makeGeneratedCloudConfigs(
1276             resources=resources,
1277             job=job
1278         )
1279         all_users = list(booking.collaborators.all())
1280         all_users.append(booking.owner)
1281         cls.makeAccessConfig(
1282             users=all_users,
1283             access_type="vpn",
1284             revoke=False,
1285             job=job
1286         )
1287         for user in all_users:
1288             try:
1289                 cls.makeAccessConfig(
1290                     users=[user],
1291                     access_type="ssh",
1292                     revoke=False,
1293                     job=job,
1294                     context={
1295                         "key": user.userprofile.ssh_public_key.open().read().decode(encoding="UTF-8"),
1296                         "hosts": [r.labid for r in resources]
1297                     }
1298                 )
1299             except Exception:
1300                 continue
1301
1302     @classmethod
1303     def makeGeneratedCloudConfigs(cls, resources=[], job=Job()):
1304         for res in resources:
1305             cif = GeneratedCloudConfig.objects.create(resource_id=res.labid, booking=job.booking, rconfig=res.config)
1306             cif.save()
1307
1308             cif = CloudInitFile.create(priority=0, text=cif.serialize())
1309             cif.save()
1310
1311             res.config.cloud_init_files.add(cif)
1312             res.config.save()
1313
1314     @classmethod
1315     def makeHardwareConfigs(cls, resources=[], job=Job()):
1316         """
1317         Create and save HardwareConfig.
1318
1319         Helper function to create the tasks related to
1320         configuring the hardware
1321         """
1322         for res in resources:
1323             hardware_config = None
1324             try:
1325                 hardware_config = HardwareConfig.objects.get(relation__resource_id=res.labid)
1326             except Exception:
1327                 hardware_config = HardwareConfig()
1328
1329             relation = HostHardwareRelation()
1330             relation.resource_id = res.labid
1331             relation.job = job
1332             relation.config = hardware_config
1333             relation.config.save()
1334             relation.config = relation.config
1335             relation.save()
1336
1337             hardware_config.set("id", "image", "hostname", "power", "ipmi_create")
1338             hardware_config.save()
1339
1340     @classmethod
1341     def makeAccessConfig(cls, users, access_type, revoke=False, job=Job(), context=False):
1342         """
1343         Create and save AccessConfig.
1344
1345         Helper function to create the tasks related to
1346         configuring the VPN, SSH, etc access for users
1347         """
1348         for user in users:
1349             relation = AccessRelation()
1350             relation.job = job
1351             config = AccessConfig()
1352             config.access_type = access_type
1353             config.user = user
1354             config.save()
1355             relation.config = config
1356             relation.save()
1357             config.clear_delta()
1358             if context:
1359                 config.set_context(context)
1360             config.set_access_type(access_type)
1361             config.set_revoke(revoke)
1362             config.set_user(user)
1363             config.save()
1364
1365     @classmethod
1366     def makeNetworkConfigs(cls, resources=[], job=Job()):
1367         """
1368         Create and save NetworkConfig.
1369
1370         Helper function to create the tasks related to
1371         configuring the networking
1372         """
1373         for res in resources:
1374             network_config = None
1375             try:
1376                 network_config = NetworkConfig.objects.get(relation__host=res)
1377             except Exception:
1378                 network_config = NetworkConfig.objects.create()
1379
1380             relation = HostNetworkRelation()
1381             relation.resource_id = res.labid
1382             relation.job = job
1383             network_config.save()
1384             relation.config = network_config
1385             relation.save()
1386             network_config.clear_delta()
1387
1388             # TODO: use get_interfaces() on resource
1389             for interface in res.interfaces.all():
1390                 network_config.add_interface(interface)
1391             network_config.save()
1392
1393     @classmethod
1394     def make_bridge_config(cls, booking):
1395         if len(booking.resource.get_resources()) < 2:
1396             return None
1397         try:
1398             jumphost_config = ResourceOPNFVConfig.objects.filter(
1399                 role__name__iexact="jumphost"
1400             )
1401             jumphost = ResourceQuery.filter(
1402                 bundle=booking.resource,
1403                 config=jumphost_config.resource_config
1404             )[0]
1405         except Exception:
1406             return None
1407         br_config = BridgeConfig.objects.create(opnfv_config=booking.opnfv_config)
1408         for iface in jumphost.interfaces.all():
1409             br_config.interfaces.add(iface)
1410         return br_config
1411
1412     @classmethod
1413     def makeSoftware(cls, booking=None, job=Job()):
1414         """
1415         Create and save SoftwareConfig.
1416
1417         Helper function to create the tasks related to
1418         configuring the desired software, e.g. an OPNFV deployment
1419         """
1420         if not booking.opnfv_config:
1421             return None
1422
1423         opnfv_api_config = OpnfvApiConfig.objects.create(
1424             opnfv_config=booking.opnfv_config,
1425             installer=booking.opnfv_config.installer.name,
1426             scenario=booking.opnfv_config.scenario.name,
1427             bridge_config=cls.make_bridge_config(booking)
1428         )
1429
1430         opnfv_api_config.set_xdf(booking, False)
1431         opnfv_api_config.save()
1432
1433         for host in booking.resource.get_resources():
1434             opnfv_api_config.roles.add(host)
1435         software_config = SoftwareConfig.objects.create(opnfv=opnfv_api_config)
1436         software_relation = SoftwareRelation.objects.create(job=job, config=software_config)
1437         return software_relation
1438
1439
1440 JOB_TASK_CLASSLIST = [
1441     HostHardwareRelation,
1442     AccessRelation,
1443     HostNetworkRelation,
1444     SoftwareRelation,
1445     SnapshotRelation,
1446     ActiveUsersRelation
1447 ]
1448
1449
1450 class JobTaskQuery(AbstractModelQuery):
1451     model_list = JOB_TASK_CLASSLIST