1 ##############################################################################
2 # Copyright (c) 2018 Sawyer Bergeron, Parker Berberian, and others.
4 # All rights reserved. This program and the accompanying materials
5 # are made available under the terms of the Apache License, Version 2.0
6 # which accompanies this distribution, and is available at
7 # http://www.apache.org/licenses/LICENSE-2.0
8 ##############################################################################
11 from django.contrib.auth.models import User
12 from django.db import models
13 from django.core.exceptions import PermissionDenied
14 from django.shortcuts import get_object_or_404
15 from django.urls import reverse
16 from django.utils import timezone
21 from booking.models import Booking
22 from resource_inventory.models import (
33 from resource_inventory.idf_templater import IDFTemplater
34 from resource_inventory.pdf_templater import PDFTemplater
35 from account.models import Downtime
38 class JobStatus(object):
40 A poor man's enum for a job's status.
42 A job is NEW if it has not been started or recognized by the Lab
43 A job is CURRENT if it has been started by the lab but it is not yet completed
44 a job is DONE if all the tasks are complete and the booking is ready to use
53 class LabManagerTracker(object):
56 def get(cls, lab_name, token):
60 Takes in a lab name (from a url path)
61 returns a lab manager instance for that lab, if it exists
62 Also checks that the given API token is correct
65 lab = Lab.objects.get(name=lab_name)
67 raise PermissionDenied("Lab not found")
68 if lab.api_token == token:
69 return LabManager(lab)
70 raise PermissionDenied("Lab not authorized")
73 class LabManager(object):
75 Handles all lab REST calls.
77 handles jobs, inventory, status, etc
78 may need to create helper classes
81 def __init__(self, lab):
84 def get_downtime(self):
85 return Downtime.objects.filter(start__lt=timezone.now(), end__gt=timezone.now(), lab=self.lab)
87 def get_downtime_json(self):
88 downtime = self.get_downtime().first() # should only be one item in queryset
92 "start": downtime.start,
94 "description": downtime.description
96 return {"is_down": False}
98 def create_downtime(self, form):
100 Create a downtime event.
102 Takes in a dictionary that describes the model.
104 "start": utc timestamp
106 "description": human text (optional)
108 For timestamp structure, https://docs.djangoproject.com/en/2.2/ref/forms/fields/#datetimefield
110 Downtime.objects.create(
111 start=form.cleaned_data['start'],
112 end=form.cleaned_data['end'],
113 description=form.cleaned_data['description'],
116 return self.get_downtime_json()
118 def update_host_remote_info(self, data, host_id):
119 host = get_object_or_404(Host, labid=host_id, lab=self.lab)
122 info['address'] = data['address']
123 info['mac_address'] = data['mac_address']
124 info['password'] = data['password']
125 info['user'] = data['user']
126 info['type'] = data['type']
127 info['versions'] = json.dumps(data['versions'])
128 except Exception as e:
129 return {"error": "invalid arguement: " + str(e)}
130 remote_info = host.remote_management
131 if "default" in remote_info.mac_address:
132 remote_info = RemoteInfo()
133 remote_info.address = info['address']
134 remote_info.mac_address = info['mac_address']
135 remote_info.password = info['password']
136 remote_info.user = info['user']
137 remote_info.type = info['type']
138 remote_info.versions = info['versions']
140 host.remote_management = remote_info
142 booking = Booking.objects.get(resource=host.bundle)
143 self.update_xdf(booking)
144 return {"status": "success"}
146 def update_xdf(self, booking):
147 booking.pdf = PDFTemplater.makePDF(booking)
148 booking.idf = IDFTemplater().makeIDF(booking)
151 def get_pdf(self, booking_id):
152 booking = get_object_or_404(Booking, pk=booking_id, lab=self.lab)
155 def get_idf(self, booking_id):
156 booking = get_object_or_404(Booking, pk=booking_id, lab=self.lab)
159 def get_profile(self):
161 prof['name'] = self.lab.name
163 "phone": self.lab.contact_phone,
164 "email": self.lab.contact_email
166 prof['host_count'] = []
167 for host in HostProfile.objects.filter(labs=self.lab):
168 count = Host.objects.filter(profile=host, lab=self.lab).count()
169 prof['host_count'].append(
177 def get_inventory(self):
179 hosts = Host.objects.filter(lab=self.lab)
180 images = Image.objects.filter(from_lab=self.lab)
181 profiles = HostProfile.objects.filter(labs=self.lab)
182 inventory['hosts'] = self.serialize_hosts(hosts)
183 inventory['images'] = self.serialize_images(images)
184 inventory['host_types'] = self.serialize_host_profiles(profiles)
187 def get_host(self, hostname):
188 host = get_object_or_404(Host, labid=hostname, lab=self.lab)
190 "booked": host.booked,
191 "working": host.working,
192 "type": host.profile.name
195 def update_host(self, hostname, data):
196 host = get_object_or_404(Host, labid=hostname, lab=self.lab)
197 if "working" in data:
198 working = data['working'] == "true"
199 host.working = working
201 return self.get_host(hostname)
203 def get_status(self):
204 return {"status": self.lab.status}
206 def set_status(self, payload):
209 def get_current_jobs(self):
210 jobs = Job.objects.filter(booking__lab=self.lab)
212 return self.serialize_jobs(jobs, status=JobStatus.CURRENT)
214 def get_new_jobs(self):
215 jobs = Job.objects.filter(booking__lab=self.lab)
217 return self.serialize_jobs(jobs, status=JobStatus.NEW)
219 def get_done_jobs(self):
220 jobs = Job.objects.filter(booking__lab=self.lab)
222 return self.serialize_jobs(jobs, status=JobStatus.DONE)
224 def get_job(self, jobid):
225 return Job.objects.get(pk=jobid).to_dict()
227 def update_job(self, jobid, data):
230 def serialize_jobs(self, jobs, status=JobStatus.NEW):
233 jsonized_job = job.get_delta(status)
234 if len(jsonized_job['payload']) < 1:
236 job_ser.append(jsonized_job)
240 def serialize_hosts(self, hosts):
245 h['hostname'] = host.name
246 h['host_type'] = host.profile.name
247 for iface in host.interfaces.all():
249 eth['mac'] = iface.mac_address
250 eth['busaddr'] = iface.bus_address
251 eth['name'] = iface.name
252 eth['switchport'] = {"switch_name": iface.switch_name, "port_name": iface.port_name}
253 h['interfaces'].append(eth)
256 def serialize_images(self, images):
262 "lab_id": image.lab_id,
263 "dashboard_id": image.id
268 def serialize_host_profiles(self, profiles):
270 for profile in profiles:
273 "cores": profile.cpuprofile.first().cores,
274 "arch": profile.cpuprofile.first().architecture,
275 "cpus": profile.cpuprofile.first().cpus,
278 for disk in profile.storageprofile.all():
281 "type": disk.media_type,
285 p['description'] = profile.description
287 for iface in profile.interfaceprofile.all():
288 p['interfaces'].append(
290 "speed": iface.speed,
295 p['ram'] = {"amount": profile.ramprofile.first().amount}
296 p['name'] = profile.name
297 profile_ser.append(p)
301 class Job(models.Model):
303 A Job to be performed by the Lab.
305 The API uses Jobs and Tasks to communicate actions that need to be taken to the Lab
306 that is hosting a booking. A booking from a user has an associated Job which tells
307 the lab how to configure the hardware, networking, etc to fulfill the booking
309 This is the class that is serialized and put into the api
312 booking = models.OneToOneField(Booking, on_delete=models.CASCADE, null=True)
313 status = models.IntegerField(default=JobStatus.NEW)
314 complete = models.BooleanField(default=False)
318 for relation in self.get_tasklist():
319 if relation.job_key not in d:
320 d[relation.job_key] = {}
321 d[relation.job_key][relation.task_id] = relation.config.to_dict()
323 return {"id": self.id, "payload": d}
325 def get_tasklist(self, status="all"):
328 HostHardwareRelation,
336 tasklist += list(cls.objects.filter(job=self))
339 tasklist += list(cls.objects.filter(job=self).filter(status=status))
342 def is_fulfilled(self):
344 If a job has been completed by the lab.
346 This method should return true if all of the job's tasks are done,
349 my_tasks = self.get_tasklist()
350 for task in my_tasks:
351 if task.status != JobStatus.DONE:
355 def get_delta(self, status):
357 for relation in self.get_tasklist(status=status):
358 if relation.job_key not in d:
359 d[relation.job_key] = {}
360 d[relation.job_key][relation.task_id] = relation.config.get_delta()
362 return {"id": self.id, "payload": d}
365 return json.dumps(self.to_dict())
368 class TaskConfig(models.Model):
369 state = models.IntegerField(default=ConfigState.CLEAN)
371 keys = set() # TODO: This needs to be an instance variable, not a class variable
372 delta_keys_list = models.CharField(max_length=200, default="[]")
375 def delta_keys(self):
376 return list(set(json.loads(self.delta_keys_list)))
379 def delta_keys(self, keylist):
380 self.delta_keys_list = json.dumps(keylist)
383 raise NotImplementedError
386 raise NotImplementedError
388 def format_delta(self, config, token):
389 delta = {k: config[k] for k in self.delta_keys}
390 delta['lab_token'] = token
394 return json.dumps(self.to_dict())
396 def clear_delta(self):
399 def set(self, *args):
400 dkeys = self.delta_keys
404 self.delta_keys = dkeys
407 class BridgeConfig(models.Model):
408 """Displays mapping between jumphost interfaces and bridges."""
410 interfaces = models.ManyToManyField(Interface)
411 opnfv_config = models.ForeignKey(OPNFVConfig, on_delete=models.CASCADE)
415 hid = self.interfaces.first().host.labid
417 for interface in self.interfaces.all():
418 d[hid][interface.mac_address] = []
419 for vlan in interface.config.all():
420 network_role = self.opnfv_model.networks().filter(network=vlan.network)
421 bridge = IDFTemplater.bridge_names[network_role.name]
423 "vlan_id": vlan.vlan_id,
424 "tagged": vlan.tagged,
427 d[hid][interface.mac_address].append(br_config)
431 return json.dumps(self.to_dict())
434 class OpnfvApiConfig(models.Model):
436 installer = models.CharField(max_length=200)
437 scenario = models.CharField(max_length=300)
438 roles = models.ManyToManyField(Host)
439 # pdf and idf are url endpoints, not the actual file
440 pdf = models.CharField(max_length=100)
441 idf = models.CharField(max_length=100)
442 bridge_config = models.OneToOneField(BridgeConfig, on_delete=models.CASCADE, null=True)
443 delta = models.TextField()
444 opnfv_config = models.ForeignKey(OPNFVConfig, null=True, on_delete=models.SET_NULL)
448 if not self.opnfv_config:
451 d['installer'] = self.installer
453 d['scenario'] = self.scenario
458 if self.bridge_config:
459 d['bridged_interfaces'] = self.bridge_config.to_dict()
461 hosts = self.roles.all()
466 host.labid: self.opnfv_config.host_opnfv_config.get(
467 host_config__pk=host.config.pk
474 return json.dumps(self.to_dict())
476 def set_installer(self, installer):
477 self.installer = installer
478 d = json.loads(self.delta)
479 d['installer'] = installer
480 self.delta = json.dumps(d)
482 def set_scenario(self, scenario):
483 self.scenario = scenario
484 d = json.loads(self.delta)
485 d['scenario'] = scenario
486 self.delta = json.dumps(d)
488 def set_xdf(self, booking, update_delta=True):
489 kwargs = {'lab_name': booking.lab.name, 'booking_id': booking.id}
490 self.pdf = reverse('get-pdf', kwargs=kwargs)
491 self.idf = reverse('get-idf', kwargs=kwargs)
493 d = json.loads(self.delta)
496 self.delta = json.dumps(d)
498 def add_role(self, host):
500 d = json.loads(self.delta)
503 d['roles'].append({host.labid: host.config.opnfvRole.name})
504 self.delta = json.dumps(d)
506 def clear_delta(self):
511 self.delta = self.to_json()
513 return json.loads(self.delta)
516 class AccessConfig(TaskConfig):
517 access_type = models.CharField(max_length=50)
518 user = models.ForeignKey(User, on_delete=models.CASCADE)
519 revoke = models.BooleanField(default=False)
520 context = models.TextField(default="")
521 delta = models.TextField(default="{}")
525 d['access_type'] = self.access_type
526 d['user'] = self.user.id
527 d['revoke'] = self.revoke
529 d['context'] = json.loads(self.context)
536 self.delta = self.to_json()
538 d = json.loads(self.delta)
539 d["lab_token"] = self.accessrelation.lab_token
544 return json.dumps(self.to_dict())
546 def clear_delta(self):
548 d["lab_token"] = self.accessrelation.lab_token
549 self.delta = json.dumps(d)
551 def set_access_type(self, access_type):
552 self.access_type = access_type
553 d = json.loads(self.delta)
554 d['access_type'] = access_type
555 self.delta = json.dumps(d)
557 def set_user(self, user):
559 d = json.loads(self.delta)
560 d['user'] = self.user.id
561 self.delta = json.dumps(d)
563 def set_revoke(self, revoke):
565 d = json.loads(self.delta)
567 self.delta = json.dumps(d)
569 def set_context(self, context):
570 self.context = json.dumps(context)
571 d = json.loads(self.delta)
572 d['context'] = context
573 self.delta = json.dumps(d)
576 class SoftwareConfig(TaskConfig):
577 """Handles software installations, such as OPNFV or ONAP."""
579 opnfv = models.ForeignKey(OpnfvApiConfig, on_delete=models.CASCADE)
584 d['opnfv'] = self.opnfv.to_dict()
586 d["lab_token"] = self.softwarerelation.lab_token
587 self.delta = json.dumps(d)
593 d['opnfv'] = self.opnfv.get_delta()
594 d['lab_token'] = self.softwarerelation.lab_token
598 def clear_delta(self):
599 self.opnfv.clear_delta()
602 return json.dumps(self.to_dict())
605 class HardwareConfig(TaskConfig):
606 """Describes the desired configuration of the hardware."""
608 image = models.CharField(max_length=100, default="defimage")
609 power = models.CharField(max_length=100, default="off")
610 hostname = models.CharField(max_length=100, default="hostname")
611 ipmi_create = models.BooleanField(default=False)
612 delta = models.TextField()
614 keys = set(["id", "image", "power", "hostname", "ipmi_create"])
617 return self.format_delta(
618 self.hosthardwarerelation.host.get_configuration(self.state),
619 self.hosthardwarerelation.lab_token)
622 class NetworkConfig(TaskConfig):
623 """Handles network configuration."""
625 interfaces = models.ManyToManyField(Interface)
626 delta = models.TextField()
630 hid = self.hostnetworkrelation.host.labid
632 for interface in self.interfaces.all():
633 d[hid][interface.mac_address] = []
634 for vlan in interface.config.all():
635 d[hid][interface.mac_address].append({"vlan_id": vlan.vlan_id, "tagged": vlan.tagged})
640 return json.dumps(self.to_dict())
644 self.delta = self.to_json()
646 d = json.loads(self.delta)
647 d['lab_token'] = self.hostnetworkrelation.lab_token
650 def clear_delta(self):
651 self.delta = json.dumps(self.to_dict())
654 def add_interface(self, interface):
655 self.interfaces.add(interface)
656 d = json.loads(self.delta)
657 hid = self.hostnetworkrelation.host.labid
660 d[hid][interface.mac_address] = []
661 for vlan in interface.config.all():
662 d[hid][interface.mac_address].append({"vlan_id": vlan.vlan_id, "tagged": vlan.tagged})
663 self.delta = json.dumps(d)
666 class SnapshotConfig(TaskConfig):
668 host = models.ForeignKey(Host, null=True, on_delete=models.DO_NOTHING)
669 image = models.IntegerField(null=True)
670 dashboard_id = models.IntegerField()
671 delta = models.TextField(default="{}")
676 d['host'] = self.host.labid
678 d['image'] = self.image
679 d['dashboard_id'] = self.dashboard_id
683 return json.dumps(self.to_dict())
687 self.delta = self.to_json()
690 d = json.loads(self.delta)
693 def clear_delta(self):
694 self.delta = json.dumps(self.to_dict())
697 def set_host(self, host):
699 d = json.loads(self.delta)
700 d['host'] = host.labid
701 self.delta = json.dumps(d)
703 def set_image(self, image):
705 d = json.loads(self.delta)
706 d['image'] = self.image
707 self.delta = json.dumps(d)
709 def clear_image(self):
711 d = json.loads(self.delta)
713 self.delta = json.dumps(d)
715 def set_dashboard_id(self, dash):
716 self.dashboard_id = dash
717 d = json.loads(self.delta)
718 d['dashboard_id'] = self.dashboard_id
719 self.delta = json.dumps(d)
722 def get_task(task_id):
723 for taskclass in [AccessRelation, SoftwareRelation, HostHardwareRelation, HostNetworkRelation, SnapshotRelation]:
725 ret = taskclass.objects.get(task_id=task_id)
727 except taskclass.DoesNotExist:
729 from django.core.exceptions import ObjectDoesNotExist
730 raise ObjectDoesNotExist("Could not find matching TaskRelation instance")
734 return str(uuid.uuid4())
737 class TaskRelation(models.Model):
739 Relates a Job to a TaskConfig.
741 superclass that relates a Job to tasks anc maintains information
742 like status and messages from the lab
745 status = models.IntegerField(default=JobStatus.NEW)
746 job = models.ForeignKey(Job, on_delete=models.CASCADE)
747 config = models.OneToOneField(TaskConfig, on_delete=models.CASCADE)
748 task_id = models.CharField(default=get_task_uuid, max_length=37)
749 lab_token = models.CharField(default="null", max_length=50)
750 message = models.TextField(default="")
754 def delete(self, *args, **kwargs):
756 return super(self.__class__, self).delete(*args, **kwargs)
759 return "Generic Task"
765 class AccessRelation(TaskRelation):
766 config = models.OneToOneField(AccessConfig, on_delete=models.CASCADE)
772 def delete(self, *args, **kwargs):
774 return super(self.__class__, self).delete(*args, **kwargs)
777 class SoftwareRelation(TaskRelation):
778 config = models.OneToOneField(SoftwareConfig, on_delete=models.CASCADE)
782 return "Software Configuration Task"
784 def delete(self, *args, **kwargs):
786 return super(self.__class__, self).delete(*args, **kwargs)
789 class HostHardwareRelation(TaskRelation):
790 host = models.ForeignKey(Host, on_delete=models.CASCADE)
791 config = models.OneToOneField(HardwareConfig, on_delete=models.CASCADE)
795 return "Hardware Configuration Task"
798 return self.config.to_dict()
800 def delete(self, *args, **kwargs):
802 return super(self.__class__, self).delete(*args, **kwargs)
805 class HostNetworkRelation(TaskRelation):
806 host = models.ForeignKey(Host, on_delete=models.CASCADE)
807 config = models.OneToOneField(NetworkConfig, on_delete=models.CASCADE)
811 return "Network Configuration Task"
813 def delete(self, *args, **kwargs):
815 return super(self.__class__, self).delete(*args, **kwargs)
818 class SnapshotRelation(TaskRelation):
819 snapshot = models.ForeignKey(Image, on_delete=models.CASCADE)
820 config = models.OneToOneField(SnapshotConfig, on_delete=models.CASCADE)
824 return "Snapshot Task"
827 return self.config.to_dict()
829 def delete(self, *args, **kwargs):
831 return super(self.__class__, self).delete(*args, **kwargs)
834 class JobFactory(object):
835 """This class creates all the API models (jobs, tasks, etc) needed to fulfill a booking."""
838 def reimageHost(cls, new_image, booking, host):
839 """Modify an existing job to reimage the given host."""
840 job = Job.objects.get(booking=booking)
841 # make hardware task new
842 hardware_relation = HostHardwareRelation.objects.get(host=host, job=job)
843 hardware_relation.config.set_image(new_image.lab_id)
844 hardware_relation.config.save()
845 hardware_relation.status = JobStatus.NEW
847 # re-apply networking after host is reset
848 net_relation = HostNetworkRelation.objects.get(host=host, job=job)
849 net_relation.status = JobStatus.NEW
851 # re-apply ssh access after host is reset
852 for relation in AccessRelation.objects.filter(job=job, config__access_type="ssh"):
853 relation.status = JobStatus.NEW
856 hardware_relation.save()
860 def makeSnapshotTask(cls, image, booking, host):
861 relation = SnapshotRelation()
862 job = Job.objects.get(booking=booking)
863 config = SnapshotConfig.objects.create(dashboard_id=image.id)
866 relation.config = config
867 relation.config.save()
868 relation.config = relation.config
869 relation.snapshot = image
873 config.set_host(host)
877 def makeCompleteJob(cls, booking):
878 """Create everything that is needed to fulfill the given booking."""
879 hosts = Host.objects.filter(bundle=booking.resource)
882 job = Job.objects.get(booking=booking)
884 job = Job.objects.create(status=JobStatus.NEW, booking=booking)
885 cls.makeHardwareConfigs(
889 cls.makeNetworkConfigs(
897 all_users = list(booking.collaborators.all())
898 all_users.append(booking.owner)
899 cls.makeAccessConfig(
905 for user in all_users:
907 cls.makeAccessConfig(
913 "key": user.userprofile.ssh_public_key.open().read().decode(encoding="UTF-8"),
914 "hosts": [host.labid for host in hosts]
921 def makeHardwareConfigs(cls, hosts=[], job=Job()):
923 Create and save HardwareConfig.
925 Helper function to create the tasks related to
926 configuring the hardware
929 hardware_config = None
931 hardware_config = HardwareConfig.objects.get(relation__host=host)
933 hardware_config = HardwareConfig()
935 relation = HostHardwareRelation()
938 relation.config = hardware_config
939 relation.config.save()
940 relation.config = relation.config
943 hardware_config.set("image", "hostname", "power", "ipmi_create")
944 hardware_config.save()
947 def makeAccessConfig(cls, users, access_type, revoke=False, job=Job(), context=False):
949 Create and save AccessConfig.
951 Helper function to create the tasks related to
952 configuring the VPN, SSH, etc access for users
955 relation = AccessRelation()
957 config = AccessConfig()
958 config.access_type = access_type
961 relation.config = config
965 config.set_context(context)
966 config.set_access_type(access_type)
967 config.set_revoke(revoke)
968 config.set_user(user)
972 def makeNetworkConfigs(cls, hosts=[], job=Job()):
974 Create and save NetworkConfig.
976 Helper function to create the tasks related to
977 configuring the networking
980 network_config = None
982 network_config = NetworkConfig.objects.get(relation__host=host)
984 network_config = NetworkConfig.objects.create()
986 relation = HostNetworkRelation()
989 network_config.save()
990 relation.config = network_config
992 network_config.clear_delta()
994 for interface in host.interfaces.all():
995 network_config.add_interface(interface)
996 network_config.save()
999 def make_bridge_config(cls, booking):
1000 if booking.resource.hosts.count() < 2:
1003 jumphost_config = HostOPNFVConfig.objects.filter(
1004 role__name__iexact="jumphost"
1006 jumphost = Host.objects.get(
1007 bundle=booking.resource,
1008 config=jumphost_config.host_config
1012 br_config = BridgeConfig.objects.create(opnfv_config=booking.opnfv_config)
1013 for iface in jumphost.interfaces.all():
1014 br_config.interfaces.add(iface)
1018 def makeSoftware(cls, booking=None, job=Job()):
1020 Create and save SoftwareConfig.
1022 Helper function to create the tasks related to
1023 configuring the desired software, e.g. an OPNFV deployment
1025 if not booking.opnfv_config:
1028 opnfv_api_config = OpnfvApiConfig.objects.create(
1029 opnfv_config=booking.opnfv_config,
1030 installer=booking.opnfv_config.installer.name,
1031 scenario=booking.opnfv_config.scenario.name,
1032 bridge_config=cls.make_bridge_config(booking)
1035 opnfv_api_config.set_xdf(booking, False)
1036 opnfv_api_config.save()
1038 for host in booking.resource.hosts.all():
1039 opnfv_api_config.roles.add(host)
1040 software_config = SoftwareConfig.objects.create(opnfv=opnfv_api_config)
1041 software_relation = SoftwareRelation.objects.create(job=job, config=software_config)
1042 return software_relation