from django.db import models
from django.core.exceptions import PermissionDenied, ValidationError
from django.shortcuts import get_object_or_404
+from django.contrib.postgres.fields import JSONField
from django.http import HttpResponseNotFound
from django.urls import reverse
from django.utils import timezone
import json
import uuid
+import yaml
+import re
from booking.models import Booking
from resource_inventory.models import (
Lab,
ResourceProfile,
Image,
+ Opsys,
Interface,
ResourceOPNFVConfig,
RemoteInfo,
OPNFVConfig,
ConfigState,
- ResourceQuery
+ ResourceQuery,
+ ResourceConfiguration,
+ CloudInitFile
)
from resource_inventory.idf_templater import IDFTemplater
from resource_inventory.pdf_templater import PDFTemplater
-from account.models import Downtime
+from account.models import Downtime, UserProfile
from dashboard.utils import AbstractModelQuery
-class JobStatus(object):
+class JobStatus:
"""
A poor man's enum for a job's status.
ERROR = 300
-class LabManagerTracker(object):
+class LabManagerTracker:
@classmethod
def get(cls, lab_name, token):
raise PermissionDenied("Lab not authorized")
-class LabManager(object):
+class LabManager:
"""
Handles all lab REST calls.
def __init__(self, lab):
self.lab = lab
+ def get_opsyss(self):
+ return Opsys.objects.filter(from_lab=self.lab)
+
+ def get_images(self):
+ return Image.objects.filter(from_lab=self.lab)
+
+ def get_image(self, image_id):
+ return Image.objects.filter(from_lab=self.lab, lab_id=image_id)
+
+ def get_opsys(self, opsys_id):
+ return Opsys.objects.filter(from_lab=self.lab, lab_id=opsys_id)
+
def get_downtime(self):
return Downtime.objects.filter(start__lt=timezone.now(), end__gt=timezone.now(), lab=self.lab)
for profile in ResourceProfile.objects.filter(labs=self.lab)]
return prof
+ def format_user(self, userprofile):
+ return {
+ "id": userprofile.user.id,
+ "username": userprofile.user.username,
+ "email": userprofile.email_addr,
+ "first_name": userprofile.user.first_name,
+ "last_name": userprofile.user.last_name,
+ "company": userprofile.company
+ }
+
+ def get_users(self):
+ userlist = [self.format_user(profile) for profile in UserProfile.objects.select_related("user").all()]
+
+ return json.dumps({"users": userlist})
+
+ def get_user(self, user_id):
+ user = User.objects.get(pk=user_id)
+
+ profile = get_object_or_404(UserProfile, user=user)
+
+ return json.dumps(self.format_user(profile))
+
def get_inventory(self):
inventory = {}
resources = ResourceQuery.filter(lab=self.lab)
return self.serialize_jobs(jobs, status=JobStatus.DONE)
+ def get_analytics_job(self):
+ """ Get analytics job with status new """
+ jobs = Job.objects.filter(
+ booking__lab=self.lab,
+ job_type='DATA'
+ )
+
+ return self.serialize_jobs(jobs, status=JobStatus.NEW)
+
def get_job(self, jobid):
return Job.objects.get(pk=jobid).to_dict()
return profile_ser
+class GeneratedCloudConfig(models.Model):
+ resource_id = models.CharField(max_length=200)
+ booking = models.ForeignKey(Booking, on_delete=models.CASCADE)
+ rconfig = models.ForeignKey(ResourceConfiguration, on_delete=models.CASCADE)
+ text = models.TextField(null=True, blank=True)
+
+ def _normalize_username(self, username: str) -> str:
+ # TODO: make usernames posix compliant
+ s = re.sub(r'\W+', '', username)
+ return s
+
+ def _get_ssh_string(self, username: str) -> str:
+ user = User.objects.get(username=username)
+ uprofile = user.userprofile
+
+ ssh_file = uprofile.ssh_public_key
+
+ escaped_file = ssh_file.open().read().decode(encoding="UTF-8").replace("\n", " ")
+
+ return escaped_file
+
+ def _serialize_users(self):
+ """
+ returns the dictionary to be placed behind the `users` field of the toplevel c-i dict
+ """
+ # conserves distro default user
+ user_array = ["default"]
+
+ users = list(self.booking.collaborators.all())
+ users.append(self.booking.owner)
+ for collaborator in users:
+ userdict = {}
+
+ # TODO: validate if usernames are valid as linux usernames (and provide an override potentially)
+ userdict['name'] = self._normalize_username(collaborator.username)
+
+ userdict['groups'] = "sudo"
+ userdict['sudo'] = "ALL=(ALL) NOPASSWD:ALL"
+
+ userdict['ssh_authorized_keys'] = [self._get_ssh_string(collaborator.username)]
+
+ user_array.append(userdict)
+
+ # user_array.append({
+ # "name": "opnfv",
+ # "passwd": "$6$k54L.vim1cLaEc4$5AyUIrufGlbtVBzuCWOlA1yV6QdD7Gr2MzwIs/WhuYR9ebSfh3Qlb7djkqzjwjxpnSAonK1YOabPP6NxUDccu.",
+ # "ssh_redirect_user": True,
+ # "sudo": "ALL=(ALL) NOPASSWD:ALL",
+ # "groups": "sudo",
+ # })
+
+ return user_array
+
+ # TODO: make this configurable
+ def _serialize_sysinfo(self):
+ defuser = {}
+ defuser['name'] = 'opnfv'
+ defuser['plain_text_passwd'] = 'OPNFV_HOST'
+ defuser['home'] = '/home/opnfv'
+ defuser['shell'] = '/bin/bash'
+ defuser['lock_passwd'] = True
+ defuser['gecos'] = 'Lab Manager User'
+ defuser['groups'] = 'sudo'
+
+ return {'default_user': defuser}
+
+ # TODO: make this configurable
+ def _serialize_runcmds(self):
+ cmdlist = []
+
+ # have hosts run dhcp on boot
+ cmdlist.append(['sudo', 'dhclient', '-r'])
+ cmdlist.append(['sudo', 'dhclient'])
+
+ return cmdlist
+
+ def _serialize_netconf_v1(self):
+ # interfaces = {} # map from iface_name => dhcp_config
+ # vlans = {} # map from vlan_id => dhcp_config
+
+ config_arr = []
+
+ for interface in self._resource().interfaces.all():
+ interface_name = interface.profile.name
+ interface_mac = interface.mac_address
+
+ iface_dict_entry = {
+ "type": "physical",
+ "name": interface_name,
+ "mac_address": interface_mac,
+ }
+
+ for vlan in interface.config.all():
+ if vlan.tagged:
+ vlan_dict_entry = {'type': 'vlan'}
+ vlan_dict_entry['name'] = str(interface_name) + "." + str(vlan.vlan_id)
+ vlan_dict_entry['vlan_link'] = str(interface_name)
+ vlan_dict_entry['vlan_id'] = int(vlan.vlan_id)
+ vlan_dict_entry['mac_address'] = str(interface_mac)
+ if vlan.public:
+ vlan_dict_entry["subnets"] = [{"type": "dhcp"}]
+ config_arr.append(vlan_dict_entry)
+ if (not vlan.tagged) and vlan.public:
+ iface_dict_entry["subnets"] = [{"type": "dhcp"}]
+
+ # vlan_dict_entry['mtu'] = # TODO, determine override MTU if needed
+
+ config_arr.append(iface_dict_entry)
+
+ ns_dict = {
+ 'type': 'nameserver',
+ 'address': ['10.64.0.1', '8.8.8.8']
+ }
+
+ config_arr.append(ns_dict)
+
+ full_dict = {'version': 1, 'config': config_arr}
+
+ return full_dict
+
+ @classmethod
+ def get(cls, booking_id: int, resource_lab_id: str, file_id: int):
+ return GeneratedCloudConfig.objects.get(resource_id=resource_lab_id, booking__id=booking_id, file_id=file_id)
+
+ def _resource(self):
+ return ResourceQuery.get(labid=self.resource_id, lab=self.booking.lab)
+
+ # def _get_facts(self):
+ # resource = self._resource()
+
+ # hostname = self.rconfig.name
+ # iface_configs = for_config.interface_configs.all()
+
+ def _to_dict(self):
+ main_dict = {}
+
+ main_dict['users'] = self._serialize_users()
+ main_dict['network'] = self._serialize_netconf_v1()
+ main_dict['hostname'] = self.rconfig.name
+
+ # add first startup commands
+ main_dict['runcmd'] = self._serialize_runcmds()
+
+ # configure distro default user
+ main_dict['system_info'] = self._serialize_sysinfo()
+
+ return main_dict
+
+ def serialize(self) -> str:
+ return yaml.dump(self._to_dict(), width=float("inf"))
+
+
+class APILog(models.Model):
+ user = models.ForeignKey(User, on_delete=models.PROTECT)
+ call_time = models.DateTimeField(auto_now=True)
+ method = models.CharField(null=True, max_length=6)
+ endpoint = models.CharField(null=True, max_length=300)
+ ip_addr = models.GenericIPAddressField(protocol="both", null=True, unpack_ipv4=False)
+ body = JSONField(null=True)
+
+ def __str__(self):
+ return "Call to {} at {} by {}".format(
+ self.endpoint,
+ self.call_time,
+ self.user.username
+ )
+
+
+class AutomationAPIManager:
+ @staticmethod
+ def serialize_booking(booking):
+ sbook = {}
+ sbook['id'] = booking.pk
+ sbook['owner'] = booking.owner.username
+ sbook['collaborators'] = [user.username for user in booking.collaborators.all()]
+ sbook['start'] = booking.start
+ sbook['end'] = booking.end
+ sbook['lab'] = AutomationAPIManager.serialize_lab(booking.lab)
+ sbook['purpose'] = booking.purpose
+ sbook['resourceBundle'] = AutomationAPIManager.serialize_bundle(booking.resource)
+ return sbook
+
+ @staticmethod
+ def serialize_lab(lab):
+ slab = {}
+ slab['id'] = lab.pk
+ slab['name'] = lab.name
+ return slab
+
+ @staticmethod
+ def serialize_bundle(bundle):
+ sbundle = {}
+ sbundle['id'] = bundle.pk
+ sbundle['resources'] = [
+ AutomationAPIManager.serialize_server(server)
+ for server in bundle.get_resources()]
+ return sbundle
+
+ @staticmethod
+ def serialize_server(server):
+ sserver = {}
+ sserver['id'] = server.pk
+ sserver['name'] = server.name
+ return sserver
+
+ @staticmethod
+ def serialize_resource_profile(profile):
+ sprofile = {}
+ sprofile['id'] = profile.pk
+ sprofile['name'] = profile.name
+ return sprofile
+
+ @staticmethod
+ def serialize_template(rec_temp_and_count):
+ template = rec_temp_and_count[0]
+ count = rec_temp_and_count[1]
+
+ stemplate = {}
+ stemplate['id'] = template.pk
+ stemplate['name'] = template.name
+ stemplate['count_available'] = count
+ stemplate['resourceProfiles'] = [
+ AutomationAPIManager.serialize_resource_profile(config.profile)
+ for config in template.getConfigs()
+ ]
+ return stemplate
+
+ @staticmethod
+ def serialize_image(image):
+ simage = {}
+ simage['id'] = image.pk
+ simage['name'] = image.name
+ return simage
+
+ @staticmethod
+ def serialize_userprofile(up):
+ sup = {}
+ sup['id'] = up.pk
+ sup['username'] = up.user.username
+ return sup
+
+
class Job(models.Model):
"""
A Job to be performed by the Lab.
This is the class that is serialized and put into the api
"""
+ JOB_TYPES = (
+ ('BOOK', 'Booking'),
+ ('DATA', 'Analytics')
+ )
+
booking = models.OneToOneField(Booking, on_delete=models.CASCADE, null=True)
status = models.IntegerField(default=JobStatus.NEW)
complete = models.BooleanField(default=False)
+ job_type = models.CharField(
+ max_length=4,
+ choices=JOB_TYPES,
+ default='BOOK'
+ )
def to_dict(self):
d = {}
return {"id": self.id, "payload": d}
def get_tasklist(self, status="all"):
- if status == "all":
+ if status != "all":
return JobTaskQuery.filter(job=self, status=status)
return JobTaskQuery.filter(job=self)
class TaskConfig(models.Model):
- state = models.IntegerField(default=ConfigState.CLEAN)
+ state = models.IntegerField(default=ConfigState.NEW)
keys = set() # TODO: This needs to be an instance variable, not a class variable
delta_keys_list = models.CharField(max_length=200, default="[]")
def to_dict(self):
d = {}
- hid = self.interfaces.first().host.labid
+ hid = ResourceQuery.get(interface__pk=self.interfaces.first().pk).labid
d[hid] = {}
for interface in self.interfaces.all():
d[hid][interface.mac_address] = []
return json.dumps(self.to_dict())
+class ActiveUsersConfig(models.Model):
+ """
+ Task for getting active VPN users
+
+ StackStorm needs no information to run this job
+ so this task is very bare, but neccessary to fit
+ job creation convention.
+ """
+
+ def clear_delta(self):
+ self.delta = '{}'
+
+ def get_delta(self):
+ return json.loads(self.to_json())
+
+ def to_json(self):
+ return json.dumps(self.to_dict())
+
+ def to_dict(self):
+ return {}
+
+
class OpnfvApiConfig(models.Model):
installer = models.CharField(max_length=200)
self.delta = '{}'
def get_delta(self):
- if not self.delta:
- self.delta = self.to_json()
- self.save()
- return json.loads(self.delta)
+ return json.loads(self.to_json())
class AccessConfig(TaskConfig):
return d
def get_delta(self):
- if not self.delta:
- self.delta = self.to_json()
- self.save()
- d = json.loads(self.delta)
+ d = json.loads(self.to_json())
d["lab_token"] = self.accessrelation.lab_token
return d
keys = set(["id", "image", "power", "hostname", "ipmi_create"])
+ def to_dict(self):
+ return self.get_delta()
+
def get_delta(self):
+ # TODO: grab the GeneratedCloudConfig urls from self.hosthardwarerelation.get_resource()
return self.format_delta(
- self.hosthardwarerelation.host.get_configuration(self.state),
+ self.hosthardwarerelation.get_resource().get_configuration(self.state),
self.hosthardwarerelation.lab_token)
def to_dict(self):
d = {}
- hid = self.hostnetworkrelation.host.labid
+ hid = self.hostnetworkrelation.resource_id
d[hid] = {}
for interface in self.interfaces.all():
d[hid][interface.mac_address] = []
- for vlan in interface.config.all():
- # TODO: should this come from the interface?
- # e.g. will different interfaces for different resources need different configs?
- d[hid][interface.mac_address].append({"vlan_id": vlan.vlan_id, "tagged": vlan.tagged})
+ if self.state != ConfigState.CLEAN:
+ for vlan in interface.config.all():
+ # TODO: should this come from the interface?
+ # e.g. will different interfaces for different resources need different configs?
+ d[hid][interface.mac_address].append({"vlan_id": vlan.vlan_id, "tagged": vlan.tagged})
return d
return json.dumps(self.to_dict())
def get_delta(self):
- if not self.delta:
- self.delta = self.to_json()
- self.save()
- d = json.loads(self.delta)
+ d = json.loads(self.to_json())
d['lab_token'] = self.hostnetworkrelation.lab_token
return d
def add_interface(self, interface):
self.interfaces.add(interface)
d = json.loads(self.delta)
- hid = self.hostnetworkrelation.host.labid
+ hid = self.hostnetworkrelation.resource_id
if hid not in d:
d[hid] = {}
d[hid][interface.mac_address] = []
class SnapshotConfig(TaskConfig):
resource_id = models.CharField(max_length=200, default="default_id")
- image = models.IntegerField(null=True)
+ image = models.CharField(max_length=200, null=True) # cobbler ID
dashboard_id = models.IntegerField()
delta = models.TextField(default="{}")
return json.dumps(self.to_dict())
def get_delta(self):
- if not self.delta:
- self.delta = self.to_json()
- self.save()
-
- d = json.loads(self.delta)
+ d = json.loads(self.to_json())
return d
def clear_delta(self):
raise ValidationError("resource_id " + str(self.resource_id) + " does not refer to a single resource")
super().save(*args, **kwargs)
+ def get_resource(self):
+ return ResourceQuery.get(labid=self.resource_id)
+
class HostNetworkRelation(TaskRelation):
resource_id = models.CharField(max_length=200, default="default_id")
raise ValidationError("resource_id " + str(self.resource_id) + " does not refer to a single resource")
super().save(*args, **kwargs)
+ def get_resource(self):
+ return ResourceQuery.get(labid=self.resource_id)
+
class SnapshotRelation(TaskRelation):
snapshot = models.ForeignKey(Image, on_delete=models.CASCADE)
return super(self.__class__, self).delete(*args, **kwargs)
+class ActiveUsersRelation(TaskRelation):
+ config = models.OneToOneField(ActiveUsersConfig, on_delete=models.CASCADE)
+ job_key = "active users task"
+
+ def type_str(self):
+ return "Active Users Task"
+
+
class JobFactory(object):
"""This class creates all the API models (jobs, tasks, etc) needed to fulfill a booking."""
"""Modify an existing job to reimage the given host."""
job = Job.objects.get(booking=booking)
# make hardware task new
- hardware_relation = HostHardwareRelation.objects.get(host=host, job=job)
- hardware_relation.config.set_image(new_image.lab_id)
+ hardware_relation = HostHardwareRelation.objects.get(resource_id=host, job=job)
+ hardware_relation.config.image = new_image.lab_id
hardware_relation.config.save()
hardware_relation.status = JobStatus.NEW
# re-apply networking after host is reset
- net_relation = HostNetworkRelation.objects.get(host=host, job=job)
+ net_relation = HostNetworkRelation.objects.get(resource_id=host, job=job)
net_relation.status = JobStatus.NEW
# re-apply ssh access after host is reset
config.set_host(host)
config.save()
+ @classmethod
+ def makeActiveUsersTask(cls):
+ """ Append active users task to analytics job """
+ config = ActiveUsersConfig()
+ relation = ActiveUsersRelation()
+ job = Job.objects.get(job_type='DATA')
+
+ job.status = JobStatus.NEW
+
+ relation.job = job
+ relation.config = config
+ relation.config.save()
+ relation.config = relation.config
+ relation.save()
+ config.save()
+
+ @classmethod
+ def makeAnalyticsJob(cls, booking):
+ """
+ Create the analytics job
+
+ This will only run once since there will only be one analytics job.
+ All analytics tasks get appended to analytics job.
+ """
+
+ if len(Job.objects.filter(job_type='DATA')) > 0:
+ raise Exception("Cannot have more than one analytics job")
+
+ if booking.resource:
+ raise Exception("Booking is not marker for analytics job, has resoure")
+
+ job = Job()
+ job.booking = booking
+ job.job_type = 'DATA'
+ job.save()
+
+ cls.makeActiveUsersTask()
+
@classmethod
def makeCompleteJob(cls, booking):
"""Create everything that is needed to fulfill the given booking."""
booking=booking,
job=job
)
+ cls.makeGeneratedCloudConfigs(
+ resources=resources,
+ job=job
+ )
all_users = list(booking.collaborators.all())
all_users.append(booking.owner)
cls.makeAccessConfig(
except Exception:
continue
+ @classmethod
+ def makeGeneratedCloudConfigs(cls, resources=[], job=Job()):
+ for res in resources:
+ cif = GeneratedCloudConfig.objects.create(resource_id=res.labid, booking=job.booking, rconfig=res.config)
+ cif.save()
+
+ cif = CloudInitFile.create(priority=0, text=cif.serialize())
+ cif.save()
+
+ res.config.cloud_init_files.add(cif)
+ res.config.save()
+
@classmethod
def makeHardwareConfigs(cls, resources=[], job=Job()):
"""
relation.config = relation.config
relation.save()
- hardware_config.set("image", "hostname", "power", "ipmi_create")
+ hardware_config.set("id", "image", "hostname", "power", "ipmi_create")
hardware_config.save()
@classmethod
@classmethod
def make_bridge_config(cls, booking):
- if booking.resource.hosts.count() < 2:
+ if len(booking.resource.get_resources()) < 2:
return None
try:
jumphost_config = ResourceOPNFVConfig.objects.filter(
opnfv_api_config.set_xdf(booking, False)
opnfv_api_config.save()
- for host in booking.resource.hosts.all():
+ for host in booking.resource.get_resources():
opnfv_api_config.roles.add(host)
software_config = SoftwareConfig.objects.create(opnfv=opnfv_api_config)
software_relation = SoftwareRelation.objects.create(job=job, config=software_config)
AccessRelation,
HostNetworkRelation,
SoftwareRelation,
- SnapshotRelation
+ SnapshotRelation,
+ ActiveUsersRelation
]
Code Review / laas.git / blobdiff