X-Git-Url: https://gerrit.opnfv.org/gerrit/gitweb?a=blobdiff_plain;f=src%2Fbooking%2Fquick_deployer.py;h=4b85d76e37212d05b8d2429c1980575d4b95cc39;hb=0d3dd290aa6e7f39e7b0b3cbe448b6622f924240;hp=ac69c8c4e589a34cd1730e52ea2f159e4f52221e;hpb=63bec7d84cbf1acd3a9a357b58b47584b1701229;p=laas.git diff --git a/src/booking/quick_deployer.py b/src/booking/quick_deployer.py index ac69c8c..4b85d76 100644 --- a/src/booking/quick_deployer.py +++ b/src/booking/quick_deployer.py @@ -9,191 +9,155 @@ import json -import uuid -import re +import yaml from django.db.models import Q +from django.db import transaction from datetime import timedelta from django.utils import timezone -from account.models import Lab +from django.core.exceptions import ValidationError +from account.models import Lab, UserProfile from resource_inventory.models import ( - Installer, + ResourceTemplate, Image, - GenericResourceBundle, - ConfigBundle, - Host, - HostProfile, - HostConfiguration, - GenericResource, - GenericHost, - GenericInterface, OPNFVRole, OPNFVConfig, - Network, + ResourceOPNFVConfig, + ResourceConfiguration, NetworkConnection, - NetworkRole, - HostOPNFVConfig, + InterfaceConfiguration, + Network, + CloudInitFile, ) from resource_inventory.resource_manager import ResourceManager from resource_inventory.pdf_templater import PDFTemplater from notifier.manager import NotificationHandler from booking.models import Booking -from dashboard.exceptions import ( - InvalidHostnameException, - ResourceAvailabilityException, - ModelValidationException, - BookingLengthException -) +from dashboard.exceptions import BookingLengthException from api.models import JobFactory -# model validity exceptions -class IncompatibleInstallerForOS(Exception): - pass - - -class IncompatibleScenarioForInstaller(Exception): - pass - - -class IncompatibleImageForHost(Exception): - pass - - -class ImageOwnershipInvalid(Exception): - pass - - -class ImageNotAvailableAtLab(Exception): - pass - - -class LabDNE(Exception): - pass - - -class HostProfileDNE(Exception): - pass - - -class HostNotAvailable(Exception): - pass - - -class NoLabSelectedError(Exception): - pass - - -class OPNFVRoleDNE(Exception): - pass - - -class NoRemainingPublicNetwork(Exception): - pass - - -class BookingPermissionException(Exception): - pass - - -def parse_host_field(host_field_contents): - host_json = json.loads(host_field_contents) - lab_dict = host_json['labs'][0] - lab_id = list(lab_dict.keys())[0] - lab_user_id = int(lab_id.split("_")[-1]) - lab = Lab.objects.get(lab_user__id=lab_user_id) - - host_dict = host_json['hosts'][0] - profile_id = list(host_dict.keys())[0] - profile_id = int(profile_id.split("_")[-1]) - profile = HostProfile.objects.get(id=profile_id) - - # check validity of field data before trying to apply to models - if len(host_json['labs']) != 1: - raise NoLabSelectedError("No lab was selected") - if not lab: - raise LabDNE("Lab with provided ID does not exist") - if not profile: - raise HostProfileDNE("Host type with provided ID does not exist") - - return lab, profile - - -def check_available_matching_host(lab, hostprofile): - available_host_types = ResourceManager.getInstance().getAvailableHostTypes(lab) - if hostprofile not in available_host_types: - # TODO: handle deleting generic resource in this instance along with grb - raise HostNotAvailable('Requested host type is not available. Please try again later. Host availability can be viewed in the "Hosts" tab to the left.') - - hostset = Host.objects.filter(lab=lab, profile=hostprofile).filter(booked=False).filter(working=True) - if not hostset.exists(): - raise HostNotAvailable("Couldn't find any matching unbooked hosts") - - return True - - -def generate_grb(owner, lab, common_id): - grbundle = GenericResourceBundle(owner=owner) - grbundle.lab = lab - grbundle.name = "grbundle for quick booking with uid " + common_id - grbundle.description = "grbundle created for quick-deploy booking" - grbundle.save() - - return grbundle - - -def generate_gresource(bundle, hostname): - if not re.match(r"(?=^.{1,253}$)(^([A-Za-z0-9-_]{1,62}\.)*[A-Za-z0-9-_]{1,63})$", hostname): - raise InvalidHostnameException("Hostname must comply to RFC 952 and all extensions to it until this point") - gresource = GenericResource(bundle=bundle, name=hostname) - gresource.save() - - return gresource - - -def generate_ghost(generic_resource, host_profile): - ghost = GenericHost() - ghost.resource = generic_resource - ghost.profile = host_profile - ghost.save() - - return ghost - - -def generate_config_bundle(owner, common_id, grbundle): - cbundle = ConfigBundle() - cbundle.owner = owner - cbundle.name = "configbundle for quick booking with uid " + common_id - cbundle.description = "configbundle created for quick-deploy booking" - cbundle.bundle = grbundle - cbundle.save() - - return cbundle - - -def generate_opnfvconfig(scenario, installer, config_bundle): - opnfvconfig = OPNFVConfig() - opnfvconfig.scenario = scenario - opnfvconfig.installer = installer - opnfvconfig.bundle = config_bundle - opnfvconfig.save() - - return opnfvconfig - +def parse_resource_field(resource_json): + """ + Parse the json from the frontend. + + returns a reference to the selected Lab and ResourceTemplate objects + """ + lab, template = (None, None) + lab_dict = resource_json['lab'] + for lab_info in lab_dict.values(): + if lab_info['selected']: + lab = Lab.objects.get(lab_user__id=lab_info['id']) + + resource_dict = resource_json['resource'] + for resource_info in resource_dict.values(): + if resource_info['selected']: + template = ResourceTemplate.objects.get(pk=resource_info['id']) + + if lab is None: + raise ValidationError("No lab was selected") + if template is None: + raise ValidationError("No Host was selected") + + return lab, template + + +def update_template(old_template, image, hostname, user, global_cloud_config=None): + """ + Duplicate a template to the users account and update configured fields. + + The dashboard presents users with preconfigured resource templates, + but the user might want to make small modifications, e.g hostname and + linux distro. So we copy the public template and create a private version + to the user's profile, and mark it temporary. When the booking ends the + new template is deleted + """ + name = user.username + "'s Copy of '" + old_template.name + "'" + num_copies = ResourceTemplate.objects.filter(name__startswith=name).count() + template = ResourceTemplate.objects.create( + name=name if num_copies == 0 else name + " (" + str(num_copies) + ")", + xml=old_template.xml, + owner=user, + lab=old_template.lab, + description=old_template.description, + public=False, + temporary=True, + private_vlan_pool=old_template.private_vlan_pool, + public_vlan_pool=old_template.public_vlan_pool, + copy_of=old_template + ) -def generate_hostconfig(generic_host, image, config_bundle): - hconf = HostConfiguration() - hconf.host = generic_host - hconf.image = image - hconf.bundle = config_bundle - hconf.is_head_node = True - hconf.save() + for old_network in old_template.networks.all(): + Network.objects.create( + name=old_network.name, + bundle=template, + is_public=old_network.is_public + ) + # We are assuming there is only one opnfv config per public resource template + old_opnfv = template.opnfv_config.first() + if old_opnfv: + opnfv_config = OPNFVConfig.objects.create( + installer=old_opnfv.installer, + scenario=old_opnfv.installer, + template=template, + name=old_opnfv.installer, + ) + # I am explicitly leaving opnfv_config.networks empty to avoid + # problems with duplicated / shared networks. In the quick deploy, + # there is never multiple networks anyway. This may have to change in the future + + for old_config in old_template.getConfigs(): + image_to_set = image + if not image: + image_to_set = old_config.image + + config = ResourceConfiguration.objects.create( + profile=old_config.profile, + image=image_to_set, + template=template, + is_head_node=old_config.is_head_node, + name=hostname if len(old_template.getConfigs()) == 1 else old_config.name, + # cloud_init_files=old_config.cloud_init_files.set() + ) - return hconf + for file in old_config.cloud_init_files.all(): + config.cloud_init_files.add(file) + + if global_cloud_config: + config.cloud_init_files.add(global_cloud_config) + config.save() + + for old_iface_config in old_config.interface_configs.all(): + iface_config = InterfaceConfiguration.objects.create( + profile=old_iface_config.profile, + resource_config=config + ) + + for old_connection in old_iface_config.connections.all(): + iface_config.connections.add(NetworkConnection.objects.create( + network=template.networks.get(name=old_connection.network.name), + vlan_is_tagged=old_connection.vlan_is_tagged + )) + + for old_res_opnfv in old_config.resource_opnfv_config.all(): + if old_opnfv: + ResourceOPNFVConfig.objects.create( + role=old_opnfv.role, + resource_config=config, + opnfv_config=opnfv_config + ) + return template + + +def generate_opnfvconfig(scenario, installer, template): + return OPNFVConfig.objects.create( + scenario=scenario, + installer=installer, + template=template + ) def generate_hostopnfv(hostconfig, opnfvconfig): - config = HostOPNFVConfig() role = None try: role = OPNFVRole.objects.get(name="Jumphost") @@ -202,125 +166,138 @@ def generate_hostopnfv(hostconfig, opnfvconfig): name="Jumphost", description="Single server jumphost role" ) - config.role = role - config.host_config = hostconfig - config.opnfv_config = opnfvconfig - config.save() - return config + return ResourceOPNFVConfig.objects.create( + role=role, + host_config=hostconfig, + opnfv_config=opnfvconfig + ) -def generate_resource_bundle(generic_resource_bundle, config_bundle): # warning: requires cleanup - try: - resource_manager = ResourceManager.getInstance() - resource_bundle = resource_manager.convertResourceBundle(generic_resource_bundle, config=config_bundle) - return resource_bundle - except ResourceAvailabilityException: - raise ResourceAvailabilityException("Requested resources not available") - except ModelValidationException: - raise ModelValidationException("Encountered error while saving grbundle") +def generate_resource_bundle(template): + resource_manager = ResourceManager.getInstance() + resource_bundle = resource_manager.instantiateTemplate(template) + return resource_bundle -def check_invariants(request, **kwargs): - installer = kwargs['installer'] +def check_invariants(**kwargs): + # TODO: This should really happen in the BookingForm validation methods image = kwargs['image'] - scenario = kwargs['scenario'] lab = kwargs['lab'] - host_profile = kwargs['host_profile'] length = kwargs['length'] # check that image os is compatible with installer - if installer in image.os.sup_installers.all(): - # if installer not here, we can omit that and not check for scenario - if not scenario: - raise IncompatibleScenarioForInstaller("An OPNFV Installer needs a scenario to be chosen to work properly") - if scenario not in installer.sup_scenarios.all(): - raise IncompatibleScenarioForInstaller("The chosen installer does not support the chosen scenario") - if image.from_lab != lab: - raise ImageNotAvailableAtLab("The chosen image is not available at the chosen hosting lab") - if image.host_type != host_profile: - raise IncompatibleImageForHost("The chosen image is not available for the chosen host type") - if not image.public and image.owner != request.user: - raise ImageOwnershipInvalid("You are not the owner of the chosen private image") + if image: + if image.from_lab != lab: + raise ValidationError("The chosen image is not available at the chosen hosting lab") + # TODO + # if image.host_type != host_profile: + # raise ValidationError("The chosen image is not available for the chosen host type") + if not image.public and image.owner != kwargs['owner']: + raise ValidationError("You are not the owner of the chosen private image") if length < 1 or length > 21: raise BookingLengthException("Booking must be between 1 and 21 days long") -def configure_networking(grb, config): - # create network - net = Network.objects.create(name="public", bundle=grb, is_public=True) - # connect network to generic host - grb.getHosts()[0].generic_interfaces.first().connections.add( - NetworkConnection.objects.create(network=net, vlan_is_tagged=False) - ) - # asign network role - role = NetworkRole.objects.create(name="public", network=net) - opnfv_config = config.opnfv_config.first() - if opnfv_config: - opnfv_config.networks.add(role) - - def create_from_form(form, request): - quick_booking_id = str(uuid.uuid4()) + """ + Parse data from QuickBookingForm to create booking + """ + resource_field = form.cleaned_data['filter_field'] + # users_field = form.cleaned_data['users'] + hostname = 'opnfv_host' if not form.cleaned_data['hostname'] else form.cleaned_data['hostname'] - host_field = form.cleaned_data['filter_field'] - purpose_field = form.cleaned_data['purpose'] - project_field = form.cleaned_data['project'] - users_field = form.cleaned_data['users'] - hostname = form.cleaned_data['hostname'] - length = form.cleaned_data['length'] + global_cloud_config = None if not form.cleaned_data['global_cloud_config'] else form.cleaned_data['global_cloud_config'] - image = form.cleaned_data['image'] - scenario = form.cleaned_data['scenario'] - installer = form.cleaned_data['installer'] + if global_cloud_config: + form.cleaned_data['global_cloud_config'] = create_ci_file(global_cloud_config) - lab, host_profile = parse_host_field(host_field) + # image = form.cleaned_data['image'] + # scenario = form.cleaned_data['scenario'] + # installer = form.cleaned_data['installer'] + + lab, resource_template = parse_resource_field(resource_field) data = form.cleaned_data + data['hostname'] = hostname data['lab'] = lab - data['host_profile'] = host_profile - check_invariants(request, **data) + data['resource_template'] = resource_template + data['owner'] = request.user + + return _create_booking(data) + + +def create_from_API(body, user): + """ + Parse data from Automation API to create booking + """ + booking_info = json.loads(body.decode('utf-8')) + + data = {} + data['purpose'] = booking_info['purpose'] + data['project'] = booking_info['project'] + data['users'] = [UserProfile.objects.get(user__username=username) + for username in booking_info['collaborators']] + data['hostname'] = booking_info['hostname'] + data['length'] = booking_info['length'] + data['installer'] = None + data['scenario'] = None + + data['image'] = Image.objects.get(pk=booking_info['imageLabID']) + + data['resource_template'] = ResourceTemplate.objects.get(pk=booking_info['templateID']) + data['lab'] = data['resource_template'].lab + data['owner'] = user + + if 'global_cloud_config' in data.keys(): + data['global_cloud_config'] = CloudInitFile.objects.get(id=data['global_cloud_config']) + + return _create_booking(data) - # check booking privileges - if Booking.objects.filter(owner=request.user, end__gt=timezone.now()).count() >= 3 and not request.user.userprofile.booking_privledge: - raise BookingPermissionException("You do not have permission to have more than 3 bookings at a time.") - check_available_matching_host(lab, host_profile) # requires cleanup if failure after this point +def create_ci_file(data: str) -> CloudInitFile: + try: + d = yaml.load(data) + if not (type(d) is dict): + raise Exception("CI file was valid yaml but was not a dict") + except Exception: + raise ValidationError("The provided Cloud Config is not valid yaml, please refer to the Cloud Init documentation for expected structure") + print("about to create global cloud config") + config = CloudInitFile.create(text=data, priority=CloudInitFile.objects.count()) + print("made global cloud config") + + return config + - grbundle = generate_grb(request.user, lab, quick_booking_id) - gresource = generate_gresource(grbundle, hostname) - ghost = generate_ghost(gresource, host_profile) - cbundle = generate_config_bundle(request.user, quick_booking_id, grbundle) - hconf = generate_hostconfig(ghost, image, cbundle) +@transaction.atomic +def _create_booking(data): + check_invariants(**data) - # if no installer provided, just create blank host - opnfv_config = None - if installer: - opnfv_config = generate_opnfvconfig(scenario, installer, cbundle) - generate_hostopnfv(hconf, opnfv_config) + # check booking privileges + # TODO: use the canonical booking_allowed method because now template might have multiple + # machines + if Booking.objects.filter(owner=data['owner'], end__gt=timezone.now()).count() >= 3 and not data['owner'].userprofile.booking_privledge: + raise PermissionError("You do not have permission to have more than 3 bookings at a time.") - # construct generic interfaces - for interface_profile in host_profile.interfaceprofile.all(): - generic_interface = GenericInterface.objects.create(profile=interface_profile, host=ghost) - generic_interface.save() + ResourceManager.getInstance().templateIsReservable(data['resource_template']) - configure_networking(grbundle, cbundle) + resource_template = update_template(data['resource_template'], data['image'], data['hostname'], data['owner'], global_cloud_config=data['global_cloud_config']) # generate resource bundle - resource_bundle = generate_resource_bundle(grbundle, cbundle) + resource_bundle = generate_resource_bundle(resource_template) # generate booking booking = Booking.objects.create( - purpose=purpose_field, - project=project_field, - lab=lab, - owner=request.user, + purpose=data['purpose'], + project=data['project'], + lab=data['lab'], + owner=data['owner'], start=timezone.now(), - end=timezone.now() + timedelta(days=int(length)), + end=timezone.now() + timedelta(days=int(data['length'])), resource=resource_bundle, - config_bundle=cbundle, - opnfv_config=opnfv_config + opnfv_config=None ) + booking.pdf = PDFTemplater.makePDF(booking) - for collaborator in users_field: # list of UserProfiles + for collaborator in data['users']: # list of Users (not UserProfile) booking.collaborators.add(collaborator.user) booking.save() @@ -329,28 +306,38 @@ def create_from_form(form, request): JobFactory.makeCompleteJob(booking) NotificationHandler.notify_new_booking(booking) + return booking + def drop_filter(user): - installer_filter = {} - for image in Image.objects.all(): - installer_filter[image.id] = {} - for installer in image.os.sup_installers.all(): - installer_filter[image.id][installer.id] = 1 + """ + Return a dictionary that contains filters. + Only certain installlers are supported on certain images, etc + so the image filter indexed at [imageid][installerid] is truthy if + that installer is supported on that image + """ + installer_filter = {} scenario_filter = {} - for installer in Installer.objects.all(): - scenario_filter[installer.id] = {} - for scenario in installer.sup_scenarios.all(): - scenario_filter[installer.id][scenario.id] = 1 images = Image.objects.filter(Q(public=True) | Q(owner=user)) image_filter = {} for image in images: - image_filter[image.id] = {} - image_filter[image.id]['lab'] = 'lab_' + str(image.from_lab.lab_user.id) - image_filter[image.id]['host_profile'] = 'host_' + str(image.host_type.id) - image_filter[image.id]['name'] = image.name - - return {'installer_filter': json.dumps(installer_filter), - 'scenario_filter': json.dumps(scenario_filter), - 'image_filter': json.dumps(image_filter)} + image_filter[image.id] = { + 'lab': 'lab_' + str(image.from_lab.lab_user.id), + 'architecture': str(image.architecture), + 'name': image.name + } + + resource_filter = {} + templates = ResourceTemplate.objects.filter(Q(public=True) | Q(owner=user)) + for rt in templates: + profiles = [conf.profile for conf in rt.getConfigs()] + resource_filter["resource_" + str(rt.id)] = [str(p.architecture) for p in profiles] + + return { + 'installer_filter': json.dumps(installer_filter), + 'scenario_filter': json.dumps(scenario_filter), + 'image_filter': json.dumps(image_filter), + 'resource_profile_map': json.dumps(resource_filter), + }