1 ##############################################################################
2 # Copyright (c) 2021 Sawyer Bergeron and others.
4 # All rights reserved. This program and the accompanying materials
5 # are made available under the terms of the Apache License, Version 2.0
6 # which accompanies this distribution, and is available at
7 # http://www.apache.org/licenses/LICENSE-2.0
8 ##############################################################################
10 from resource_inventory.models import (
18 ResourceConfiguration,
20 InterfaceConfiguration,
35 from django.contrib.auth.models import User
37 from account.models import (
42 from resource_inventory.resource_manager import ResourceManager
43 from resource_inventory.pdf_templater import PDFTemplater
45 from booking.quick_deployer import update_template
47 from datetime import timedelta, date, datetime, timezone
49 from booking.models import Booking
50 from notifier.manager import NotificationHandler
51 from api.models import JobFactory
53 from api.models import JobStatus, Job, GeneratedCloudConfig
58 Utility function for printing dividers, does nothing directly useful as a utility
63 def book_host(owner_username, host_labid, lab_username, hostname, image_id, template_name, length_days=21, collaborator_usernames=[], purpose="internal", project="LaaS"):
65 creates a quick booking using the given host
67 @owner_username is the simple username for the user who will own the resulting booking.
68 Do not set this to a lab username!
70 @image_id is the django id of the image in question, NOT the labid of the image.
71 Query Image objects by their public status and compatible host types
73 @host_labid is usually of the form `hpe3` or similar, is the labid of the Server (subtype of Resource) object
75 @lab_username for iol is `unh_iol`, other labs will be documented here
77 @hostname the hostname that the resulting host should have set
79 @template_name the name of the (public, or user accessible) template to use for this booking
81 @length_days how long the booking should be, no hard limit currently
83 @collaborator_usernames a list of usernames for collaborators to the booking
85 @purpose what this booking will be used for
87 @project what project/group this booking is on behalf of or the owner represents
89 lab = Lab.objects.get(lab_user__username=lab_username)
90 host = Server.objects.filter(lab=lab).get(labid=host_labid)
92 print("Can't book host, already marked as booked")
98 template = ResourceTemplate.objects.filter(public=True).get(name=template_name)
99 image = Image.objects.get(id=image_id)
101 owner = User.objects.get(username=owner_username)
103 new_template = update_template(template, image, hostname, owner)
105 rmanager = ResourceManager.getInstance()
107 vlan_map = rmanager.get_vlans(new_template)
109 # only a single host so can reuse var for iter here
110 resource_bundle = ResourceBundle.objects.create(template=new_template)
111 res_configs = new_template.getConfigs()
113 for config in res_configs:
115 host.bundle = resource_bundle
117 rmanager.configureNetworking(resource_bundle, host, vlan_map)
122 print("Failed to book host due to error configuring it")
127 booking = Booking.objects.create(
132 start=timezone.now(),
133 end=timezone.now() + timedelta(days=int(length_days)),
134 resource=resource_bundle,
138 booking.pdf = PDFTemplater.makePDF(booking)
142 for collaborator_username in collaborator_usernames:
144 user = User.objects.get(username=collaborator_username)
145 booking.collaborators.add(user)
147 print("couldn't add user with username ", collaborator_username)
151 JobFactory.makeCompleteJob(booking)
152 NotificationHandler.notify_new_booking(booking)
155 def mark_working(host_labid, lab_username, working=True):
157 Mark a host working/not working so that it is either bookable or hidden in the dashboard.
159 @host_labid is usually of the form `hpe3` or similar, is the labid of the Server (subtype of Resource) object
161 @lab_username: param of the form `unh_iol` or similar
163 @working: bool, whether by the end of execution the host should be considered working or not working
166 lab = Lab.objects.get(lab_user__username=lab_username)
167 server = Server.objects.filter(lab=lab).get(labid=host_labid)
168 print("changing server working status from ", server.working, "to", working)
169 server.working = working
173 def mark_booked(host_labid, lab_username, booked=True):
175 Mark a host as booked/unbooked
177 @host_labid is usually of the form `hpe3` or similar, is the labid of the Server (subtype of Resource) object
179 @lab_username: param of the form `unh_iol` or similar
181 @working: bool, whether by the end of execution the host should be considered booked or not booked
184 lab = Lab.objects.get(lab_user__username=lab_username)
185 server = Server.objects.filter(lab=lab).get(labid=host_labid)
186 print("changing server booked status from ", server.booked, "to", booked)
187 server.booked = booked
191 def get_host(host_labid, lab_username):
193 Returns host filtered by lab and then unique id within lab
195 @host_labid is usually of the form `hpe3` or similar, is the labid of the Server (subtype of Resource) object
197 @lab_username: param of the form `unh_iol` or similar
199 lab = Lab.objects.get(lab_user__username=lab_username)
200 return Server.objects.filter(lab=lab).get(labid=host_labid)
203 def get_info(host_labid, lab_username):
205 Returns various information on the host queried by the given parameters
207 @host_labid is usually of the form `hpe3` or similar, is the labid of the Server (subtype of Resource) object
209 @lab_username: param of the form `unh_iol` or similar
212 host = get_host(host_labid, lab_username)
213 info['host_labid'] = host_labid
214 info['booked'] = host.booked
215 info['working'] = host.working
216 info['profile'] = str(host.profile)
219 info['bundle'] = binfo
222 info['config'] = cinfo
227 class CumulativeData:
232 def __init__(self, file_writer):
233 self.file_writer = file_writer
235 def account(self, booking, usage_days):
236 self.count_bookings += 1
237 self.count_extensions += booking.ext_count
238 self.use_days += usage_days
240 def write_cumulative(self):
241 self.file_writer.writerow([])
242 self.file_writer.writerow([])
243 self.file_writer.writerow(['Lab Use Days', 'Count of Bookings', 'Total Extensions Used'])
244 self.file_writer.writerow([self.use_days, self.count_bookings, (self.count_bookings * 2) - self.count_extensions])
247 def get_years_booking_data(start_year=None, end_year=None):
249 Outputs yearly booking information from the past 'start_year' years (default: current year)
250 until the last day of the end year (default current year) as a csv file.
252 if start_year is None and end_year is None:
253 start = datetime.combine(date(datetime.now().year, 1, 1), datetime.min.time()).replace(tzinfo=timezone.utc)
254 end = datetime.combine(date(start.year + 1, 1, 1), datetime.min.time()).replace(tzinfo=timezone.utc)
255 elif end_year is None:
256 start = datetime.combine(date(start_year, 1, 1), datetime.min.time()).replace(tzinfo=timezone.utc)
257 end = datetime.combine(date(datetime.now().year, 1, 1), datetime.min.time()).replace(tzinfo=timezone.utc)
259 start = datetime.combine(date(start_year, 1, 1), datetime.min.time()).replace(tzinfo=timezone.utc)
260 end = datetime.combine(date(end_year + 1, 1, 1), datetime.min.time()).replace(tzinfo=timezone.utc)
262 if (start.year == end.year - 1):
263 file_name = "yearly_booking_data_" + str(start.year) + ".csv"
265 file_name = "yearly_booking_data_" + str(start.year) + "-" + str(end.year - 1) + ".csv"
267 with open(file_name, "w", newline="") as file:
268 file_writer = csv.writer(file)
269 cumulative_data = CumulativeData(file_writer)
270 file_writer.writerow(
284 for booking in Booking.objects.filter(start__gte=start, start__lte=end):
286 booking_filter = [279]
287 user_filter = ["ParkerBerberian", "ssmith", "ahassick", "sbergeron", "jhodgdon", "rhodgdon", "aburch", "jspewock"]
288 user = booking.owner.username if booking.owner.username is not None else "None"
290 for b in booking_filter:
294 for u in user_filter:
297 # trims time delta to the the specified year(s) if between years
298 usage_days = ((end if booking.end > end else booking.end) - (start if booking.start < start else booking.start)).days
301 for c in booking.collaborators.all():
302 collaborators.append(c.username)
305 cumulative_data.account(booking, usage_days)
306 file_writer.writerow([
308 str(booking.project),
309 str(booking.purpose),
310 str(booking.owner.username),
311 ','.join(collaborators),
312 str(booking.ext_count),
317 cumulative_data.write_cumulative()
320 def map_cntt_interfaces(labid: str):
322 Use this during cntt migrations, call it with a host labid and it will change profiles for this host
323 as well as mapping its interfaces across. interface ens1f2 should have the mac address of interface eno50
324 as an invariant before calling this function
326 host = get_host(labid, "unh_iol")
327 host.profile = ResourceProfile.objects.get(name="HPE x86 CNTT")
329 host = get_host(labid, "unh_iol")
331 for iface in host.interfaces.all():
333 if iface.profile.name == "ens1f2":
334 new_ifprofile = InterfaceProfile.objects.get(host=host.profile, name="eno50")
336 new_ifprofile = InterfaceProfile.objects.get(host=host.profile, name=iface.profile.name)
338 iface.profile = new_ifprofile
343 def detect_leaked_hosts(labid="unh_iol"):
345 Use this to try to detect leaked hosts.
346 These hosts may still be in the process of unprovisioning,
347 but if they are not (or unprovisioning is frozen) then
348 these hosts are instead leaked
350 working_servers = Server.objects.filter(working=True, lab__lab_user__username=labid)
351 booked = working_servers.filter(booked=True)
355 for booking in Booking.objects.filter(end__gte=timezone.now()):
356 res_for_booking = booking.resource.get_resources()
357 print(res_for_booking)
358 for resource in res_for_booking:
359 filtered = filtered.exclude(id=resource.id)
361 print("Possibly leaked:")
362 for host in filtered:
368 def booking_for_host(host_labid: str, lab_username="unh_iol"):
370 Returns the booking that this server is a part of, if any.
371 Fails with an exception if no such booking exists
373 @host_labid is usually of the form `hpe3` or similar, is the labid of the Server (subtype of Resource) object
375 @lab_username: param of the form `unh_iol` or similar
377 server = Server.objects.get(lab__lab_user__username=lab_username, labid=host_labid)
378 booking = server.bundle.booking_set.first()
381 print("id:", booking.id)
382 print("owner:", booking.owner)
383 print("job (id):", booking.job, "(" + str(booking.job.id) + ")")
388 def force_release_booking(booking_id: int):
390 Takes a booking id and forces the booking to end whether or not the tasks have
393 Use with caution! Hosts may or may not be released depending on other underlying issues
395 @booking_id: the id of the Booking object to be released
397 booking = Booking.objects.get(id=booking_id)
399 tasks = job.get_tasklist()
401 task.status = JobStatus.DONE
405 def free_leaked_public_vlans(safety_buffer_days=2):
406 for lab in Lab.objects.all():
407 current_booking_set = Booking.objects.filter(end__gte=timezone.now() + timedelta(days=safety_buffer_days))
411 for booking in current_booking_set:
412 for network in get_network_metadata(booking.id):
413 marked_nets.add(network["vlan_id"])
415 for net in PublicNetwork.objects.filter(lab=lab).filter(in_use=True):
416 if net.vlan not in marked_nets:
417 lab.vlan_manager.release_public_vlan(net.vlan)
420 def get_network_metadata(booking_id: int):
422 Takes a booking id and prints all (known) networks that are owned by it.
423 Returns an object of the form {<network name>: {"vlan_id": int, "netname": str <network name>, "public": bool <whether network is public/routable}}
425 @booking_id: the id of the Booking object to be queried
427 booking = Booking.objects.get(id=booking_id)
428 bundle = booking.resource
429 pnets = PhysicalNetwork.objects.filter(bundle=bundle).all()
432 net = pnet.generic_network
433 mdata = {"vlan_id": pnet.vlan_id, "netname": net.name, "public": net.is_public}
434 metadata[net.name] = mdata
438 def print_dict_pretty(a_dict):
440 admin_utils internal function
443 print(json.dumps(a_dict, sort_keys=True, indent=4))
446 def add_profile(data):
448 Used for adding a host profile to the dashboard
450 schema (of dict passed as "data" param):
460 media_type: str ("SSD" or "HDD")
461 interface: str ("sata", "sas", "ssd", "nvme", "scsi", or "iscsi")
467 "nic_type": str ("onboard" or "pcie")
468 "order": int (compared to the other interfaces, indicates the "order" that the ports are laid out)
472 cores: int (hardware threads count)
473 architecture: str (x86_64" or "aarch64")
474 cpus: int (number of sockets)
483 base_profile = ResourceProfile.objects.create(name=data['name'], description=data['description'])
486 for lab_username in data['labs']:
487 lab = Lab.objects.get(lab_user__username=lab_username)
489 base_profile.labs.add(lab)
492 for diskname in data['disks'].keys():
493 disk = data['disks'][diskname]
495 disk_profile = DiskProfile.objects.create(name=diskname, size=disk['capacity'], media_type=disk['media_type'], interface=disk['interface'], host=base_profile)
498 for ifacename in data['interfaces'].keys():
499 iface = data['interfaces'][ifacename]
501 iface_profile = InterfaceProfile.objects.create(name=ifacename, speed=iface['speed'], nic_type=iface['nic_type'], order=iface['order'], host=base_profile)
505 cpu_prof = CpuProfile.objects.create(cores=cpu['cores'], architecture=cpu['architecture'], cpus=cpu['cpus'], cflags=cpu['cflags'], host=base_profile)
508 ram_prof = RamProfile.objects.create(amount=data['ram']['amount'], channels=data['ram']['channels'], host=base_profile)
512 def make_default_template(resource_profile, image_id=None, template_name=None, connected_interface_names=None, interfaces_tagged=False, connected_interface_tagged=False, owner_username="root", lab_username="unh_iol", public=True, temporary=False, description=""):
514 Do not call this function without reading the related source code, it may have unintended effects.
516 Used for creating a default template from some host profile
519 if not resource_profile:
520 raise Exception("No viable continuation from none resource_profile")
522 if not template_name:
523 template_name = resource_profile.name
525 if not connected_interface_names:
526 connected_interface_names = [InterfaceProfile.objects.filter(host=resource_profile).first().name]
527 print("setting connected interface names to", connected_interface_names)
530 image_id = Image.objects.filter(host_type=resource_profile).first().id
532 image = Image.objects.get(id=image_id)
534 base = ResourceTemplate.objects.create(
537 owner=User.objects.get(username=owner_username),
538 lab=Lab.objects.get(lab_user__username=lab_username), description=description,
539 public=public, temporary=temporary, copy_of=None)
541 rconf = ResourceConfiguration.objects.create(profile=resource_profile, image=image, template=base, is_head_node=True, name="opnfv_host")
544 connected_interfaces = []
546 for iface_prof in InterfaceProfile.objects.filter(host=resource_profile).all():
547 iface_conf = InterfaceConfiguration.objects.create(profile=iface_prof, resource_config=rconf)
549 if iface_prof.name in connected_interface_names:
550 connected_interfaces.append(iface_conf)
552 network = Network.objects.create(name="public", bundle=base, is_public=True)
554 for iface in connected_interfaces:
555 connection = NetworkConnection.objects.create(network=network, vlan_is_tagged=interfaces_tagged)
558 iface.connections.add(connection)
559 print("adding connection to iface ", iface)
564 def add_server(profile, name, interfaces, lab_username="unh_iol", vendor="unknown", model="unknown"):
566 Used to enroll a new host of some profile
568 @profile: the ResourceProfile in question (by reference to a model object)
570 @name: the unique name of the server, currently indistinct from labid
572 @interfaces: interfaces should be dict from interface name (eg ens1f0) to dict of schema:
574 mac_address: <mac addr>,
575 bus_addr: <bus addr>, //this field is optional, "" is default
578 @lab_username: username of the lab to be added to
580 @vendor: vendor name of the host, such as "HPE" or "Gigabyte"
582 @model: specific model of the host, such as "DL380 Gen 9"
585 server = Server.objects.create(
593 lab=Lab.objects.get(lab_user__username=lab_username),
597 for iface_prof in InterfaceProfile.objects.filter(host=profile).all():
598 mac_addr = interfaces[iface_prof.name]["mac_address"]
600 if "bus_addr" in interfaces[iface_prof.name].keys():
601 bus_addr = interfaces[iface_prof.name]["bus_addr"]
603 iface = Interface.objects.create(acts_as=None, profile=iface_prof, mac_address=mac_addr, bus_address=bus_addr)
606 server.interfaces.add(iface)
610 def extend_booking(booking_id, days=0, hours=0, minutes=0, weeks=0):
612 Extend a booking by n <days, hours, minutes, weeks>
614 @booking_id: id of the booking
616 @days/@hours/@minutes/@weeks: the cumulative amount of delta to add to the length of the booking
619 booking = Booking.objects.get(id=booking_id)
620 booking.end = booking.end + timedelta(days=days, hours=hours, minutes=minutes, weeks=weeks)
624 def regenerate_cloud_configs(booking_id):
625 b = Booking.objects.get(id=booking_id)
626 for res in b.resource.get_resources():
627 res.config.cloud_init_files.set(res.config.cloud_init_files.filter(generated=False)) # careful!
629 cif = GeneratedCloudConfig.objects.create(resource_id=res.labid, booking=b, rconfig=res.config)
631 cif = CloudInitFile.create(priority=0, text=cif.serialize())
633 res.config.cloud_init_files.add(cif)
637 def set_job_new(job_id):
638 j = Job.objects.get(id=job_id)
640 regenerate_cloud_configs(b.id)
641 for task in j.get_tasklist():
642 task.status = JobStatus.NEW
644 j.status = JobStatus.NEW
648 def docs(function=None, fulltext=False):
650 Print documentation for a given function in admin_utils.
651 Call without arguments for more information
656 if isinstance(function, str):
658 fn = globals()[function]
660 print("Couldn't find a function by the given name")
662 elif callable(function):
665 print("docs(function: callable | str, fulltext: bool) was called with a 'function' that was neither callable nor a string name of a function")
666 print("usage: docs('some_function_in_admin_utils', fulltext=True)")
667 print("The 'fulltext' argument is used to choose if you want the complete source of the function printed. If this argument is false then you will only see the pydoc rendered documentation for the function")
671 print("couldn't find a function by that name")
674 print("Pydoc documents the function as such:")
675 print(pydoc.render_doc(fn))
677 print("The full source of the function is this:")
678 print(inspect.getsource(fn))
681 def admin_functions():
683 List functions available to call within admin_utils
686 return [name for name, func in inspect.getmembers(sys.modules[__name__]) if (inspect.isfunction(func) and func.__module__ == __name__)]
689 print("Hint: call `docs(<function name>)` or `admin_functions()` for help on using the admin utils")
690 print("docs(<function name>) displays documentation on a given function")
691 print("admin_functions() lists all functions available to call within this module")