added util to get booking data in a range of years
[laas.git] / src / dashboard / admin_utils.py
1 ##############################################################################
2 # Copyright (c) 2021 Sawyer Bergeron and others.
3 #
4 # All rights reserved. This program and the accompanying materials
5 # are made available under the terms of the Apache License, Version 2.0
6 # which accompanies this distribution, and is available at
7 # http://www.apache.org/licenses/LICENSE-2.0
8 ##############################################################################
9
10 from resource_inventory.models import (
11     ResourceTemplate,
12     Image,
13     Server,
14     ResourceBundle,
15     ResourceProfile,
16     InterfaceProfile,
17     PhysicalNetwork,
18     ResourceConfiguration,
19     NetworkConnection,
20     InterfaceConfiguration,
21     Network,
22     DiskProfile,
23     CpuProfile,
24     RamProfile,
25     Interface,
26     CloudInitFile,
27 )
28
29 import json
30 import sys
31 import inspect
32 import pydoc
33 import csv
34
35 from django.contrib.auth.models import User
36
37 from account.models import (
38     Lab,
39     PublicNetwork
40 )
41
42 from resource_inventory.resource_manager import ResourceManager
43 from resource_inventory.pdf_templater import PDFTemplater
44
45 from booking.quick_deployer import update_template
46
47 from datetime import timedelta, date, datetime, timezone
48
49 from booking.models import Booking
50 from notifier.manager import NotificationHandler
51 from api.models import JobFactory
52
53 from api.models import JobStatus, Job, GeneratedCloudConfig
54
55
56 def print_div():
57     """
58     Utility function for printing dividers, does nothing directly useful as a utility
59     """
60     print("=" * 68)
61
62
63 def book_host(owner_username, host_labid, lab_username, hostname, image_id, template_name, length_days=21, collaborator_usernames=[], purpose="internal", project="LaaS"):
64     """
65     creates a quick booking using the given host
66
67     @owner_username is the simple username for the user who will own the resulting booking.
68     Do not set this to a lab username!
69
70     @image_id is the django id of the image in question, NOT the labid of the image.
71     Query Image objects by their public status and compatible host types
72
73     @host_labid is usually of the form `hpe3` or similar, is the labid of the Server (subtype of Resource) object
74
75     @lab_username for iol is `unh_iol`, other labs will be documented here
76
77     @hostname the hostname that the resulting host should have set
78
79     @template_name the name of the (public, or user accessible) template to use for this booking
80
81     @length_days how long the booking should be, no hard limit currently
82
83     @collaborator_usernames a list of usernames for collaborators to the booking
84
85     @purpose what this booking will be used for
86
87     @project what project/group this booking is on behalf of or the owner represents
88     """
89     lab = Lab.objects.get(lab_user__username=lab_username)
90     host = Server.objects.filter(lab=lab).get(labid=host_labid)
91     if host.booked:
92         print("Can't book host, already marked as booked")
93         return
94     else:
95         host.booked = True
96         host.save()
97
98     template = ResourceTemplate.objects.filter(public=True).get(name=template_name)
99     image = Image.objects.get(id=image_id)
100
101     owner = User.objects.get(username=owner_username)
102
103     new_template = update_template(template, image, hostname, owner)
104
105     rmanager = ResourceManager.getInstance()
106
107     vlan_map = rmanager.get_vlans(new_template)
108
109     # only a single host so can reuse var for iter here
110     resource_bundle = ResourceBundle.objects.create(template=new_template)
111     res_configs = new_template.getConfigs()
112
113     for config in res_configs:
114         try:
115             host.bundle = resource_bundle
116             host.config = config
117             rmanager.configureNetworking(resource_bundle, host, vlan_map)
118             host.save()
119         except Exception:
120             host.booked = False
121             host.save()
122             print("Failed to book host due to error configuring it")
123             return
124
125     new_template.save()
126
127     booking = Booking.objects.create(
128         purpose=purpose,
129         project=project,
130         lab=lab,
131         owner=owner,
132         start=timezone.now(),
133         end=timezone.now() + timedelta(days=int(length_days)),
134         resource=resource_bundle,
135         opnfv_config=None
136     )
137
138     booking.pdf = PDFTemplater.makePDF(booking)
139
140     booking.save()
141
142     for collaborator_username in collaborator_usernames:
143         try:
144             user = User.objects.get(username=collaborator_username)
145             booking.collaborators.add(user)
146         except Exception:
147             print("couldn't add user with username ", collaborator_username)
148
149     booking.save()
150
151     JobFactory.makeCompleteJob(booking)
152     NotificationHandler.notify_new_booking(booking)
153
154
155 def mark_working(host_labid, lab_username, working=True):
156     """
157     Mark a host working/not working so that it is either bookable or hidden in the dashboard.
158
159     @host_labid is usually of the form `hpe3` or similar, is the labid of the Server (subtype of Resource) object
160
161     @lab_username: param of the form `unh_iol` or similar
162
163     @working: bool, whether by the end of execution the host should be considered working or not working
164     """
165
166     lab = Lab.objects.get(lab_user__username=lab_username)
167     server = Server.objects.filter(lab=lab).get(labid=host_labid)
168     print("changing server working status from ", server.working, "to", working)
169     server.working = working
170     server.save()
171
172
173 def mark_booked(host_labid, lab_username, booked=True):
174     """
175     Mark a host as booked/unbooked
176
177     @host_labid is usually of the form `hpe3` or similar, is the labid of the Server (subtype of Resource) object
178
179     @lab_username: param of the form `unh_iol` or similar
180
181     @working: bool, whether by the end of execution the host should be considered booked or not booked
182     """
183
184     lab = Lab.objects.get(lab_user__username=lab_username)
185     server = Server.objects.filter(lab=lab).get(labid=host_labid)
186     print("changing server booked status from ", server.booked, "to", booked)
187     server.booked = booked
188     server.save()
189
190
191 def get_host(host_labid, lab_username):
192     """
193     Returns host filtered by lab and then unique id within lab
194
195     @host_labid is usually of the form `hpe3` or similar, is the labid of the Server (subtype of Resource) object
196
197     @lab_username: param of the form `unh_iol` or similar
198     """
199     lab = Lab.objects.get(lab_user__username=lab_username)
200     return Server.objects.filter(lab=lab).get(labid=host_labid)
201
202
203 def get_info(host_labid, lab_username):
204     """
205     Returns various information on the host queried by the given parameters
206
207     @host_labid is usually of the form `hpe3` or similar, is the labid of the Server (subtype of Resource) object
208
209     @lab_username: param of the form `unh_iol` or similar
210     """
211     info = {}
212     host = get_host(host_labid, lab_username)
213     info['host_labid'] = host_labid
214     info['booked'] = host.booked
215     info['working'] = host.working
216     info['profile'] = str(host.profile)
217     if host.bundle:
218         binfo = {}
219         info['bundle'] = binfo
220     if host.config:
221         cinfo = {}
222         info['config'] = cinfo
223
224     return info
225
226
227 class CumulativeData:
228     use_days = 0
229     count_bookings = 0
230     count_extensions = 0
231
232     def __init__(self, file_writer):
233         self.file_writer = file_writer
234
235     def account(self, booking, usage_days):
236         self.count_bookings += 1
237         self.count_extensions += booking.ext_count
238         self.use_days += usage_days
239
240     def write_cumulative(self):
241         self.file_writer.writerow([])
242         self.file_writer.writerow([])
243         self.file_writer.writerow(['Lab Use Days', 'Count of Bookings', 'Total Extensions Used'])
244         self.file_writer.writerow([self.use_days, self.count_bookings, (self.count_bookings * 2) - self.count_extensions])
245
246
247 def get_years_booking_data(start_year=None, end_year=None):
248     """
249     Outputs yearly booking information from the past 'start_year' years (default: current year)
250     until the last day of the end year (default current year) as a csv file.
251     """
252     if start_year is None and end_year is None:
253         start = datetime.combine(date(datetime.now().year, 1, 1), datetime.min.time()).replace(tzinfo=timezone.utc)
254         end = datetime.combine(date(start.year + 1, 1, 1), datetime.min.time()).replace(tzinfo=timezone.utc)
255     elif end_year is None:
256         start = datetime.combine(date(start_year, 1, 1), datetime.min.time()).replace(tzinfo=timezone.utc)
257         end = datetime.combine(date(datetime.now().year, 1, 1), datetime.min.time()).replace(tzinfo=timezone.utc)
258     else:
259         start = datetime.combine(date(start_year, 1, 1), datetime.min.time()).replace(tzinfo=timezone.utc)
260         end = datetime.combine(date(end_year + 1, 1, 1), datetime.min.time()).replace(tzinfo=timezone.utc)
261
262     if (start.year == end.year - 1):
263         file_name = "yearly_booking_data_" + str(start.year) + ".csv"
264     else:
265         file_name = "yearly_booking_data_" + str(start.year) + "-" + str(end.year - 1) + ".csv"
266
267     with open(file_name, "w", newline="") as file:
268         file_writer = csv.writer(file)
269         cumulative_data = CumulativeData(file_writer)
270         file_writer.writerow(
271             [
272                 'ID',
273                 'Project',
274                 'Purpose',
275                 'User',
276                 'Collaborators',
277                 'Extensions Left',
278                 'Usage Days',
279                 'Start',
280                 'End'
281             ]
282         )
283
284         for booking in Booking.objects.filter(start__gte=start, start__lte=end):
285             filtered = False
286             booking_filter = [279]
287             user_filter = ["ParkerBerberian", "ssmith", "ahassick", "sbergeron", "jhodgdon", "rhodgdon", "aburch", "jspewock"]
288             user = booking.owner.username if booking.owner.username is not None else "None"
289
290             for b in booking_filter:
291                 if b == booking.id:
292                     filtered = True
293
294             for u in user_filter:
295                 if u == user:
296                     filtered = True
297             # trims time delta to the the specified year(s) if between years
298             usage_days = ((end if booking.end > end else booking.end) - (start if booking.start < start else booking.start)).days
299             collaborators = []
300
301             for c in booking.collaborators.all():
302                 collaborators.append(c.username)
303
304             if (not filtered):
305                 cumulative_data.account(booking, usage_days)
306                 file_writer.writerow([
307                     str(booking.id),
308                     str(booking.project),
309                     str(booking.purpose),
310                     str(booking.owner.username),
311                     ','.join(collaborators),
312                     str(booking.ext_count),
313                     str(usage_days),
314                     str(booking.start),
315                     str(booking.end)
316                 ])
317         cumulative_data.write_cumulative()
318
319
320 def map_cntt_interfaces(labid: str):
321     """
322     Use this during cntt migrations, call it with a host labid and it will change profiles for this host
323     as well as mapping its interfaces across. interface ens1f2 should have the mac address of interface eno50
324     as an invariant before calling this function
325     """
326     host = get_host(labid, "unh_iol")
327     host.profile = ResourceProfile.objects.get(name="HPE x86 CNTT")
328     host.save()
329     host = get_host(labid, "unh_iol")
330
331     for iface in host.interfaces.all():
332         new_ifprofile = None
333         if iface.profile.name == "ens1f2":
334             new_ifprofile = InterfaceProfile.objects.get(host=host.profile, name="eno50")
335         else:
336             new_ifprofile = InterfaceProfile.objects.get(host=host.profile, name=iface.profile.name)
337
338         iface.profile = new_ifprofile
339
340         iface.save()
341
342
343 def detect_leaked_hosts(labid="unh_iol"):
344     """
345     Use this to try to detect leaked hosts.
346     These hosts may still be in the process of unprovisioning,
347     but if they are not (or unprovisioning is frozen) then
348     these hosts are instead leaked
349     """
350     working_servers = Server.objects.filter(working=True, lab__lab_user__username=labid)
351     booked = working_servers.filter(booked=True)
352     filtered = booked
353     print_div()
354     print("In use now:")
355     for booking in Booking.objects.filter(end__gte=timezone.now()):
356         res_for_booking = booking.resource.get_resources()
357         print(res_for_booking)
358         for resource in res_for_booking:
359             filtered = filtered.exclude(id=resource.id)
360     print_div()
361     print("Possibly leaked:")
362     for host in filtered:
363         print(host)
364     print_div()
365     return filtered
366
367
368 def booking_for_host(host_labid: str, lab_username="unh_iol"):
369     """
370     Returns the booking that this server is a part of, if any.
371     Fails with an exception if no such booking exists
372
373     @host_labid is usually of the form `hpe3` or similar, is the labid of the Server (subtype of Resource) object
374
375     @lab_username: param of the form `unh_iol` or similar
376     """
377     server = Server.objects.get(lab__lab_user__username=lab_username, labid=host_labid)
378     booking = server.bundle.booking_set.first()
379     print_div()
380     print(booking)
381     print("id:", booking.id)
382     print("owner:", booking.owner)
383     print("job (id):", booking.job, "(" + str(booking.job.id) + ")")
384     print_div()
385     return booking
386
387
388 def force_release_booking(booking_id: int):
389     """
390     Takes a booking id and forces the booking to end whether or not the tasks have
391     completed normally.
392
393     Use with caution! Hosts may or may not be released depending on other underlying issues
394
395     @booking_id: the id of the Booking object to be released
396     """
397     booking = Booking.objects.get(id=booking_id)
398     job = booking.job
399     tasks = job.get_tasklist()
400     for task in tasks:
401         task.status = JobStatus.DONE
402         task.save()
403
404
405 def free_leaked_public_vlans(safety_buffer_days=2):
406     for lab in Lab.objects.all():
407         current_booking_set = Booking.objects.filter(end__gte=timezone.now() + timedelta(days=safety_buffer_days))
408
409         marked_nets = set()
410
411         for booking in current_booking_set:
412             for network in get_network_metadata(booking.id):
413                 marked_nets.add(network["vlan_id"])
414
415         for net in PublicNetwork.objects.filter(lab=lab).filter(in_use=True):
416             if net.vlan not in marked_nets:
417                 lab.vlan_manager.release_public_vlan(net.vlan)
418
419
420 def get_network_metadata(booking_id: int):
421     """
422     Takes a booking id and prints all (known) networks that are owned by it.
423     Returns an object of the form {<network name>: {"vlan_id": int, "netname": str <network name>, "public": bool <whether network is public/routable}}
424
425     @booking_id: the id of the Booking object to be queried
426     """
427     booking = Booking.objects.get(id=booking_id)
428     bundle = booking.resource
429     pnets = PhysicalNetwork.objects.filter(bundle=bundle).all()
430     metadata = {}
431     for pnet in pnets:
432         net = pnet.generic_network
433         mdata = {"vlan_id": pnet.vlan_id, "netname": net.name, "public": net.is_public}
434         metadata[net.name] = mdata
435     return metadata
436
437
438 def print_dict_pretty(a_dict):
439     """
440     admin_utils internal function
441     """
442
443     print(json.dumps(a_dict, sort_keys=True, indent=4))
444
445
446 def add_profile(data):
447     """
448     Used for adding a host profile to the dashboard
449
450     schema (of dict passed as "data" param):
451     {
452         "name": str
453         "description": str
454         "labs": [
455             str (lab username)
456         ]
457         "disks": {
458             <diskname> : {
459                 capacity: int (GiB)
460                 media_type: str ("SSD" or "HDD")
461                 interface: str ("sata", "sas", "ssd", "nvme", "scsi", or "iscsi")
462             }
463         }
464         interfaces: {
465             <intname>: {
466                 "speed": int (mbit)
467                 "nic_type": str ("onboard" or "pcie")
468                 "order": int (compared to the other interfaces, indicates the "order" that the ports are laid out)
469             }
470         }
471         cpus: {
472             cores: int (hardware threads count)
473             architecture: str (x86_64" or "aarch64")
474             cpus: int (number of sockets)
475             cflags: str
476         }
477         ram: {
478             amount: int (GiB)
479             channels: int
480         }
481     }
482     """
483     base_profile = ResourceProfile.objects.create(name=data['name'], description=data['description'])
484     base_profile.save()
485
486     for lab_username in data['labs']:
487         lab = Lab.objects.get(lab_user__username=lab_username)
488
489         base_profile.labs.add(lab)
490         base_profile.save()
491
492     for diskname in data['disks'].keys():
493         disk = data['disks'][diskname]
494
495         disk_profile = DiskProfile.objects.create(name=diskname, size=disk['capacity'], media_type=disk['media_type'], interface=disk['interface'], host=base_profile)
496         disk_profile.save()
497
498     for ifacename in data['interfaces'].keys():
499         iface = data['interfaces'][ifacename]
500
501         iface_profile = InterfaceProfile.objects.create(name=ifacename, speed=iface['speed'], nic_type=iface['nic_type'], order=iface['order'], host=base_profile)
502         iface_profile.save()
503
504     cpu = data['cpus']
505     cpu_prof = CpuProfile.objects.create(cores=cpu['cores'], architecture=cpu['architecture'], cpus=cpu['cpus'], cflags=cpu['cflags'], host=base_profile)
506     cpu_prof.save()
507
508     ram_prof = RamProfile.objects.create(amount=data['ram']['amount'], channels=data['ram']['channels'], host=base_profile)
509     ram_prof.save()
510
511
512 def make_default_template(resource_profile, image_id=None, template_name=None, connected_interface_names=None, interfaces_tagged=False, connected_interface_tagged=False, owner_username="root", lab_username="unh_iol", public=True, temporary=False, description=""):
513     """
514     Do not call this function without reading the related source code, it may have unintended effects.
515
516     Used for creating a default template from some host profile
517     """
518
519     if not resource_profile:
520         raise Exception("No viable continuation from none resource_profile")
521
522     if not template_name:
523         template_name = resource_profile.name
524
525     if not connected_interface_names:
526         connected_interface_names = [InterfaceProfile.objects.filter(host=resource_profile).first().name]
527         print("setting connected interface names to", connected_interface_names)
528
529     if not image_id:
530         image_id = Image.objects.filter(host_type=resource_profile).first().id
531
532     image = Image.objects.get(id=image_id)
533
534     base = ResourceTemplate.objects.create(
535         name=template_name,
536         xml="",
537         owner=User.objects.get(username=owner_username),
538         lab=Lab.objects.get(lab_user__username=lab_username), description=description,
539         public=public, temporary=temporary, copy_of=None)
540
541     rconf = ResourceConfiguration.objects.create(profile=resource_profile, image=image, template=base, is_head_node=True, name="opnfv_host")
542     rconf.save()
543
544     connected_interfaces = []
545
546     for iface_prof in InterfaceProfile.objects.filter(host=resource_profile).all():
547         iface_conf = InterfaceConfiguration.objects.create(profile=iface_prof, resource_config=rconf)
548
549         if iface_prof.name in connected_interface_names:
550             connected_interfaces.append(iface_conf)
551
552     network = Network.objects.create(name="public", bundle=base, is_public=True)
553
554     for iface in connected_interfaces:
555         connection = NetworkConnection.objects.create(network=network, vlan_is_tagged=interfaces_tagged)
556         connection.save()
557
558         iface.connections.add(connection)
559         print("adding connection to iface ", iface)
560         iface.save()
561         connection.save()
562
563
564 def add_server(profile, name, interfaces, lab_username="unh_iol", vendor="unknown", model="unknown"):
565     """
566     Used to enroll a new host of some profile
567
568     @profile: the ResourceProfile in question (by reference to a model object)
569
570     @name: the unique name of the server, currently indistinct from labid
571
572     @interfaces: interfaces should be dict from interface name (eg ens1f0) to dict of schema:
573         {
574             mac_address: <mac addr>,
575             bus_addr: <bus addr>, //this field is optional, "" is default
576         }
577
578     @lab_username: username of the lab to be added to
579
580     @vendor: vendor name of the host, such as "HPE" or "Gigabyte"
581
582     @model: specific model of the host, such as "DL380 Gen 9"
583
584     """
585     server = Server.objects.create(
586         bundle=None,
587         profile=profile,
588         config=None,
589         working=True,
590         vendor=vendor,
591         model=model,
592         labid=name,
593         lab=Lab.objects.get(lab_user__username=lab_username),
594         name=name,
595         booked=False)
596
597     for iface_prof in InterfaceProfile.objects.filter(host=profile).all():
598         mac_addr = interfaces[iface_prof.name]["mac_address"]
599         bus_addr = "unknown"
600         if "bus_addr" in interfaces[iface_prof.name].keys():
601             bus_addr = interfaces[iface_prof.name]["bus_addr"]
602
603         iface = Interface.objects.create(acts_as=None, profile=iface_prof, mac_address=mac_addr, bus_address=bus_addr)
604         iface.save()
605
606         server.interfaces.add(iface)
607         server.save()
608
609
610 def extend_booking(booking_id, days=0, hours=0, minutes=0, weeks=0):
611     """
612     Extend a booking by n <days, hours, minutes, weeks>
613
614     @booking_id: id of the booking
615
616     @days/@hours/@minutes/@weeks: the cumulative amount of delta to add to the length of the booking
617     """
618
619     booking = Booking.objects.get(id=booking_id)
620     booking.end = booking.end + timedelta(days=days, hours=hours, minutes=minutes, weeks=weeks)
621     booking.save()
622
623
624 def regenerate_cloud_configs(booking_id):
625     b = Booking.objects.get(id=booking_id)
626     for res in b.resource.get_resources():
627         res.config.cloud_init_files.set(res.config.cloud_init_files.filter(generated=False))  # careful!
628         res.config.save()
629         cif = GeneratedCloudConfig.objects.create(resource_id=res.labid, booking=b, rconfig=res.config)
630         cif.save()
631         cif = CloudInitFile.create(priority=0, text=cif.serialize())
632         cif.save()
633         res.config.cloud_init_files.add(cif)
634         res.config.save()
635
636
637 def set_job_new(job_id):
638     j = Job.objects.get(id=job_id)
639     b = j.booking
640     regenerate_cloud_configs(b.id)
641     for task in j.get_tasklist():
642         task.status = JobStatus.NEW
643         task.save()
644     j.status = JobStatus.NEW
645     j.save()
646
647
648 def docs(function=None, fulltext=False):
649     """
650     Print documentation for a given function in admin_utils.
651     Call without arguments for more information
652     """
653
654     fn = None
655
656     if isinstance(function, str):
657         try:
658             fn = globals()[function]
659         except KeyError:
660             print("Couldn't find a function by the given name")
661             return
662     elif callable(function):
663         fn = function
664     else:
665         print("docs(function: callable | str, fulltext: bool) was called with a 'function' that was neither callable nor a string name of a function")
666         print("usage: docs('some_function_in_admin_utils', fulltext=True)")
667         print("The 'fulltext' argument is used to choose if you want the complete source of the function printed. If this argument is false then you will only see the pydoc rendered documentation for the function")
668         return
669
670     if not fn:
671         print("couldn't find a function by that name")
672
673     if not fulltext:
674         print("Pydoc documents the function as such:")
675         print(pydoc.render_doc(fn))
676     else:
677         print("The full source of the function is this:")
678         print(inspect.getsource(fn))
679
680
681 def admin_functions():
682     """
683     List functions available to call within admin_utils
684     """
685
686     return [name for name, func in inspect.getmembers(sys.modules[__name__]) if (inspect.isfunction(func) and func.__module__ == __name__)]
687
688
689 print("Hint: call `docs(<function name>)` or `admin_functions()` for help on using the admin utils")
690 print("docs(<function name>) displays documentation on a given function")
691 print("admin_functions() lists all functions available to call within this module")