Manually merge CI files

Signed-off-by: Sawyer Bergeron <sbergeron@iol.unh.edu>
Change-Id: Ic63d5da699578007ef2f2cc373350ded06c66971

diff --git a/requirements.txt b/requirements.txt
index b34dd1e..d93ac7c 100644 (file)
--- a/requirements.txt
+++ b/requirements.txt
@@ -16,3 +16,4 @@ django-fernet-fields==0.6
 pyyaml==3.13
 pytz==2018.5
 mozilla-django-oidc==1.2.3
+deepmerge==0.3

diff --git a/src/api/views.py b/src/api/views.py
index 79da84c..5af3d69 100644 (file)
--- a/src/api/views.py
+++ b/src/api/views.py
@@ -37,6 +37,7 @@ from resource_inventory.models import (
 )
 
 import json
+from deepmerge import Merger
 
 """
 API views.
@@ -263,7 +264,26 @@ def resource_ci_userdata(request, lab_name="", job_id="", resource_id="", file_i
     except ObjectDoesNotExist:
         return HttpResponseNotFound("Could not find a matching resource by id " + str(resource_id))
 
-    return HttpResponse(cifile.text, status=200)
+    text = cifile.text
+
+    prepended_text = "#cloud-config\n"
+    #mstrat = CloudInitFile.merge_strategy()
+    #prepended_text = prepended_text + yaml.dump({"merge_strategy": mstrat}) + "\n"
+    #print("in cloudinitfile create")
+    text = prepended_text + text
+    cloud_dict = {
+            "datasource": {
+                "None": {
+                    "metadata": {
+                        "instance-id": str(uuid.uuid4())
+                    },
+                    "userdata_raw": text,
+                },
+            },
+            "datasource_list": ["None"],
+        }
+
+    return HttpResponse(yaml.dump(cloud_dict), status=200)
 
 @csrf_exempt
 def resource_ci_metadata(request, lab_name="", job_id="", resource_id="", file_id=0):
@@ -276,7 +296,36 @@ def resource_ci_userdata_directory(request, lab_name="", job_id="", resource_id=
     files = resource.config.cloud_init_files
     files = [{"id": file.id, "priority": file.priority} for file in files.order_by("priority").all()]
 
-    return HttpResponse(json.dumps(files), status=200)
+    d = {
+            'merge_failures': []
+        }
+
+    merger = Merger(
+            [
+                (list, ["append"]),
+                (dict, ["merge"]),
+            ],
+            ["override"], # fallback
+            ["override"], # if types conflict (shouldn't happen in CI, but handle case)
+        )
+
+    for file in files.order_by("priority").all():
+        try:
+            other_dict = yaml.load(file.text)
+            if not (type(d) is dict):
+                raise Exception("CI file was valid yaml but was not a dict")
+
+            merger.merge(d, other_dict)
+        except Exception as e:
+            # if fail to merge, then just skip
+            print("Failed to merge file in, as it had invalid content:", file.id)
+            print("File text was:")
+            print(file.text)
+            d['merge_failures'].append({file.id: str(e)})
+
+    file = CloudInitFile.create(text=yaml.dump(d), priority=0)
+
+    return HttpResponse(json.dumps([{"id": file.id, "priority": file.priority}]), status=200)
 
 
 def new_jobs(request, lab_name=""):

diff --git a/src/booking/quick_deployer.py b/src/booking/quick_deployer.py
index 2ab18a6..261b095 100644 (file)
--- a/src/booking/quick_deployer.py
+++ b/src/booking/quick_deployer.py
@@ -219,6 +219,12 @@ def create_from_form(form, request):
     global_cloud_config = None if not form.cleaned_data['global_cloud_config'] else form.cleaned_data['global_cloud_config']
 
     if global_cloud_config:
+        try:
+            d = yaml.load(global_cloud_config)
+            if not (type(d) is dict):
+                raise Exception("CI file was valid yaml but was not a dict")
+        except Exception as e:
+            raise ValidationError("The provided Cloud Config is not valid yaml, please refer to the Cloud Init documentation for expected structure")
         print("about to create global cloud config")
         global_cloud_config = CloudInitFile.create(text=global_cloud_config, priority=CloudInitFile.objects.count())
         print("made global cloud config")

diff --git a/src/resource_inventory/models.py b/src/resource_inventory/models.py
index 71046a8..6117035 100644 (file)
--- a/src/resource_inventory/models.py
+++ b/src/resource_inventory/models.py
@@ -169,23 +169,23 @@ class CloudInitFile(models.Model):
 
     @classmethod
     def create(cls, text="", priority=0):
-        prepended_text = "#cloud-config\n"
-        mstrat = CloudInitFile.merge_strategy()
-        prepended_text = prepended_text + yaml.dump({"merge_strategy": mstrat}) + "\n"
-        print("in cloudinitfile create")
-        text = prepended_text + text
-        cloud_dict = {
-                "datasource": {
-                    "None": {
-                        "metadata": {
-                            "instance-id": str(uuid.uuid4())
-                        },
-                        "userdata_raw": text,
-                    },
-                },
-                "datasource_list": ["None"],
-            }
-        return CloudInitFile.objects.create(priority=priority, text=yaml.dump(cloud_dict))
+        #prepended_text = "#cloud-config\n"
+        #mstrat = CloudInitFile.merge_strategy()
+        #prepended_text = prepended_text + yaml.dump({"merge_strategy": mstrat}) + "\n"
+        #print("in cloudinitfile create")
+        #text = prepended_text + text
+        #cloud_dict = {
+        #        "datasource": {
+        #            "None": {
+        #                "metadata": {
+        #                    "instance-id": str(uuid.uuid4())
+        #                },
+        #                "userdata_raw": text,
+        #            },
+        #        },
+        #        "datasource_list": ["None"],
+        #    }
+        return CloudInitFile.objects.create(priority=priority, text=text)
 
 class ResourceTemplate(models.Model):
     """