some more refactoring, now working

return the factoring removed in latest commit
add refactoring of image download code
fixed ldap config file text writing

Change-Id: I80fadf5c2eaba53fe12271f50dd156604d175030
Signed-off-by: Yaron Yogev <yaronyogev@gmail.com>

diff --git a/app/install/calipso-installer.py b/app/install/calipso-installer.py
index f16e31b..56f381e 100644 (file)
--- a/app/install/calipso-installer.py
+++ b/app/install/calipso-installer.py
@@ -17,7 +17,22 @@ import dockerpycreds
 import time
 import json
 
+
+C_MONGO_CONFIG = "/local_dir/calipso_mongo_access.conf"
+H_MONGO_CONFIG = "/home/calipso/calipso_mongo_access.conf"
+PYTHONPATH = "/home/scan/calipso_prod/app"
+C_LDAP_CONFIG = "/local_dir/ldap.conf"
+H_LDAP_CONFIG = "/home/calipso/ldap.conf"
+
+
 calipso_volume = {'/home/calipso': {'bind': '/local_dir', 'mode': 'rw'}}
+RESTART_POLICY = {"Name": "always"}
+
+# environment variables definitions
+PYTHON_PATH = "PYTHONPATH=" + PYTHONPATH
+MONGO_CONFIG = "MONGO_CONFIG=" + C_MONGO_CONFIG
+LDAP_CONFIG = "LDAP_CONFIG=" + C_LDAP_CONFIG
+LOG_LEVEL = "LOG_LEVEL=DEBUG"
 
 
 class MongoComm:
@@ -63,7 +78,8 @@ class MongoComm:
 
         def update(self, coll, doc, upsert=False):
             collection = self.client.calipso[coll]
-            doc_id = collection.update_one({'_id': doc['_id']},{'$set': doc},
+            doc_id = collection.update_one({'_id': doc['_id']},
+                                           {'$set': doc},
                                            upsert=upsert)
             return doc_id
 
@@ -78,6 +94,7 @@ DockerClient = docker.from_env()
 # DockerClient = \
 # docker.DockerClient(base_url='tcp://korlev-calipso-testing.cisco.com:2375')
 
+
 def copy_file(filename):
     c = MongoComm(args.hostname, args.dbuser, args.dbpassword, args.dbport)
     txt = open('db/'+filename+'.json')
@@ -87,11 +104,6 @@ def copy_file(filename):
     print("Copied", filename, "mongo doc_ids:\n\n", doc_id, "\n\n")
     time.sleep(1)
 
-C_MONGO_CONFIG = "/local_dir/calipso_mongo_access.conf"
-H_MONGO_CONFIG = "/home/calipso/calipso_mongo_access.conf"
-PYTHONPATH = "/home/scan/calipso_prod/app"
-C_LDAP_CONFIG = "/local_dir/ldap.conf"
-H_LDAP_CONFIG = "/home/calipso/ldap.conf"
 
 def container_started(name: str, print_message=True):
     found = DockerClient.containers.list(all=True, filters={"name": name})
@@ -101,26 +113,35 @@ def container_started(name: str, print_message=True):
               .format(name))
     return bool(found)
 
+
+def download_image(image_name):
+    image = DockerClient.images.list(all=True, name=image_name)
+
+    if image:
+        print(image, "exists...not downloading...")
+        return
+
+    print("image {} missing, "
+          "hold on while downloading first...\n"
+          .format(image_name))
+    image = DockerClient.images.pull(image_name)
+    print("Downloaded", image, "\n\n")
+
+
 # functions to check and start calipso containers:
 def start_mongo(dbport):
     name = "calipso-mongo"
     if container_started(name):
         return
     print("\nstarting container {}, please wait...\n".format(name))
-    image = DockerClient.images.list(all=True,
-                                     name="korenlev/calipso:mongo")
-    if image:
-        print(image, "exists...not downloading...")
-    else:
-        print("image korenlev/calipso:mongo missing, "
-              "hold on while downloading first...\n")
-        image = DockerClient.images.pull("korenlev/calipso:mongo")
-        print("Downloaded", image, "\n\n")
-    DockerClient.containers.run('korenlev/calipso:mongo',
+    image_name = "korenlev/calipso:mongo"
+    download_image(image_name)
+    mongo_ports = {'27017/tcp': dbport, '28017/tcp': 28017}
+    DockerClient.containers.run(image_name,
                                 detach=True,
                                 name=name,
-                                ports={'27017/tcp': dbport, '28017/tcp': 28017},
-                                restart_policy={"Name": "always"})
+                                ports=mongo_ports,
+                                restart_policy=RESTART_POLICY)
     # wait a bit till mongoDB is up before starting to copy the json files
     # from 'db' folder:
     time.sleep(5)
@@ -158,143 +179,112 @@ def start_mongo(dbport):
     # some other docs are filled later by scanning, logging
     # and monitoring
 
+
 def start_listen():
     name = "calipso-listen"
     if container_started(name):
         return
     print("\nstarting container {}...\n".format(name))
-    image = DockerClient.images.list(all=True,
-                                     name="korenlev/calipso:listen")
-    if image:
-        print(image, "exists...not downloading...")
-    else:
-        print("image korenlev/calipso:listen missing, "
-              "hold on while downloading first...\n")
-        image = DockerClient.images.pull("korenlev/calipso:listen")
-        print("Downloaded", image, "\n\n")
-    listencontainer = DockerClient.containers.run('korenlev/calipso:listen',
-                                                  detach=True,
-                                                  name=name,
-                                                  ports={'22/tcp': 50022},
-                                                  restart_policy={"Name": "always"},
-                                                  environment=["PYTHONPATH=" + PYTHONPATH,
-                                                               "MONGO_CONFIG=" + C_MONGO_CONFIG],
-                                                  volumes=calipso_volume)
+    image_name = "korenlev/calipso:listen"
+    download_image(image_name)
+    ports = {'22/tcp': 50022}
+    DockerClient.containers.run(image_name,
+                                detach=True,
+                                name=name,
+                                ports=ports,
+                                restart_policy=RESTART_POLICY,
+                                environment=[PYTHON_PATH, MONGO_CONFIG],
+                                volumes=calipso_volume)
+
 
 def start_ldap():
     name = "calipso-ldap"
     if container_started(name):
         return
     print("\nstarting container {}...\n".format(name))
-    image = DockerClient.images.list(all=True,
-                                     name="korenlev/calipso:ldap")
-    if image:
-        print(image, "exists...not downloading...")
-    else:
-        print("image korenlev/calipso:ldap missing, "
-              "hold on while downloading first...\n")
-        image = DockerClient.images.pull("korenlev/calipso:ldap")
-        print("Downloaded", image, "\n\n")
-    ldapcontainer = DockerClient.containers.run('korenlev/calipso:ldap',
-                                                detach=True,
-                                                name=name,
-                                                ports={'389/tcp': 389, '389/udp': 389},
-                                                restart_policy={"Name": "always"},
-                                                volumes=calipso_volume)
+    image_name = "korenlev/calipso:ldap"
+    download_image(image_name)
+    ports = {'389/tcp': 389, '389/udp': 389}
+    DockerClient.containers.run(image_name,
+                                detach=True,
+                                name=name,
+                                ports=ports,
+                                restart_policy=RESTART_POLICY,
+                                volumes=calipso_volume)
+
 
 def start_api():
     name = "calipso-api"
     if container_started(name):
         return
     print("\nstarting container {}...\n".format(name))
-    image = DockerClient.images.list(all=True,
-                                     name="korenlev/calipso:api")
-    if image:
-        print(image, "exists...not downloading...")
-    else:
-        print("image korenlev/calipso:api missing,"
-              " hold on while downloading first...\n")
-        image = DockerClient.images.pull("korenlev/calipso:api")
-        print("Downloaded", image, "\n\n")
-    apicontainer = DockerClient.containers.run('korenlev/calipso:api',
-                                               detach=True,
-                                               name=name,
-                                               ports={'8000/tcp': 8000, '22/tcp': 40022},
-                                               restart_policy={"Name": "always"},
-                                               environment=["PYTHONPATH=" + PYTHONPATH,
-                                                            "MONGO_CONFIG=" + C_MONGO_CONFIG,
-                                                            "LDAP_CONFIG=" + C_LDAP_CONFIG,
-                                                            "LOG_LEVEL=DEBUG"],
-                                               volumes=calipso_volume)
+    image_name = "korenlev/calipso:api"
+    download_image(image_name)
+    api_ports = {'8000/tcp': 8000, '22/tcp': 40022}
+    DockerClient.containers.run(image_name,
+                                detach=True,
+                                name=name,
+                                ports=api_ports,
+                                restart_policy=RESTART_POLICY,
+                                environment=[PYTHON_PATH, MONGO_CONFIG,
+                                             LDAP_CONFIG,
+                                             LOG_LEVEL],
+                                volumes=calipso_volume)
+
 
 def start_scan():
     name = "calipso-scan"
     if container_started(name):
         return
     print("\nstarting container {}...\n".format(name))
-    image = DockerClient.images.list(all=True,
-                                     name="korenlev/calipso:scan")
-    if image:
-        print(image, "exists...not downloading...")
-    else:
-        print("image korenlev/calipso:scan missing, "
-              "hold on while downloading first...\n")
-        image = DockerClient.images.pull("korenlev/calipso:scan")
-        print("Downloaded", image, "\n\n")
-    scancontainer = DockerClient.containers.run('korenlev/calipso:scan',
-                                                detach=True,
-                                                name=name,
-                                                ports={'22/tcp': 30022},
-                                                restart_policy={"Name": "always"},
-                                                environment=["PYTHONPATH=" + PYTHONPATH,
-                                                             "MONGO_CONFIG=" + C_MONGO_CONFIG],
-                                                volumes=calipso_volume)
+    image_name = "korenlev/calipso:scan"
+    download_image(image_name)
+    ports = {'22/tcp': 30022}
+    DockerClient.containers.run(image_name,
+                                detach=True,
+                                name=name,
+                                ports=ports,
+                                restart_policy=RESTART_POLICY,
+                                environment=[PYTHON_PATH, MONGO_CONFIG],
+                                volumes=calipso_volume)
+
 
 def start_sensu():
     name = "calipso-sensu"
     if container_started(name):
         return
     print("\nstarting container {}...\n".format(name))
-    image = DockerClient.images.list(all=True,
-                                     name="korenlev/calipso:sensu")
-    if image:
-        print(image, "exists...not downloading...")
-    else:
-        print("image korenlev/calipso:sensu missing,"
-              " hold on while downloading first...\n")
-        image = DockerClient.images.pull("korenlev/calipso:sensu")
-        print("Downloaded", image, "\n\n")
-    sensucontainer = DockerClient.containers.run('korenlev/calipso:sensu',
-                                                 detach=True,
-                                                 name=name,
-                                                 ports={'22/tcp': 20022, '3000/tcp': 3000, '4567/tcp': 4567,
-                                                        '5671/tcp': 5671, '15672/tcp': 15672},
-                                                 restart_policy={"Name": "always"},
-                                                 environment=["PYTHONPATH=" + PYTHONPATH],
-                                                 volumes=calipso_volume)
+    image_name = "korenlev/calipso:sensu"
+    download_image(image_name)
+    sensu_ports = {'22/tcp': 20022, '3000/tcp': 3000, '4567/tcp': 4567,
+                   '5671/tcp': 5671, '15672/tcp': 15672}
+    DockerClient.containers.run(image_name,
+                                detach=True,
+                                name=name,
+                                ports=sensu_ports,
+                                restart_policy=RESTART_POLICY,
+                                environment=[PYTHON_PATH],
+                                volumes=calipso_volume)
+
 
 def start_ui(host, dbuser, dbpassword, webport, dbport):
     name = "calipso-ui"
     if container_started(name):
         return
     print("\nstarting container {}...\n".format(name))
-    image = DockerClient.images.list(all=True, name="korenlev/calipso:ui")
-    if image:
-        print(image, "exists...not downloading...")
-    else:
-        print("image korenlev/calipso:ui missing, "
-              "hold on while downloading first...\n")
-        image = DockerClient.images.pull("korenlev/calipso:ui")
-        print("Downloaded", image, "\n\n")
-    uicontainer = DockerClient.containers.run('korenlev/calipso:ui',
-                                              detach=True,
-                                              name=name,
-                                              ports={'3000/tcp': webport},
-                                              restart_policy={"Name": "always"},
-                                              environment=["ROOT_URL=http://{}:{}".format(host, str(webport)),
-                                                           "MONGO_URL=mongodb://{}:{}@{}:{}/calipso".format(
-                                                               dbuser, dbpassword, host, str(dbport)),
-                                                           "LDAP_CONFIG=" + C_LDAP_CONFIG])
+    image_name = "korenlev/calipso:ui"
+    download_image(image_name)
+    root_url = "ROOT_URL=http://{}:{}".format(host, str(webport))
+    mongo_url = "MONGO_URL=mongodb://{}:{}@{}:{}/calipso"\
+                .format(dbuser, dbpassword, host, str(dbport))
+    ports = {'3000/tcp': webport}
+    DockerClient.containers.run(image_name,
+                                detach=True,
+                                name=name,
+                                ports=ports,
+                                restart_policy=RESTART_POLICY,
+                                environment=[root_url, mongo_url, LDAP_CONFIG])
+
 
 # check and stop a calipso container by given name
 def container_stop(container_name):
@@ -312,6 +302,7 @@ def container_stop(container_name):
     print("removing container name", c.name, "...\n")
     c.remove()
 
+
 # parser for getting optional command arguments:
 parser = argparse.ArgumentParser()
 parser.add_argument("--hostname",
@@ -365,20 +356,27 @@ while container != "all" and container not in container_names:
 if action == "start":
     # building /home/calipso/calipso_mongo_access.conf and
     # /home/calipso/ldap.conf files, per the arguments:
-    calipso_mongo_access_text =\
-        "server " + args.hostname +\
-        "\nuser " + args.dbuser +\
-        "\npwd " + args.dbpassword +\
-        "\nauth_db calipso"
-    ldap_text =\
-        "user admin" +\
-        "\npassword password" +\
-        "\nurl ldap://" + args.hostname + ":389" +\
-        "\nuser_id_attribute CN" + "\nuser_pass_attribute userpassword" +\
-        "\nuser_objectclass inetOrgPerson" +\
-        "\nuser_tree_dn OU=Users,DC=openstack,DC=org" + "\nquery_scope one" +\
-        "\ntls_req_cert allow" +\
-        "\ngroup_member_attribute member"
+    calipso_mongo_access_text = \
+        "server {}\n" \
+        "user {}\n" \
+        "pwd {}\n" \
+        "auth_db calipso" \
+        .format(args.hostname, args.dbuser, args.dbpassword)
+    LDAP_PWD_ATTRIBUTE = "password password"
+    LDAP_USER_PWD_ATTRIBUTE = "password"
+    ldap_text = \
+        "user admin\n" + \
+        "{}\n" + \
+        "url ldap://{}:389\n" + \
+        "user_id_attribute CN\n" + \
+        "user_pass_attribute {}\n" + \
+        "user_objectclass inetOrgPerson\n" + \
+        "user_tree_dn OU=Users,DC=openstack,DC=org\n" + \
+        "query_scope one\n" + \
+        "tls_req_cert allow\n" + \
+        "group_member_attribute member"
+    ldap_text = ldap_text.format(LDAP_PWD_ATTRIBUTE, args.hostname,
+                                 LDAP_USER_PWD_ATTRIBUTE)
     print("creating default", H_MONGO_CONFIG, "file...\n")
     calipso_mongo_access_file = open(H_MONGO_CONFIG, "w+")
     time.sleep(1)