Add nginx services control functionality

- Modify nginx server template to use clover-controller
in clover-system namespace for file upload
- Add ability to set nginx server paths and move uploaded files
in clover-controller and set/get file upload metadata
- Add cloverctl commands to set nginx server and lb configurations
- Add example yaml files to configure nginx server and
lb v1/v2 from cloverctl
- Modify service definition for http-lb versions to distinguish
the two versions for GRPC messaging from clover-controller in SDC
deployment yaml

Change-Id: I5c6866c5ff3de358939c58ea8c0bde64a69c6eca
Signed-off-by: earrage <eddie.arrage@huawei.com>

diff --git a/clover/cloverctl/src/cloverctl/cmd/set_nginx.go b/clover/cloverctl/src/cloverctl/cmd/set_nginx.go
new file mode 100644 (file)
index 0000000..e7e65c2
--- /dev/null
+++ b/clover/cloverctl/src/cloverctl/cmd/set_nginx.go
@@ -0,0 +1,76 @@
+// Copyright (c) Authors of Clover
+//
+// All rights reserved. This program and the accompanying materials
+// are made available under the terms of the Apache License, Version 2.0
+// which accompanies this distribution, and is available at
+// http://www.apache.org/licenses/LICENSE-2.0
+
+package cmd
+
+import (
+    "fmt"
+    "io/ioutil"
+    "gopkg.in/resty.v1"
+    "github.com/spf13/cobra"
+    "github.com/ghodss/yaml"
+)
+
+
+var setserverCmd = &cobra.Command{
+    Use:   "server",
+    Short: "Modify nginx server configuration",
+    Long: ``,
+    Run: func(cmd *cobra.Command, args []string) {
+        setNginx("server")
+    },
+}
+
+var setlbCmd = &cobra.Command{
+    Use:   "lb",
+    Short: "Modify nginx lb configuration",
+    Long: ``,
+    Run: func(cmd *cobra.Command, args []string) {
+        setNginx("lb")
+    },
+}
+
+func init() {
+    setCmd.AddCommand(setserverCmd)
+    setserverCmd.Flags().StringVarP(&cloverFile, "file", "f", "", "Input yaml file for server config")
+    setserverCmd.MarkFlagRequired("file")
+
+    setCmd.AddCommand(setlbCmd)
+    setlbCmd.Flags().StringVarP(&cloverFile, "file", "f", "", "Input yaml file for lb config")
+    setlbCmd.MarkFlagRequired("file")
+
+}
+
+func setNginx(nginx_service string) {
+
+    url := ""
+    if nginx_service == "server" {
+        url = controllerIP + "/nginx/server"
+    } else {
+        url = controllerIP + "/nginx/lb"
+    }
+
+    in, err := ioutil.ReadFile(cloverFile)
+    if err != nil {
+        fmt.Println("Please specify a valid yaml file")
+        return
+    }
+    out_json, err := yaml.YAMLToJSON(in)
+    if err != nil {
+        panic(err.Error())
+    }
+    resp, err := resty.R().
+    SetHeader("Content-Type", "application/json").
+    SetBody(out_json).
+    Post(url)
+    if err != nil {
+        panic(err.Error())
+    }
+    fmt.Printf("\n%v\n", resp)
+
+
+}

diff --git a/clover/cloverctl/src/cloverctl/yaml/lbv1.yaml b/clover/cloverctl/src/cloverctl/yaml/lbv1.yaml
new file mode 100644 (file)
index 0000000..3bb841d
--- /dev/null
+++ b/clover/cloverctl/src/cloverctl/yaml/lbv1.yaml
@@ -0,0 +1,9 @@
+server_port: "9180"
+server_name: "http-lb-v1"
+lb_name: "http-lb-v1.default"
+lb_group: "cloverlb"
+lb_path: "/"
+lb_list:
+    - url: "clover-server1:9180"
+    - url: "clover-server2:9180"
+    - url: "clover-server3:9180"

diff --git a/clover/cloverctl/src/cloverctl/yaml/lbv2.yaml b/clover/cloverctl/src/cloverctl/yaml/lbv2.yaml
new file mode 100644 (file)
index 0000000..7df9376
--- /dev/null
+++ b/clover/cloverctl/src/cloverctl/yaml/lbv2.yaml
@@ -0,0 +1,8 @@
+server_port: "9180"
+server_name: "http-lb-v2"
+lb_name: "http-lb-v2.default"
+lb_group: "cloverlb"
+lb_path: "/"
+lb_list:
+    - url: "clover-server4:9180"
+    - url: "clover-server5:9180"

diff --git a/clover/cloverctl/src/cloverctl/yaml/server.yaml b/clover/cloverctl/src/cloverctl/yaml/server.yaml
new file mode 100644 (file)
index 0000000..b44779a
--- /dev/null
+++ b/clover/cloverctl/src/cloverctl/yaml/server.yaml
@@ -0,0 +1,16 @@
+server_port: "9180"
+server_name: "clover-server1"
+site_root: "/var/www/html"
+site_index: index.html
+upload_path_config: "/upload"
+upload_path_test: "/upload_test"
+locations:
+    - uri_match: "/clover/testurl"
+      directive: "try_files $uri @default2"
+      path: "/clover/testurl"
+    - uri_match: "/test"
+      directive: "try_files $uri @default1"
+      path: "/test"
+files:
+    - src_file: "/var/www/html/upload/0000000001"
+      dest_file: "var/www/html/clover/testurl/file1.png"

diff --git a/clover/controller/__init__.py b/clover/controller/__init__.py
index d67a6c0..e69de29 100644 (file)
--- a/clover/controller/__init__.py
+++ b/clover/controller/__init__.py
@@ -1,11 +0,0 @@
-from flask import Flask, Response
-
-
-app = Flask(__name__)
-
-@app.route("/")
-def index():
-    return Response("It works!"), 200
-
-if __name__ == "__main__":
-    app.run(debug=True)

diff --git a/clover/controller/control/__init__.py b/clover/controller/control/__init__.py
index d67a6c0..e69de29 100644 (file)
--- a/clover/controller/control/__init__.py
+++ b/clover/controller/control/__init__.py
@@ -1,11 +0,0 @@
-from flask import Flask, Response
-
-
-app = Flask(__name__)
-
-@app.route("/")
-def index():
-    return Response("It works!"), 200
-
-if __name__ == "__main__":
-    app.run(debug=True)

diff --git a/clover/controller/control/api/file_upload.py b/clover/controller/control/api/file_upload.py
index a479c30..a532bc8 100644 (file)
--- a/clover/controller/control/api/file_upload.py
+++ b/clover/controller/control/api/file_upload.py
@@ -5,24 +5,50 @@
 # which accompanies this distribution, and is available at
 # http://www.apache.org/licenses/LICENSE-2.0
 
-from flask import Blueprint, request, Response
+from flask import Blueprint, request, Response, jsonify
 import redis
 import logging
 
 file_upload = Blueprint('file_upload', __name__)
 
-HOST_IP = 'redis'
+HOST_IP = 'redis.default'
 
 
 @file_upload.route("/upload", methods=['GET', 'POST'])
-def upload_meta():
+def set_upload_metadata():
     try:
+        response = "Uploaded file(s) successfully"
         content = request.form
+        logging.debug(content)
         r = redis.StrictRedis(host=HOST_IP, port=6379, db=0)
-        response = content.get('upload.name')
-        r.set('upload_meta', response)
+        # Try various variable names
+        upload_var_names = ['upload', 'file1', 'file2', 'file3',
+                            'file4', 'file5', 'file6']
+        for n in upload_var_names:
+            try:
+                param_name = n + '.name'
+                meta_name = content.get(param_name)
+                if meta_name:
+                    param_path = n + '.path'
+                    param_server = n + '.server'
+                    meta_path = content.get(param_path)
+                    meta_server = content.get(param_server)
+                    entry = meta_name + ':' + meta_path + ':' + meta_server
+                    r.sadd('upload_metadata', entry)
+            except Exception as e:
+                print("no metadata")
     except Exception as e:
         logging.debug(e)
-        r.set('upload_meta', "failure")
         return Response('Unable to write file metadata to redis', status=400)
     return response
+
+
+@file_upload.route("/upload/get", methods=['GET', 'POST'])
+def get_upload_metadata():
+    try:
+        r = redis.StrictRedis(host=HOST_IP, port=6379, db=0)
+        response = jsonify(list(r.smembers('upload_metadata')))
+    except Exception as e:
+        logging.debug(e)
+        return Response('Unable to retrieve upload metadata', status=400)
+    return response

diff --git a/clover/controller/control/api/nginx.py b/clover/controller/control/api/nginx.py
index ba99b94..a7d902d 100644 (file)
--- a/clover/controller/control/api/nginx.py
+++ b/clover/controller/control/api/nginx.py
@@ -15,28 +15,28 @@ import logging
 nginx = Blueprint('nginx', __name__)
 
 
-@nginx.route("/nginx/slb", methods=['GET', 'POST'])
-def slblist():
+@nginx.route("/nginx/lb", methods=['GET', 'POST'])
+def modify_lb():
     grpc_port = '50054'
     try:
         p = request.json
         try:
-            slb_name = p['slb_name']
-            nginx_grpc = slb_name + ':' + grpc_port
+            lb_name = p['lb_name']
+            nginx_grpc = lb_name + ':' + grpc_port
             channel = grpc.insecure_channel(nginx_grpc)
             stub = nginx_pb2_grpc.ControllerStub(channel)
 
             s_list = []
-            for s in p['slb_list']:
+            for s in p['lb_list']:
                 s_list.append(s['url'])
-            slb_list = pickle.dumps(s_list)
+            lb_list = pickle.dumps(s_list)
             response = stub.ModifyLB(nginx_pb2.ConfigLB(
                 server_port=p['server_port'], server_name=p['server_name'],
-                slb_list=slb_list,
-                slb_group=p['slb_group'], lb_path=p['lb_path']))
+                slb_list=lb_list,
+                slb_group=p['lb_group'], lb_path=p['lb_path']))
         except (KeyError, ValueError) as e:
             logging.debug(e)
-            return Response('Invalid value in test plan json/yaml', status=400)
+            return Response('Invalid value in LB json/yaml', status=400)
     except Exception as e:
         logging.debug(e)
         if e.__class__.__name__ == "_Rendezvous":
@@ -46,6 +46,37 @@ def slblist():
     return response.message
 
 
+@nginx.route("/nginx/server", methods=['GET', 'POST'])
+def modify_server():
+    grpc_port = '50054'
+    try:
+        p = request.json
+        try:
+            server_name = p['server_name']
+            nginx_grpc = server_name + ':' + grpc_port
+            channel = grpc.insecure_channel(nginx_grpc)
+            stub = nginx_pb2_grpc.ControllerStub(channel)
+
+            locations = pickle.dumps(p['locations'])
+            files = pickle.dumps(p['files'])
+            response = stub.ModifyServer(nginx_pb2.ConfigServer(
+                server_port=p['server_port'], server_name=p['server_name'],
+                site_root=p['site_root'], locations=locations, files=files,
+                site_index=p['site_index'],
+                upload_path_config=p['upload_path_config'],
+                upload_path_test=p['upload_path_test']))
+        except (KeyError, ValueError) as e:
+            logging.debug(e)
+            return Response('Invalid value in server json/yaml', status=400)
+    except Exception as e:
+        logging.debug(e)
+        if e.__class__.__name__ == "_Rendezvous":
+            return Response("Error connecting to server via gRPC", status=400)
+        else:
+            return Response("Error modifying server", status=400)
+    return response.message
+
+
 @nginx.route("/nginx/test")
 def test():
     return "<h1 style='color:blue'>Nginx API Test Response</h1>"

diff --git a/samples/scenarios/service_delivery_controller.yaml b/samples/scenarios/service_delivery_controller.yaml
index b9c3506..34e2df0 100644 (file)
--- a/samples/scenarios/service_delivery_controller.yaml
+++ b/samples/scenarios/service_delivery_controller.yaml
@@ -226,6 +226,7 @@ spec:
       labels:
         app: http-lb
         version: v1
+        name: http-lb-v1
     spec:
       containers:
         - name: http-lb
@@ -247,6 +248,7 @@ spec:
       labels:
         app: http-lb
         version: v2
+        name: http-lb-v2
     spec:
       containers:
         - name: http-lb
@@ -263,13 +265,33 @@ metadata:
     app: http-lb
 spec:
   ports:
-  - port: 50054
-    name: grpc
   - port: 9180
     name: http
   selector:
     app: http-lb
 ---
+apiVersion: v1
+kind: Service
+metadata:
+  name: http-lb-v1
+spec:
+  ports:
+  - port: 50054
+    name: grpc
+  selector:
+    name: http-lb-v1
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: http-lb-v2
+spec:
+  ports:
+  - port: 50054
+    name: grpc
+  selector:
+    name: http-lb-v2
+---
 apiVersion: extensions/v1beta1
 kind: Deployment
 metadata:

diff --git a/samples/scenarios/service_delivery_controller_opnfv.yaml b/samples/scenarios/service_delivery_controller_opnfv.yaml
index ceba36f..cb4743f 100644 (file)
--- a/samples/scenarios/service_delivery_controller_opnfv.yaml
+++ b/samples/scenarios/service_delivery_controller_opnfv.yaml
@@ -226,6 +226,7 @@ spec:
       labels:
         app: http-lb
         version: v1
+        name: http-lb-v1
     spec:
       containers:
         - name: http-lb
@@ -247,6 +248,7 @@ spec:
       labels:
         app: http-lb
         version: v2
+        name: http-lb-v2
     spec:
       containers:
         - name: http-lb
@@ -263,13 +265,33 @@ metadata:
     app: http-lb
 spec:
   ports:
-  - port: 50054
-    name: grpc
   - port: 9180
     name: http
   selector:
     app: http-lb
 ---
+apiVersion: v1
+kind: Service
+metadata:
+  name: http-lb-v1
+spec:
+  ports:
+  - port: 50054
+    name: grpc
+  selector:
+    name: http-lb-v1
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: http-lb-v2
+spec:
+  ports:
+  - port: 50054
+    name: grpc
+  selector:
+    name: http-lb-v2
+---
 apiVersion: extensions/v1beta1
 kind: Deployment
 metadata:

diff --git a/samples/services/nginx/docker/grpc/templates/server.template b/samples/services/nginx/docker/grpc/templates/server.template
index c1582fa..2ca5410 100644 (file)
--- a/samples/services/nginx/docker/grpc/templates/server.template
+++ b/samples/services/nginx/docker/grpc/templates/server.template
@@ -116,7 +116,7 @@ http {
             }
 
             location @return_config {
-                proxy_pass http://clover-controller:80;
+                proxy_pass http://clover-controller.clover-system:80;
                 proxy_http_version 1.1;
             }