Fix Nginx lb in k8s/istio

- Provide workaround to make nginx lb work properly
- nginx_client sample can modify default load balancing
from three to two servers at runtime
- Ensure port 9180 is used for default deploy for lb and
servers
- Modify render_yaml to specify deploy_name so that
clover-server1, 2, 3 can be used for default lb config
- Ensure proxy template is aligned to lb to allow the
source IP from originating host to be propagated to final
destination

- Fix default nginx proxy server_name to 'proxy-access-control'
and change default proxy destination to 'http-lb'
- Split lb service_type to 'lbv1' and 'lbv2' to provide an example
of how to modify the run-time configuration of the load balancer
after deployment - modify http-lb-v2 to use clover-server4/5 instead
of the defualt clover-server1/2/3 - modify http-lb-v1 to use
clover-server1/2 instead of 1/2/3
- Aligned pod IP retrival method with nginx_client.py

Change-Id: I73fa60a69c93ae1e82a477ef6283c00f67a21360
Signed-off-by: Eddie Arrage <eddie.arrage@huawei.com>
(cherry picked from commit 5e213108dfade163a85cff9b9156de9bd2c18887)

diff --git a/samples/services/nginx/docker/grpc/nginx_client.py b/samples/services/nginx/docker/grpc/nginx_client.py
index dfefb08..7a07464 100644 (file)
--- a/samples/services/nginx/docker/grpc/nginx_client.py
+++ b/samples/services/nginx/docker/grpc/nginx_client.py
@@ -16,18 +16,24 @@ import nginx_pb2
 import nginx_pb2_grpc
 
 
-def run(args):
+def run(args, grpc_port='50054'):
     # get pod ip for grpc
     pod_ip = get_podip(args['service_name'])
     if pod_ip == '':
-        return "Cant find service with name: {}".format(args['service_name'])
-    nginx_grpc = pod_ip + ':50054'
+        return "Cant find service: {}".format(args['service_name'])
+    nginx_grpc = pod_ip + ':' + grpc_port
     channel = grpc.insecure_channel(nginx_grpc)
     stub = nginx_pb2_grpc.ControllerStub(channel)
 
     # modify config
-    if args['service_type'] == 'lb':
-        modify_lb(stub)
+    if args['service_type'] == 'lbv1':
+        slb_list = pickle.dumps(
+                    ['clover-server1:9180', 'clover-server2:9180'])
+        modify_lb(stub, slb_list)
+    if args['service_type'] == 'lbv2':
+        slb_list = pickle.dumps(
+                    ['clover-server4:9180', 'clover-server5:9180'])
+        modify_lb(stub, slb_list)
     elif args['service_type'] == 'proxy':
         modify_proxy(stub)
     elif args['service_type'] == 'server':
@@ -38,14 +44,16 @@ def run(args):
 
 
 def get_podip(pod_name):
-    config.load_kube_config()
-    v1 = client.CoreV1Api()
-    ret = v1.list_pod_for_all_namespaces(watch=False)
     ip = ''
-    for i in ret.items:
-        if i.metadata.name.lower().find(pod_name.lower()) != -1:
-            print(i.status.pod_ip)
-            ip = i.status.pod_ip
+    if pod_name != '':
+        config.load_kube_config()
+        v1 = client.CoreV1Api()
+        ret = v1.list_pod_for_all_namespaces(watch=False)
+        for i in ret.items:
+            if i.metadata.name.lower().find(pod_name.lower()) != -1:
+                print("Pod IP: {}".format(i.status.pod_ip))
+                ip = i.status.pod_ip
+                return str(ip)
     return str(ip)
 
 
@@ -64,11 +72,9 @@ def modify_server(stub):
     print(response.message)
 
 
-def modify_lb(stub):
-    slb_list = pickle.dumps(
-                    ['clover-server1', 'clover-server2', 'clover-server3'])
+def modify_lb(stub, slb_list):
     response = stub.ModifyLB(nginx_pb2.ConfigLB(
-            server_port='9188', server_name='http-lb',
+            server_port='9180', server_name='http-lb',
             slb_list=slb_list,
             slb_group='cloverlb', lb_path='/'))
     print(response.message)

diff --git a/samples/services/nginx/docker/grpc/nginx_grpc_server.py b/samples/services/nginx/docker/grpc/nginx_grpc_server.py
index 6f2de0f..1dfe708 100644 (file)
--- a/samples/services/nginx/docker/grpc/nginx_grpc_server.py
+++ b/samples/services/nginx/docker/grpc/nginx_grpc_server.py
@@ -33,8 +33,8 @@ class Controller(nginx_pb2_grpc.ControllerServicer):
             # self.template_file = 'templates/proxy.template'
             self.template_file = '/grpc/templates/proxy.template'
             self.ModifyProxy(nginx_pb2.ConfigProxy(
-                server_port='9180', server_name='http-proxy',
-                location_path='/', proxy_path='http://clover-server:9180',
+                server_port='9180', server_name='proxy-access-control',
+                location_path='/', proxy_path='http://http-lb:9180',
                 mirror_path='http://snort-ids:80'), "")
         if service_type == "server":
             # self.template_file = 'templates/server.template'
@@ -47,7 +47,8 @@ class Controller(nginx_pb2_grpc.ControllerServicer):
             # self.template_file = 'templates/lb.template'
             self.template_file = '/grpc/templates/lb.template'
             slb_list = pickle.dumps(
-                    ['clover-server1', 'clover-server2', 'clover-server3'])
+                    ['clover-server1:9180', 'clover-server2:9180',
+                        'clover-server3:9180'])
             self.ModifyLB(nginx_pb2.ConfigLB(
                 server_port='9180', server_name='http-lb',
                 slb_list=slb_list,

diff --git a/samples/services/nginx/docker/grpc/templates/lb.template b/samples/services/nginx/docker/grpc/templates/lb.template
index 4866408..37de1af 100644 (file)
--- a/samples/services/nginx/docker/grpc/templates/lb.template
+++ b/samples/services/nginx/docker/grpc/templates/lb.template
@@ -61,10 +61,20 @@ http {
 
     upstream {{ slb_group }} {
         {%- for item in slb_list %}
-            server {{ item }};
+            server 127.0.0.1:800{{ loop.index }};
         {%- endfor %}
     }
 
+    {%- for item in slb_list %}
+    server {
+        listen 800{{ loop.index }};
+        location / {
+           proxy_http_version 1.1;
+           proxy_pass http://{{ item }};
+        }
+    }
+    {%- endfor %}
+
     server {
         listen {{ server_port }};
         server_name {{ server_name }};
@@ -73,6 +83,9 @@ http {
             proxy_http_version 1.1;
             proxy_set_header Upgrade $http_upgrade;
             proxy_set_header Connection "upgrade";
+            proxy_set_header X-Forwarded-For $remote_addr;
+            proxy_set_header X-Forwarded-Proto http;
+            proxy_set_header X-Real-IP $remote_addr;
             proxy_pass http://{{ slb_group }};
         }
 

diff --git a/samples/services/nginx/docker/grpc/templates/proxy.template b/samples/services/nginx/docker/grpc/templates/proxy.template
index 72d611e..e7305b3 100644 (file)
--- a/samples/services/nginx/docker/grpc/templates/proxy.template
+++ b/samples/services/nginx/docker/grpc/templates/proxy.template
@@ -67,6 +67,9 @@ http {
             proxy_http_version 1.1;
             proxy_set_header Upgrade $http_upgrade;
             proxy_set_header Connection "upgrade";
+            proxy_set_header X-Forwarded-For $remote_addr;
+            proxy_set_header X-Forwarded-Proto http;
+            proxy_set_header X-Real-IP $remote_addr;
             proxy_pass {{ proxy_path }};
 
             post_action @post_ids;
@@ -76,6 +79,9 @@ http {
             proxy_http_version 1.1;
             proxy_set_header Upgrade $http_upgrade;
             proxy_set_header Connection "upgrade";
+            proxy_set_header X-Forwarded-For $remote_addr;
+            proxy_set_header X-Forwarded-Proto http;
+            proxy_set_header X-Real-IP $remote_addr;
             proxy_pass {{ mirror_path }};
         }
 

diff --git a/samples/services/nginx/yaml/render_yaml.py b/samples/services/nginx/yaml/render_yaml.py
index 527ba8d..850f715 100644 (file)
--- a/samples/services/nginx/yaml/render_yaml.py
+++ b/samples/services/nginx/yaml/render_yaml.py
@@ -16,13 +16,16 @@ def render_yaml(args):
     grpc_port = '50054'
     if args['service_type'] == 'lb':
         out_file = 'lb.yaml'
-        deploy_name = 'http-lb'
+        if args['deploy_name'] == 'default':
+            args['deploy_name'] = 'http-lb'
     elif args['service_type'] == 'proxy':
         out_file = 'proxy.yaml'
-        deploy_name = 'proxy-access-control'
+        if args['deploy_name'] == 'default':
+            args['deploy_name'] = 'proxy-access-control'
     elif args['service_type'] == 'server':
         out_file = 'server.yaml'
-        deploy_name = 'clover-server'
+        if args['deploy_name'] == 'default':
+            args['deploy_name'] = 'clover-server'
     else:
         return "Invalid service type: {}".format(args['service_type'])
 
@@ -33,7 +36,7 @@ def render_yaml(args):
             image_path=args['image_path'],
             image_name=args['image_name'],
             image_tag=args['image_tag'],
-            deploy_name=deploy_name,
+            deploy_name=args['deploy_name'],
             server_port=server_port,
             grpc_port=grpc_port
         )
@@ -60,5 +63,8 @@ if __name__ == '__main__':
     parser.add_argument(
             '--image_tag', default='latest',
             help='The image tag to use')
+    parser.add_argument(
+            '--deploy_name', default='default',
+            help='The deployment name to use')
     args = parser.parse_args()
     print(render_yaml(vars(args)))