Update cnf_conformance to v0.9.4

It removes all hacks bypassing a few former bugs (e.g. [1]) and adapts
to the new cnf-conformance result output logic.

It runs all workload tests as it's now proposed (k8s conformance
is already part of smoke) and select the latest coredns
cnf-conformance.yml taking latest helm repo changes into account.

It sets criteria 95 as all steps are considered as passed and a
overall result 100/105.

[1] https://github.com/cncf/cnf-conformance/issues/388

Change-Id: I95b7889b21b86961e7a1d0c84b13e0ae367cae44
Signed-off-by: Cédric Ollivier <cedric.ollivier@orange.com>
(cherry picked from commit 854f164998793000fcf8019214b259b7283f5194)

diff --git a/docker/cnf/Dockerfile b/docker/cnf/Dockerfile
index 4a1ef1f..dff67d0 100644 (file)
--- a/docker/cnf/Dockerfile
+++ b/docker/cnf/Dockerfile
@@ -1,6 +1,6 @@
 FROM opnfv/functest-kubernetes-core:iruya
 
-ARG CNF_CONFORMANCE_TAG=v0.7.2-beta1
+ARG CNF_CONFORMANCE_TAG=v0.9.4
 ARG HELM_TAG=v3.3.1
 
 RUN apk --no-cache add --update wget curl libc6-compat && \
@@ -15,13 +15,16 @@ RUN apk --no-cache add --update wget curl libc6-compat && \
     chmod +x /usr/local/bin/kubectl /usr/local/bin/helm && \
     rm -r /src/helm-$HELM_TAG-linux-$ARCH.tar.gz /src/linux-$ARCH && \
     case $(uname -m) in x86_64) \
-        curl https://github.com/cncf/cnf-conformance/releases/download/$CNF_CONFORMANCE_TAG/cnf-conformance-$(echo $CNF_CONFORMANCE_TAG |cut -d\- -f 1)-$ARCH-static \
-            -Ls --output /usr/local/bin/cnf-conformance && \
-            chmod +x /usr/local/bin/cnf-conformance ;; esac && \
-    mkdir -p /src/cnf-conformance && \
-    curl -Ls https://raw.githubusercontent.com/cncf/cnf-conformance/$CNF_CONFORMANCE_TAG/example-cnfs/coredns/cnf-conformance.yml \
-        -s --output /src/cnf-conformance/cnf-conformance.yml && \
-    curl -Ls https://raw.githubusercontent.com/cncf/cnf-conformance/$CNF_CONFORMANCE_TAG/points.yml \
-        -s --output /src/cnf-conformance/points.yml
+        curl https://github.com/cncf/cnf-conformance/releases/download/$CNF_CONFORMANCE_TAG/cnf-conformance-$CNF_CONFORMANCE_TAG.tar.gz \
+            -Ls --output /src/cnf-conformance-$CNF_CONFORMANCE_TAG.tar.gz && \
+        tar zxf /src/cnf-conformance-$CNF_CONFORMANCE_TAG.tar.gz ./cnf-conformance -C /usr/local/bin && \
+        chmod +x /usr/local/bin/cnf-conformance && \
+        mkdir /src/cnf-conformance && \
+        curl -Ls https://raw.githubusercontent.com/cncf/cnf-conformance/e636baff623dd934eadfce77891fc54da08a8134/example-cnfs/coredns/cnf-conformance.yml \
+            --output /src/cnf-conformance/cnf-conformance.yml && \
+        curl -Ls https://raw.githubusercontent.com/cncf/cnf-conformance/$CNF_CONFORMANCE_TAG/points.yml \
+            --output /src/cnf-conformance/points.yml && \
+        helm repo add stable https://cncf.gitlab.io/stable && \
+        rm /src/cnf-conformance-$CNF_CONFORMANCE_TAG.tar.gz ;; esac
 COPY testcases.yaml /usr/lib/python3.6/site-packages/xtesting/ci/testcases.yaml
 CMD ["run_tests", "-t", "all"]

diff --git a/docker/cnf/testcases.yaml b/docker/cnf/testcases.yaml
index f04eb6f..62ae54b 100644 (file)
--- a/docker/cnf/testcases.yaml
+++ b/docker/cnf/testcases.yaml
@@ -34,8 +34,7 @@ tiers:
             -
                 case_name: cnf_conformance
                 project_name: functest
-                enabled: false
-                criteria: 75
+                criteria: 95
                 blocking: false
                 description: >-
                     Enable interoperability of Cloud native Network

diff --git a/functest_kubernetes/cnf_conformance/conformance.py b/functest_kubernetes/cnf_conformance/conformance.py
index f342fcc..564f279 100644 (file)
--- a/functest_kubernetes/cnf_conformance/conformance.py
+++ b/functest_kubernetes/cnf_conformance/conformance.py
@@ -16,7 +16,7 @@ different vendors [1].
 
 from __future__ import division
 
-import fnmatch
+import glob
 import logging
 import os
 import re
@@ -38,7 +38,7 @@ class CNFConformance(testcase.TestCase):
 
     src_dir = '/src/cnf-conformance'
     bin_dir = '/usr/local/bin'
-    default_tag = 'all'
+    default_tag = 'workload'
 
     __logger = logging.getLogger(__name__)
 
@@ -64,12 +64,6 @@ class CNFConformance(testcase.TestCase):
         shutil.copy2(
             os.path.join(self.src_dir, 'cnf-conformance.yml'), self.res_dir)
         os.chdir(self.res_dir)
-        # cnf-conformance must be in the working dir
-        # https://github.com/cncf/cnf-conformance/issues/388
-        if not os.path.exists(os.path.join(self.res_dir, 'cnf-conformance')):
-            os.symlink(
-                os.path.join(self.bin_dir, 'cnf-conformance'),
-                os.path.join(self.res_dir, 'cnf-conformance'))
         cmd = ['cnf-conformance', 'setup']
         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
         self.__logger.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
@@ -80,25 +74,26 @@ class CNFConformance(testcase.TestCase):
 
     def run_conformance(self, **kwargs):
         """Run CNF Conformance"""
-        # a previous results.yml leads to interactive mode
-        if os.path.exists(os.path.join(self.res_dir, 'results.yml')):
-            os.remove(os.path.join(self.res_dir, 'results.yml'))
         cmd = ['cnf-conformance', kwargs.get("tag", self.default_tag)]
         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
         self.__logger.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
-        for lfile in os.listdir(self.res_dir):
-            if fnmatch.fnmatch(lfile, 'cnf-conformance-results-*.yml'):
-                with open(os.path.join(self.res_dir, lfile)) as yfile:
-                    self.details = yaml.safe_load(yfile)
-                    msg = prettytable.PrettyTable(
-                        header_style='upper', padding_width=5,
-                        field_names=['name', 'status'])
-                    for item in self.details['items']:
-                        msg.add_row([item['name'], item['status']])
-                    self.__logger.info("\n\n%s\n", msg.get_string())
-        grp = re.search(r'Final score: (\d+) of (\d+)', output.decode("utf-8"))
+        lfiles = glob.glob(os.path.join(
+            self.res_dir, 'results', 'cnf-conformance-results-*.yml'))
+        results = max(lfiles, key=os.path.getmtime)
+        with open(os.path.join(self.res_dir, 'results', results)) as yfile:
+            self.details = yaml.safe_load(yfile)
+            msg = prettytable.PrettyTable(
+                header_style='upper', padding_width=5,
+                field_names=['name', 'status'])
+            for item in self.details['items']:
+                msg.add_row([item['name'], item['status']])
+            self.__logger.info("\n\n%s\n", msg.get_string())
+        grp = re.search(
+            r'Final .* score: (\d+) of (\d+)', output.decode("utf-8"))
         if grp:
             self.result = int(grp.group(1)) / int(grp.group(2)) * 100
+        shutil.copy2(
+            os.path.join(self.res_dir, 'results', results), self.res_dir)
 
     def run(self, **kwargs):
         """"Running the test with example CNF"""
@@ -117,6 +112,5 @@ class CNFConformance(testcase.TestCase):
         self.__logger.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
         shutil.rmtree(os.path.join(self.res_dir, 'tools'), ignore_errors=True)
         shutil.rmtree(os.path.join(self.res_dir, 'cnfs'), ignore_errors=True)
-        for lfile in os.listdir(self.res_dir):
-            if not fnmatch.fnmatch(lfile, 'cnf-conformance-results-*.yml'):
-                os.remove(os.path.join(self.res_dir, lfile))
+        shutil.rmtree(
+            os.path.join(self.res_dir, 'results'), ignore_errors=True)