Use cnf certifaction criteria

[functest-kubernetes.git] / functest_kubernetes / cnf_conformance / conformance.py

diff --git a/functest_kubernetes/cnf_conformance/conformance.py b/functest_kubernetes/cnf_conformance/conformance.py
index 1ef42ad..bfb0c1b 100644 (file)
--- a/functest_kubernetes/cnf_conformance/conformance.py
+++ b/functest_kubernetes/cnf_conformance/conformance.py
@@ -38,12 +38,12 @@ class CNFConformance(testcase.TestCase):
 
     src_dir = '/src/cnf-testsuite'
     bin_dir = '/usr/local/bin'
-    default_tag = 'workload'
+    default_tag = 'cert'
 
     __logger = logging.getLogger(__name__)
 
     def __init__(self, **kwargs):
-        super(CNFConformance, self).__init__(**kwargs)
+        super().__init__(**kwargs)
         self.output_log_name = 'functest-kubernetes.log'
         self.output_debug_log_name = 'functest-kubernetes.debug.log'
 
@@ -57,24 +57,14 @@ class CNFConformance(testcase.TestCase):
 
     def setup(self):
         """Implement initialization and pre-reqs steps"""
-        if os.path.exists(self.res_dir):
-            shutil.rmtree(self.res_dir)
-        os.makedirs(self.res_dir)
-        shutil.copy2(os.path.join(self.src_dir, 'points.yml'), self.res_dir)
-        shutil.copy2(
-            os.path.join(self.src_dir, 'cnf-conformance.yml'), self.res_dir)
-        os.makedirs(os.path.join(self.res_dir, 'spec/fixtures'))
-        for cfile in ["chaos_network_loss.yml", "chaos_cpu_hog.yml",
-                      "chaos_container_kill.yml"]:
-            shutil.copy2(
-                os.path.join(self.src_dir, 'spec/fixtures', cfile),
-                os.path.join(self.res_dir, 'spec/fixtures', cfile))
-        os.chdir(self.res_dir)
+        if os.path.exists(os.path.join(self.src_dir, "results")):
+            shutil.rmtree(os.path.join(self.src_dir, "results"))
+        os.chdir(self.src_dir)
         cmd = ['cnf-testsuite', 'setup']
         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
         self.__logger.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
         cmd = ['cnf-testsuite', 'cnf_setup',
-               'cnf-config=cnf-conformance.yml']
+               'cnf-config=cnf-testsuite.yml']
         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
         self.__logger.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
 
@@ -84,22 +74,35 @@ class CNFConformance(testcase.TestCase):
         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
         self.__logger.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
         lfiles = glob.glob(os.path.join(
-            self.res_dir, 'results', 'cnf-testsuite-results-*.yml'))
+            self.src_dir, 'results', 'cnf-testsuite-results-*.yml'))
         results = max(lfiles, key=os.path.getmtime)
-        with open(os.path.join(self.res_dir, 'results', results)) as yfile:
+        with open(os.path.join(
+                self.src_dir, 'results', results), encoding='utf-8') as yfile:
             self.details = yaml.safe_load(yfile)
             msg = prettytable.PrettyTable(
                 header_style='upper', padding_width=5,
                 field_names=['name', 'status'])
+            item_criteria = 0
             for item in self.details['items']:
                 msg.add_row([item['name'], item['status']])
+                if item['status'] == "passed":
+                    item_criteria += 1
+                elif item['status'] == "failed":
+                    self.__logger.warning(
+                        "%s %s", item['name'], item['status'])
             self.__logger.info("\n\n%s\n", msg.get_string())
         grp = re.search(
-            r'Final .* score: (\d+) of (\d+)', output.decode("utf-8"))
+            r'(\d+) of (\d+) essential tests passed', output.decode("utf-8"))
         if grp:
-            self.result = int(grp.group(1)) / int(grp.group(2)) * 100
+            # https://github.com/cncf/cnf-certification/blob/main/reviewing.md
+            self.result = int(grp.group(1))
+        else:
+            self.result = 0
+        if not os.path.exists(self.res_dir):
+            os.makedirs(self.res_dir)
         shutil.copy2(
-            os.path.join(self.res_dir, 'results', results), self.res_dir)
+            os.path.join(self.src_dir, 'results', results),
+            os.path.join(self.res_dir, 'cnf-testsuite-results.yml'))
 
     def run(self, **kwargs):
         """"Running the test with example CNF"""
@@ -107,16 +110,18 @@ class CNFConformance(testcase.TestCase):
         try:
             self.setup()
             self.run_conformance(**kwargs)
+        except subprocess.CalledProcessError as exc:
+            self.__logger.warning(
+                "Can not run CNT Conformance: \n%s\n%s\n",
+                " ".join(exc.cmd), exc.output.decode("utf-8"))
         except Exception:  # pylint: disable=broad-except
-            self.__logger.exception("Can not run CNF Conformance")
+            self.__logger.warning("CNF Conformance exited with errors")
         self.stop_time = time.time()
 
     def clean(self):
-        cmd = ['cnf-testsuite', 'cnf_cleanup',
-               'cnf-config=cnf-conformance.yml']
-        output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
-        self.__logger.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
-        shutil.rmtree(os.path.join(self.res_dir, 'tools'), ignore_errors=True)
-        shutil.rmtree(os.path.join(self.res_dir, 'cnfs'), ignore_errors=True)
-        shutil.rmtree(
-            os.path.join(self.res_dir, 'results'), ignore_errors=True)
+        for clean_cmd in ['uninstall_falco',
+                          'cnf_cleanup']:
+            cmd = ['cnf-testsuite', clean_cmd,
+                   'cnf-config=cnf-testsuite.yml']
+            output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
+            self.__logger.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))