Add new function to get indivial test score for aarch64

[releng.git] / utils / test / reporting / reporting / functest / reporting-status.py

diff --git a/utils/test/reporting/reporting/functest/reporting-status.py b/utils/test/reporting/reporting/functest/reporting-status.py
index 808c841..592f929 100755 (executable)
--- a/utils/test/reporting/reporting/functest/reporting-status.py
+++ b/utils/test/reporting/reporting/functest/reporting-status.py
@@ -13,8 +13,8 @@ import time
 
 import jinja2
 
-import testCase as tc
-import scenarioResult as sr
+import reporting.functest.testCase as tc
+import reporting.functest.scenarioResult as sr
 import reporting.utils.reporting_utils as rp_utils
 
 """
@@ -172,8 +172,13 @@ for version in versions:
                             nb_test_runnable_for_this_scenario += 1
                             LOGGER.info(" Searching results for case %s ",
                                         displayName)
-                            result = rp_utils.getResult(name, installer,
-                                                        s, version)
+                            if "fuel" in installer:
+                                result = rp_utils.getCaseScoreFromBuildTag(
+                                    name,
+                                    s_result)
+                            else:
+                                result = rp_utils.getCaseScore(name, installer,
+                                                               s, version)
                             # if no result set the value to 0
                             if result < 0:
                                 result = 0
@@ -204,8 +209,13 @@ for version in versions:
                             project = test_case.getProject()
                             LOGGER.info(" Searching results for case %s ",
                                         displayName)
-                            result = rp_utils.getResult(name, installer,
-                                                        s, version)
+                            if "fuel" in installer:
+                                result = rp_utils.getCaseScoreFromBuildTag(
+                                    name,
+                                    s_result)
+                            else:
+                                result = rp_utils.getCaseScore(name, installer,
+                                                               s, version)
                             # at least 1 result for the test
                             if result > -1:
                                 test_case.setCriteria(result)
@@ -221,7 +231,7 @@ for version in versions:
                                 LOGGER.debug("No results found")
 
                         items[s] = testCases2BeDisplayed
-                except Exception:
+                except Exception:  # pylint: disable=broad-except
                     LOGGER.error("Error installer %s, version %s, scenario %s",
                                  installer, version, s)
                     LOGGER.error("No data available: %s", sys.exc_info()[0])
@@ -230,12 +240,28 @@ for version in versions:
                 # Evaluate the results for scenario validation
                 # **********************************************
                 # the validation criteria = nb runnable tests x 3
-                # because each test case = 0,1,2 or 3
-                scenario_criteria = nb_test_runnable_for_this_scenario * 3
-                # if 0 runnable tests set criteria at a high value
-                if scenario_criteria < 1:
-                    scenario_criteria = 50  # conf.MAX_SCENARIO_CRITERIA
+                # because each test case can get
+                # 0 point (never PASS)
+                # 1 point at least (PASS once over the time window)
+                # 2 points (PASS more than once but 1 FAIL on the last 4)
+                # 3 points PASS on the last 4 iterations
+                # e.g. 1 scenario = 10 cases
+                # 1 iteration : max score = 10 (10x1)
+                # 2 iterations : max score = 20 (10x2)
+                # 3 iterations : max score = 20
+                # 4 or more iterations : max score = 30 (1x30)
+                LOGGER.info("Number of iterations for this scenario: %s",
+                            len(s_result))
+                if len(s_result) > 3:
+                    k_score = 3
+                elif len(s_result) < 2:
+                    k_score = 1
+                else:
+                    k_score = 2
+
+                scenario_criteria = nb_test_runnable_for_this_scenario*k_score
 
+                # score for reporting
                 s_score = str(scenario_score) + "/" + str(scenario_criteria)
                 s_score_percent = rp_utils.getScenarioPercent(
                     scenario_score,