Fix security issues reported by the security audit
[releng.git] / utils / test / reporting / functest / reporting-status.py
1 #!/usr/bin/python
2 #
3 # This program and the accompanying materials
4 # are made available under the terms of the Apache License, Version 2.0
5 # which accompanies this distribution, and is available at
6 #
7 # http://www.apache.org/licenses/LICENSE-2.0
8 #
9 import datetime
10 import jinja2
11 import requests
12 import sys
13 import time
14 import yaml
15
16 import reportingUtils as utils
17 import reportingConf as conf
18 import testCase as tc
19 import scenarioResult as sr
20
21 # Logger
22 logger = utils.getLogger("Status")
23
24 # Initialization
25 testValid = []
26 otherTestCases = []
27
28 # init just tempest to get the list of scenarios
29 # as all the scenarios run Tempest
30 tempest = tc.TestCase("tempest_smoke_serial", "functest", -1)
31
32 # Retrieve the Functest configuration to detect which tests are relevant
33 # according to the installer, scenario
34 cf = conf.TEST_CONF
35 response = requests.get(cf)
36
37 functest_yaml_config = yaml.load(response.text)
38
39 logger.info("*******************************************")
40 logger.info("*   Generating reporting scenario status  *")
41 logger.info("*   Data retention = %s days              *" % conf.PERIOD)
42 logger.info("*                                         *")
43 logger.info("*******************************************")
44
45 # Retrieve test cases of Tier 1 (smoke)
46 config_tiers = functest_yaml_config.get("tiers")
47
48 # we consider Tier 1 (smoke),2 (sdn suites) and 3 (features)
49 # to validate scenarios
50 # Tier > 4 are not used to validate scenarios but we display the results anyway
51 # tricky thing for the API as some tests are Functest tests
52 # other tests are declared directly in the feature projects
53 for tier in config_tiers:
54     if tier['order'] > 0 and tier['order'] < 3:
55         for case in tier['testcases']:
56             if case['name'] not in conf.blacklist:
57                 testValid.append(tc.TestCase(case['name'],
58                                              "functest",
59                                              case['dependencies']))
60     elif tier['order'] == 3:
61         for case in tier['testcases']:
62             if case['name'] not in conf.blacklist:
63                 testValid.append(tc.TestCase(case['name'],
64                                              case['name'],
65                                              case['dependencies']))
66     elif tier['order'] > 3:
67         for case in tier['testcases']:
68             if case['name'] not in conf.blacklist:
69                 otherTestCases.append(tc.TestCase(case['name'],
70                                                   "functest",
71                                                   case['dependencies']))
72
73 # For all the versions
74 for version in conf.versions:
75     # For all the installers
76     for installer in conf.installers:
77         # get scenarios
78         scenario_results = utils.getScenarios(tempest, installer, version)
79         scenario_stats = utils.getScenarioStats(scenario_results)
80         items = {}
81         scenario_result_criteria = {}
82
83         # For all the scenarios get results
84         for s, s_result in scenario_results.items():
85             # Green or Red light for a given scenario
86             nb_test_runnable_for_this_scenario = 0
87             scenario_score = 0
88
89             testCases2BeDisplayed = []
90             # Check if test case is runnable / installer, scenario
91             # for the test case used for Scenario validation
92             try:
93                 logger.info("---------------------------------")
94                 logger.info("installer %s, version %s, scenario %s:" %
95                             (installer, version, s))
96
97                 # 1) Manage the test cases for the scenario validation
98                 # concretely Tiers 0-3
99                 for test_case in testValid:
100                     test_case.checkRunnable(installer, s,
101                                             test_case.getConstraints())
102                     logger.debug("testcase %s (%s) is %s" %
103                                  (test_case.getDisplayName(),
104                                   test_case.getName(),
105                                   test_case.isRunnable))
106                     time.sleep(1)
107                     if test_case.isRunnable:
108                         dbName = test_case.getDbName()
109                         name = test_case.getName()
110                         displayName = test_case.getDisplayName()
111                         project = test_case.getProject()
112                         nb_test_runnable_for_this_scenario += 1
113                         logger.info(" Searching results for case %s " %
114                                     (displayName))
115                         result = utils.getResult(dbName, installer, s, version)
116                         # if no result set the value to 0
117                         if result < 0:
118                             result = 0
119                         logger.info(" >>>> Test score = " + str(result))
120                         test_case.setCriteria(result)
121                         test_case.setIsRunnable(True)
122                         testCases2BeDisplayed.append(tc.TestCase(name,
123                                                                  project,
124                                                                  "",
125                                                                  result,
126                                                                  True,
127                                                                  1))
128                         scenario_score = scenario_score + result
129
130                 # 2) Manage the test cases for the scenario qualification
131                 # concretely Tiers > 3
132                 for test_case in otherTestCases:
133                     test_case.checkRunnable(installer, s,
134                                             test_case.getConstraints())
135                     logger.debug("testcase %s (%s) is %s" %
136                                  (test_case.getDisplayName(),
137                                   test_case.getName(),
138                                   test_case.isRunnable))
139                     time.sleep(1)
140                     if test_case.isRunnable:
141                         dbName = test_case.getDbName()
142                         name = test_case.getName()
143                         displayName = test_case.getDisplayName()
144                         project = test_case.getProject()
145                         logger.info(" Searching results for case %s " %
146                                     (displayName))
147                         result = utils.getResult(dbName, installer, s, version)
148                         # at least 1 result for the test
149                         if result > -1:
150                             test_case.setCriteria(result)
151                             test_case.setIsRunnable(True)
152                             testCases2BeDisplayed.append(tc.TestCase(name,
153                                                                      project,
154                                                                      "",
155                                                                      result,
156                                                                      True,
157                                                                      4))
158                         else:
159                             logger.debug("No results found")
160
161                     items[s] = testCases2BeDisplayed
162             except:
163                 logger.error("Error: installer %s, version %s, scenario %s" %
164                              (installer, version, s))
165                 logger.error("No data available: %s " % (sys.exc_info()[0]))
166
167             # **********************************************
168             # Evaluate the results for scenario validation
169             # **********************************************
170             # the validation criteria = nb runnable tests x 3
171             # because each test case = 0,1,2 or 3
172             scenario_criteria = nb_test_runnable_for_this_scenario * 3
173             # if 0 runnable tests set criteria at a high value
174             if scenario_criteria < 1:
175                 scenario_criteria = conf.MAX_SCENARIO_CRITERIA
176
177             s_score = str(scenario_score) + "/" + str(scenario_criteria)
178             s_status = "KO"
179             if scenario_score < scenario_criteria:
180                 logger.info(">>>> scenario not OK, score = %s/%s" %
181                             (scenario_score, scenario_criteria))
182                 s_status = "KO"
183             else:
184                 logger.info(">>>>> scenario OK, save the information")
185                 s_status = "OK"
186                 path_validation_file = (conf.REPORTING_PATH + "/release/" + version +
187                                         "/validated_scenario_history.txt")
188                 with open(path_validation_file, "a") as f:
189                     time_format = "%Y-%m-%d %H:%M"
190                     info = (datetime.datetime.now().strftime(time_format) +
191                             ";" + installer + ";" + s + "\n")
192                     f.write(info)
193
194             scenario_result_criteria[s] = sr.ScenarioResult(s_status, s_score)
195             logger.info("--------------------------")
196
197         templateLoader = jinja2.FileSystemLoader(conf.REPORTING_PATH)
198         templateEnv = jinja2.Environment(loader=templateLoader, autoescape=True)
199
200         TEMPLATE_FILE = "/template/index-status-tmpl.html"
201         template = templateEnv.get_template(TEMPLATE_FILE)
202
203         outputText = template.render(scenario_stats=scenario_stats,
204                                      scenario_results=scenario_result_criteria,
205                                      items=items,
206                                      installer=installer,
207                                      period=conf.PERIOD,
208                                      version=version)
209
210         with open(conf.REPORTING_PATH + "/release/" + version +
211                   "/index-status-" + installer + ".html", "wb") as fh:
212             fh.write(outputText)