2 # Copyright 2021 Orange
4 # Licensed under the Apache License, Version 2.0 (the "License"); you may
5 # not use this file except in compliance with the License. You may obtain
6 # a copy of the License at
8 # http://www.apache.org/licenses/LICENSE-2.0
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13 # License for the specific language governing permissions and limitations
22 __test__ = False # Hint for pytest: TestapiClient is not a test class.
24 def __init__(self, testapi_url: str):
27 testapi_url: testapi URL as a string, for instance
28 "http://172.20.73.203:8000/api/v1/results"
30 self._base_url = testapi_url
31 self._logger = logging.getLogger("behave_tests")
33 def find_last_result(self, testapi_params, scenario_tag: str, nfvbench_test_input):
34 """Search testapi database and return latest result matching filters.
36 Look for the most recent testapi result matching testapi params, behave
37 scenario tag and nfvbench test input params, and return that result as a
41 testapi_params: dict holding the parameters of the testapi request. See
42 `build_testapi_url()` for the list of supported keys.
44 scenario_tag: Behave scenario tag to filter results. One of
45 "throughput" or "latency".
47 nfvbench_test_input: dict holding nfvbench test parameters and used
48 to filter the testapi results. The following keys are currently
50 - mandatory keys: 'duration_sec', 'frame_sizes', 'flow_count', 'rate'
51 - optional keys: 'user_label'
54 None if no result matching the filters can be found, else a dictionary
55 built from testapi JSON test result.
58 self._logger.info(f"find_last_result: filter on scenario tag: {scenario_tag}")
59 nfvbench_input_str = nfvbench_input_to_str(nfvbench_test_input)
60 self._logger.info(f"find_last_result: filter on test conditions: {nfvbench_input_str}")
63 while True: # While there are results pages to read
64 url = self._build_testapi_url(testapi_params, page)
65 self._logger.info("find_last_result: GET " + url)
66 last_results = self._do_testapi_request(url)
68 for result in last_results["results"]:
69 for tagged_result in result["details"]["results"][scenario_tag]:
70 if tagged_result["output"]["status"] != "OK":
71 # Drop result if nfvbench status is not OK
72 # (such result should not have been put in database by behave_tests,
73 # but let's be cautious)
75 if equal_test_conditions(tagged_result["input"], nfvbench_test_input):
78 if page >= last_results["pagination"]["total_pages"]:
84 def _build_testapi_url(self, testapi_params, page=1):
85 """Build URL for testapi request.
87 Build a URL for a testapi HTTP GET request using the provided parameters and
88 limiting the results to the tests whose criteria equals "PASS".
91 testapi_params: dictionary holding the parameters of the testapi
93 - mandatory keys: "project_name", "case_name"
94 - optional keys: "installer", "pod_name"
95 - ignored keys: "build_tag", "scenario", "version", "criteria".
97 page: (Optional) number of the results page to get.
101 url += f"?project={testapi_params['project_name']}"
102 url += f"&case={testapi_params['case_name']}"
104 if "installer" in testapi_params.keys():
105 url += f"&installer={testapi_params['installer']}"
106 if "pod_name" in testapi_params.keys():
107 url += f"&pod={testapi_params['pod_name']}"
109 url += '&criteria=PASS'
110 url += f"&page={page}"
114 def _do_testapi_request(self, testapi_url):
115 """Perform HTTP GET request on testapi.
117 Perform an HTTP GET request on testapi, check status code and return JSON
118 results as dictionary.
121 testapi_url: a complete URL to request testapi results (with base
122 endpoint and parameters)
125 The JSON document from testapi as a Python dictionary
128 * requests.exceptions.ConnectionError in case of network problem
129 when trying to establish a connection with the TestAPI database
130 (DNS failure, refused connection, ...)
132 * requests.exceptions.ConnectTimeout in case of timeout during the
135 * requests.exception.HTTPError if the HTTP request returned an
136 unsuccessful status code.
138 * another exception derived from requests.exceptions.RequestException
139 in case of problem during the HTTP request.
141 response = requests.get(testapi_url)
142 # raise an HTTPError if the HTTP request returned an unsuccessful status code:
143 response.raise_for_status()
144 return response.json()
147 def equal_test_conditions(testapi_input, nfvbench_input):
148 """Check test conditions in behave scenario results record.
150 Check whether a behave scenario results record from testapi matches a given
151 nfvbench input, ie whether the record comes from a test done under the same
152 conditions (frame size, flow count, ...)
155 testapi_input: dict holding the test conditions of a behave scenario
156 results record from testapi
158 nfvbench_input: dict of nfvbench test parameters (reference)
160 The following dict keys are currently supported:
161 - mandatory keys: 'duration_sec', 'frame_sizes', 'flow_count', 'rate'
162 - optional keys: 'user_label'
164 Optional keys are taken into account only when they can be found in
165 `nfvbench_input`, else they are ignored.
168 True if test conditions match, else False.
171 # Select required keys (other keys can be not set or unconsistent between scenarios)
172 required_keys = ['duration_sec', 'frame_sizes', 'flow_count', 'rate']
173 if 'user_label' in nfvbench_input:
174 required_keys.append('user_label')
177 testapi_subset = {k: testapi_input[k] for k in required_keys}
178 nfvbench_subset = {k: nfvbench_input[k] for k in required_keys}
179 return testapi_subset == nfvbench_subset
181 # Fail the comparison if a required key is missing from one of the dicts
185 def nfvbench_input_to_str(nfvbench_input: dict) -> str:
186 """Build string showing nfvbench input parameters used for results search
189 nfvbench_input: dict of nfvbench test parameters
192 for key in ['user_label', 'frame_sizes', 'flow_count', 'rate', 'duration_sec']:
193 if key in nfvbench_input:
194 string += f"{key}={nfvbench_input[key]} "