Rather override request_checksum_calculation by user's config
[functest-xtesting.git] / xtesting / core / testcase.py
1 #!/usr/bin/env python
2
3 # Copyright (c) 2016 Orange and others.
4 #
5 # All rights reserved. This program and the accompanying materials
6 # are made available under the terms of the Apache License, Version 2.0
7 # which accompanies this distribution, and is available at
8 # http://www.apache.org/licenses/LICENSE-2.0
9
10 """Define the parent class of all Xtesting TestCases."""
11
12 import abc
13 from datetime import datetime
14 import json
15 import logging
16 import mimetypes
17 import os
18 import re
19
20 from urllib.parse import urlparse
21 import boto3
22 from boto3.s3.transfer import TransferConfig
23 import botocore
24 import prettytable
25 import requests
26
27 from xtesting.utils import decorators
28 from xtesting.utils import env
29 from xtesting.utils import constants
30
31 __author__ = "Cedric Ollivier <cedric.ollivier@orange.com>"
32
33
34 class TestCase(metaclass=abc.ABCMeta):
35     # pylint: disable=too-many-instance-attributes
36     """Base model for single test case."""
37
38     EX_OK = os.EX_OK
39     """everything is OK"""
40
41     EX_RUN_ERROR = os.EX_SOFTWARE
42     """run() failed"""
43
44     EX_PUSH_TO_DB_ERROR = os.EX_SOFTWARE - 1
45     """push_to_db() failed"""
46
47     EX_TESTCASE_FAILED = os.EX_SOFTWARE - 2
48     """results are false"""
49
50     EX_TESTCASE_SKIPPED = os.EX_SOFTWARE - 3
51     """requirements are unmet"""
52
53     EX_PUBLISH_ARTIFACTS_ERROR = os.EX_SOFTWARE - 4
54     """publish_artifacts() failed"""
55
56     dir_results = constants.RESULTS_DIR
57     _job_name_rule = "(dai|week)ly-(.+?)-[0-9]*"
58     headers = {'Content-Type': 'application/json'}
59     __logger = logging.getLogger(__name__)
60     __test__ = False
61
62     def __init__(self, **kwargs):
63         self.details = {}
64         self.project_name = os.environ.get(
65             'PROJECT_NAME', kwargs.get('project_name', 'xtesting'))
66         self.case_name = kwargs.get('case_name', '')
67         self.criteria = kwargs.get('criteria', 100)
68         self.result = 0
69         self.start_time = 0
70         self.stop_time = 0
71         self.is_skipped = False
72         self.output_log_name = os.path.basename(constants.LOG_PATH)
73         self.output_debug_log_name = os.path.basename(constants.DEBUG_LOG_PATH)
74         self.res_dir = os.path.join(self.dir_results, self.case_name)
75
76     def __str__(self):
77         try:
78             assert self.project_name
79             assert self.case_name
80             if self.is_skipped:
81                 result = 'SKIP'
82             else:
83                 result = 'PASS' if (self.is_successful(
84                     ) == TestCase.EX_OK) else 'FAIL'
85             msg = prettytable.PrettyTable(
86                 header_style='upper', padding_width=5,
87                 field_names=['test case', 'project', 'duration',
88                              'result'])
89             msg.add_row([self.case_name, self.project_name,
90                          self.get_duration(), result])
91             return msg.get_string()
92         except AssertionError:
93             self.__logger.error("We cannot print invalid objects")
94             return super().__str__()
95
96     def get_duration(self):
97         """Return the duration of the test case.
98
99         Returns:
100             duration if start_time and stop_time are set
101             "XX:XX" otherwise.
102         """
103         try:
104             if self.is_skipped:
105                 return "00:00"
106             assert self.start_time
107             assert self.stop_time
108             if self.stop_time < self.start_time:
109                 return "XX:XX"
110             return (
111                 f"{str(int(self.stop_time - self.start_time) // 60).zfill(2)}:"
112                 f"{str(int(self.stop_time - self.start_time) % 60).zfill(2)}")
113
114         except Exception:  # pylint: disable=broad-except
115             self.__logger.error("Please run test before getting the duration")
116             return "XX:XX"
117
118     def is_successful(self):
119         """Interpret the result of the test case.
120
121         It allows getting the result of TestCase. It completes run()
122         which only returns the execution status.
123
124         It can be overriden if checking result is not suitable.
125
126         Returns:
127             TestCase.EX_OK if result is 'PASS'.
128             TestCase.EX_TESTCASE_SKIPPED if test case is skipped.
129             TestCase.EX_TESTCASE_FAILED otherwise.
130         """
131         try:
132             if self.is_skipped:
133                 return TestCase.EX_TESTCASE_SKIPPED
134             assert self.criteria
135             assert self.result is not None
136             if (not isinstance(self.result, str) and
137                     not isinstance(self.criteria, str)):
138                 if self.result >= self.criteria:
139                     return TestCase.EX_OK
140             else:
141                 # Backward compatibility
142                 # It must be removed as soon as TestCase subclasses
143                 # stop setting result = 'PASS' or 'FAIL'.
144                 # In this case criteria is unread.
145                 self.__logger.warning(
146                     "Please update result which must be an int!")
147                 if self.result == 'PASS':
148                     return TestCase.EX_OK
149         except AssertionError:
150             self.__logger.error("Please run test before checking the results")
151         return TestCase.EX_TESTCASE_FAILED
152
153     def check_requirements(self):
154         """Check the requirements of the test case.
155
156         It can be overriden on purpose.
157         """
158         self.is_skipped = False
159
160     @abc.abstractmethod
161     def run(self, **kwargs):
162         """Run the test case.
163
164         It allows running TestCase and getting its execution
165         status.
166
167         The subclasses must override the default implementation which
168         is false on purpose.
169
170         The new implementation must set the following attributes to
171         push the results to DB:
172
173             * result,
174             * start_time,
175             * stop_time.
176
177         Args:
178             kwargs: Arbitrary keyword arguments.
179         """
180
181     @decorators.can_dump_request_to_file
182     def push_to_db(self):
183         """Push the results of the test case to the DB.
184
185         It allows publishing the results and checking the status.
186
187         It could be overriden if the common implementation is not
188         suitable.
189
190         The following attributes must be set before pushing the results to DB:
191
192             * project_name,
193             * case_name,
194             * result,
195             * start_time,
196             * stop_time.
197
198         The next vars must be set in env:
199
200             * TEST_DB_URL,
201             * INSTALLER_TYPE,
202             * DEPLOY_SCENARIO,
203             * NODE_NAME,
204             * BUILD_TAG.
205
206         Returns:
207             TestCase.EX_OK if results were pushed to DB.
208             TestCase.EX_PUSH_TO_DB_ERROR otherwise.
209         """
210         try:
211             if self.is_skipped:
212                 return TestCase.EX_PUSH_TO_DB_ERROR
213             assert self.project_name
214             assert self.case_name
215             assert self.start_time
216             assert self.stop_time
217             url = env.get('TEST_DB_URL')
218             data = {"project_name": self.project_name,
219                     "case_name": self.case_name,
220                     "details": self.details}
221             data["installer"] = env.get('INSTALLER_TYPE')
222             data["scenario"] = env.get('DEPLOY_SCENARIO')
223             data["pod_name"] = env.get('NODE_NAME')
224             data["build_tag"] = env.get('BUILD_TAG')
225             data["criteria"] = 'PASS' if self.is_successful(
226                 ) == TestCase.EX_OK else 'FAIL'
227             data["start_date"] = datetime.fromtimestamp(
228                 self.start_time).strftime('%Y-%m-%d %H:%M:%S')
229             data["stop_date"] = datetime.fromtimestamp(
230                 self.stop_time).strftime('%Y-%m-%d %H:%M:%S')
231             try:
232                 data["version"] = re.search(
233                     TestCase._job_name_rule,
234                     env.get('BUILD_TAG')).group(2)
235             except Exception:  # pylint: disable=broad-except
236                 data["version"] = "unknown"
237             req = requests.post(
238                 url, data=json.dumps(data, sort_keys=True),
239                 headers=self.headers,
240                 timeout=10)
241             req.raise_for_status()
242             if urlparse(url).scheme != "file":
243                 # href must be postprocessed as OPNFV testapi is misconfigured
244                 # (localhost is returned)
245                 uid = re.sub(r'^.*/api/v1/results/*', '', req.json()["href"])
246                 netloc = env.get('TEST_DB_EXT_URL') if env.get(
247                     'TEST_DB_EXT_URL') else env.get('TEST_DB_URL')
248                 self.__logger.info(
249                     "The results were successfully pushed to DB: \n\n%s\n",
250                     os.path.join(netloc, uid))
251         except AssertionError:
252             self.__logger.exception(
253                 "Please run test before publishing the results")
254             return TestCase.EX_PUSH_TO_DB_ERROR
255         except requests.exceptions.HTTPError:
256             self.__logger.exception("The HTTP request raises issues")
257             return TestCase.EX_PUSH_TO_DB_ERROR
258         except Exception:  # pylint: disable=broad-except
259             self.__logger.exception("The results cannot be pushed to DB")
260             return TestCase.EX_PUSH_TO_DB_ERROR
261         return TestCase.EX_OK
262
263     def publish_artifacts(self):  # pylint: disable=too-many-locals
264         """Push the artifacts to the S3 repository.
265
266         It allows publishing the artifacts.
267
268         It could be overriden if the common implementation is not
269         suitable.
270
271         The credentials must be configured before publishing the artifacts:
272
273             * fill ~/.aws/credentials or ~/.boto,
274             * set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in env.
275
276         The next vars must be set in env:
277
278             * S3_ENDPOINT_URL (http://127.0.0.1:9000),
279             * S3_DST_URL (s3://xtesting/prefix),
280             * HTTP_DST_URL (http://127.0.0.1/prefix).
281
282         Returns:
283             TestCase.EX_OK if artifacts were published to repository.
284             TestCase.EX_PUBLISH_ARTIFACTS_ERROR otherwise.
285         """
286         try:
287             b3resource = boto3.resource(
288                 's3', endpoint_url=os.environ["S3_ENDPOINT_URL"])
289             dst_s3_url = os.environ["S3_DST_URL"]
290             multipart_threshold = 5 * 1024 ** 5 if "google" in os.environ[
291                 "S3_ENDPOINT_URL"] else 8 * 1024 * 1024
292             config = TransferConfig(multipart_threshold=multipart_threshold)
293             bucket_name = urlparse(dst_s3_url).netloc
294             try:
295                 b3resource.meta.client.head_bucket(Bucket=bucket_name)
296             except botocore.exceptions.ClientError as exc:
297                 error_code = exc.response['Error']['Code']
298                 if error_code == '404':
299                     # pylint: disable=no-member
300                     b3resource.create_bucket(Bucket=bucket_name)
301                 else:
302                     raise exc
303             except Exception as exc:  # pylint: disable=broad-except
304                 raise exc
305             path = urlparse(dst_s3_url).path.strip("/")
306             dst_http_url = os.environ["HTTP_DST_URL"]
307             output_str = "\n"
308             # protects if test cases return details as None
309             self.details = self.details or {}
310             self.details["links"] = []
311             for log_file in [self.output_log_name, self.output_debug_log_name]:
312                 if os.path.exists(os.path.join(self.dir_results, log_file)):
313                     abs_file = os.path.join(self.dir_results, log_file)
314                     mime_type = mimetypes.guess_type(abs_file)
315                     self.__logger.debug(
316                         "Publishing %s %s", abs_file, mime_type)
317                     # pylint: disable=no-member
318                     b3resource.Bucket(bucket_name).upload_file(
319                         abs_file, os.path.join(path, log_file), Config=config,
320                         ExtraArgs={'ContentType': mime_type[
321                             0] or 'application/octet-stream'})
322                     link = os.path.join(dst_http_url, log_file)
323                     output_str += f"\n{link}"
324                     self.details["links"].append(link)
325             for root, _, files in os.walk(self.res_dir):
326                 for pub_file in files:
327                     abs_file = os.path.join(root, pub_file)
328                     mime_type = mimetypes.guess_type(abs_file)
329                     self.__logger.debug(
330                         "Publishing %s %s", abs_file, mime_type)
331                     # pylint: disable=no-member
332                     b3resource.Bucket(bucket_name).upload_file(
333                         abs_file,
334                         os.path.join(path, os.path.relpath(
335                             os.path.join(root, pub_file),
336                             start=self.dir_results)),
337                         Config=config,
338                         ExtraArgs={'ContentType': mime_type[
339                             0] or 'application/octet-stream'})
340                     link = os.path.join(dst_http_url, os.path.relpath(
341                         os.path.join(root, pub_file),
342                         start=self.dir_results))
343                     output_str += f"\n{link}"
344                     self.details["links"].append(link)
345             self.__logger.info(
346                 "All artifacts were successfully published: %s\n", output_str)
347             return TestCase.EX_OK
348         except KeyError as ex:
349             self.__logger.error("Please check env var: %s", str(ex))
350             return TestCase.EX_PUBLISH_ARTIFACTS_ERROR
351         except botocore.exceptions.NoCredentialsError:
352             self.__logger.error(
353                 "Please fill ~/.aws/credentials, ~/.boto or set "
354                 "AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in env")
355             return TestCase.EX_PUBLISH_ARTIFACTS_ERROR
356         except Exception:  # pylint: disable=broad-except
357             self.__logger.exception("Cannot publish the artifacts")
358             return TestCase.EX_PUBLISH_ARTIFACTS_ERROR
359
360     def clean(self):
361         """Clean the resources.
362
363         It can be overriden if resources must be deleted after
364         running the test case.
365         """