xtesting/core/testcase.py

   1 #!/usr/bin/env python
   2
   3 # Copyright (c) 2016 Orange and others.
   4 #
   5 # All rights reserved. This program and the accompanying materials
   6 # are made available under the terms of the Apache License, Version 2.0
   7 # which accompanies this distribution, and is available at
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9
  10 """Define the parent class of all Xtesting TestCases."""
  11
  12 import abc
  13 from datetime import datetime
  14 import json
  15 import logging
  16 import mimetypes
  17 import os
  18 import re
  19
  20 from urllib.parse import urlparse
  21 import boto3
  22 from boto3.s3.transfer import TransferConfig
  23 import botocore
  24 import prettytable
  25 import requests
  26
  27 from xtesting.utils import decorators
  28 from xtesting.utils import env
  29 from xtesting.utils import constants
  30
  31 __author__ = "Cedric Ollivier <cedric.ollivier@orange.com>"
  32
  33
  34 class TestCase(metaclass=abc.ABCMeta):
  35     # pylint: disable=too-many-instance-attributes
  36     """Base model for single test case."""
  37
  38     EX_OK = os.EX_OK
  39     """everything is OK"""
  40
  41     EX_RUN_ERROR = os.EX_SOFTWARE
  42     """run() failed"""
  43
  44     EX_PUSH_TO_DB_ERROR = os.EX_SOFTWARE - 1
  45     """push_to_db() failed"""
  46
  47     EX_TESTCASE_FAILED = os.EX_SOFTWARE - 2
  48     """results are false"""
  49
  50     EX_TESTCASE_SKIPPED = os.EX_SOFTWARE - 3
  51     """requirements are unmet"""
  52
  53     EX_PUBLISH_ARTIFACTS_ERROR = os.EX_SOFTWARE - 4
  54     """publish_artifacts() failed"""
  55
  56     dir_results = constants.RESULTS_DIR
  57     _job_name_rule = "(dai|week)ly-(.+?)-[0-9]*"
  58     headers = {'Content-Type': 'application/json'}
  59     __logger = logging.getLogger(__name__)
  60
  61     def __init__(self, **kwargs):
  62         self.details = {}
  63         self.project_name = os.environ.get(
  64             'PROJECT_NAME', kwargs.get('project_name', 'xtesting'))
  65         self.case_name = kwargs.get('case_name', '')
  66         self.criteria = kwargs.get('criteria', 100)
  67         self.result = 0
  68         self.start_time = 0
  69         self.stop_time = 0
  70         self.is_skipped = False
  71         self.output_log_name = os.path.basename(constants.LOG_PATH)
  72         self.output_debug_log_name = os.path.basename(constants.DEBUG_LOG_PATH)
  73         self.res_dir = os.path.join(self.dir_results, self.case_name)
  74
  75     def __str__(self):
  76         try:
  77             assert self.project_name
  78             assert self.case_name
  79             if self.is_skipped:
  80                 result = 'SKIP'
  81             else:
  82                 result = 'PASS' if(self.is_successful(
  83                     ) == TestCase.EX_OK) else 'FAIL'
  84             msg = prettytable.PrettyTable(
  85                 header_style='upper', padding_width=5,
  86                 field_names=['test case', 'project', 'duration',
  87                              'result'])
  88             msg.add_row([self.case_name, self.project_name,
  89                          self.get_duration(), result])
  90             return msg.get_string()
  91         except AssertionError:
  92             self.__logger.error("We cannot print invalid objects")
  93             return super().__str__()
  94
  95     def get_duration(self):
  96         """Return the duration of the test case.
  97
  98         Returns:
  99             duration if start_time and stop_time are set
 100             "XX:XX" otherwise.
 101         """
 102         try:
 103             if self.is_skipped:
 104                 return "00:00"
 105             assert self.start_time
 106             assert self.stop_time
 107             if self.stop_time < self.start_time:
 108                 return "XX:XX"
 109             return(
 110                 f"{str(int(self.stop_time - self.start_time) // 60).zfill(2)}:"
 111                 f"{str(int(self.stop_time - self.start_time) % 60).zfill(2)}")
 112
 113         except Exception:  # pylint: disable=broad-except
 114             self.__logger.error("Please run test before getting the duration")
 115             return "XX:XX"
 116
 117     def is_successful(self):
 118         """Interpret the result of the test case.
 119
 120         It allows getting the result of TestCase. It completes run()
 121         which only returns the execution status.
 122
 123         It can be overriden if checking result is not suitable.
 124
 125         Returns:
 126             TestCase.EX_OK if result is 'PASS'.
 127             TestCase.EX_TESTCASE_SKIPPED if test case is skipped.
 128             TestCase.EX_TESTCASE_FAILED otherwise.
 129         """
 130         try:
 131             if self.is_skipped:
 132                 return TestCase.EX_TESTCASE_SKIPPED
 133             assert self.criteria
 134             assert self.result is not None
 135             if (not isinstance(self.result, str) and
 136                     not isinstance(self.criteria, str)):
 137                 if self.result >= self.criteria:
 138                     return TestCase.EX_OK
 139             else:
 140                 # Backward compatibility
 141                 # It must be removed as soon as TestCase subclasses
 142                 # stop setting result = 'PASS' or 'FAIL'.
 143                 # In this case criteria is unread.
 144                 self.__logger.warning(
 145                     "Please update result which must be an int!")
 146                 if self.result == 'PASS':
 147                     return TestCase.EX_OK
 148         except AssertionError:
 149             self.__logger.error("Please run test before checking the results")
 150         return TestCase.EX_TESTCASE_FAILED
 151
 152     def check_requirements(self):
 153         """Check the requirements of the test case.
 154
 155         It can be overriden on purpose.
 156         """
 157         self.is_skipped = False
 158
 159     @abc.abstractmethod
 160     def run(self, **kwargs):
 161         """Run the test case.
 162
 163         It allows running TestCase and getting its execution
 164         status.
 165
 166         The subclasses must override the default implementation which
 167         is false on purpose.
 168
 169         The new implementation must set the following attributes to
 170         push the results to DB:
 171
 172             * result,
 173             * start_time,
 174             * stop_time.
 175
 176         Args:
 177             kwargs: Arbitrary keyword arguments.
 178         """
 179
 180     @decorators.can_dump_request_to_file
 181     def push_to_db(self):
 182         """Push the results of the test case to the DB.
 183
 184         It allows publishing the results and checking the status.
 185
 186         It could be overriden if the common implementation is not
 187         suitable.
 188
 189         The following attributes must be set before pushing the results to DB:
 190
 191             * project_name,
 192             * case_name,
 193             * result,
 194             * start_time,
 195             * stop_time.
 196
 197         The next vars must be set in env:
 198
 199             * TEST_DB_URL,
 200             * INSTALLER_TYPE,
 201             * DEPLOY_SCENARIO,
 202             * NODE_NAME,
 203             * BUILD_TAG.
 204
 205         Returns:
 206             TestCase.EX_OK if results were pushed to DB.
 207             TestCase.EX_PUSH_TO_DB_ERROR otherwise.
 208         """
 209         try:
 210             if self.is_skipped:
 211                 return TestCase.EX_PUSH_TO_DB_ERROR
 212             assert self.project_name
 213             assert self.case_name
 214             assert self.start_time
 215             assert self.stop_time
 216             url = env.get('TEST_DB_URL')
 217             data = {"project_name": self.project_name,
 218                     "case_name": self.case_name,
 219                     "details": self.details}
 220             data["installer"] = env.get('INSTALLER_TYPE')
 221             data["scenario"] = env.get('DEPLOY_SCENARIO')
 222             data["pod_name"] = env.get('NODE_NAME')
 223             data["build_tag"] = env.get('BUILD_TAG')
 224             data["criteria"] = 'PASS' if self.is_successful(
 225                 ) == TestCase.EX_OK else 'FAIL'
 226             data["start_date"] = datetime.fromtimestamp(
 227                 self.start_time).strftime('%Y-%m-%d %H:%M:%S')
 228             data["stop_date"] = datetime.fromtimestamp(
 229                 self.stop_time).strftime('%Y-%m-%d %H:%M:%S')
 230             try:
 231                 data["version"] = re.search(
 232                     TestCase._job_name_rule,
 233                     env.get('BUILD_TAG')).group(2)
 234             except Exception:  # pylint: disable=broad-except
 235                 data["version"] = "unknown"
 236             req = requests.post(
 237                 url, data=json.dumps(data, sort_keys=True),
 238                 headers=self.headers,
 239                 timeout=10)
 240             req.raise_for_status()
 241             if urlparse(url).scheme != "file":
 242                 # href must be postprocessed as OPNFV testapi is misconfigured
 243                 # (localhost is returned)
 244                 uid = re.sub(r'^.*/api/v1/results/*', '', req.json()["href"])
 245                 netloc = env.get('TEST_DB_EXT_URL') if env.get(
 246                     'TEST_DB_EXT_URL') else env.get('TEST_DB_URL')
 247                 self.__logger.info(
 248                     "The results were successfully pushed to DB: \n\n%s\n",
 249                     os.path.join(netloc, uid))
 250         except AssertionError:
 251             self.__logger.exception(
 252                 "Please run test before publishing the results")
 253             return TestCase.EX_PUSH_TO_DB_ERROR
 254         except requests.exceptions.HTTPError:
 255             self.__logger.exception("The HTTP request raises issues")
 256             return TestCase.EX_PUSH_TO_DB_ERROR
 257         except Exception:  # pylint: disable=broad-except
 258             self.__logger.exception("The results cannot be pushed to DB")
 259             return TestCase.EX_PUSH_TO_DB_ERROR
 260         return TestCase.EX_OK
 261
 262     def publish_artifacts(self):  # pylint: disable=too-many-locals
 263         """Push the artifacts to the S3 repository.
 264
 265         It allows publishing the artifacts.
 266
 267         It could be overriden if the common implementation is not
 268         suitable.
 269
 270         The credentials must be configured before publishing the artifacts:
 271
 272             * fill ~/.aws/credentials or ~/.boto,
 273             * set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in env.
 274
 275         The next vars must be set in env:
 276
 277             * S3_ENDPOINT_URL (http://127.0.0.1:9000),
 278             * S3_DST_URL (s3://xtesting/prefix),
 279             * HTTP_DST_URL (http://127.0.0.1/prefix).
 280
 281         Returns:
 282             TestCase.EX_OK if artifacts were published to repository.
 283             TestCase.EX_PUBLISH_ARTIFACTS_ERROR otherwise.
 284         """
 285         try:
 286             b3resource = boto3.resource(
 287                 's3', endpoint_url=os.environ["S3_ENDPOINT_URL"])
 288             dst_s3_url = os.environ["S3_DST_URL"]
 289             multipart_threshold = 5 * 1024 ** 5 if "google" in os.environ[
 290                 "S3_ENDPOINT_URL"] else 8 * 1024 * 1024
 291             config = TransferConfig(multipart_threshold=multipart_threshold)
 292             bucket_name = urlparse(dst_s3_url).netloc
 293             try:
 294                 b3resource.meta.client.head_bucket(Bucket=bucket_name)
 295             except botocore.exceptions.ClientError as exc:
 296                 error_code = exc.response['Error']['Code']
 297                 if error_code == '404':
 298                     # pylint: disable=no-member
 299                     b3resource.create_bucket(Bucket=bucket_name)
 300                 else:
 301                     raise exc
 302             except Exception as exc:  # pylint: disable=broad-except
 303                 raise exc
 304             path = urlparse(dst_s3_url).path.strip("/")
 305             dst_http_url = os.environ["HTTP_DST_URL"]
 306             output_str = "\n"
 307             # protects if test cases return details as None
 308             self.details = self.details or {}
 309             self.details["links"] = []
 310             for log_file in [self.output_log_name, self.output_debug_log_name]:
 311                 if os.path.exists(os.path.join(self.dir_results, log_file)):
 312                     abs_file = os.path.join(self.dir_results, log_file)
 313                     mime_type = mimetypes.guess_type(abs_file)
 314                     self.__logger.debug(
 315                         "Publishing %s %s", abs_file, mime_type)
 316                     # pylint: disable=no-member
 317                     b3resource.Bucket(bucket_name).upload_file(
 318                         abs_file, os.path.join(path, log_file), Config=config,
 319                         ExtraArgs={'ContentType': mime_type[
 320                             0] or 'application/octet-stream'})
 321                     link = os.path.join(dst_http_url, log_file)
 322                     output_str += f"\n{link}"
 323                     self.details["links"].append(link)
 324             for root, _, files in os.walk(self.res_dir):
 325                 for pub_file in files:
 326                     abs_file = os.path.join(root, pub_file)
 327                     mime_type = mimetypes.guess_type(abs_file)
 328                     self.__logger.debug(
 329                         "Publishing %s %s", abs_file, mime_type)
 330                     # pylint: disable=no-member
 331                     b3resource.Bucket(bucket_name).upload_file(
 332                         abs_file,
 333                         os.path.join(path, os.path.relpath(
 334                             os.path.join(root, pub_file),
 335                             start=self.dir_results)),
 336                         Config=config,
 337                         ExtraArgs={'ContentType': mime_type[
 338                             0] or 'application/octet-stream'})
 339                     link = os.path.join(dst_http_url, os.path.relpath(
 340                         os.path.join(root, pub_file),
 341                         start=self.dir_results))
 342                     output_str += f"\n{link}"
 343                     self.details["links"].append(link)
 344             self.__logger.info(
 345                 "All artifacts were successfully published: %s\n", output_str)
 346             return TestCase.EX_OK
 347         except KeyError as ex:
 348             self.__logger.error("Please check env var: %s", str(ex))
 349             return TestCase.EX_PUBLISH_ARTIFACTS_ERROR
 350         except botocore.exceptions.NoCredentialsError:
 351             self.__logger.error(
 352                 "Please fill ~/.aws/credentials, ~/.boto or set "
 353                 "AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in env")
 354             return TestCase.EX_PUBLISH_ARTIFACTS_ERROR
 355         except Exception:  # pylint: disable=broad-except
 356             self.__logger.exception("Cannot publish the artifacts")
 357             return TestCase.EX_PUBLISH_ARTIFACTS_ERROR
 358
 359     def clean(self):
 360         """Clean the resources.
 361
 362         It can be overriden if resources must be deleted after
 363         running the test case.
 364         """