xtesting/core/testcase.py

   1 #!/usr/bin/env python
   2
   3 # Copyright (c) 2016 Orange and others.
   4 #
   5 # All rights reserved. This program and the accompanying materials
   6 # are made available under the terms of the Apache License, Version 2.0
   7 # which accompanies this distribution, and is available at
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9
  10 """Define the parent class of all Xtesting TestCases."""
  11
  12 import abc
  13 from datetime import datetime
  14 import json
  15 import logging
  16 import mimetypes
  17 import os
  18 import re
  19 import sys
  20
  21 import boto3
  22 from boto3.s3.transfer import TransferConfig
  23 import botocore
  24 import prettytable
  25 import requests
  26 import six
  27 from six.moves import urllib
  28
  29 from xtesting.utils import decorators
  30 from xtesting.utils import env
  31
  32 __author__ = "Cedric Ollivier <cedric.ollivier@orange.com>"
  33
  34
  35 @six.add_metaclass(abc.ABCMeta)
  36 class TestCase():
  37     # pylint: disable=too-many-instance-attributes
  38     """Base model for single test case."""
  39
  40     EX_OK = os.EX_OK
  41     """everything is OK"""
  42
  43     EX_RUN_ERROR = os.EX_SOFTWARE
  44     """run() failed"""
  45
  46     EX_PUSH_TO_DB_ERROR = os.EX_SOFTWARE - 1
  47     """push_to_db() failed"""
  48
  49     EX_TESTCASE_FAILED = os.EX_SOFTWARE - 2
  50     """results are false"""
  51
  52     EX_TESTCASE_SKIPPED = os.EX_SOFTWARE - 3
  53     """requirements are unmet"""
  54
  55     EX_PUBLISH_ARTIFACTS_ERROR = os.EX_SOFTWARE - 4
  56     """publish_artifacts() failed"""
  57
  58     dir_results = "/var/lib/xtesting/results"
  59     _job_name_rule = "(dai|week)ly-(.+?)-[0-9]*"
  60     _headers = {'Content-Type': 'application/json'}
  61     __logger = logging.getLogger(__name__)
  62
  63     def __init__(self, **kwargs):
  64         self.details = {}
  65         self.project_name = kwargs.get('project_name', 'xtesting')
  66         self.case_name = kwargs.get('case_name', '')
  67         self.criteria = kwargs.get('criteria', 100)
  68         self.result = 0
  69         self.start_time = 0
  70         self.stop_time = 0
  71         self.is_skipped = False
  72         self.output_log_name = 'xtesting.log'
  73         self.output_debug_log_name = 'xtesting.debug.log'
  74         self.res_dir = "{}/{}".format(self.dir_results, self.case_name)
  75
  76     def __str__(self):
  77         try:
  78             assert self.project_name
  79             assert self.case_name
  80             if self.is_skipped:
  81                 result = 'SKIP'
  82             else:
  83                 result = 'PASS' if(self.is_successful(
  84                     ) == TestCase.EX_OK) else 'FAIL'
  85             msg = prettytable.PrettyTable(
  86                 header_style='upper', padding_width=5,
  87                 field_names=['test case', 'project', 'duration',
  88                              'result'])
  89             msg.add_row([self.case_name, self.project_name,
  90                          self.get_duration(), result])
  91             return msg.get_string()
  92         except AssertionError:
  93             self.__logger.error("We cannot print invalid objects")
  94             return super(TestCase, self).__str__()
  95
  96     def get_duration(self):
  97         """Return the duration of the test case.
  98
  99         Returns:
 100             duration if start_time and stop_time are set
 101             "XX:XX" otherwise.
 102         """
 103         try:
 104             if self.is_skipped:
 105                 return "00:00"
 106             assert self.start_time
 107             assert self.stop_time
 108             if self.stop_time < self.start_time:
 109                 return "XX:XX"
 110             return "{0[0]:02.0f}:{0[1]:02.0f}".format(divmod(
 111                 self.stop_time - self.start_time, 60))
 112         except Exception:  # pylint: disable=broad-except
 113             self.__logger.error("Please run test before getting the duration")
 114             return "XX:XX"
 115
 116     def is_successful(self):
 117         """Interpret the result of the test case.
 118
 119         It allows getting the result of TestCase. It completes run()
 120         which only returns the execution status.
 121
 122         It can be overriden if checking result is not suitable.
 123
 124         Returns:
 125             TestCase.EX_OK if result is 'PASS'.
 126             TestCase.EX_TESTCASE_SKIPPED if test case is skipped.
 127             TestCase.EX_TESTCASE_FAILED otherwise.
 128         """
 129         try:
 130             if self.is_skipped:
 131                 return TestCase.EX_TESTCASE_SKIPPED
 132             assert self.criteria
 133             assert self.result is not None
 134             if (not isinstance(self.result, str) and
 135                     not isinstance(self.criteria, str)):
 136                 if self.result >= self.criteria:
 137                     return TestCase.EX_OK
 138             else:
 139                 # Backward compatibility
 140                 # It must be removed as soon as TestCase subclasses
 141                 # stop setting result = 'PASS' or 'FAIL'.
 142                 # In this case criteria is unread.
 143                 self.__logger.warning(
 144                     "Please update result which must be an int!")
 145                 if self.result == 'PASS':
 146                     return TestCase.EX_OK
 147         except AssertionError:
 148             self.__logger.error("Please run test before checking the results")
 149         return TestCase.EX_TESTCASE_FAILED
 150
 151     def check_requirements(self):  # pylint: disable=no-self-use
 152         """Check the requirements of the test case.
 153
 154         It can be overriden on purpose.
 155         """
 156         self.is_skipped = False
 157
 158     @abc.abstractmethod
 159     def run(self, **kwargs):
 160         """Run the test case.
 161
 162         It allows running TestCase and getting its execution
 163         status.
 164
 165         The subclasses must override the default implementation which
 166         is false on purpose.
 167
 168         The new implementation must set the following attributes to
 169         push the results to DB:
 170
 171             * result,
 172             * start_time,
 173             * stop_time.
 174
 175         Args:
 176             kwargs: Arbitrary keyword arguments.
 177         """
 178
 179     @decorators.can_dump_request_to_file
 180     def push_to_db(self):
 181         """Push the results of the test case to the DB.
 182
 183         It allows publishing the results and checking the status.
 184
 185         It could be overriden if the common implementation is not
 186         suitable.
 187
 188         The following attributes must be set before pushing the results to DB:
 189
 190             * project_name,
 191             * case_name,
 192             * result,
 193             * start_time,
 194             * stop_time.
 195
 196         The next vars must be set in env:
 197
 198             * TEST_DB_URL,
 199             * INSTALLER_TYPE,
 200             * DEPLOY_SCENARIO,
 201             * NODE_NAME,
 202             * BUILD_TAG.
 203
 204         Returns:
 205             TestCase.EX_OK if results were pushed to DB.
 206             TestCase.EX_PUSH_TO_DB_ERROR otherwise.
 207         """
 208         try:
 209             if self.is_skipped:
 210                 return TestCase.EX_PUSH_TO_DB_ERROR
 211             assert self.project_name
 212             assert self.case_name
 213             assert self.start_time
 214             assert self.stop_time
 215             url = env.get('TEST_DB_URL')
 216             data = {"project_name": self.project_name,
 217                     "case_name": self.case_name,
 218                     "details": self.details}
 219             data["installer"] = env.get('INSTALLER_TYPE')
 220             data["scenario"] = env.get('DEPLOY_SCENARIO')
 221             data["pod_name"] = env.get('NODE_NAME')
 222             data["build_tag"] = env.get('BUILD_TAG')
 223             data["criteria"] = 'PASS' if self.is_successful(
 224                 ) == TestCase.EX_OK else 'FAIL'
 225             data["start_date"] = datetime.fromtimestamp(
 226                 self.start_time).strftime('%Y-%m-%d %H:%M:%S')
 227             data["stop_date"] = datetime.fromtimestamp(
 228                 self.stop_time).strftime('%Y-%m-%d %H:%M:%S')
 229             try:
 230                 data["version"] = re.search(
 231                     TestCase._job_name_rule,
 232                     env.get('BUILD_TAG')).group(2)
 233             except Exception:  # pylint: disable=broad-except
 234                 data["version"] = "unknown"
 235             req = requests.post(
 236                 url, data=json.dumps(data, sort_keys=True),
 237                 headers=self._headers)
 238             req.raise_for_status()
 239             if urllib.parse.urlparse(url).scheme != "file":
 240                 res_url = req.json()["href"]
 241                 if env.get('TEST_DB_EXT_URL'):
 242                     res_url = res_url.replace(
 243                         env.get('TEST_DB_URL'), env.get('TEST_DB_EXT_URL'))
 244                 self.__logger.info(
 245                     "The results were successfully pushed to DB: \n\n%s\n",
 246                     res_url)
 247         except AssertionError:
 248             self.__logger.exception(
 249                 "Please run test before publishing the results")
 250             return TestCase.EX_PUSH_TO_DB_ERROR
 251         except requests.exceptions.HTTPError:
 252             self.__logger.exception("The HTTP request raises issues")
 253             return TestCase.EX_PUSH_TO_DB_ERROR
 254         except Exception:  # pylint: disable=broad-except
 255             self.__logger.exception("The results cannot be pushed to DB")
 256             return TestCase.EX_PUSH_TO_DB_ERROR
 257         return TestCase.EX_OK
 258
 259     def publish_artifacts(self):  # pylint: disable=too-many-locals
 260         """Push the artifacts to the S3 repository.
 261
 262         It allows publishing the artifacts.
 263
 264         It could be overriden if the common implementation is not
 265         suitable.
 266
 267         The credentials must be configured before publishing the artifacts:
 268
 269             * fill ~/.aws/credentials or ~/.boto,
 270             * set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in env.
 271
 272         The next vars must be set in env:
 273
 274             * S3_ENDPOINT_URL (http://127.0.0.1:9000),
 275             * S3_DST_URL (s3://xtesting/prefix),
 276             * HTTP_DST_URL (http://127.0.0.1/prefix).
 277
 278         Returns:
 279             TestCase.EX_OK if artifacts were published to repository.
 280             TestCase.EX_PUBLISH_ARTIFACTS_ERROR otherwise.
 281         """
 282         try:
 283             b3resource = boto3.resource(
 284                 's3', endpoint_url=os.environ["S3_ENDPOINT_URL"])
 285             dst_s3_url = os.environ["S3_DST_URL"]
 286             multipart_threshold = 5 * 1024 ** 5 if "google" in os.environ[
 287                 "S3_ENDPOINT_URL"] else 8 * 1024 * 1024
 288             config = TransferConfig(multipart_threshold=multipart_threshold)
 289             bucket_name = urllib.parse.urlparse(dst_s3_url).netloc
 290             try:
 291                 b3resource.meta.client.head_bucket(Bucket=bucket_name)
 292             except botocore.exceptions.ClientError as exc:
 293                 error_code = exc.response['Error']['Code']
 294                 if error_code == '404':
 295                     # pylint: disable=no-member
 296                     b3resource.create_bucket(Bucket=bucket_name)
 297                 else:
 298                     typ, value, traceback = sys.exc_info()
 299                     six.reraise(typ, value, traceback)
 300             except Exception:  # pylint: disable=broad-except
 301                 typ, value, traceback = sys.exc_info()
 302                 six.reraise(typ, value, traceback)
 303             path = urllib.parse.urlparse(dst_s3_url).path.strip("/")
 304             dst_http_url = os.environ["HTTP_DST_URL"]
 305             output_str = "\n"
 306             self.details["links"] = []
 307             for log_file in [self.output_log_name, self.output_debug_log_name]:
 308                 if os.path.exists(os.path.join(self.dir_results, log_file)):
 309                     abs_file = os.path.join(self.dir_results, log_file)
 310                     mime_type = mimetypes.guess_type(abs_file)
 311                     self.__logger.debug(
 312                         "Publishing %s %s", abs_file, mime_type)
 313                     # pylint: disable=no-member
 314                     b3resource.Bucket(bucket_name).upload_file(
 315                         abs_file, os.path.join(path, log_file), Config=config,
 316                         ExtraArgs={'ContentType': mime_type[
 317                             0] or 'application/octet-stream'})
 318                     link = os.path.join(dst_http_url, log_file)
 319                     output_str += "\n{}".format(link)
 320                     self.details["links"].append(link)
 321             for root, _, files in os.walk(self.res_dir):
 322                 for pub_file in files:
 323                     abs_file = os.path.join(root, pub_file)
 324                     mime_type = mimetypes.guess_type(abs_file)
 325                     self.__logger.debug(
 326                         "Publishing %s %s", abs_file, mime_type)
 327                     # pylint: disable=no-member
 328                     b3resource.Bucket(bucket_name).upload_file(
 329                         abs_file,
 330                         os.path.join(path, os.path.relpath(
 331                             os.path.join(root, pub_file),
 332                             start=self.dir_results)),
 333                         Config=config,
 334                         ExtraArgs={'ContentType': mime_type[
 335                             0] or 'application/octet-stream'})
 336                     link = os.path.join(dst_http_url, os.path.relpath(
 337                         os.path.join(root, pub_file),
 338                         start=self.dir_results))
 339                     output_str += "\n{}".format(link)
 340                     self.details["links"].append(link)
 341             self.__logger.info(
 342                 "All artifacts were successfully published: %s\n", output_str)
 343             return TestCase.EX_OK
 344         except KeyError as ex:
 345             self.__logger.error("Please check env var: %s", str(ex))
 346             return TestCase.EX_PUBLISH_ARTIFACTS_ERROR
 347         except botocore.exceptions.NoCredentialsError:
 348             self.__logger.error(
 349                 "Please fill ~/.aws/credentials, ~/.boto or set "
 350                 "AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in env")
 351             return TestCase.EX_PUBLISH_ARTIFACTS_ERROR
 352         except Exception:  # pylint: disable=broad-except
 353             self.__logger.exception("Cannot publish the artifacts")
 354             return TestCase.EX_PUBLISH_ARTIFACTS_ERROR
 355
 356     def clean(self):
 357         """Clean the resources.
 358
 359         It can be overriden if resources must be deleted after
 360         running the test case.
 361         """