xtesting/core/testcase.py

   1 #!/usr/bin/env python
   2
   3 # Copyright (c) 2016 Orange and others.
   4 #
   5 # All rights reserved. This program and the accompanying materials
   6 # are made available under the terms of the Apache License, Version 2.0
   7 # which accompanies this distribution, and is available at
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9
  10 """Define the parent class of all Xtesting TestCases."""
  11
  12 import abc
  13 from datetime import datetime
  14 import json
  15 import logging
  16 import mimetypes
  17 import os
  18 import re
  19 import sys
  20
  21 import boto3
  22 from boto3.s3.transfer import TransferConfig
  23 import botocore
  24 import prettytable
  25 import requests
  26 import six
  27 from six.moves import urllib
  28
  29 from xtesting.utils import decorators
  30 from xtesting.utils import env
  31
  32 __author__ = "Cedric Ollivier <cedric.ollivier@orange.com>"
  33
  34
  35 @six.add_metaclass(abc.ABCMeta)
  36 class TestCase():
  37     # pylint: disable=too-many-instance-attributes
  38     """Base model for single test case."""
  39
  40     EX_OK = os.EX_OK
  41     """everything is OK"""
  42
  43     EX_RUN_ERROR = os.EX_SOFTWARE
  44     """run() failed"""
  45
  46     EX_PUSH_TO_DB_ERROR = os.EX_SOFTWARE - 1
  47     """push_to_db() failed"""
  48
  49     EX_TESTCASE_FAILED = os.EX_SOFTWARE - 2
  50     """results are false"""
  51
  52     EX_TESTCASE_SKIPPED = os.EX_SOFTWARE - 3
  53     """requirements are unmet"""
  54
  55     EX_PUBLISH_ARTIFACTS_ERROR = os.EX_SOFTWARE - 4
  56     """publish_artifacts() failed"""
  57
  58     dir_results = "/var/lib/xtesting/results"
  59     _job_name_rule = "(dai|week)ly-(.+?)-[0-9]*"
  60     _headers = {'Content-Type': 'application/json'}
  61     __logger = logging.getLogger(__name__)
  62
  63     def __init__(self, **kwargs):
  64         self.details = {}
  65         self.project_name = kwargs.get('project_name', 'xtesting')
  66         self.case_name = kwargs.get('case_name', '')
  67         self.criteria = kwargs.get('criteria', 100)
  68         self.result = 0
  69         self.start_time = 0
  70         self.stop_time = 0
  71         self.is_skipped = False
  72         self.output_log_name = 'xtesting.log'
  73         self.output_debug_log_name = 'xtesting.debug.log'
  74         self.res_dir = "{}/{}".format(self.dir_results, self.case_name)
  75
  76     def __str__(self):
  77         try:
  78             assert self.project_name
  79             assert self.case_name
  80             if self.is_skipped:
  81                 result = 'SKIP'
  82             else:
  83                 result = 'PASS' if(self.is_successful(
  84                     ) == TestCase.EX_OK) else 'FAIL'
  85             msg = prettytable.PrettyTable(
  86                 header_style='upper', padding_width=5,
  87                 field_names=['test case', 'project', 'duration',
  88                              'result'])
  89             msg.add_row([self.case_name, self.project_name,
  90                          self.get_duration(), result])
  91             return msg.get_string()
  92         except AssertionError:
  93             self.__logger.error("We cannot print invalid objects")
  94             return super(TestCase, self).__str__()
  95
  96     def get_duration(self):
  97         """Return the duration of the test case.
  98
  99         Returns:
 100             duration if start_time and stop_time are set
 101             "XX:XX" otherwise.
 102         """
 103         try:
 104             if self.is_skipped:
 105                 return "00:00"
 106             assert self.start_time
 107             assert self.stop_time
 108             if self.stop_time < self.start_time:
 109                 return "XX:XX"
 110             return "{0[0]:02.0f}:{0[1]:02.0f}".format(divmod(
 111                 self.stop_time - self.start_time, 60))
 112         except Exception:  # pylint: disable=broad-except
 113             self.__logger.error("Please run test before getting the duration")
 114             return "XX:XX"
 115
 116     def is_successful(self):
 117         """Interpret the result of the test case.
 118
 119         It allows getting the result of TestCase. It completes run()
 120         which only returns the execution status.
 121
 122         It can be overriden if checking result is not suitable.
 123
 124         Returns:
 125             TestCase.EX_OK if result is 'PASS'.
 126             TestCase.EX_TESTCASE_SKIPPED if test case is skipped.
 127             TestCase.EX_TESTCASE_FAILED otherwise.
 128         """
 129         try:
 130             if self.is_skipped:
 131                 return TestCase.EX_TESTCASE_SKIPPED
 132             assert self.criteria
 133             assert self.result is not None
 134             if (not isinstance(self.result, str) and
 135                     not isinstance(self.criteria, str)):
 136                 if self.result >= self.criteria:
 137                     return TestCase.EX_OK
 138             else:
 139                 # Backward compatibility
 140                 # It must be removed as soon as TestCase subclasses
 141                 # stop setting result = 'PASS' or 'FAIL'.
 142                 # In this case criteria is unread.
 143                 self.__logger.warning(
 144                     "Please update result which must be an int!")
 145                 if self.result == 'PASS':
 146                     return TestCase.EX_OK
 147         except AssertionError:
 148             self.__logger.error("Please run test before checking the results")
 149         return TestCase.EX_TESTCASE_FAILED
 150
 151     def check_requirements(self):  # pylint: disable=no-self-use
 152         """Check the requirements of the test case.
 153
 154         It can be overriden on purpose.
 155         """
 156         self.is_skipped = False
 157
 158     @abc.abstractmethod
 159     def run(self, **kwargs):
 160         """Run the test case.
 161
 162         It allows running TestCase and getting its execution
 163         status.
 164
 165         The subclasses must override the default implementation which
 166         is false on purpose.
 167
 168         The new implementation must set the following attributes to
 169         push the results to DB:
 170
 171             * result,
 172             * start_time,
 173             * stop_time.
 174
 175         Args:
 176             kwargs: Arbitrary keyword arguments.
 177         """
 178
 179     @decorators.can_dump_request_to_file
 180     def push_to_db(self):
 181         """Push the results of the test case to the DB.
 182
 183         It allows publishing the results and checking the status.
 184
 185         It could be overriden if the common implementation is not
 186         suitable.
 187
 188         The following attributes must be set before pushing the results to DB:
 189
 190             * project_name,
 191             * case_name,
 192             * result,
 193             * start_time,
 194             * stop_time.
 195
 196         The next vars must be set in env:
 197
 198             * TEST_DB_URL,
 199             * INSTALLER_TYPE,
 200             * DEPLOY_SCENARIO,
 201             * NODE_NAME,
 202             * BUILD_TAG.
 203
 204         Returns:
 205             TestCase.EX_OK if results were pushed to DB.
 206             TestCase.EX_PUSH_TO_DB_ERROR otherwise.
 207         """
 208         try:
 209             if self.is_skipped:
 210                 return TestCase.EX_PUSH_TO_DB_ERROR
 211             assert self.project_name
 212             assert self.case_name
 213             assert self.start_time
 214             assert self.stop_time
 215             url = env.get('TEST_DB_URL')
 216             data = {"project_name": self.project_name,
 217                     "case_name": self.case_name,
 218                     "details": self.details}
 219             data["installer"] = env.get('INSTALLER_TYPE')
 220             data["scenario"] = env.get('DEPLOY_SCENARIO')
 221             data["pod_name"] = env.get('NODE_NAME')
 222             data["build_tag"] = env.get('BUILD_TAG')
 223             data["criteria"] = 'PASS' if self.is_successful(
 224                 ) == TestCase.EX_OK else 'FAIL'
 225             data["start_date"] = datetime.fromtimestamp(
 226                 self.start_time).strftime('%Y-%m-%d %H:%M:%S')
 227             data["stop_date"] = datetime.fromtimestamp(
 228                 self.stop_time).strftime('%Y-%m-%d %H:%M:%S')
 229             try:
 230                 data["version"] = re.search(
 231                     TestCase._job_name_rule,
 232                     env.get('BUILD_TAG')).group(2)
 233             except Exception:  # pylint: disable=broad-except
 234                 data["version"] = "unknown"
 235             req = requests.post(
 236                 url, data=json.dumps(data, sort_keys=True),
 237                 headers=self._headers)
 238             req.raise_for_status()
 239             if urllib.parse.urlparse(url).scheme != "file":
 240                 # href must be postprocessed as OPNFV testapi is misconfigured
 241                 # (localhost is returned)
 242                 uid = re.sub(r'^.*/api/v1/results/*', '', req.json()["href"])
 243                 netloc = env.get('TEST_DB_EXT_URL') if env.get(
 244                     'TEST_DB_EXT_URL') else env.get('TEST_DB_URL')
 245                 self.__logger.info(
 246                     "The results were successfully pushed to DB: \n\n%s\n",
 247                     os.path.join(netloc, uid))
 248         except AssertionError:
 249             self.__logger.exception(
 250                 "Please run test before publishing the results")
 251             return TestCase.EX_PUSH_TO_DB_ERROR
 252         except requests.exceptions.HTTPError:
 253             self.__logger.exception("The HTTP request raises issues")
 254             return TestCase.EX_PUSH_TO_DB_ERROR
 255         except Exception:  # pylint: disable=broad-except
 256             self.__logger.exception("The results cannot be pushed to DB")
 257             return TestCase.EX_PUSH_TO_DB_ERROR
 258         return TestCase.EX_OK
 259
 260     def publish_artifacts(self):  # pylint: disable=too-many-locals
 261         """Push the artifacts to the S3 repository.
 262
 263         It allows publishing the artifacts.
 264
 265         It could be overriden if the common implementation is not
 266         suitable.
 267
 268         The credentials must be configured before publishing the artifacts:
 269
 270             * fill ~/.aws/credentials or ~/.boto,
 271             * set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in env.
 272
 273         The next vars must be set in env:
 274
 275             * S3_ENDPOINT_URL (http://127.0.0.1:9000),
 276             * S3_DST_URL (s3://xtesting/prefix),
 277             * HTTP_DST_URL (http://127.0.0.1/prefix).
 278
 279         Returns:
 280             TestCase.EX_OK if artifacts were published to repository.
 281             TestCase.EX_PUBLISH_ARTIFACTS_ERROR otherwise.
 282         """
 283         try:
 284             b3resource = boto3.resource(
 285                 's3', endpoint_url=os.environ["S3_ENDPOINT_URL"])
 286             dst_s3_url = os.environ["S3_DST_URL"]
 287             multipart_threshold = 5 * 1024 ** 5 if "google" in os.environ[
 288                 "S3_ENDPOINT_URL"] else 8 * 1024 * 1024
 289             config = TransferConfig(multipart_threshold=multipart_threshold)
 290             bucket_name = urllib.parse.urlparse(dst_s3_url).netloc
 291             try:
 292                 b3resource.meta.client.head_bucket(Bucket=bucket_name)
 293             except botocore.exceptions.ClientError as exc:
 294                 error_code = exc.response['Error']['Code']
 295                 if error_code == '404':
 296                     # pylint: disable=no-member
 297                     b3resource.create_bucket(Bucket=bucket_name)
 298                 else:
 299                     typ, value, traceback = sys.exc_info()
 300                     six.reraise(typ, value, traceback)
 301             except Exception:  # pylint: disable=broad-except
 302                 typ, value, traceback = sys.exc_info()
 303                 six.reraise(typ, value, traceback)
 304             path = urllib.parse.urlparse(dst_s3_url).path.strip("/")
 305             dst_http_url = os.environ["HTTP_DST_URL"]
 306             output_str = "\n"
 307             self.details["links"] = []
 308             for log_file in [self.output_log_name, self.output_debug_log_name]:
 309                 if os.path.exists(os.path.join(self.dir_results, log_file)):
 310                     abs_file = os.path.join(self.dir_results, log_file)
 311                     mime_type = mimetypes.guess_type(abs_file)
 312                     self.__logger.debug(
 313                         "Publishing %s %s", abs_file, mime_type)
 314                     # pylint: disable=no-member
 315                     b3resource.Bucket(bucket_name).upload_file(
 316                         abs_file, os.path.join(path, log_file), Config=config,
 317                         ExtraArgs={'ContentType': mime_type[
 318                             0] or 'application/octet-stream'})
 319                     link = os.path.join(dst_http_url, log_file)
 320                     output_str += "\n{}".format(link)
 321                     self.details["links"].append(link)
 322             for root, _, files in os.walk(self.res_dir):
 323                 for pub_file in files:
 324                     abs_file = os.path.join(root, pub_file)
 325                     mime_type = mimetypes.guess_type(abs_file)
 326                     self.__logger.debug(
 327                         "Publishing %s %s", abs_file, mime_type)
 328                     # pylint: disable=no-member
 329                     b3resource.Bucket(bucket_name).upload_file(
 330                         abs_file,
 331                         os.path.join(path, os.path.relpath(
 332                             os.path.join(root, pub_file),
 333                             start=self.dir_results)),
 334                         Config=config,
 335                         ExtraArgs={'ContentType': mime_type[
 336                             0] or 'application/octet-stream'})
 337                     link = os.path.join(dst_http_url, os.path.relpath(
 338                         os.path.join(root, pub_file),
 339                         start=self.dir_results))
 340                     output_str += "\n{}".format(link)
 341                     self.details["links"].append(link)
 342             self.__logger.info(
 343                 "All artifacts were successfully published: %s\n", output_str)
 344             return TestCase.EX_OK
 345         except KeyError as ex:
 346             self.__logger.error("Please check env var: %s", str(ex))
 347             return TestCase.EX_PUBLISH_ARTIFACTS_ERROR
 348         except botocore.exceptions.NoCredentialsError:
 349             self.__logger.error(
 350                 "Please fill ~/.aws/credentials, ~/.boto or set "
 351                 "AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in env")
 352             return TestCase.EX_PUBLISH_ARTIFACTS_ERROR
 353         except Exception:  # pylint: disable=broad-except
 354             self.__logger.exception("Cannot publish the artifacts")
 355             return TestCase.EX_PUBLISH_ARTIFACTS_ERROR
 356
 357     def clean(self):
 358         """Clean the resources.
 359
 360         It can be overriden if resources must be deleted after
 361         running the test case.
 362         """