xtesting/core/testcase.py

   1 #!/usr/bin/env python
   2
   3 # Copyright (c) 2016 Orange and others.
   4 #
   5 # All rights reserved. This program and the accompanying materials
   6 # are made available under the terms of the Apache License, Version 2.0
   7 # which accompanies this distribution, and is available at
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9
  10 """Define the parent class of all Xtesting TestCases."""
  11
  12 import abc
  13 from datetime import datetime
  14 import json
  15 import logging
  16 import mimetypes
  17 import os
  18 import re
  19 import sys
  20
  21 import boto3
  22 from boto3.s3.transfer import TransferConfig
  23 import botocore
  24 import prettytable
  25 import requests
  26 import six
  27 from six.moves import urllib
  28
  29 from xtesting.utils import decorators
  30 from xtesting.utils import env
  31
  32 __author__ = "Cedric Ollivier <cedric.ollivier@orange.com>"
  33
  34
  35 @six.add_metaclass(abc.ABCMeta)
  36 class TestCase():
  37     # pylint: disable=too-many-instance-attributes
  38     """Base model for single test case."""
  39
  40     EX_OK = os.EX_OK
  41     """everything is OK"""
  42
  43     EX_RUN_ERROR = os.EX_SOFTWARE
  44     """run() failed"""
  45
  46     EX_PUSH_TO_DB_ERROR = os.EX_SOFTWARE - 1
  47     """push_to_db() failed"""
  48
  49     EX_TESTCASE_FAILED = os.EX_SOFTWARE - 2
  50     """results are false"""
  51
  52     EX_TESTCASE_SKIPPED = os.EX_SOFTWARE - 3
  53     """requirements are unmet"""
  54
  55     EX_PUBLISH_ARTIFACTS_ERROR = os.EX_SOFTWARE - 4
  56     """publish_artifacts() failed"""
  57
  58     dir_results = "/var/lib/xtesting/results"
  59     _job_name_rule = "(dai|week)ly-(.+?)-[0-9]*"
  60     headers = {'Content-Type': 'application/json'}
  61     __logger = logging.getLogger(__name__)
  62
  63     def __init__(self, **kwargs):
  64         self.details = {}
  65         self.project_name = kwargs.get('project_name', 'xtesting')
  66         self.case_name = kwargs.get('case_name', '')
  67         self.criteria = kwargs.get('criteria', 100)
  68         self.result = 0
  69         self.start_time = 0
  70         self.stop_time = 0
  71         self.is_skipped = False
  72         self.output_log_name = 'xtesting.log'
  73         self.output_debug_log_name = 'xtesting.debug.log'
  74         self.res_dir = "{}/{}".format(self.dir_results, self.case_name)
  75
  76     def __str__(self):
  77         try:
  78             assert self.project_name
  79             assert self.case_name
  80             if self.is_skipped:
  81                 result = 'SKIP'
  82             else:
  83                 result = 'PASS' if(self.is_successful(
  84                     ) == TestCase.EX_OK) else 'FAIL'
  85             msg = prettytable.PrettyTable(
  86                 header_style='upper', padding_width=5,
  87                 field_names=['test case', 'project', 'duration',
  88                              'result'])
  89             msg.add_row([self.case_name, self.project_name,
  90                          self.get_duration(), result])
  91             return msg.get_string()
  92         except AssertionError:
  93             self.__logger.error("We cannot print invalid objects")
  94             return super(TestCase, self).__str__()
  95
  96     def get_duration(self):
  97         """Return the duration of the test case.
  98
  99         Returns:
 100             duration if start_time and stop_time are set
 101             "XX:XX" otherwise.
 102         """
 103         try:
 104             if self.is_skipped:
 105                 return "00:00"
 106             assert self.start_time
 107             assert self.stop_time
 108             if self.stop_time < self.start_time:
 109                 return "XX:XX"
 110             return "{}:{}".format(
 111                 str(int(self.stop_time - self.start_time) // 60).zfill(2),
 112                 str(int(self.stop_time - self.start_time) % 60).zfill(2))
 113         except Exception:  # pylint: disable=broad-except
 114             self.__logger.error("Please run test before getting the duration")
 115             return "XX:XX"
 116
 117     def is_successful(self):
 118         """Interpret the result of the test case.
 119
 120         It allows getting the result of TestCase. It completes run()
 121         which only returns the execution status.
 122
 123         It can be overriden if checking result is not suitable.
 124
 125         Returns:
 126             TestCase.EX_OK if result is 'PASS'.
 127             TestCase.EX_TESTCASE_SKIPPED if test case is skipped.
 128             TestCase.EX_TESTCASE_FAILED otherwise.
 129         """
 130         try:
 131             if self.is_skipped:
 132                 return TestCase.EX_TESTCASE_SKIPPED
 133             assert self.criteria
 134             assert self.result is not None
 135             if (not isinstance(self.result, str) and
 136                     not isinstance(self.criteria, str)):
 137                 if self.result >= self.criteria:
 138                     return TestCase.EX_OK
 139             else:
 140                 # Backward compatibility
 141                 # It must be removed as soon as TestCase subclasses
 142                 # stop setting result = 'PASS' or 'FAIL'.
 143                 # In this case criteria is unread.
 144                 self.__logger.warning(
 145                     "Please update result which must be an int!")
 146                 if self.result == 'PASS':
 147                     return TestCase.EX_OK
 148         except AssertionError:
 149             self.__logger.error("Please run test before checking the results")
 150         return TestCase.EX_TESTCASE_FAILED
 151
 152     def check_requirements(self):  # pylint: disable=no-self-use
 153         """Check the requirements of the test case.
 154
 155         It can be overriden on purpose.
 156         """
 157         self.is_skipped = False
 158
 159     @abc.abstractmethod
 160     def run(self, **kwargs):
 161         """Run the test case.
 162
 163         It allows running TestCase and getting its execution
 164         status.
 165
 166         The subclasses must override the default implementation which
 167         is false on purpose.
 168
 169         The new implementation must set the following attributes to
 170         push the results to DB:
 171
 172             * result,
 173             * start_time,
 174             * stop_time.
 175
 176         Args:
 177             kwargs: Arbitrary keyword arguments.
 178         """
 179
 180     @decorators.can_dump_request_to_file
 181     def push_to_db(self):
 182         """Push the results of the test case to the DB.
 183
 184         It allows publishing the results and checking the status.
 185
 186         It could be overriden if the common implementation is not
 187         suitable.
 188
 189         The following attributes must be set before pushing the results to DB:
 190
 191             * project_name,
 192             * case_name,
 193             * result,
 194             * start_time,
 195             * stop_time.
 196
 197         The next vars must be set in env:
 198
 199             * TEST_DB_URL,
 200             * INSTALLER_TYPE,
 201             * DEPLOY_SCENARIO,
 202             * NODE_NAME,
 203             * BUILD_TAG.
 204
 205         Returns:
 206             TestCase.EX_OK if results were pushed to DB.
 207             TestCase.EX_PUSH_TO_DB_ERROR otherwise.
 208         """
 209         try:
 210             if self.is_skipped:
 211                 return TestCase.EX_PUSH_TO_DB_ERROR
 212             assert self.project_name
 213             assert self.case_name
 214             assert self.start_time
 215             assert self.stop_time
 216             url = env.get('TEST_DB_URL')
 217             data = {"project_name": self.project_name,
 218                     "case_name": self.case_name,
 219                     "details": self.details}
 220             data["installer"] = env.get('INSTALLER_TYPE')
 221             data["scenario"] = env.get('DEPLOY_SCENARIO')
 222             data["pod_name"] = env.get('NODE_NAME')
 223             data["build_tag"] = env.get('BUILD_TAG')
 224             data["criteria"] = 'PASS' if self.is_successful(
 225                 ) == TestCase.EX_OK else 'FAIL'
 226             data["start_date"] = datetime.fromtimestamp(
 227                 self.start_time).strftime('%Y-%m-%d %H:%M:%S')
 228             data["stop_date"] = datetime.fromtimestamp(
 229                 self.stop_time).strftime('%Y-%m-%d %H:%M:%S')
 230             try:
 231                 data["version"] = re.search(
 232                     TestCase._job_name_rule,
 233                     env.get('BUILD_TAG')).group(2)
 234             except Exception:  # pylint: disable=broad-except
 235                 data["version"] = "unknown"
 236             req = requests.post(
 237                 url, data=json.dumps(data, sort_keys=True),
 238                 headers=self.headers)
 239             req.raise_for_status()
 240             if urllib.parse.urlparse(url).scheme != "file":
 241                 # href must be postprocessed as OPNFV testapi is misconfigured
 242                 # (localhost is returned)
 243                 uid = re.sub(r'^.*/api/v1/results/*', '', req.json()["href"])
 244                 netloc = env.get('TEST_DB_EXT_URL') if env.get(
 245                     'TEST_DB_EXT_URL') else env.get('TEST_DB_URL')
 246                 self.__logger.info(
 247                     "The results were successfully pushed to DB: \n\n%s\n",
 248                     os.path.join(netloc, uid))
 249         except AssertionError:
 250             self.__logger.exception(
 251                 "Please run test before publishing the results")
 252             return TestCase.EX_PUSH_TO_DB_ERROR
 253         except requests.exceptions.HTTPError:
 254             self.__logger.exception("The HTTP request raises issues")
 255             return TestCase.EX_PUSH_TO_DB_ERROR
 256         except Exception:  # pylint: disable=broad-except
 257             self.__logger.exception("The results cannot be pushed to DB")
 258             return TestCase.EX_PUSH_TO_DB_ERROR
 259         return TestCase.EX_OK
 260
 261     def publish_artifacts(self):  # pylint: disable=too-many-locals
 262         """Push the artifacts to the S3 repository.
 263
 264         It allows publishing the artifacts.
 265
 266         It could be overriden if the common implementation is not
 267         suitable.
 268
 269         The credentials must be configured before publishing the artifacts:
 270
 271             * fill ~/.aws/credentials or ~/.boto,
 272             * set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in env.
 273
 274         The next vars must be set in env:
 275
 276             * S3_ENDPOINT_URL (http://127.0.0.1:9000),
 277             * S3_DST_URL (s3://xtesting/prefix),
 278             * HTTP_DST_URL (http://127.0.0.1/prefix).
 279
 280         Returns:
 281             TestCase.EX_OK if artifacts were published to repository.
 282             TestCase.EX_PUBLISH_ARTIFACTS_ERROR otherwise.
 283         """
 284         try:
 285             b3resource = boto3.resource(
 286                 's3', endpoint_url=os.environ["S3_ENDPOINT_URL"])
 287             dst_s3_url = os.environ["S3_DST_URL"]
 288             multipart_threshold = 5 * 1024 ** 5 if "google" in os.environ[
 289                 "S3_ENDPOINT_URL"] else 8 * 1024 * 1024
 290             config = TransferConfig(multipart_threshold=multipart_threshold)
 291             bucket_name = urllib.parse.urlparse(dst_s3_url).netloc
 292             try:
 293                 b3resource.meta.client.head_bucket(Bucket=bucket_name)
 294             except botocore.exceptions.ClientError as exc:
 295                 error_code = exc.response['Error']['Code']
 296                 if error_code == '404':
 297                     # pylint: disable=no-member
 298                     b3resource.create_bucket(Bucket=bucket_name)
 299                 else:
 300                     typ, value, traceback = sys.exc_info()
 301                     six.reraise(typ, value, traceback)
 302             except Exception:  # pylint: disable=broad-except
 303                 typ, value, traceback = sys.exc_info()
 304                 six.reraise(typ, value, traceback)
 305             path = urllib.parse.urlparse(dst_s3_url).path.strip("/")
 306             dst_http_url = os.environ["HTTP_DST_URL"]
 307             output_str = "\n"
 308             self.details["links"] = []
 309             for log_file in [self.output_log_name, self.output_debug_log_name]:
 310                 if os.path.exists(os.path.join(self.dir_results, log_file)):
 311                     abs_file = os.path.join(self.dir_results, log_file)
 312                     mime_type = mimetypes.guess_type(abs_file)
 313                     self.__logger.debug(
 314                         "Publishing %s %s", abs_file, mime_type)
 315                     # pylint: disable=no-member
 316                     b3resource.Bucket(bucket_name).upload_file(
 317                         abs_file, os.path.join(path, log_file), Config=config,
 318                         ExtraArgs={'ContentType': mime_type[
 319                             0] or 'application/octet-stream'})
 320                     link = os.path.join(dst_http_url, log_file)
 321                     output_str += "\n{}".format(link)
 322                     self.details["links"].append(link)
 323             for root, _, files in os.walk(self.res_dir):
 324                 for pub_file in files:
 325                     abs_file = os.path.join(root, pub_file)
 326                     mime_type = mimetypes.guess_type(abs_file)
 327                     self.__logger.debug(
 328                         "Publishing %s %s", abs_file, mime_type)
 329                     # pylint: disable=no-member
 330                     b3resource.Bucket(bucket_name).upload_file(
 331                         abs_file,
 332                         os.path.join(path, os.path.relpath(
 333                             os.path.join(root, pub_file),
 334                             start=self.dir_results)),
 335                         Config=config,
 336                         ExtraArgs={'ContentType': mime_type[
 337                             0] or 'application/octet-stream'})
 338                     link = os.path.join(dst_http_url, os.path.relpath(
 339                         os.path.join(root, pub_file),
 340                         start=self.dir_results))
 341                     output_str += "\n{}".format(link)
 342                     self.details["links"].append(link)
 343             self.__logger.info(
 344                 "All artifacts were successfully published: %s\n", output_str)
 345             return TestCase.EX_OK
 346         except KeyError as ex:
 347             self.__logger.error("Please check env var: %s", str(ex))
 348             return TestCase.EX_PUBLISH_ARTIFACTS_ERROR
 349         except botocore.exceptions.NoCredentialsError:
 350             self.__logger.error(
 351                 "Please fill ~/.aws/credentials, ~/.boto or set "
 352                 "AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in env")
 353             return TestCase.EX_PUBLISH_ARTIFACTS_ERROR
 354         except Exception:  # pylint: disable=broad-except
 355             self.__logger.exception("Cannot publish the artifacts")
 356             return TestCase.EX_PUBLISH_ARTIFACTS_ERROR
 357
 358     def clean(self):
 359         """Clean the resources.
 360
 361         It can be overriden if resources must be deleted after
 362         running the test case.
 363         """