xtesting/core/campaign.py

   1 #!/usr/bin/env python
   2
   3 # Copyright (c) 2019 Orange and others.
   4 #
   5 # All rights reserved. This program and the accompanying materials
   6 # are made available under the terms of the Apache License, Version 2.0
   7 # which accompanies this distribution, and is available at
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9
  10 """Dump DB and artifacts for third-party certifications."""
  11
  12 import json
  13 import logging
  14 import logging.config
  15 import mimetypes
  16 import os
  17 import re
  18 import urllib
  19 import zipfile
  20
  21 import boto3
  22 from boto3.s3.transfer import TransferConfig
  23 import botocore
  24 import requests
  25
  26 from xtesting.core import testcase
  27 from xtesting.utils import env
  28 from xtesting.utils import config
  29 from xtesting.utils import constants
  30
  31 __author__ = "Cedric Ollivier <cedric.ollivier@orange.com>"
  32
  33
  34 class Campaign():
  35     "Dump, archive and publish all results and artifacts from a campaign."
  36
  37     EX_OK = os.EX_OK
  38     """everything is OK"""
  39
  40     EX_DUMP_FROM_DB_ERROR = os.EX_SOFTWARE - 5
  41     """dump_db() failed"""
  42
  43     EX_DUMP_ARTIFACTS_ERROR = os.EX_SOFTWARE - 6
  44     """dump_artifacts() failed"""
  45
  46     EX_ZIP_CAMPAIGN_FILES_ERROR = os.EX_SOFTWARE - 7
  47     """dump_artifacts() failed"""
  48
  49     __logger = logging.getLogger(__name__)
  50
  51     @staticmethod
  52     def dump_db():
  53         """Dump all test campaign results from the DB.
  54
  55         It allows collecting all the results from the DB.
  56
  57         It could be overriden if the common implementation is not
  58         suitable.
  59
  60         The next vars must be set in env:
  61
  62             * TEST_DB_URL,
  63             * BUILD_TAG.
  64
  65         Returns:
  66             Campaign.EX_OK if results were collected from DB.
  67             Campaign.EX_DUMP_FROM_DB_ERROR otherwise.
  68         """
  69         try:
  70             url = env.get('TEST_DB_URL')
  71             req = requests.get(
  72                 f"{url}?build_tag={env.get('BUILD_TAG')}",
  73                 headers=testcase.TestCase.headers)
  74             req.raise_for_status()
  75             output = req.json()
  76             Campaign.__logger.debug("data from DB: \n%s", output)
  77             for i, _ in enumerate(output["results"]):
  78                 for j, _ in enumerate(
  79                         output["results"][i]["details"]["links"]):
  80                     output["results"][i]["details"]["links"][j] = re.sub(
  81                         "^{os.environ['HTTP_DST_URL']}/*", '',
  82                         output["results"][i]["details"]["links"][j])
  83             Campaign.__logger.debug("data to archive: \n%s", output)
  84             with open(f"{env.get('BUILD_TAG')}.json", "w",
  85                       encoding='utf-8') as dfile:
  86                 json.dump(output, dfile)
  87         except Exception:  # pylint: disable=broad-except
  88             Campaign.__logger.exception(
  89                 "The results cannot be collected from DB")
  90             return Campaign.EX_DUMP_FROM_DB_ERROR
  91         return Campaign.EX_OK
  92
  93     @staticmethod
  94     def dump_artifacts():
  95         """Dump all test campaign artifacts from the S3 repository.
  96
  97         It allows collecting all the artifacts from the S3 repository.
  98
  99         It could be overriden if the common implementation is not
 100         suitable.
 101
 102         The credentials must be configured before publishing the artifacts:
 103
 104             * fill ~/.aws/credentials or ~/.boto,
 105             * set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in env.
 106
 107         The next vars must be set in env:
 108
 109             * S3_ENDPOINT_URL (http://127.0.0.1:9000),
 110             * S3_DST_URL (s3://xtesting/prefix),
 111
 112         Returns:
 113             Campaign.EX_OK if artifacts were published to repository.
 114             Campaign.EX_DUMP_ARTIFACTS_ERROR otherwise.
 115         """
 116         try:
 117             build_tag = env.get('BUILD_TAG')
 118             b3resource = boto3.resource(
 119                 's3', endpoint_url=os.environ["S3_ENDPOINT_URL"])
 120             dst_s3_url = os.environ["S3_DST_URL"]
 121             multipart_threshold = 5 * 1024 ** 5 if "google" in os.environ[
 122                 "S3_ENDPOINT_URL"] else 8 * 1024 * 1024
 123             tconfig = TransferConfig(multipart_threshold=multipart_threshold)
 124             bucket_name = urllib.parse.urlparse(dst_s3_url).netloc
 125             s3path = re.search(
 126                 '^/*(.*)/*$', urllib.parse.urlparse(dst_s3_url).path).group(1)
 127             prefix = os.path.join(s3path, build_tag)
 128             # pylint: disable=no-member
 129             for s3_object in b3resource.Bucket(bucket_name).objects.filter(
 130                     Prefix=f"{prefix}/"):
 131                 path, _ = os.path.split(
 132                     urllib.parse.unquote_plus(s3_object.key))
 133                 lpath = re.sub(f'^{s3path}/*', '', path)
 134                 if lpath and not os.path.exists(lpath):
 135                     os.makedirs(lpath)
 136                 Campaign.__logger.info(
 137                     "Downloading %s",
 138                     re.sub(f'^{s3path}/*', '',
 139                            urllib.parse.unquote_plus(s3_object.key)))
 140                 # pylint: disable=no-member
 141                 b3resource.Bucket(bucket_name).download_file(
 142                     urllib.parse.unquote_plus(s3_object.key),
 143                     re.sub(f'^{s3path}/*', '',
 144                            urllib.parse.unquote_plus(s3_object.key)),
 145                     Config=tconfig)
 146             return Campaign.EX_OK
 147         except Exception:  # pylint: disable=broad-except
 148             Campaign.__logger.exception("Cannot publish the artifacts")
 149             return Campaign.EX_DUMP_ARTIFACTS_ERROR
 150
 151     @staticmethod
 152     def zip_campaign_files():  # pylint: disable=too-many-locals
 153         """Archive and publish all test campaign data to the S3 repository.
 154
 155         It allows collecting all the artifacts from the S3 repository.
 156
 157         It could be overriden if the common implementation is not
 158         suitable.
 159
 160         The credentials must be configured before publishing the artifacts:
 161
 162             * fill ~/.aws/credentials or ~/.boto,
 163             * set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in env.
 164
 165         The next vars must be set in env:
 166
 167             * S3_ENDPOINT_URL (http://127.0.0.1:9000),
 168             * S3_DST_URL (s3://xtesting/prefix),
 169
 170         Returns:
 171             Campaign.EX_OK if artifacts were published to repository.
 172             Campaign.EX_DUMP_ARTIFACTS_ERROR otherwise.
 173         """
 174         try:
 175             build_tag = env.get('BUILD_TAG')
 176             assert Campaign.dump_db() == Campaign.EX_OK
 177             assert Campaign.dump_artifacts() == Campaign.EX_OK
 178             with zipfile.ZipFile(f'{build_tag}.zip',
 179                                  'w', zipfile.ZIP_DEFLATED) as zfile:
 180                 zfile.write(f"{build_tag}.json")
 181                 for root, _, files in os.walk(build_tag):
 182                     for filename in files:
 183                         zfile.write(os.path.join(root, filename))
 184             b3resource = boto3.resource(
 185                 's3', endpoint_url=os.environ["S3_ENDPOINT_URL"])
 186             dst_s3_url = os.environ["S3_DST_URL"]
 187             multipart_threshold = 5 * 1024 ** 5 if "google" in os.environ[
 188                 "S3_ENDPOINT_URL"] else 8 * 1024 * 1024
 189             tconfig = TransferConfig(multipart_threshold=multipart_threshold)
 190             bucket_name = urllib.parse.urlparse(dst_s3_url).netloc
 191             mime_type = mimetypes.guess_type(f'{build_tag}.zip')
 192             path = urllib.parse.urlparse(dst_s3_url).path.strip("/")
 193             # pylint: disable=no-member
 194             b3resource.Bucket(bucket_name).upload_file(
 195                 f'{build_tag}.zip',
 196                 os.path.join(path, f'{build_tag}.zip'),
 197                 Config=tconfig,
 198                 ExtraArgs={'ContentType': mime_type[
 199                     0] or 'application/octet-stream'})
 200             dst_http_url = os.environ["HTTP_DST_URL"]
 201             link = os.path.join(dst_http_url, f'{build_tag}.zip')
 202             Campaign.__logger.info(
 203                 "All data were successfully published:\n\n%s", link)
 204             return Campaign.EX_OK
 205         except KeyError as ex:
 206             Campaign.__logger.error("Please check env var: %s", str(ex))
 207             return Campaign.EX_ZIP_CAMPAIGN_FILES_ERROR
 208         except botocore.exceptions.NoCredentialsError:
 209             Campaign.__logger.error(
 210                 "Please fill ~/.aws/credentials, ~/.boto or set "
 211                 "AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in env")
 212             return Campaign.EX_ZIP_CAMPAIGN_FILES_ERROR
 213         except Exception:  # pylint: disable=broad-except
 214             Campaign.__logger.exception("Cannot publish the artifacts")
 215             return Campaign.EX_ZIP_CAMPAIGN_FILES_ERROR
 216
 217
 218 def main():
 219     """Entry point for Campaign.zip_campaign_files()."""
 220     if not os.path.exists(testcase.TestCase.dir_results):
 221         os.makedirs(testcase.TestCase.dir_results)
 222     if env.get('DEBUG').lower() == 'true':
 223         logging.config.fileConfig(config.get_xtesting_config(
 224             'logging.debug.ini', constants.DEBUG_INI_PATH_DEFAULT))
 225     else:
 226         logging.config.fileConfig(config.get_xtesting_config(
 227             'logging.ini', constants.INI_PATH_DEFAULT))
 228     logging.captureWarnings(True)
 229     os.chdir(testcase.TestCase.dir_results)
 230     Campaign.zip_campaign_files()