xtesting/core/campaign.py

   1 #!/usr/bin/env python
   2
   3 # Copyright (c) 2019 Orange and others.
   4 #
   5 # All rights reserved. This program and the accompanying materials
   6 # are made available under the terms of the Apache License, Version 2.0
   7 # which accompanies this distribution, and is available at
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9
  10 """Dump DB and artifacts for third-party certifications."""
  11
  12 import json
  13 import logging
  14 import logging.config
  15 import mimetypes
  16 import os
  17 import re
  18 import urllib
  19 import zipfile
  20
  21 import boto3
  22 from boto3.s3.transfer import TransferConfig
  23 import botocore
  24 import requests
  25
  26 from xtesting.core import testcase
  27 from xtesting.utils import env
  28 from xtesting.utils import config
  29 from xtesting.utils import constants
  30
  31 __author__ = "Cedric Ollivier <cedric.ollivier@orange.com>"
  32
  33
  34 class Campaign():
  35     "Dump, archive and publish all results and artifacts from a campaign."
  36
  37     EX_OK = os.EX_OK
  38     """everything is OK"""
  39
  40     EX_DUMP_FROM_DB_ERROR = os.EX_SOFTWARE - 5
  41     """dump_db() failed"""
  42
  43     EX_DUMP_ARTIFACTS_ERROR = os.EX_SOFTWARE - 6
  44     """dump_artifacts() failed"""
  45
  46     EX_ZIP_CAMPAIGN_FILES_ERROR = os.EX_SOFTWARE - 7
  47     """dump_artifacts() failed"""
  48
  49     __logger = logging.getLogger(__name__)
  50
  51     @staticmethod
  52     def dump_db():
  53         """Dump all test campaign results from the DB.
  54
  55         It allows collecting all the results from the DB.
  56
  57         It could be overriden if the common implementation is not
  58         suitable.
  59
  60         The next vars must be set in env:
  61
  62             * TEST_DB_URL,
  63             * BUILD_TAG.
  64
  65         Returns:
  66             Campaign.EX_OK if results were collected from DB.
  67             Campaign.EX_DUMP_FROM_DB_ERROR otherwise.
  68         """
  69         try:
  70             url = env.get('TEST_DB_URL')
  71             req = requests.get(
  72                 f"{url}?build_tag={env.get('BUILD_TAG')}",
  73                 headers=testcase.TestCase.headers,
  74                 timeout=10)
  75             req.raise_for_status()
  76             output = req.json()
  77             Campaign.__logger.debug("data from DB: \n%s", output)
  78             for i, _ in enumerate(output["results"]):
  79                 for j, _ in enumerate(
  80                         output["results"][i]["details"]["links"]):
  81                     output["results"][i]["details"]["links"][j] = re.sub(
  82                         "^{os.environ['HTTP_DST_URL']}/*", '',
  83                         output["results"][i]["details"]["links"][j])
  84             Campaign.__logger.debug("data to archive: \n%s", output)
  85             with open(f"{env.get('BUILD_TAG')}.json", "w",
  86                       encoding='utf-8') as dfile:
  87                 json.dump(output, dfile)
  88         except Exception:  # pylint: disable=broad-except
  89             Campaign.__logger.exception(
  90                 "The results cannot be collected from DB")
  91             return Campaign.EX_DUMP_FROM_DB_ERROR
  92         return Campaign.EX_OK
  93
  94     @staticmethod
  95     def dump_artifacts():
  96         """Dump all test campaign artifacts from the S3 repository.
  97
  98         It allows collecting all the artifacts from the S3 repository.
  99
 100         It could be overriden if the common implementation is not
 101         suitable.
 102
 103         The credentials must be configured before publishing the artifacts:
 104
 105             * fill ~/.aws/credentials or ~/.boto,
 106             * set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in env.
 107
 108         The next vars must be set in env:
 109
 110             * S3_ENDPOINT_URL (http://127.0.0.1:9000),
 111             * S3_DST_URL (s3://xtesting/prefix),
 112
 113         Returns:
 114             Campaign.EX_OK if artifacts were published to repository.
 115             Campaign.EX_DUMP_ARTIFACTS_ERROR otherwise.
 116         """
 117         try:
 118             build_tag = env.get('BUILD_TAG')
 119             b3resource = boto3.resource(
 120                 's3', endpoint_url=os.environ["S3_ENDPOINT_URL"])
 121             dst_s3_url = os.environ["S3_DST_URL"]
 122             multipart_threshold = 5 * 1024 ** 5 if "google" in os.environ[
 123                 "S3_ENDPOINT_URL"] else 8 * 1024 * 1024
 124             tconfig = TransferConfig(multipart_threshold=multipart_threshold)
 125             bucket_name = urllib.parse.urlparse(dst_s3_url).netloc
 126             s3path = re.search(
 127                 '^/*(.*)/*$', urllib.parse.urlparse(dst_s3_url).path).group(1)
 128             prefix = os.path.join(s3path, build_tag)
 129             # pylint: disable=no-member
 130             for s3_object in b3resource.Bucket(bucket_name).objects.filter(
 131                     Prefix=f"{prefix}/"):
 132                 path, _ = os.path.split(
 133                     urllib.parse.unquote_plus(s3_object.key))
 134                 lpath = re.sub(f'^{s3path}/*', '', path)
 135                 if lpath and not os.path.exists(lpath):
 136                     os.makedirs(lpath)
 137                 Campaign.__logger.info(
 138                     "Downloading %s",
 139                     re.sub(f'^{s3path}/*', '',
 140                            urllib.parse.unquote_plus(s3_object.key)))
 141                 # pylint: disable=no-member
 142                 b3resource.Bucket(bucket_name).download_file(
 143                     urllib.parse.unquote_plus(s3_object.key),
 144                     re.sub(f'^{s3path}/*', '',
 145                            urllib.parse.unquote_plus(s3_object.key)),
 146                     Config=tconfig)
 147             return Campaign.EX_OK
 148         except Exception:  # pylint: disable=broad-except
 149             Campaign.__logger.exception("Cannot publish the artifacts")
 150             return Campaign.EX_DUMP_ARTIFACTS_ERROR
 151
 152     @staticmethod
 153     def zip_campaign_files():  # pylint: disable=too-many-locals
 154         """Archive and publish all test campaign data to the S3 repository.
 155
 156         It allows collecting all the artifacts from the S3 repository.
 157
 158         It could be overriden if the common implementation is not
 159         suitable.
 160
 161         The credentials must be configured before publishing the artifacts:
 162
 163             * fill ~/.aws/credentials or ~/.boto,
 164             * set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in env.
 165
 166         The next vars must be set in env:
 167
 168             * S3_ENDPOINT_URL (http://127.0.0.1:9000),
 169             * S3_DST_URL (s3://xtesting/prefix),
 170
 171         Returns:
 172             Campaign.EX_OK if artifacts were published to repository.
 173             Campaign.EX_DUMP_ARTIFACTS_ERROR otherwise.
 174         """
 175         try:
 176             build_tag = env.get('BUILD_TAG')
 177             assert Campaign.dump_db() == Campaign.EX_OK
 178             assert Campaign.dump_artifacts() == Campaign.EX_OK
 179             with zipfile.ZipFile(f'{build_tag}.zip',
 180                                  'w', zipfile.ZIP_DEFLATED) as zfile:
 181                 zfile.write(f"{build_tag}.json")
 182                 for root, _, files in os.walk(build_tag):
 183                     for filename in files:
 184                         zfile.write(os.path.join(root, filename))
 185             b3resource = boto3.resource(
 186                 's3', endpoint_url=os.environ["S3_ENDPOINT_URL"])
 187             dst_s3_url = os.environ["S3_DST_URL"]
 188             multipart_threshold = 5 * 1024 ** 5 if "google" in os.environ[
 189                 "S3_ENDPOINT_URL"] else 8 * 1024 * 1024
 190             tconfig = TransferConfig(multipart_threshold=multipart_threshold)
 191             bucket_name = urllib.parse.urlparse(dst_s3_url).netloc
 192             mime_type = mimetypes.guess_type(f'{build_tag}.zip')
 193             path = urllib.parse.urlparse(dst_s3_url).path.strip("/")
 194             # pylint: disable=no-member
 195             b3resource.Bucket(bucket_name).upload_file(
 196                 f'{build_tag}.zip',
 197                 os.path.join(path, f'{build_tag}.zip'),
 198                 Config=tconfig,
 199                 ExtraArgs={'ContentType': mime_type[
 200                     0] or 'application/octet-stream'})
 201             dst_http_url = os.environ["HTTP_DST_URL"]
 202             link = os.path.join(dst_http_url, f'{build_tag}.zip')
 203             Campaign.__logger.info(
 204                 "All data were successfully published:\n\n%s", link)
 205             return Campaign.EX_OK
 206         except KeyError as ex:
 207             Campaign.__logger.error("Please check env var: %s", str(ex))
 208             return Campaign.EX_ZIP_CAMPAIGN_FILES_ERROR
 209         except botocore.exceptions.NoCredentialsError:
 210             Campaign.__logger.error(
 211                 "Please fill ~/.aws/credentials, ~/.boto or set "
 212                 "AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in env")
 213             return Campaign.EX_ZIP_CAMPAIGN_FILES_ERROR
 214         except Exception:  # pylint: disable=broad-except
 215             Campaign.__logger.exception("Cannot publish the artifacts")
 216             return Campaign.EX_ZIP_CAMPAIGN_FILES_ERROR
 217
 218
 219 def main():
 220     """Entry point for Campaign.zip_campaign_files()."""
 221     if not os.path.exists(testcase.TestCase.dir_results):
 222         os.makedirs(testcase.TestCase.dir_results)
 223     if env.get('DEBUG').lower() == 'true':
 224         logging.config.fileConfig(config.get_xtesting_config(
 225             'logging.debug.ini', constants.DEBUG_INI_PATH_DEFAULT))
 226     else:
 227         logging.config.fileConfig(config.get_xtesting_config(
 228             'logging.ini', constants.INI_PATH_DEFAULT))
 229     logging.captureWarnings(True)
 230     os.chdir(testcase.TestCase.dir_results)
 231     Campaign.zip_campaign_files()