xtesting/core/campaign.py

   1 #!/usr/bin/env python
   2
   3 # Copyright (c) 2019 Orange and others.
   4 #
   5 # All rights reserved. This program and the accompanying materials
   6 # are made available under the terms of the Apache License, Version 2.0
   7 # which accompanies this distribution, and is available at
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9
  10 """Dump DB and artifacts for third-party certifications."""
  11
  12 import json
  13 import logging
  14 import logging.config
  15 import mimetypes
  16 import os
  17 import re
  18 import zipfile
  19
  20 from urllib.parse import urlparse
  21 import boto3
  22 from boto3.s3.transfer import TransferConfig
  23 import botocore
  24 import pkg_resources
  25 import requests
  26
  27 from xtesting.core import testcase
  28 from xtesting.utils import env
  29 from xtesting.utils import constants
  30
  31 __author__ = "Cedric Ollivier <cedric.ollivier@orange.com>"
  32
  33
  34 class Campaign():
  35     "Dump, archive and publish all results and artifacts from a campaign."
  36
  37     EX_OK = os.EX_OK
  38     """everything is OK"""
  39
  40     EX_DUMP_FROM_DB_ERROR = os.EX_SOFTWARE - 5
  41     """dump_db() failed"""
  42
  43     EX_DUMP_ARTIFACTS_ERROR = os.EX_SOFTWARE - 6
  44     """dump_artifacts() failed"""
  45
  46     EX_ZIP_CAMPAIGN_FILES_ERROR = os.EX_SOFTWARE - 7
  47     """dump_artifacts() failed"""
  48
  49     __logger = logging.getLogger(__name__)
  50
  51     @staticmethod
  52     def dump_db():
  53         """Dump all test campaign results from the DB.
  54
  55         It allows collecting all the results from the DB.
  56
  57         It could be overriden if the common implementation is not
  58         suitable.
  59
  60         The next vars must be set in env:
  61
  62             * TEST_DB_URL,
  63             * BUILD_TAG.
  64
  65         Returns:
  66             Campaign.EX_OK if results were collected from DB.
  67             Campaign.EX_DUMP_FROM_DB_ERROR otherwise.
  68         """
  69         try:
  70             url = env.get('TEST_DB_URL')
  71             req = requests.get(
  72                 f"{url}?build_tag={env.get('BUILD_TAG')}",
  73                 headers=testcase.TestCase.headers)
  74             req.raise_for_status()
  75             output = req.json()
  76             Campaign.__logger.debug("data from DB: \n%s", output)
  77             for i, _ in enumerate(output["results"]):
  78                 for j, _ in enumerate(
  79                         output["results"][i]["details"]["links"]):
  80                     output["results"][i]["details"]["links"][j] = re.sub(
  81                         "^{os.environ['HTTP_DST_URL']}/*", '',
  82                         output["results"][i]["details"]["links"][j])
  83             Campaign.__logger.debug("data to archive: \n%s", output)
  84             with open(f"{env.get('BUILD_TAG')}.json", "w",
  85                       encoding='utf-8') as dfile:
  86                 json.dump(output, dfile)
  87         except Exception:  # pylint: disable=broad-except
  88             Campaign.__logger.exception(
  89                 "The results cannot be collected from DB")
  90             return Campaign.EX_DUMP_FROM_DB_ERROR
  91         return Campaign.EX_OK
  92
  93     @staticmethod
  94     def dump_artifacts():
  95         """Dump all test campaign artifacts from the S3 repository.
  96
  97         It allows collecting all the artifacts from the S3 repository.
  98
  99         It could be overriden if the common implementation is not
 100         suitable.
 101
 102         The credentials must be configured before publishing the artifacts:
 103
 104             * fill ~/.aws/credentials or ~/.boto,
 105             * set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in env.
 106
 107         The next vars must be set in env:
 108
 109             * S3_ENDPOINT_URL (http://127.0.0.1:9000),
 110             * S3_DST_URL (s3://xtesting/prefix),
 111
 112         Returns:
 113             Campaign.EX_OK if artifacts were published to repository.
 114             Campaign.EX_DUMP_ARTIFACTS_ERROR otherwise.
 115         """
 116         try:
 117             build_tag = env.get('BUILD_TAG')
 118             b3resource = boto3.resource(
 119                 's3', endpoint_url=os.environ["S3_ENDPOINT_URL"])
 120             dst_s3_url = os.environ["S3_DST_URL"]
 121             multipart_threshold = 5 * 1024 ** 5 if "google" in os.environ[
 122                 "S3_ENDPOINT_URL"] else 8 * 1024 * 1024
 123             config = TransferConfig(multipart_threshold=multipart_threshold)
 124             bucket_name = urlparse(dst_s3_url).netloc
 125             s3path = re.search(
 126                 '^/*(.*)/*$', urlparse(dst_s3_url).path).group(1)
 127             prefix = os.path.join(s3path, build_tag)
 128             # pylint: disable=no-member
 129             for s3_object in b3resource.Bucket(bucket_name).objects.filter(
 130                     Prefix=f"{prefix}/"):
 131                 path, _ = os.path.split(s3_object.key)
 132                 lpath = re.sub(f'^{s3path}/*', '', path)
 133                 if lpath and not os.path.exists(lpath):
 134                     os.makedirs(lpath)
 135                 # pylint: disable=no-member
 136                 b3resource.Bucket(bucket_name).download_file(
 137                     s3_object.key,
 138                     re.sub(f'^{s3path}/*', '', s3_object.key),
 139                     Config=config)
 140                 Campaign.__logger.info(
 141                     "Downloading %s",
 142                     re.sub(f'^{s3path}/*', '', s3_object.key))
 143             return Campaign.EX_OK
 144         except Exception:  # pylint: disable=broad-except
 145             Campaign.__logger.exception("Cannot publish the artifacts")
 146             return Campaign.EX_DUMP_ARTIFACTS_ERROR
 147
 148     @staticmethod
 149     def zip_campaign_files():  # pylint: disable=too-many-locals
 150         """Archive and publish all test campaign data to the S3 repository.
 151
 152         It allows collecting all the artifacts from the S3 repository.
 153
 154         It could be overriden if the common implementation is not
 155         suitable.
 156
 157         The credentials must be configured before publishing the artifacts:
 158
 159             * fill ~/.aws/credentials or ~/.boto,
 160             * set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in env.
 161
 162         The next vars must be set in env:
 163
 164             * S3_ENDPOINT_URL (http://127.0.0.1:9000),
 165             * S3_DST_URL (s3://xtesting/prefix),
 166
 167         Returns:
 168             Campaign.EX_OK if artifacts were published to repository.
 169             Campaign.EX_DUMP_ARTIFACTS_ERROR otherwise.
 170         """
 171         try:
 172             build_tag = env.get('BUILD_TAG')
 173             assert Campaign.dump_db() == Campaign.EX_OK
 174             assert Campaign.dump_artifacts() == Campaign.EX_OK
 175             with zipfile.ZipFile(f'{build_tag}.zip',
 176                                  'w', zipfile.ZIP_DEFLATED) as zfile:
 177                 zfile.write(f"{build_tag}.json")
 178                 for root, _, files in os.walk(build_tag):
 179                     for filename in files:
 180                         zfile.write(os.path.join(root, filename))
 181             b3resource = boto3.resource(
 182                 's3', endpoint_url=os.environ["S3_ENDPOINT_URL"])
 183             dst_s3_url = os.environ["S3_DST_URL"]
 184             multipart_threshold = 5 * 1024 ** 5 if "google" in os.environ[
 185                 "S3_ENDPOINT_URL"] else 8 * 1024 * 1024
 186             config = TransferConfig(multipart_threshold=multipart_threshold)
 187             bucket_name = urlparse(dst_s3_url).netloc
 188             mime_type = mimetypes.guess_type(f'{build_tag}.zip')
 189             path = urlparse(dst_s3_url).path.strip("/")
 190             # pylint: disable=no-member
 191             b3resource.Bucket(bucket_name).upload_file(
 192                 f'{build_tag}.zip',
 193                 os.path.join(path, f'{build_tag}.zip'),
 194                 Config=config,
 195                 ExtraArgs={'ContentType': mime_type[
 196                     0] or 'application/octet-stream'})
 197             dst_http_url = os.environ["HTTP_DST_URL"]
 198             link = os.path.join(dst_http_url, f'{build_tag}.zip')
 199             Campaign.__logger.info(
 200                 "All data were successfully published:\n\n%s", link)
 201             return Campaign.EX_OK
 202         except KeyError as ex:
 203             Campaign.__logger.error("Please check env var: %s", str(ex))
 204             return Campaign.EX_ZIP_CAMPAIGN_FILES_ERROR
 205         except botocore.exceptions.NoCredentialsError:
 206             Campaign.__logger.error(
 207                 "Please fill ~/.aws/credentials, ~/.boto or set "
 208                 "AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in env")
 209             return Campaign.EX_ZIP_CAMPAIGN_FILES_ERROR
 210         except Exception:  # pylint: disable=broad-except
 211             Campaign.__logger.exception("Cannot publish the artifacts")
 212             return Campaign.EX_ZIP_CAMPAIGN_FILES_ERROR
 213
 214
 215 def main():
 216     """Entry point for Campaign.zip_campaign_files()."""
 217     if not os.path.exists(testcase.TestCase.dir_results):
 218         os.makedirs(testcase.TestCase.dir_results)
 219     if env.get('DEBUG').lower() == 'true':
 220         logging.config.fileConfig(pkg_resources.resource_filename(
 221             'xtesting', constants.DEBUG_INI_PATH))
 222     else:
 223         logging.config.fileConfig(pkg_resources.resource_filename(
 224             'xtesting', constants.INI_PATH))
 225     logging.captureWarnings(True)
 226     Campaign.zip_campaign_files()