xtesting/core/campaign.py

   1 #!/usr/bin/env python
   2
   3 # Copyright (c) 2019 Orange and others.
   4 #
   5 # All rights reserved. This program and the accompanying materials
   6 # are made available under the terms of the Apache License, Version 2.0
   7 # which accompanies this distribution, and is available at
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9
  10 """Dump DB and artifacts for third-party certifications."""
  11
  12 import json
  13 import logging
  14 import logging.config
  15 import mimetypes
  16 import os
  17 import re
  18 import zipfile
  19
  20 from urllib.parse import urlparse
  21 import boto3
  22 from boto3.s3.transfer import TransferConfig
  23 import botocore
  24 import pkg_resources
  25 import requests
  26
  27 from xtesting.core import testcase
  28 from xtesting.utils import env
  29 from xtesting.utils import constants
  30
  31 __author__ = "Cedric Ollivier <cedric.ollivier@orange.com>"
  32
  33
  34 class Campaign():
  35     "Dump, archive and publish all results and artifacts from a campaign."
  36
  37     EX_OK = os.EX_OK
  38     """everything is OK"""
  39
  40     EX_DUMP_FROM_DB_ERROR = os.EX_SOFTWARE - 5
  41     """dump_db() failed"""
  42
  43     EX_DUMP_ARTIFACTS_ERROR = os.EX_SOFTWARE - 6
  44     """dump_artifacts() failed"""
  45
  46     EX_ZIP_CAMPAIGN_FILES_ERROR = os.EX_SOFTWARE - 7
  47     """dump_artifacts() failed"""
  48
  49     __logger = logging.getLogger(__name__)
  50
  51     @staticmethod
  52     def dump_db():
  53         """Dump all test campaign results from the DB.
  54
  55         It allows collecting all the results from the DB.
  56
  57         It could be overriden if the common implementation is not
  58         suitable.
  59
  60         The next vars must be set in env:
  61
  62             * TEST_DB_URL,
  63             * BUILD_TAG.
  64
  65         Returns:
  66             Campaign.EX_OK if results were collected from DB.
  67             Campaign.EX_DUMP_FROM_DB_ERROR otherwise.
  68         """
  69         try:
  70             url = env.get('TEST_DB_URL')
  71             req = requests.get(
  72                 "{}?build_tag={}".format(url, env.get('BUILD_TAG')),
  73                 headers=testcase.TestCase.headers)
  74             req.raise_for_status()
  75             output = req.json()
  76             Campaign.__logger.debug("data from DB: \n%s", output)
  77             for i, _ in enumerate(output["results"]):
  78                 for j, _ in enumerate(
  79                         output["results"][i]["details"]["links"]):
  80                     output["results"][i]["details"]["links"][j] = re.sub(
  81                         "^{}/*".format(os.environ["HTTP_DST_URL"]), '',
  82                         output["results"][i]["details"]["links"][j])
  83             Campaign.__logger.debug("data to archive: \n%s", output)
  84             with open("{}.json".format(env.get('BUILD_TAG')), "w") as dfile:
  85                 json.dump(output, dfile)
  86         except Exception:  # pylint: disable=broad-except
  87             Campaign.__logger.exception(
  88                 "The results cannot be collected from DB")
  89             return Campaign.EX_DUMP_FROM_DB_ERROR
  90         return Campaign.EX_OK
  91
  92     @staticmethod
  93     def dump_artifacts():
  94         """Dump all test campaign artifacts from the S3 repository.
  95
  96         It allows collecting all the artifacts from the S3 repository.
  97
  98         It could be overriden if the common implementation is not
  99         suitable.
 100
 101         The credentials must be configured before publishing the artifacts:
 102
 103             * fill ~/.aws/credentials or ~/.boto,
 104             * set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in env.
 105
 106         The next vars must be set in env:
 107
 108             * S3_ENDPOINT_URL (http://127.0.0.1:9000),
 109             * S3_DST_URL (s3://xtesting/prefix),
 110
 111         Returns:
 112             Campaign.EX_OK if artifacts were published to repository.
 113             Campaign.EX_DUMP_ARTIFACTS_ERROR otherwise.
 114         """
 115         try:
 116             build_tag = env.get('BUILD_TAG')
 117             b3resource = boto3.resource(
 118                 's3', endpoint_url=os.environ["S3_ENDPOINT_URL"])
 119             dst_s3_url = os.environ["S3_DST_URL"]
 120             multipart_threshold = 5 * 1024 ** 5 if "google" in os.environ[
 121                 "S3_ENDPOINT_URL"] else 8 * 1024 * 1024
 122             config = TransferConfig(multipart_threshold=multipart_threshold)
 123             bucket_name = urlparse(dst_s3_url).netloc
 124             s3path = re.search(
 125                 '^/*(.*)/*$', urlparse(dst_s3_url).path).group(1)
 126             prefix = os.path.join(s3path, build_tag)
 127             # pylint: disable=no-member
 128             for s3_object in b3resource.Bucket(bucket_name).objects.filter(
 129                     Prefix="{}/".format(prefix)):
 130                 path, _ = os.path.split(s3_object.key)
 131                 lpath = re.sub('^{}/*'.format(s3path), '', path)
 132                 if lpath and not os.path.exists(lpath):
 133                     os.makedirs(lpath)
 134                 # pylint: disable=no-member
 135                 b3resource.Bucket(bucket_name).download_file(
 136                     s3_object.key,
 137                     re.sub('^{}/*'.format(s3path), '', s3_object.key),
 138                     Config=config)
 139                 Campaign.__logger.info(
 140                     "Downloading %s",
 141                     re.sub('^{}/*'.format(s3path), '', s3_object.key))
 142             return Campaign.EX_OK
 143         except Exception:  # pylint: disable=broad-except
 144             Campaign.__logger.exception("Cannot publish the artifacts")
 145             return Campaign.EX_DUMP_ARTIFACTS_ERROR
 146
 147     @staticmethod
 148     def zip_campaign_files():  # pylint: disable=too-many-locals
 149         """Archive and publish all test campaign data to the S3 repository.
 150
 151         It allows collecting all the artifacts from the S3 repository.
 152
 153         It could be overriden if the common implementation is not
 154         suitable.
 155
 156         The credentials must be configured before publishing the artifacts:
 157
 158             * fill ~/.aws/credentials or ~/.boto,
 159             * set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in env.
 160
 161         The next vars must be set in env:
 162
 163             * S3_ENDPOINT_URL (http://127.0.0.1:9000),
 164             * S3_DST_URL (s3://xtesting/prefix),
 165
 166         Returns:
 167             Campaign.EX_OK if artifacts were published to repository.
 168             Campaign.EX_DUMP_ARTIFACTS_ERROR otherwise.
 169         """
 170         try:
 171             build_tag = env.get('BUILD_TAG')
 172             assert Campaign.dump_db() == Campaign.EX_OK
 173             assert Campaign.dump_artifacts() == Campaign.EX_OK
 174             with zipfile.ZipFile('{}.zip'.format(build_tag),
 175                                  'w', zipfile.ZIP_DEFLATED) as zfile:
 176                 zfile.write("{}.json".format(build_tag))
 177                 for root, _, files in os.walk(build_tag):
 178                     for filename in files:
 179                         zfile.write(os.path.join(root, filename))
 180             b3resource = boto3.resource(
 181                 's3', endpoint_url=os.environ["S3_ENDPOINT_URL"])
 182             dst_s3_url = os.environ["S3_DST_URL"]
 183             multipart_threshold = 5 * 1024 ** 5 if "google" in os.environ[
 184                 "S3_ENDPOINT_URL"] else 8 * 1024 * 1024
 185             config = TransferConfig(multipart_threshold=multipart_threshold)
 186             bucket_name = urlparse(dst_s3_url).netloc
 187             mime_type = mimetypes.guess_type('{}.zip'.format(build_tag))
 188             path = urlparse(dst_s3_url).path.strip("/")
 189             # pylint: disable=no-member
 190             b3resource.Bucket(bucket_name).upload_file(
 191                 '{}.zip'.format(build_tag),
 192                 os.path.join(path, '{}.zip'.format(build_tag)),
 193                 Config=config,
 194                 ExtraArgs={'ContentType': mime_type[
 195                     0] or 'application/octet-stream'})
 196             dst_http_url = os.environ["HTTP_DST_URL"]
 197             link = os.path.join(dst_http_url, '{}.zip'.format(build_tag))
 198             Campaign.__logger.info(
 199                 "All data were successfully published:\n\n%s", link)
 200             return Campaign.EX_OK
 201         except KeyError as ex:
 202             Campaign.__logger.error("Please check env var: %s", str(ex))
 203             return Campaign.EX_ZIP_CAMPAIGN_FILES_ERROR
 204         except botocore.exceptions.NoCredentialsError:
 205             Campaign.__logger.error(
 206                 "Please fill ~/.aws/credentials, ~/.boto or set "
 207                 "AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in env")
 208             return Campaign.EX_ZIP_CAMPAIGN_FILES_ERROR
 209         except Exception:  # pylint: disable=broad-except
 210             Campaign.__logger.exception("Cannot publish the artifacts")
 211             return Campaign.EX_ZIP_CAMPAIGN_FILES_ERROR
 212
 213
 214 def main():
 215     """Entry point for Campaign.zip_campaign_files()."""
 216     if not os.path.exists(testcase.TestCase.dir_results):
 217         os.makedirs(testcase.TestCase.dir_results)
 218     if env.get('DEBUG').lower() == 'true':
 219         logging.config.fileConfig(pkg_resources.resource_filename(
 220             'xtesting', constants.DEBUG_INI_PATH))
 221     else:
 222         logging.config.fileConfig(pkg_resources.resource_filename(
 223             'xtesting', constants.INI_PATH))
 224     logging.captureWarnings(True)
 225     Campaign.zip_campaign_files()