xtesting/core/campaign.py

   1 #!/usr/bin/env python
   2
   3 # Copyright (c) 2019 Orange and others.
   4 #
   5 # All rights reserved. This program and the accompanying materials
   6 # are made available under the terms of the Apache License, Version 2.0
   7 # which accompanies this distribution, and is available at
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9
  10 """Dump DB and artifacts for third-party certifications."""
  11
  12 import json
  13 import logging
  14 import logging.config
  15 import mimetypes
  16 import os
  17 import re
  18 import zipfile
  19
  20 import boto3
  21 from boto3.s3.transfer import TransferConfig
  22 import botocore
  23 import pkg_resources
  24 import requests
  25 from six.moves import urllib
  26
  27 from xtesting.core import testcase
  28 from xtesting.utils import env
  29
  30 __author__ = "Cedric Ollivier <cedric.ollivier@orange.com>"
  31
  32
  33 class Campaign():
  34     "Dump, archive and publish all results and artifacts from a campaign."
  35
  36     EX_OK = os.EX_OK
  37     """everything is OK"""
  38
  39     EX_DUMP_FROM_DB_ERROR = os.EX_SOFTWARE - 5
  40     """dump_db() failed"""
  41
  42     EX_DUMP_ARTIFACTS_ERROR = os.EX_SOFTWARE - 6
  43     """dump_artifacts() failed"""
  44
  45     EX_ZIP_CAMPAIGN_FILES_ERROR = os.EX_SOFTWARE - 7
  46     """dump_artifacts() failed"""
  47
  48     __logger = logging.getLogger(__name__)
  49
  50     @staticmethod
  51     def dump_db():
  52         """Dump all test campaign results from the DB.
  53
  54         It allows collecting all the results from the DB.
  55
  56         It could be overriden if the common implementation is not
  57         suitable.
  58
  59         The next vars must be set in env:
  60
  61             * TEST_DB_URL,
  62             * BUILD_TAG.
  63
  64         Returns:
  65             Campaign.EX_OK if results were collected from DB.
  66             Campaign.EX_DUMP_FROM_DB_ERROR otherwise.
  67         """
  68         try:
  69             url = env.get('TEST_DB_URL')
  70             req = requests.get(
  71                 "{}?build_tag={}".format(url, env.get('BUILD_TAG')),
  72                 headers=testcase.TestCase.headers)
  73             req.raise_for_status()
  74             output = req.json()
  75             Campaign.__logger.debug("data from DB: \n%s", output)
  76             for i, _ in enumerate(output["results"]):
  77                 for j, _ in enumerate(
  78                         output["results"][i]["details"]["links"]):
  79                     output["results"][i]["details"]["links"][j] = re.sub(
  80                         "^{}/*".format(os.environ["HTTP_DST_URL"]), '',
  81                         output["results"][i]["details"]["links"][j])
  82             Campaign.__logger.debug("data to archive: \n%s", output)
  83             with open("{}.json".format(env.get('BUILD_TAG')), "w") as dfile:
  84                 json.dump(output, dfile)
  85         except Exception:  # pylint: disable=broad-except
  86             Campaign.__logger.exception(
  87                 "The results cannot be collected from DB")
  88             return Campaign.EX_DUMP_FROM_DB_ERROR
  89         return Campaign.EX_OK
  90
  91     @staticmethod
  92     def dump_artifacts():
  93         """Dump all test campaign artifacts from the S3 repository.
  94
  95         It allows collecting all the artifacts from the S3 repository.
  96
  97         It could be overriden if the common implementation is not
  98         suitable.
  99
 100         The credentials must be configured before publishing the artifacts:
 101
 102             * fill ~/.aws/credentials or ~/.boto,
 103             * set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in env.
 104
 105         The next vars must be set in env:
 106
 107             * S3_ENDPOINT_URL (http://127.0.0.1:9000),
 108             * S3_DST_URL (s3://xtesting/prefix),
 109
 110         Returns:
 111             Campaign.EX_OK if artifacts were published to repository.
 112             Campaign.EX_DUMP_ARTIFACTS_ERROR otherwise.
 113         """
 114         try:
 115             build_tag = env.get('BUILD_TAG')
 116             b3resource = boto3.resource(
 117                 's3', endpoint_url=os.environ["S3_ENDPOINT_URL"])
 118             dst_s3_url = os.environ["S3_DST_URL"]
 119             multipart_threshold = 5 * 1024 ** 5 if "google" in os.environ[
 120                 "S3_ENDPOINT_URL"] else 8 * 1024 * 1024
 121             config = TransferConfig(multipart_threshold=multipart_threshold)
 122             bucket_name = urllib.parse.urlparse(dst_s3_url).netloc
 123             s3path = re.search(
 124                 '^/*(.*)/*$', urllib.parse.urlparse(dst_s3_url).path).group(1)
 125             prefix = os.path.join(s3path, build_tag)
 126             # pylint: disable=no-member
 127             for s3_object in b3resource.Bucket(bucket_name).objects.filter(
 128                     Prefix="{}/".format(prefix)):
 129                 path, _ = os.path.split(s3_object.key)
 130                 lpath = re.sub('^{}/*'.format(s3path), '', path)
 131                 if lpath and not os.path.exists(lpath):
 132                     os.makedirs(lpath)
 133                 # pylint: disable=no-member
 134                 b3resource.Bucket(bucket_name).download_file(
 135                     s3_object.key,
 136                     re.sub('^{}/*'.format(s3path), '', s3_object.key),
 137                     Config=config)
 138                 Campaign.__logger.info(
 139                     "Downloading %s",
 140                     re.sub('^{}/*'.format(s3path), '', s3_object.key))
 141             return Campaign.EX_OK
 142         except Exception:  # pylint: disable=broad-except
 143             Campaign.__logger.exception("Cannot publish the artifacts")
 144             return Campaign.EX_DUMP_ARTIFACTS_ERROR
 145
 146     @staticmethod
 147     def zip_campaign_files():  # pylint: disable=too-many-locals
 148         """Archive and publish all test campaign data to the S3 repository.
 149
 150         It allows collecting all the artifacts from the S3 repository.
 151
 152         It could be overriden if the common implementation is not
 153         suitable.
 154
 155         The credentials must be configured before publishing the artifacts:
 156
 157             * fill ~/.aws/credentials or ~/.boto,
 158             * set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in env.
 159
 160         The next vars must be set in env:
 161
 162             * S3_ENDPOINT_URL (http://127.0.0.1:9000),
 163             * S3_DST_URL (s3://xtesting/prefix),
 164
 165         Returns:
 166             Campaign.EX_OK if artifacts were published to repository.
 167             Campaign.EX_DUMP_ARTIFACTS_ERROR otherwise.
 168         """
 169         try:
 170             build_tag = env.get('BUILD_TAG')
 171             assert Campaign.dump_db() == Campaign.EX_OK
 172             assert Campaign.dump_artifacts() == Campaign.EX_OK
 173             with zipfile.ZipFile('{}.zip'.format(build_tag),
 174                                  'w', zipfile.ZIP_DEFLATED) as zfile:
 175                 zfile.write("{}.json".format(build_tag))
 176                 for root, _, files in os.walk(build_tag):
 177                     for filename in files:
 178                         zfile.write(os.path.join(root, filename))
 179             b3resource = boto3.resource(
 180                 's3', endpoint_url=os.environ["S3_ENDPOINT_URL"])
 181             dst_s3_url = os.environ["S3_DST_URL"]
 182             multipart_threshold = 5 * 1024 ** 5 if "google" in os.environ[
 183                 "S3_ENDPOINT_URL"] else 8 * 1024 * 1024
 184             config = TransferConfig(multipart_threshold=multipart_threshold)
 185             bucket_name = urllib.parse.urlparse(dst_s3_url).netloc
 186             mime_type = mimetypes.guess_type('{}.zip'.format(build_tag))
 187             path = urllib.parse.urlparse(dst_s3_url).path.strip("/")
 188             # pylint: disable=no-member
 189             b3resource.Bucket(bucket_name).upload_file(
 190                 '{}.zip'.format(build_tag),
 191                 os.path.join(path, '{}.zip'.format(build_tag)),
 192                 Config=config,
 193                 ExtraArgs={'ContentType': mime_type[
 194                     0] or 'application/octet-stream'})
 195             dst_http_url = os.environ["HTTP_DST_URL"]
 196             link = os.path.join(dst_http_url, '{}.zip'.format(build_tag))
 197             Campaign.__logger.info(
 198                 "All data were successfully published:\n\n%s", link)
 199             return Campaign.EX_OK
 200         except KeyError as ex:
 201             Campaign.__logger.error("Please check env var: %s", str(ex))
 202             return Campaign.EX_ZIP_CAMPAIGN_FILES_ERROR
 203         except botocore.exceptions.NoCredentialsError:
 204             Campaign.__logger.error(
 205                 "Please fill ~/.aws/credentials, ~/.boto or set "
 206                 "AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in env")
 207             return Campaign.EX_ZIP_CAMPAIGN_FILES_ERROR
 208         except Exception:  # pylint: disable=broad-except
 209             Campaign.__logger.exception("Cannot publish the artifacts")
 210             return Campaign.EX_ZIP_CAMPAIGN_FILES_ERROR
 211
 212
 213 def main():
 214     """Entry point for Campaign.zip_campaign_files()."""
 215     if not os.path.exists(testcase.TestCase.dir_results):
 216         os.makedirs(testcase.TestCase.dir_results)
 217     if env.get('DEBUG').lower() == 'true':
 218         logging.config.fileConfig(pkg_resources.resource_filename(
 219             'xtesting', 'ci/logging.debug.ini'))
 220     else:
 221         logging.config.fileConfig(pkg_resources.resource_filename(
 222             'xtesting', 'ci/logging.ini'))
 223     logging.captureWarnings(True)
 224     Campaign.zip_campaign_files()