xtesting/core/campaign.py

   1 #!/usr/bin/env python
   2
   3 # Copyright (c) 2019 Orange and others.
   4 #
   5 # All rights reserved. This program and the accompanying materials
   6 # are made available under the terms of the Apache License, Version 2.0
   7 # which accompanies this distribution, and is available at
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9
  10 """Dump DB and artifacts for third-party certifications."""
  11
  12 import json
  13 import logging
  14 import logging.config
  15 import mimetypes
  16 import os
  17 import re
  18 import zipfile
  19
  20 import boto3
  21 from boto3.s3.transfer import TransferConfig
  22 import botocore
  23 import pkg_resources
  24 import requests
  25 from six.moves import urllib
  26
  27 from xtesting.core import testcase
  28 from xtesting.utils import env
  29
  30 __author__ = "Cedric Ollivier <cedric.ollivier@orange.com>"
  31
  32
  33 class Campaign():
  34     "Dump, archive and publish all results and artifacts from a campaign."
  35
  36     EX_OK = os.EX_OK
  37     """everything is OK"""
  38
  39     EX_DUMP_FROM_DB_ERROR = os.EX_SOFTWARE - 5
  40     """dump_db() failed"""
  41
  42     EX_DUMP_ARTIFACTS_ERROR = os.EX_SOFTWARE - 6
  43     """dump_artifacts() failed"""
  44
  45     EX_ZIP_CAMPAIGN_FILES_ERROR = os.EX_SOFTWARE - 7
  46     """dump_artifacts() failed"""
  47
  48     __logger = logging.getLogger(__name__)
  49
  50     @staticmethod
  51     def dump_db():
  52         """Dump all test campaign results from the DB.
  53
  54         It allows collecting all the results from the DB.
  55
  56         It could be overriden if the common implementation is not
  57         suitable.
  58
  59         The next vars must be set in env:
  60
  61             * TEST_DB_URL,
  62             * BUILD_TAG.
  63
  64         Returns:
  65             Campaign.EX_OK if results were collected from DB.
  66             Campaign.EX_DUMP_FROM_DB_ERROR otherwise.
  67         """
  68         try:
  69             url = env.get('TEST_DB_URL')
  70             req = requests.get(
  71                 "{}?build_tag={}".format(url, env.get('BUILD_TAG')),
  72                 headers=testcase.TestCase.headers)
  73             req.raise_for_status()
  74             Campaign.__logger.debug("data from DB: \n%s", req.json())
  75             with open("{}.json".format(env.get('BUILD_TAG')), "w") as dfile:
  76                 json.dump(req.json(), dfile)
  77         except Exception:  # pylint: disable=broad-except
  78             Campaign.__logger.exception(
  79                 "The results cannot be collected from DB")
  80             return Campaign.EX_DUMP_FROM_DB_ERROR
  81         return Campaign.EX_OK
  82
  83     @staticmethod
  84     def dump_artifacts():
  85         """Dump all test campaign artifacts from the S3 repository.
  86
  87         It allows collecting all the artifacts from the S3 repository.
  88
  89         It could be overriden if the common implementation is not
  90         suitable.
  91
  92         The credentials must be configured before publishing the artifacts:
  93
  94             * fill ~/.aws/credentials or ~/.boto,
  95             * set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in env.
  96
  97         The next vars must be set in env:
  98
  99             * S3_ENDPOINT_URL (http://127.0.0.1:9000),
 100             * S3_DST_URL (s3://xtesting/prefix),
 101
 102         Returns:
 103             Campaign.EX_OK if artifacts were published to repository.
 104             Campaign.EX_DUMP_ARTIFACTS_ERROR otherwise.
 105         """
 106         try:
 107             build_tag = env.get('BUILD_TAG')
 108             b3resource = boto3.resource(
 109                 's3', endpoint_url=os.environ["S3_ENDPOINT_URL"])
 110             dst_s3_url = os.environ["S3_DST_URL"]
 111             multipart_threshold = 5 * 1024 ** 5 if "google" in os.environ[
 112                 "S3_ENDPOINT_URL"] else 8 * 1024 * 1024
 113             config = TransferConfig(multipart_threshold=multipart_threshold)
 114             bucket_name = urllib.parse.urlparse(dst_s3_url).netloc
 115             s3path = re.search(
 116                 '^/*(.*)/*$', urllib.parse.urlparse(dst_s3_url).path).group(1)
 117             prefix = os.path.join(s3path, build_tag)
 118             # pylint: disable=no-member
 119             for s3_object in b3resource.Bucket(bucket_name).objects.filter(
 120                     Prefix="{}/".format(prefix)):
 121                 path, _ = os.path.split(s3_object.key)
 122                 lpath = re.sub('^{}/*'.format(s3path), '', path)
 123                 if lpath and not os.path.exists(lpath):
 124                     os.makedirs(lpath)
 125                 # pylint: disable=no-member
 126                 b3resource.Bucket(bucket_name).download_file(
 127                     s3_object.key,
 128                     re.sub('^{}/*'.format(s3path), '', s3_object.key),
 129                     Config=config)
 130                 Campaign.__logger.info(
 131                     "Downloading %s",
 132                     re.sub('^{}/*'.format(s3path), '', s3_object.key))
 133             return Campaign.EX_OK
 134         except Exception:  # pylint: disable=broad-except
 135             Campaign.__logger.exception("Cannot publish the artifacts")
 136             return Campaign.EX_DUMP_ARTIFACTS_ERROR
 137
 138     @staticmethod
 139     def zip_campaign_files():  # pylint: disable=too-many-locals
 140         """Archive and publish all test campaign data to the S3 repository.
 141
 142         It allows collecting all the artifacts from the S3 repository.
 143
 144         It could be overriden if the common implementation is not
 145         suitable.
 146
 147         The credentials must be configured before publishing the artifacts:
 148
 149             * fill ~/.aws/credentials or ~/.boto,
 150             * set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in env.
 151
 152         The next vars must be set in env:
 153
 154             * S3_ENDPOINT_URL (http://127.0.0.1:9000),
 155             * S3_DST_URL (s3://xtesting/prefix),
 156
 157         Returns:
 158             Campaign.EX_OK if artifacts were published to repository.
 159             Campaign.EX_DUMP_ARTIFACTS_ERROR otherwise.
 160         """
 161         try:
 162             build_tag = env.get('BUILD_TAG')
 163             assert Campaign.dump_db() == Campaign.EX_OK
 164             assert Campaign.dump_artifacts() == Campaign.EX_OK
 165             with zipfile.ZipFile('{}.zip'.format(build_tag), 'w') as zfile:
 166                 zfile.write("{}.json".format(build_tag))
 167                 for root, _, files in os.walk(build_tag):
 168                     for filename in files:
 169                         zfile.write(os.path.join(root, filename))
 170             b3resource = boto3.resource(
 171                 's3', endpoint_url=os.environ["S3_ENDPOINT_URL"])
 172             dst_s3_url = os.environ["S3_DST_URL"]
 173             multipart_threshold = 5 * 1024 ** 5 if "google" in os.environ[
 174                 "S3_ENDPOINT_URL"] else 8 * 1024 * 1024
 175             config = TransferConfig(multipart_threshold=multipart_threshold)
 176             bucket_name = urllib.parse.urlparse(dst_s3_url).netloc
 177             mime_type = mimetypes.guess_type('{}.zip'.format(build_tag))
 178             path = urllib.parse.urlparse(dst_s3_url).path.strip("/")
 179             # pylint: disable=no-member
 180             b3resource.Bucket(bucket_name).upload_file(
 181                 '{}.zip'.format(build_tag),
 182                 os.path.join(path, '{}.zip'.format(build_tag)),
 183                 Config=config,
 184                 ExtraArgs={'ContentType': mime_type[
 185                     0] or 'application/octet-stream'})
 186             dst_http_url = os.environ["HTTP_DST_URL"]
 187             link = os.path.join(dst_http_url, '{}.zip'.format(build_tag))
 188             Campaign.__logger.info(
 189                 "All data were successfully published:\n\n%s", link)
 190             return Campaign.EX_OK
 191         except KeyError as ex:
 192             Campaign.__logger.error("Please check env var: %s", str(ex))
 193             return Campaign.EX_ZIP_CAMPAIGN_FILES_ERROR
 194         except botocore.exceptions.NoCredentialsError:
 195             Campaign.__logger.error(
 196                 "Please fill ~/.aws/credentials, ~/.boto or set "
 197                 "AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in env")
 198             return Campaign.EX_ZIP_CAMPAIGN_FILES_ERROR
 199         except Exception:  # pylint: disable=broad-except
 200             Campaign.__logger.exception("Cannot publish the artifacts")
 201             return Campaign.EX_ZIP_CAMPAIGN_FILES_ERROR
 202
 203
 204 def main():
 205     """Entry point for Campaign.zip_campaign_files()."""
 206     if not os.path.exists(testcase.TestCase.dir_results):
 207         os.makedirs(testcase.TestCase.dir_results)
 208     if env.get('DEBUG').lower() == 'true':
 209         logging.config.fileConfig(pkg_resources.resource_filename(
 210             'xtesting', 'ci/logging.debug.ini'))
 211     else:
 212         logging.config.fileConfig(pkg_resources.resource_filename(
 213             'xtesting', 'ci/logging.ini'))
 214     logging.captureWarnings(True)
 215     Campaign.zip_campaign_files()