xtesting/core/campaign.py

   1 #!/usr/bin/env python
   2
   3 # Copyright (c) 2019 Orange and others.
   4 #
   5 # All rights reserved. This program and the accompanying materials
   6 # are made available under the terms of the Apache License, Version 2.0
   7 # which accompanies this distribution, and is available at
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9
  10 """Dump DB and artifacts for third-party certifications."""
  11
  12 import json
  13 import logging
  14 import logging.config
  15 import mimetypes
  16 import os
  17 import re
  18 import zipfile
  19
  20 import boto3
  21 from boto3.s3.transfer import TransferConfig
  22 import botocore
  23 import pkg_resources
  24 import requests
  25 from six.moves import urllib
  26
  27 from xtesting.core import testcase
  28 from xtesting.utils import env
  29
  30 __author__ = "Cedric Ollivier <cedric.ollivier@orange.com>"
  31
  32
  33 class Campaign():
  34     "Dump, archive and publish all results and artifacts from a campaign."
  35
  36     EX_OK = os.EX_OK
  37     """everything is OK"""
  38
  39     EX_DUMP_FROM_DB_ERROR = os.EX_SOFTWARE - 5
  40     """dump_db() failed"""
  41
  42     EX_DUMP_ARTIFACTS_ERROR = os.EX_SOFTWARE - 6
  43     """dump_artifacts() failed"""
  44
  45     EX_ZIP_CAMPAIGN_FILES_ERROR = os.EX_SOFTWARE - 7
  46     """dump_artifacts() failed"""
  47
  48     __logger = logging.getLogger(__name__)
  49
  50     @staticmethod
  51     def dump_db():
  52         """Dump all test campaign results from the DB.
  53
  54         It allows collecting all the results from the DB.
  55
  56         It could be overriden if the common implementation is not
  57         suitable.
  58
  59         The next vars must be set in env:
  60
  61             * TEST_DB_URL,
  62             * BUILD_TAG.
  63
  64         Returns:
  65             Campaign.EX_OK if results were collected from DB.
  66             Campaign.EX_DUMP_FROM_DB_ERROR otherwise.
  67         """
  68         try:
  69             url = env.get('TEST_DB_URL')
  70             req = requests.get(
  71                 "{}?build_tag={}".format(url, env.get('BUILD_TAG')),
  72                 headers=testcase.TestCase.headers)
  73             req.raise_for_status()
  74             Campaign.__logger.debug("data from DB: \n%s", req.json())
  75             with open("{}.json".format(env.get('BUILD_TAG')), "w") as dfile:
  76                 json.dump(req.json(), dfile)
  77         except Exception:  # pylint: disable=broad-except
  78             Campaign.__logger.exception(
  79                 "The results cannot be collected from DB")
  80             return Campaign.EX_DUMP_FROM_DB_ERROR
  81         return Campaign.EX_OK
  82
  83     @staticmethod
  84     def dump_artifacts():
  85         """Dump all test campaign artifacts from the S3 repository.
  86
  87         It allows collecting all the artifacts from the S3 repository.
  88
  89         It could be overriden if the common implementation is not
  90         suitable.
  91
  92         The credentials must be configured before publishing the artifacts:
  93
  94             * fill ~/.aws/credentials or ~/.boto,
  95             * set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in env.
  96
  97         The next vars must be set in env:
  98
  99             * S3_ENDPOINT_URL (http://127.0.0.1:9000),
 100             * S3_DST_URL (s3://xtesting/prefix),
 101
 102         Returns:
 103             Campaign.EX_OK if artifacts were published to repository.
 104             Campaign.EX_DUMP_ARTIFACTS_ERROR otherwise.
 105         """
 106         try:
 107             build_tag = env.get('BUILD_TAG')
 108             b3resource = boto3.resource(
 109                 's3', endpoint_url=os.environ["S3_ENDPOINT_URL"])
 110             dst_s3_url = os.environ["S3_DST_URL"]
 111             multipart_threshold = 5 * 1024 ** 5 if "google" in os.environ[
 112                 "S3_ENDPOINT_URL"] else 8 * 1024 * 1024
 113             config = TransferConfig(multipart_threshold=multipart_threshold)
 114             bucket_name = urllib.parse.urlparse(dst_s3_url).netloc
 115             s3path = re.search(
 116                 '^/*(.*)/*$', urllib.parse.urlparse(dst_s3_url).path).group(1)
 117             prefix = os.path.join(s3path, build_tag)
 118             # pylint: disable=no-member
 119             for s3_object in b3resource.Bucket(bucket_name).objects.filter(
 120                     Prefix="{}/".format(prefix)):
 121                 path, _ = os.path.split(s3_object.key)
 122                 lpath = re.sub('^{}/*'.format(s3path), '', path)
 123                 if lpath and not os.path.exists(lpath):
 124                     os.makedirs(lpath)
 125                 # pylint: disable=no-member
 126                 b3resource.Bucket(bucket_name).download_file(
 127                     s3_object.key,
 128                     re.sub('^{}/*'.format(s3path), '', s3_object.key),
 129                     Config=config)
 130                 Campaign.__logger.info(
 131                     "Downloading %s",
 132                     re.sub('^{}/*'.format(s3path), '', s3_object.key))
 133             return Campaign.EX_OK
 134         except Exception:  # pylint: disable=broad-except
 135             Campaign.__logger.exception("Cannot publish the artifacts")
 136             return Campaign.EX_DUMP_ARTIFACTS_ERROR
 137
 138     @staticmethod
 139     def zip_campaign_files():  # pylint: disable=too-many-locals
 140         """Archive and publish all test campaign data to the S3 repository.
 141
 142         It allows collecting all the artifacts from the S3 repository.
 143
 144         It could be overriden if the common implementation is not
 145         suitable.
 146
 147         The credentials must be configured before publishing the artifacts:
 148
 149             * fill ~/.aws/credentials or ~/.boto,
 150             * set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in env.
 151
 152         The next vars must be set in env:
 153
 154             * S3_ENDPOINT_URL (http://127.0.0.1:9000),
 155             * S3_DST_URL (s3://xtesting/prefix),
 156
 157         Returns:
 158             Campaign.EX_OK if artifacts were published to repository.
 159             Campaign.EX_DUMP_ARTIFACTS_ERROR otherwise.
 160         """
 161         try:
 162             build_tag = env.get('BUILD_TAG')
 163             assert Campaign.dump_db() == Campaign.EX_OK
 164             assert Campaign.dump_artifacts() == Campaign.EX_OK
 165             with zipfile.ZipFile('{}.zip'.format(build_tag),
 166                                  'w', zipfile.ZIP_DEFLATED) as zfile:
 167                 zfile.write("{}.json".format(build_tag))
 168                 for root, _, files in os.walk(build_tag):
 169                     for filename in files:
 170                         zfile.write(os.path.join(root, filename))
 171             b3resource = boto3.resource(
 172                 's3', endpoint_url=os.environ["S3_ENDPOINT_URL"])
 173             dst_s3_url = os.environ["S3_DST_URL"]
 174             multipart_threshold = 5 * 1024 ** 5 if "google" in os.environ[
 175                 "S3_ENDPOINT_URL"] else 8 * 1024 * 1024
 176             config = TransferConfig(multipart_threshold=multipart_threshold)
 177             bucket_name = urllib.parse.urlparse(dst_s3_url).netloc
 178             mime_type = mimetypes.guess_type('{}.zip'.format(build_tag))
 179             path = urllib.parse.urlparse(dst_s3_url).path.strip("/")
 180             # pylint: disable=no-member
 181             b3resource.Bucket(bucket_name).upload_file(
 182                 '{}.zip'.format(build_tag),
 183                 os.path.join(path, '{}.zip'.format(build_tag)),
 184                 Config=config,
 185                 ExtraArgs={'ContentType': mime_type[
 186                     0] or 'application/octet-stream'})
 187             dst_http_url = os.environ["HTTP_DST_URL"]
 188             link = os.path.join(dst_http_url, '{}.zip'.format(build_tag))
 189             Campaign.__logger.info(
 190                 "All data were successfully published:\n\n%s", link)
 191             return Campaign.EX_OK
 192         except KeyError as ex:
 193             Campaign.__logger.error("Please check env var: %s", str(ex))
 194             return Campaign.EX_ZIP_CAMPAIGN_FILES_ERROR
 195         except botocore.exceptions.NoCredentialsError:
 196             Campaign.__logger.error(
 197                 "Please fill ~/.aws/credentials, ~/.boto or set "
 198                 "AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in env")
 199             return Campaign.EX_ZIP_CAMPAIGN_FILES_ERROR
 200         except Exception:  # pylint: disable=broad-except
 201             Campaign.__logger.exception("Cannot publish the artifacts")
 202             return Campaign.EX_ZIP_CAMPAIGN_FILES_ERROR
 203
 204
 205 def main():
 206     """Entry point for Campaign.zip_campaign_files()."""
 207     if not os.path.exists(testcase.TestCase.dir_results):
 208         os.makedirs(testcase.TestCase.dir_results)
 209     if env.get('DEBUG').lower() == 'true':
 210         logging.config.fileConfig(pkg_resources.resource_filename(
 211             'xtesting', 'ci/logging.debug.ini'))
 212     else:
 213         logging.config.fileConfig(pkg_resources.resource_filename(
 214             'xtesting', 'ci/logging.ini'))
 215     logging.captureWarnings(True)
 216     Campaign.zip_campaign_files()