3 # Copyright (c) 2019 Orange and others.
5 # All rights reserved. This program and the accompanying materials
6 # are made available under the terms of the Apache License, Version 2.0
7 # which accompanies this distribution, and is available at
8 # http://www.apache.org/licenses/LICENSE-2.0
10 """Dump DB and artifacts for third-party certifications."""
21 from boto3.s3.transfer import TransferConfig
25 from six.moves import urllib
27 from xtesting.core import testcase
28 from xtesting.utils import env
30 __author__ = "Cedric Ollivier <cedric.ollivier@orange.com>"
34 "Dump, archive and publish all results and artifacts from a campaign."
37 """everything is OK"""
39 EX_DUMP_FROM_DB_ERROR = os.EX_SOFTWARE - 5
40 """dump_db() failed"""
42 EX_DUMP_ARTIFACTS_ERROR = os.EX_SOFTWARE - 6
43 """dump_artifacts() failed"""
45 EX_ZIP_CAMPAIGN_FILES_ERROR = os.EX_SOFTWARE - 7
46 """dump_artifacts() failed"""
48 __logger = logging.getLogger(__name__)
52 """Dump all test campaign results from the DB.
54 It allows collecting all the results from the DB.
56 It could be overriden if the common implementation is not
59 The next vars must be set in env:
65 Campaign.EX_OK if results were collected from DB.
66 Campaign.EX_DUMP_FROM_DB_ERROR otherwise.
69 url = env.get('TEST_DB_URL')
71 "{}?build_tag={}".format(url, env.get('BUILD_TAG')),
72 headers=testcase.TestCase.headers)
73 req.raise_for_status()
74 Campaign.__logger.debug("data from DB: \n%s", req.json())
75 with open("{}.json".format(env.get('BUILD_TAG')), "w") as dfile:
76 json.dump(req.json(), dfile)
77 except Exception: # pylint: disable=broad-except
78 Campaign.__logger.exception(
79 "The results cannot be collected from DB")
80 return Campaign.EX_DUMP_FROM_DB_ERROR
85 """Dump all test campaign artifacts from the S3 repository.
87 It allows collecting all the artifacts from the S3 repository.
89 It could be overriden if the common implementation is not
92 The credentials must be configured before publishing the artifacts:
94 * fill ~/.aws/credentials or ~/.boto,
95 * set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in env.
97 The next vars must be set in env:
99 * S3_ENDPOINT_URL (http://127.0.0.1:9000),
100 * S3_DST_URL (s3://xtesting/prefix),
103 Campaign.EX_OK if artifacts were published to repository.
104 Campaign.EX_DUMP_ARTIFACTS_ERROR otherwise.
107 build_tag = env.get('BUILD_TAG')
108 b3resource = boto3.resource(
109 's3', endpoint_url=os.environ["S3_ENDPOINT_URL"])
110 dst_s3_url = os.environ["S3_DST_URL"]
111 multipart_threshold = 5 * 1024 ** 5 if "google" in os.environ[
112 "S3_ENDPOINT_URL"] else 8 * 1024 * 1024
113 config = TransferConfig(multipart_threshold=multipart_threshold)
114 bucket_name = urllib.parse.urlparse(dst_s3_url).netloc
116 '^/*(.*)/*$', urllib.parse.urlparse(dst_s3_url).path).group(1)
117 prefix = os.path.join(s3path, build_tag)
118 # pylint: disable=no-member
119 for s3_object in b3resource.Bucket(bucket_name).objects.filter(
120 Prefix="{}/".format(prefix)):
121 path, _ = os.path.split(s3_object.key)
122 lpath = re.sub('^{}/*'.format(s3path), '', path)
123 if lpath and not os.path.exists(lpath):
125 # pylint: disable=no-member
126 b3resource.Bucket(bucket_name).download_file(
128 re.sub('^{}/*'.format(s3path), '', s3_object.key),
130 Campaign.__logger.info(
132 re.sub('^{}/*'.format(s3path), '', s3_object.key))
133 return Campaign.EX_OK
134 except Exception: # pylint: disable=broad-except
135 Campaign.__logger.exception("Cannot publish the artifacts")
136 return Campaign.EX_DUMP_ARTIFACTS_ERROR
139 def zip_campaign_files(): # pylint: disable=too-many-locals
140 """Archive and publish all test campaign data to the S3 repository.
142 It allows collecting all the artifacts from the S3 repository.
144 It could be overriden if the common implementation is not
147 The credentials must be configured before publishing the artifacts:
149 * fill ~/.aws/credentials or ~/.boto,
150 * set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in env.
152 The next vars must be set in env:
154 * S3_ENDPOINT_URL (http://127.0.0.1:9000),
155 * S3_DST_URL (s3://xtesting/prefix),
158 Campaign.EX_OK if artifacts were published to repository.
159 Campaign.EX_DUMP_ARTIFACTS_ERROR otherwise.
162 build_tag = env.get('BUILD_TAG')
163 assert Campaign.dump_db() == Campaign.EX_OK
164 assert Campaign.dump_artifacts() == Campaign.EX_OK
165 with zipfile.ZipFile('{}.zip'.format(build_tag), 'w') as zfile:
166 zfile.write("{}.json".format(build_tag))
167 for root, _, files in os.walk(build_tag):
168 for filename in files:
169 zfile.write(os.path.join(root, filename))
170 b3resource = boto3.resource(
171 's3', endpoint_url=os.environ["S3_ENDPOINT_URL"])
172 dst_s3_url = os.environ["S3_DST_URL"]
173 multipart_threshold = 5 * 1024 ** 5 if "google" in os.environ[
174 "S3_ENDPOINT_URL"] else 8 * 1024 * 1024
175 config = TransferConfig(multipart_threshold=multipart_threshold)
176 bucket_name = urllib.parse.urlparse(dst_s3_url).netloc
177 mime_type = mimetypes.guess_type('{}.zip'.format(build_tag))
178 path = urllib.parse.urlparse(dst_s3_url).path.strip("/")
179 # pylint: disable=no-member
180 b3resource.Bucket(bucket_name).upload_file(
181 '{}.zip'.format(build_tag),
182 os.path.join(path, '{}.zip'.format(build_tag)),
184 ExtraArgs={'ContentType': mime_type[
185 0] or 'application/octet-stream'})
186 dst_http_url = os.environ["HTTP_DST_URL"]
187 link = os.path.join(dst_http_url, '{}.zip'.format(build_tag))
188 Campaign.__logger.info(
189 "All data were successfully published:\n\n%s", link)
190 return Campaign.EX_OK
191 except KeyError as ex:
192 Campaign.__logger.error("Please check env var: %s", str(ex))
193 return Campaign.EX_ZIP_CAMPAIGN_FILES_ERROR
194 except botocore.exceptions.NoCredentialsError:
195 Campaign.__logger.error(
196 "Please fill ~/.aws/credentials, ~/.boto or set "
197 "AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in env")
198 return Campaign.EX_ZIP_CAMPAIGN_FILES_ERROR
199 except Exception: # pylint: disable=broad-except
200 Campaign.__logger.exception("Cannot publish the artifacts")
201 return Campaign.EX_ZIP_CAMPAIGN_FILES_ERROR
205 """Entry point for Campaign.zip_campaign_files()."""
206 if not os.path.exists(testcase.TestCase.dir_results):
207 os.makedirs(testcase.TestCase.dir_results)
208 if env.get('DEBUG').lower() == 'true':
209 logging.config.fileConfig(pkg_resources.resource_filename(
210 'xtesting', 'ci/logging.debug.ini'))
212 logging.config.fileConfig(pkg_resources.resource_filename(
213 'xtesting', 'ci/logging.ini'))
214 logging.captureWarnings(True)
215 Campaign.zip_campaign_files()