3 # Copyright (c) 2019 Orange and others.
5 # All rights reserved. This program and the accompanying materials
6 # are made available under the terms of the Apache License, Version 2.0
7 # which accompanies this distribution, and is available at
8 # http://www.apache.org/licenses/LICENSE-2.0
10 """Dump DB and artifacts for third-party certifications."""
20 from urllib.parse import urlparse
22 from boto3.s3.transfer import TransferConfig
27 from xtesting.core import testcase
28 from xtesting.utils import env
29 from xtesting.utils import constants
31 __author__ = "Cedric Ollivier <cedric.ollivier@orange.com>"
35 "Dump, archive and publish all results and artifacts from a campaign."
38 """everything is OK"""
40 EX_DUMP_FROM_DB_ERROR = os.EX_SOFTWARE - 5
41 """dump_db() failed"""
43 EX_DUMP_ARTIFACTS_ERROR = os.EX_SOFTWARE - 6
44 """dump_artifacts() failed"""
46 EX_ZIP_CAMPAIGN_FILES_ERROR = os.EX_SOFTWARE - 7
47 """dump_artifacts() failed"""
49 __logger = logging.getLogger(__name__)
53 """Dump all test campaign results from the DB.
55 It allows collecting all the results from the DB.
57 It could be overriden if the common implementation is not
60 The next vars must be set in env:
66 Campaign.EX_OK if results were collected from DB.
67 Campaign.EX_DUMP_FROM_DB_ERROR otherwise.
70 url = env.get('TEST_DB_URL')
72 f"{url}?build_tag={env.get('BUILD_TAG')}",
73 headers=testcase.TestCase.headers)
74 req.raise_for_status()
76 Campaign.__logger.debug("data from DB: \n%s", output)
77 for i, _ in enumerate(output["results"]):
78 for j, _ in enumerate(
79 output["results"][i]["details"]["links"]):
80 output["results"][i]["details"]["links"][j] = re.sub(
81 "^{os.environ['HTTP_DST_URL']}/*", '',
82 output["results"][i]["details"]["links"][j])
83 Campaign.__logger.debug("data to archive: \n%s", output)
84 with open("{env.get('BUILD_TAG')}.json", "w",
85 encoding='utf-8') as dfile:
86 json.dump(output, dfile)
87 except Exception: # pylint: disable=broad-except
88 Campaign.__logger.exception(
89 "The results cannot be collected from DB")
90 return Campaign.EX_DUMP_FROM_DB_ERROR
95 """Dump all test campaign artifacts from the S3 repository.
97 It allows collecting all the artifacts from the S3 repository.
99 It could be overriden if the common implementation is not
102 The credentials must be configured before publishing the artifacts:
104 * fill ~/.aws/credentials or ~/.boto,
105 * set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in env.
107 The next vars must be set in env:
109 * S3_ENDPOINT_URL (http://127.0.0.1:9000),
110 * S3_DST_URL (s3://xtesting/prefix),
113 Campaign.EX_OK if artifacts were published to repository.
114 Campaign.EX_DUMP_ARTIFACTS_ERROR otherwise.
117 build_tag = env.get('BUILD_TAG')
118 b3resource = boto3.resource(
119 's3', endpoint_url=os.environ["S3_ENDPOINT_URL"])
120 dst_s3_url = os.environ["S3_DST_URL"]
121 multipart_threshold = 5 * 1024 ** 5 if "google" in os.environ[
122 "S3_ENDPOINT_URL"] else 8 * 1024 * 1024
123 config = TransferConfig(multipart_threshold=multipart_threshold)
124 bucket_name = urlparse(dst_s3_url).netloc
126 '^/*(.*)/*$', urlparse(dst_s3_url).path).group(1)
127 prefix = os.path.join(s3path, build_tag)
128 # pylint: disable=no-member
129 for s3_object in b3resource.Bucket(bucket_name).objects.filter(
130 Prefix=f"{prefix}/"):
131 path, _ = os.path.split(s3_object.key)
132 lpath = re.sub(f'^{s3path}/*', '', path)
133 if lpath and not os.path.exists(lpath):
135 # pylint: disable=no-member
136 b3resource.Bucket(bucket_name).download_file(
138 re.sub(f'^{s3path}/*', '', s3_object.key),
140 Campaign.__logger.info(
142 re.sub(f'^{s3path}/*', '', s3_object.key))
143 return Campaign.EX_OK
144 except Exception: # pylint: disable=broad-except
145 Campaign.__logger.exception("Cannot publish the artifacts")
146 return Campaign.EX_DUMP_ARTIFACTS_ERROR
149 def zip_campaign_files(): # pylint: disable=too-many-locals
150 """Archive and publish all test campaign data to the S3 repository.
152 It allows collecting all the artifacts from the S3 repository.
154 It could be overriden if the common implementation is not
157 The credentials must be configured before publishing the artifacts:
159 * fill ~/.aws/credentials or ~/.boto,
160 * set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in env.
162 The next vars must be set in env:
164 * S3_ENDPOINT_URL (http://127.0.0.1:9000),
165 * S3_DST_URL (s3://xtesting/prefix),
168 Campaign.EX_OK if artifacts were published to repository.
169 Campaign.EX_DUMP_ARTIFACTS_ERROR otherwise.
172 build_tag = env.get('BUILD_TAG')
173 assert Campaign.dump_db() == Campaign.EX_OK
174 assert Campaign.dump_artifacts() == Campaign.EX_OK
175 with zipfile.ZipFile(f'{build_tag}.zip',
176 'w', zipfile.ZIP_DEFLATED) as zfile:
177 zfile.write(f"{build_tag}.json")
178 for root, _, files in os.walk(build_tag):
179 for filename in files:
180 zfile.write(os.path.join(root, filename))
181 b3resource = boto3.resource(
182 's3', endpoint_url=os.environ["S3_ENDPOINT_URL"])
183 dst_s3_url = os.environ["S3_DST_URL"]
184 multipart_threshold = 5 * 1024 ** 5 if "google" in os.environ[
185 "S3_ENDPOINT_URL"] else 8 * 1024 * 1024
186 config = TransferConfig(multipart_threshold=multipart_threshold)
187 bucket_name = urlparse(dst_s3_url).netloc
188 mime_type = mimetypes.guess_type(f'{build_tag}.zip')
189 path = urlparse(dst_s3_url).path.strip("/")
190 # pylint: disable=no-member
191 b3resource.Bucket(bucket_name).upload_file(
193 os.path.join(path, f'{build_tag}.zip'),
195 ExtraArgs={'ContentType': mime_type[
196 0] or 'application/octet-stream'})
197 dst_http_url = os.environ["HTTP_DST_URL"]
198 link = os.path.join(dst_http_url, f'{build_tag}.zip')
199 Campaign.__logger.info(
200 "All data were successfully published:\n\n%s", link)
201 return Campaign.EX_OK
202 except KeyError as ex:
203 Campaign.__logger.error("Please check env var: %s", str(ex))
204 return Campaign.EX_ZIP_CAMPAIGN_FILES_ERROR
205 except botocore.exceptions.NoCredentialsError:
206 Campaign.__logger.error(
207 "Please fill ~/.aws/credentials, ~/.boto or set "
208 "AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in env")
209 return Campaign.EX_ZIP_CAMPAIGN_FILES_ERROR
210 except Exception: # pylint: disable=broad-except
211 Campaign.__logger.exception("Cannot publish the artifacts")
212 return Campaign.EX_ZIP_CAMPAIGN_FILES_ERROR
216 """Entry point for Campaign.zip_campaign_files()."""
217 if not os.path.exists(testcase.TestCase.dir_results):
218 os.makedirs(testcase.TestCase.dir_results)
219 if env.get('DEBUG').lower() == 'true':
220 logging.config.fileConfig(pkg_resources.resource_filename(
221 'xtesting', constants.DEBUG_INI_PATH))
223 logging.config.fileConfig(pkg_resources.resource_filename(
224 'xtesting', constants.INI_PATH))
225 logging.captureWarnings(True)
226 Campaign.zip_campaign_files()