3 # Copyright (c) 2019 Orange and others.
5 # All rights reserved. This program and the accompanying materials
6 # are made available under the terms of the Apache License, Version 2.0
7 # which accompanies this distribution, and is available at
8 # http://www.apache.org/licenses/LICENSE-2.0
10 """Dump DB and artifacts for third-party certifications."""
22 from boto3.s3.transfer import TransferConfig
26 from xtesting.core import testcase
27 from xtesting.utils import env
28 from xtesting.utils import config
29 from xtesting.utils import constants
31 __author__ = "Cedric Ollivier <cedric.ollivier@orange.com>"
35 "Dump, archive and publish all results and artifacts from a campaign."
38 """everything is OK"""
40 EX_DUMP_FROM_DB_ERROR = os.EX_SOFTWARE - 5
41 """dump_db() failed"""
43 EX_DUMP_ARTIFACTS_ERROR = os.EX_SOFTWARE - 6
44 """dump_artifacts() failed"""
46 EX_ZIP_CAMPAIGN_FILES_ERROR = os.EX_SOFTWARE - 7
47 """dump_artifacts() failed"""
49 __logger = logging.getLogger(__name__)
53 """Dump all test campaign results from the DB.
55 It allows collecting all the results from the DB.
57 It could be overriden if the common implementation is not
60 The next vars must be set in env:
66 Campaign.EX_OK if results were collected from DB.
67 Campaign.EX_DUMP_FROM_DB_ERROR otherwise.
70 url = env.get('TEST_DB_URL')
72 f"{url}?build_tag={env.get('BUILD_TAG')}",
73 headers=testcase.TestCase.headers)
74 req.raise_for_status()
76 Campaign.__logger.debug("data from DB: \n%s", output)
77 for i, _ in enumerate(output["results"]):
78 for j, _ in enumerate(
79 output["results"][i]["details"]["links"]):
80 output["results"][i]["details"]["links"][j] = re.sub(
81 "^{os.environ['HTTP_DST_URL']}/*", '',
82 output["results"][i]["details"]["links"][j])
83 Campaign.__logger.debug("data to archive: \n%s", output)
84 with open(f"{env.get('BUILD_TAG')}.json", "w",
85 encoding='utf-8') as dfile:
86 json.dump(output, dfile)
87 except Exception: # pylint: disable=broad-except
88 Campaign.__logger.exception(
89 "The results cannot be collected from DB")
90 return Campaign.EX_DUMP_FROM_DB_ERROR
95 """Dump all test campaign artifacts from the S3 repository.
97 It allows collecting all the artifacts from the S3 repository.
99 It could be overriden if the common implementation is not
102 The credentials must be configured before publishing the artifacts:
104 * fill ~/.aws/credentials or ~/.boto,
105 * set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in env.
107 The next vars must be set in env:
109 * S3_ENDPOINT_URL (http://127.0.0.1:9000),
110 * S3_DST_URL (s3://xtesting/prefix),
113 Campaign.EX_OK if artifacts were published to repository.
114 Campaign.EX_DUMP_ARTIFACTS_ERROR otherwise.
117 build_tag = env.get('BUILD_TAG')
118 b3resource = boto3.resource(
119 's3', endpoint_url=os.environ["S3_ENDPOINT_URL"])
120 dst_s3_url = os.environ["S3_DST_URL"]
121 multipart_threshold = 5 * 1024 ** 5 if "google" in os.environ[
122 "S3_ENDPOINT_URL"] else 8 * 1024 * 1024
123 tconfig = TransferConfig(multipart_threshold=multipart_threshold)
124 bucket_name = urllib.parse.urlparse(dst_s3_url).netloc
126 '^/*(.*)/*$', urllib.parse.urlparse(dst_s3_url).path).group(1)
127 prefix = os.path.join(s3path, build_tag)
128 # pylint: disable=no-member
129 for s3_object in b3resource.Bucket(bucket_name).objects.filter(
130 Prefix=f"{prefix}/"):
131 path, _ = os.path.split(
132 urllib.parse.unquote_plus(s3_object.key))
133 lpath = re.sub(f'^{s3path}/*', '', path)
134 if lpath and not os.path.exists(lpath):
136 Campaign.__logger.info(
138 re.sub(f'^{s3path}/*', '',
139 urllib.parse.unquote_plus(s3_object.key)))
140 # pylint: disable=no-member
141 b3resource.Bucket(bucket_name).download_file(
142 urllib.parse.unquote_plus(s3_object.key),
143 re.sub(f'^{s3path}/*', '',
144 urllib.parse.unquote_plus(s3_object.key)),
146 return Campaign.EX_OK
147 except Exception: # pylint: disable=broad-except
148 Campaign.__logger.exception("Cannot publish the artifacts")
149 return Campaign.EX_DUMP_ARTIFACTS_ERROR
152 def zip_campaign_files(): # pylint: disable=too-many-locals
153 """Archive and publish all test campaign data to the S3 repository.
155 It allows collecting all the artifacts from the S3 repository.
157 It could be overriden if the common implementation is not
160 The credentials must be configured before publishing the artifacts:
162 * fill ~/.aws/credentials or ~/.boto,
163 * set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in env.
165 The next vars must be set in env:
167 * S3_ENDPOINT_URL (http://127.0.0.1:9000),
168 * S3_DST_URL (s3://xtesting/prefix),
171 Campaign.EX_OK if artifacts were published to repository.
172 Campaign.EX_DUMP_ARTIFACTS_ERROR otherwise.
175 build_tag = env.get('BUILD_TAG')
176 assert Campaign.dump_db() == Campaign.EX_OK
177 assert Campaign.dump_artifacts() == Campaign.EX_OK
178 with zipfile.ZipFile(f'{build_tag}.zip',
179 'w', zipfile.ZIP_DEFLATED) as zfile:
180 zfile.write(f"{build_tag}.json")
181 for root, _, files in os.walk(build_tag):
182 for filename in files:
183 zfile.write(os.path.join(root, filename))
184 b3resource = boto3.resource(
185 's3', endpoint_url=os.environ["S3_ENDPOINT_URL"])
186 dst_s3_url = os.environ["S3_DST_URL"]
187 multipart_threshold = 5 * 1024 ** 5 if "google" in os.environ[
188 "S3_ENDPOINT_URL"] else 8 * 1024 * 1024
189 tconfig = TransferConfig(multipart_threshold=multipart_threshold)
190 bucket_name = urllib.parse.urlparse(dst_s3_url).netloc
191 mime_type = mimetypes.guess_type(f'{build_tag}.zip')
192 path = urllib.parse.urlparse(dst_s3_url).path.strip("/")
193 # pylint: disable=no-member
194 b3resource.Bucket(bucket_name).upload_file(
196 os.path.join(path, f'{build_tag}.zip'),
198 ExtraArgs={'ContentType': mime_type[
199 0] or 'application/octet-stream'})
200 dst_http_url = os.environ["HTTP_DST_URL"]
201 link = os.path.join(dst_http_url, f'{build_tag}.zip')
202 Campaign.__logger.info(
203 "All data were successfully published:\n\n%s", link)
204 return Campaign.EX_OK
205 except KeyError as ex:
206 Campaign.__logger.error("Please check env var: %s", str(ex))
207 return Campaign.EX_ZIP_CAMPAIGN_FILES_ERROR
208 except botocore.exceptions.NoCredentialsError:
209 Campaign.__logger.error(
210 "Please fill ~/.aws/credentials, ~/.boto or set "
211 "AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in env")
212 return Campaign.EX_ZIP_CAMPAIGN_FILES_ERROR
213 except Exception: # pylint: disable=broad-except
214 Campaign.__logger.exception("Cannot publish the artifacts")
215 return Campaign.EX_ZIP_CAMPAIGN_FILES_ERROR
219 """Entry point for Campaign.zip_campaign_files()."""
220 if not os.path.exists(testcase.TestCase.dir_results):
221 os.makedirs(testcase.TestCase.dir_results)
222 if env.get('DEBUG').lower() == 'true':
223 logging.config.fileConfig(config.get_xtesting_config(
224 'logging.debug.ini', constants.DEBUG_INI_PATH_DEFAULT))
226 logging.config.fileConfig(config.get_xtesting_config(
227 'logging.ini', constants.INI_PATH_DEFAULT))
228 logging.captureWarnings(True)
229 Campaign.zip_campaign_files()