3 # Copyright (c) 2019 Orange and others.
5 # All rights reserved. This program and the accompanying materials
6 # are made available under the terms of the Apache License, Version 2.0
7 # which accompanies this distribution, and is available at
8 # http://www.apache.org/licenses/LICENSE-2.0
10 """Dump DB and artifacts for third-party certifications."""
21 from boto3.s3.transfer import TransferConfig
25 from six.moves import urllib
27 from xtesting.core import testcase
28 from xtesting.utils import env
29 from xtesting.utils import constants
31 __author__ = "Cedric Ollivier <cedric.ollivier@orange.com>"
35 "Dump, archive and publish all results and artifacts from a campaign."
38 """everything is OK"""
40 EX_DUMP_FROM_DB_ERROR = os.EX_SOFTWARE - 5
41 """dump_db() failed"""
43 EX_DUMP_ARTIFACTS_ERROR = os.EX_SOFTWARE - 6
44 """dump_artifacts() failed"""
46 EX_ZIP_CAMPAIGN_FILES_ERROR = os.EX_SOFTWARE - 7
47 """dump_artifacts() failed"""
49 __logger = logging.getLogger(__name__)
53 """Dump all test campaign results from the DB.
55 It allows collecting all the results from the DB.
57 It could be overriden if the common implementation is not
60 The next vars must be set in env:
66 Campaign.EX_OK if results were collected from DB.
67 Campaign.EX_DUMP_FROM_DB_ERROR otherwise.
70 url = env.get('TEST_DB_URL')
72 "{}?build_tag={}".format(url, env.get('BUILD_TAG')),
73 headers=testcase.TestCase.headers)
74 req.raise_for_status()
76 Campaign.__logger.debug("data from DB: \n%s", output)
77 for i, _ in enumerate(output["results"]):
78 for j, _ in enumerate(
79 output["results"][i]["details"]["links"]):
80 output["results"][i]["details"]["links"][j] = re.sub(
81 "^{}/*".format(os.environ["HTTP_DST_URL"]), '',
82 output["results"][i]["details"]["links"][j])
83 Campaign.__logger.debug("data to archive: \n%s", output)
84 with open("{}.json".format(env.get('BUILD_TAG')), "w") as dfile:
85 json.dump(output, dfile)
86 except Exception: # pylint: disable=broad-except
87 Campaign.__logger.exception(
88 "The results cannot be collected from DB")
89 return Campaign.EX_DUMP_FROM_DB_ERROR
94 """Dump all test campaign artifacts from the S3 repository.
96 It allows collecting all the artifacts from the S3 repository.
98 It could be overriden if the common implementation is not
101 The credentials must be configured before publishing the artifacts:
103 * fill ~/.aws/credentials or ~/.boto,
104 * set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in env.
106 The next vars must be set in env:
108 * S3_ENDPOINT_URL (http://127.0.0.1:9000),
109 * S3_DST_URL (s3://xtesting/prefix),
112 Campaign.EX_OK if artifacts were published to repository.
113 Campaign.EX_DUMP_ARTIFACTS_ERROR otherwise.
116 build_tag = env.get('BUILD_TAG')
117 b3resource = boto3.resource(
118 's3', endpoint_url=os.environ["S3_ENDPOINT_URL"])
119 dst_s3_url = os.environ["S3_DST_URL"]
120 multipart_threshold = 5 * 1024 ** 5 if "google" in os.environ[
121 "S3_ENDPOINT_URL"] else 8 * 1024 * 1024
122 config = TransferConfig(multipart_threshold=multipart_threshold)
123 bucket_name = urllib.parse.urlparse(dst_s3_url).netloc
125 '^/*(.*)/*$', urllib.parse.urlparse(dst_s3_url).path).group(1)
126 prefix = os.path.join(s3path, build_tag)
127 # pylint: disable=no-member
128 for s3_object in b3resource.Bucket(bucket_name).objects.filter(
129 Prefix="{}/".format(prefix)):
130 path, _ = os.path.split(s3_object.key)
131 lpath = re.sub('^{}/*'.format(s3path), '', path)
132 if lpath and not os.path.exists(lpath):
134 # pylint: disable=no-member
135 b3resource.Bucket(bucket_name).download_file(
137 re.sub('^{}/*'.format(s3path), '', s3_object.key),
139 Campaign.__logger.info(
141 re.sub('^{}/*'.format(s3path), '', s3_object.key))
142 return Campaign.EX_OK
143 except Exception: # pylint: disable=broad-except
144 Campaign.__logger.exception("Cannot publish the artifacts")
145 return Campaign.EX_DUMP_ARTIFACTS_ERROR
148 def zip_campaign_files(): # pylint: disable=too-many-locals
149 """Archive and publish all test campaign data to the S3 repository.
151 It allows collecting all the artifacts from the S3 repository.
153 It could be overriden if the common implementation is not
156 The credentials must be configured before publishing the artifacts:
158 * fill ~/.aws/credentials or ~/.boto,
159 * set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in env.
161 The next vars must be set in env:
163 * S3_ENDPOINT_URL (http://127.0.0.1:9000),
164 * S3_DST_URL (s3://xtesting/prefix),
167 Campaign.EX_OK if artifacts were published to repository.
168 Campaign.EX_DUMP_ARTIFACTS_ERROR otherwise.
171 build_tag = env.get('BUILD_TAG')
172 assert Campaign.dump_db() == Campaign.EX_OK
173 assert Campaign.dump_artifacts() == Campaign.EX_OK
174 with zipfile.ZipFile('{}.zip'.format(build_tag),
175 'w', zipfile.ZIP_DEFLATED) as zfile:
176 zfile.write("{}.json".format(build_tag))
177 for root, _, files in os.walk(build_tag):
178 for filename in files:
179 zfile.write(os.path.join(root, filename))
180 b3resource = boto3.resource(
181 's3', endpoint_url=os.environ["S3_ENDPOINT_URL"])
182 dst_s3_url = os.environ["S3_DST_URL"]
183 multipart_threshold = 5 * 1024 ** 5 if "google" in os.environ[
184 "S3_ENDPOINT_URL"] else 8 * 1024 * 1024
185 config = TransferConfig(multipart_threshold=multipart_threshold)
186 bucket_name = urllib.parse.urlparse(dst_s3_url).netloc
187 mime_type = mimetypes.guess_type('{}.zip'.format(build_tag))
188 path = urllib.parse.urlparse(dst_s3_url).path.strip("/")
189 # pylint: disable=no-member
190 b3resource.Bucket(bucket_name).upload_file(
191 '{}.zip'.format(build_tag),
192 os.path.join(path, '{}.zip'.format(build_tag)),
194 ExtraArgs={'ContentType': mime_type[
195 0] or 'application/octet-stream'})
196 dst_http_url = os.environ["HTTP_DST_URL"]
197 link = os.path.join(dst_http_url, '{}.zip'.format(build_tag))
198 Campaign.__logger.info(
199 "All data were successfully published:\n\n%s", link)
200 return Campaign.EX_OK
201 except KeyError as ex:
202 Campaign.__logger.error("Please check env var: %s", str(ex))
203 return Campaign.EX_ZIP_CAMPAIGN_FILES_ERROR
204 except botocore.exceptions.NoCredentialsError:
205 Campaign.__logger.error(
206 "Please fill ~/.aws/credentials, ~/.boto or set "
207 "AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in env")
208 return Campaign.EX_ZIP_CAMPAIGN_FILES_ERROR
209 except Exception: # pylint: disable=broad-except
210 Campaign.__logger.exception("Cannot publish the artifacts")
211 return Campaign.EX_ZIP_CAMPAIGN_FILES_ERROR
215 """Entry point for Campaign.zip_campaign_files()."""
216 if not os.path.exists(testcase.TestCase.dir_results):
217 os.makedirs(testcase.TestCase.dir_results)
218 if env.get('DEBUG').lower() == 'true':
219 logging.config.fileConfig(pkg_resources.resource_filename(
220 'xtesting', constants.DEBUG_INI_PATH))
222 logging.config.fileConfig(pkg_resources.resource_filename(
223 'xtesting', constants.INI_PATH))
224 logging.captureWarnings(True)
225 Campaign.zip_campaign_files()