Google Storage doesn't support S3 multipart uploads
https://docs.aws.amazon.com/AmazonS3/latest/dev/mpuoverview.html
https://cloud.google.com/storage/docs/composite-objects
5 * 1024 ** 5 is the multipart upload limit.
https://docs.aws.amazon.com/AmazonS3/latest/dev/qfacts.html
Change-Id: Iec3a5cd70ea6e912272ea231eecde6ee4f51ce4f
Signed-off-by: Cédric Ollivier <cedric.ollivier@orange.com>
+from boto3.s3.transfer import TransferConfig
import botocore
import prettytable
import requests
import botocore
import prettytable
import requests
b3resource = boto3.resource(
's3', endpoint_url=os.environ["S3_ENDPOINT_URL"])
dst_s3_url = os.environ["S3_DST_URL"]
b3resource = boto3.resource(
's3', endpoint_url=os.environ["S3_ENDPOINT_URL"])
dst_s3_url = os.environ["S3_DST_URL"]
+ multipart_threshold = 5 * 1024 ** 5 if "google" in os.environ[
+ "S3_ENDPOINT_URL"] else 8 * 1024 * 1024
+ config = TransferConfig(multipart_threshold=multipart_threshold)
bucket_name = urllib.parse.urlparse(dst_s3_url).netloc
try:
b3resource.meta.client.head_bucket(Bucket=bucket_name)
bucket_name = urllib.parse.urlparse(dst_s3_url).netloc
try:
b3resource.meta.client.head_bucket(Bucket=bucket_name)
"Publishing %s %s", abs_file, mime_type)
# pylint: disable=no-member
b3resource.Bucket(bucket_name).upload_file(
"Publishing %s %s", abs_file, mime_type)
# pylint: disable=no-member
b3resource.Bucket(bucket_name).upload_file(
- abs_file,
- os.path.join(path, log_file),
+ abs_file, os.path.join(path, log_file), Config=config,
ExtraArgs={'ContentType': mime_type[
0] or 'application/octet-stream'})
link = os.path.join(dst_http_url, log_file)
ExtraArgs={'ContentType': mime_type[
0] or 'application/octet-stream'})
link = os.path.join(dst_http_url, log_file)
os.path.join(path, os.path.relpath(
os.path.join(root, pub_file),
start=self.dir_results)),
os.path.join(path, os.path.relpath(
os.path.join(root, pub_file),
start=self.dir_results)),
ExtraArgs={'ContentType': mime_type[
0] or 'application/octet-stream'})
link = os.path.join(dst_http_url, os.path.relpath(
ExtraArgs={'ContentType': mime_type[
0] or 'application/octet-stream'})
link = os.path.join(dst_http_url, os.path.relpath(
mock.call().Bucket().upload_file(
'/var/lib/xtesting/results/xtesting.log',
'prefix/xtesting.log',
mock.call().Bucket().upload_file(
'/var/lib/xtesting/results/xtesting.log',
'prefix/xtesting.log',
ExtraArgs={'ContentType': 'application/octet-stream'}),
mock.call().Bucket('xtesting'),
mock.call().Bucket().upload_file(
'/var/lib/xtesting/results/xtesting.debug.log',
'prefix/xtesting.debug.log',
ExtraArgs={'ContentType': 'application/octet-stream'}),
mock.call().Bucket('xtesting'),
mock.call().Bucket().upload_file(
'/var/lib/xtesting/results/xtesting.debug.log',
'prefix/xtesting.debug.log',
ExtraArgs={'ContentType': 'application/octet-stream'}),
mock.call().Bucket('xtesting'),
mock.call().Bucket().upload_file(
'/var/lib/xtesting/results/bar', 'prefix/bar',
ExtraArgs={'ContentType': 'application/octet-stream'}),
mock.call().Bucket('xtesting'),
mock.call().Bucket().upload_file(
'/var/lib/xtesting/results/bar', 'prefix/bar',
ExtraArgs={'ContentType': 'application/octet-stream'})]
self.assertEqual(args[1].mock_calls, expected)
ExtraArgs={'ContentType': 'application/octet-stream'})]
self.assertEqual(args[1].mock_calls, expected)
mock.call().Bucket().upload_file(
'/var/lib/xtesting/results/xtesting.log',
'prefix/xtesting.log',
mock.call().Bucket().upload_file(
'/var/lib/xtesting/results/xtesting.log',
'prefix/xtesting.log',
ExtraArgs={'ContentType': 'text/plain'}),
mock.call().Bucket('xtesting'),
mock.call().Bucket().upload_file(
'/var/lib/xtesting/results/xtesting.debug.log',
'prefix/xtesting.debug.log',
ExtraArgs={'ContentType': 'text/plain'}),
mock.call().Bucket('xtesting'),
mock.call().Bucket().upload_file(
'/var/lib/xtesting/results/xtesting.debug.log',
'prefix/xtesting.debug.log',
ExtraArgs={'ContentType': 'text/plain'}),
mock.call().Bucket('xtesting'),
mock.call().Bucket().upload_file(
'/var/lib/xtesting/results/bar', 'prefix/bar',
ExtraArgs={'ContentType': 'text/plain'}),
mock.call().Bucket('xtesting'),
mock.call().Bucket().upload_file(
'/var/lib/xtesting/results/bar', 'prefix/bar',
ExtraArgs={'ContentType': 'text/plain'})]
self.assertEqual(args[1].mock_calls, expected)
ExtraArgs={'ContentType': 'text/plain'})]
self.assertEqual(args[1].mock_calls, expected)