Forbid multipart upload if google storage

Google Storage doesn't support S3 multipart uploads
https://docs.aws.amazon.com/AmazonS3/latest/dev/mpuoverview.html
https://cloud.google.com/storage/docs/composite-objects

5 * 1024 ** 5 is the multipart upload limit.
https://docs.aws.amazon.com/AmazonS3/latest/dev/qfacts.html

Change-Id: Iec3a5cd70ea6e912272ea231eecde6ee4f51ce4f
Signed-off-by: Cédric Ollivier <cedric.ollivier@orange.com>
(cherry picked from commit f5aceb5447deed25234c5354b9dd9601115d9378)

diff --git a/xtesting/core/testcase.py b/xtesting/core/testcase.py
index c3b8f61..58de9f4 100644 (file)
--- a/xtesting/core/testcase.py
+++ b/xtesting/core/testcase.py
@@ -19,6 +19,7 @@ import re
 import sys
 
 import boto3
+from boto3.s3.transfer import TransferConfig
 import botocore
 import prettytable
 import requests
@@ -282,6 +283,9 @@ class TestCase():
             b3resource = boto3.resource(
                 's3', endpoint_url=os.environ["S3_ENDPOINT_URL"])
             dst_s3_url = os.environ["S3_DST_URL"]
+            multipart_threshold = 5 * 1024 ** 5 if "google" in os.environ[
+                "S3_ENDPOINT_URL"] else 8 * 1024 * 1024
+            config = TransferConfig(multipart_threshold=multipart_threshold)
             bucket_name = urllib.parse.urlparse(dst_s3_url).netloc
             try:
                 b3resource.meta.client.head_bucket(Bucket=bucket_name)
@@ -308,8 +312,7 @@ class TestCase():
                         "Publishing %s %s", abs_file, mime_type)
                     # pylint: disable=no-member
                     b3resource.Bucket(bucket_name).upload_file(
-                        abs_file,
-                        os.path.join(path, log_file),
+                        abs_file, os.path.join(path, log_file), Config=config,
                         ExtraArgs={'ContentType': mime_type[
                             0] or 'application/octet-stream'})
                     link = os.path.join(dst_http_url, log_file)
@@ -327,6 +330,7 @@ class TestCase():
                         os.path.join(path, os.path.relpath(
                             os.path.join(root, pub_file),
                             start=self.dir_results)),
+                        Config=config,
                         ExtraArgs={'ContentType': mime_type[
                             0] or 'application/octet-stream'})
                     link = os.path.join(dst_http_url, os.path.relpath(

diff --git a/xtesting/tests/unit/core/test_testcase.py b/xtesting/tests/unit/core/test_testcase.py
index 5d34c34..80e9d8c 100644 (file)
--- a/xtesting/tests/unit/core/test_testcase.py
+++ b/xtesting/tests/unit/core/test_testcase.py
@@ -400,15 +400,18 @@ class TestCaseTesting(unittest.TestCase):
             mock.call().Bucket().upload_file(
                 '/var/lib/xtesting/results/xtesting.log',
                 'prefix/xtesting.log',
+                Config=mock.ANY,
                 ExtraArgs={'ContentType': 'application/octet-stream'}),
             mock.call().Bucket('xtesting'),
             mock.call().Bucket().upload_file(
                 '/var/lib/xtesting/results/xtesting.debug.log',
                 'prefix/xtesting.debug.log',
+                Config=mock.ANY,
                 ExtraArgs={'ContentType': 'application/octet-stream'}),
             mock.call().Bucket('xtesting'),
             mock.call().Bucket().upload_file(
                 '/var/lib/xtesting/results/bar', 'prefix/bar',
+                Config=mock.ANY,
                 ExtraArgs={'ContentType': 'application/octet-stream'})]
         self.assertEqual(args[1].mock_calls, expected)
 
@@ -429,15 +432,18 @@ class TestCaseTesting(unittest.TestCase):
             mock.call().Bucket().upload_file(
                 '/var/lib/xtesting/results/xtesting.log',
                 'prefix/xtesting.log',
+                Config=mock.ANY,
                 ExtraArgs={'ContentType': 'text/plain'}),
             mock.call().Bucket('xtesting'),
             mock.call().Bucket().upload_file(
                 '/var/lib/xtesting/results/xtesting.debug.log',
                 'prefix/xtesting.debug.log',
+                Config=mock.ANY,
                 ExtraArgs={'ContentType': 'text/plain'}),
             mock.call().Bucket('xtesting'),
             mock.call().Bucket().upload_file(
                 '/var/lib/xtesting/results/bar', 'prefix/bar',
+                Config=mock.ANY,
                 ExtraArgs={'ContentType': 'text/plain'})]
         self.assertEqual(args[1].mock_calls, expected)