diff --git a/scripts/ci/install.sh b/scripts/ci/install.sh index 40a73a133d..dd69f6caee 100644 --- a/scripts/ci/install.sh +++ b/scripts/ci/install.sh @@ -17,6 +17,10 @@ function retry { retry pip install --upgrade pip setuptools wheel retry pip install .[all,tests] +# NOTE: waiting for https://github.com/spulec/moto/issues/2172 +pip uninstall -y moto +retry pip install git+https://github.com/efiop/moto.git@move-env-mocking + git config --global user.email "dvctester@example.com" git config --global user.name "DVC Tester" diff --git a/setup.py b/setup.py index da01f7fb96..28d85847ab 100644 --- a/setup.py +++ b/setup.py @@ -102,6 +102,7 @@ def run(self): "pydocstyle<4.0", "jaraco.windows==3.9.2", "mock-ssh-server>=0.5.0", + "moto", ] if (sys.version_info) >= (3, 6): diff --git a/tests/func/test_s3.py b/tests/func/test_s3.py index 3bcb64ba3a..9cead965fe 100644 --- a/tests/func/test_s3.py +++ b/tests/func/test_s3.py @@ -1,8 +1,32 @@ import boto3 -import pytest + +from moto import mock_s3 +from functools import wraps +import moto.s3.models as s3model from dvc.remote.s3 import RemoteS3 -from tests.func.test_data_cloud import _should_test_aws, get_aws_url +from tests.func.test_data_cloud import get_aws_url + + +# from https://github.com/spulec/moto/blob/v1.3.5/tests/test_s3/test_s3.py#L40 +REDUCED_PART_SIZE = 256 + + +def reduced_min_part_size(f): + """ speed up tests by temporarily making the multipart minimum part size + small + """ + orig_size = s3model.UPLOAD_PART_MIN_SIZE + + @wraps(f) + def wrapped(*args, **kwargs): + try: + s3model.UPLOAD_PART_MIN_SIZE = REDUCED_PART_SIZE + return f(*args, **kwargs) + finally: + s3model.UPLOAD_PART_MIN_SIZE = orig_size + + return wrapped def _get_src_dst(): @@ -10,13 +34,12 @@ def _get_src_dst(): return base_info / "from", base_info / "to" +@mock_s3 def test_copy_singlepart_preserve_etag(): from_info, to_info = _get_src_dst() - if not _should_test_aws(): - pytest.skip() - s3 = boto3.client("s3") + s3.create_bucket(Bucket=from_info.bucket) s3.put_object(Bucket=from_info.bucket, Key=from_info.path, Body="data") RemoteS3._copy(s3, from_info, to_info, {}) @@ -32,8 +55,8 @@ def _upload_multipart(s3, Bucket, Key): # NOTE: Generation parts of variable size. Part size should be at # least 5MB: # https://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadComplete.html - part_size = 10 * 1024 * 1024 + 2 ** i - body = str(i) * part_size + part_size = REDUCED_PART_SIZE + i + body = b"1" * part_size part = s3.upload_part( Bucket=Bucket, Key=Key, @@ -52,12 +75,12 @@ def _upload_multipart(s3, Bucket, Key): ) +@mock_s3 +@reduced_min_part_size def test_copy_multipart_preserve_etag(): from_info, to_info = _get_src_dst() - if not _should_test_aws(): - pytest.skip() - s3 = boto3.client("s3") + s3.create_bucket(Bucket=from_info.bucket) _upload_multipart(s3, from_info.bucket, from_info.path) RemoteS3._copy(s3, from_info, to_info, {})