Skip to content

Commit

Permalink
tests: s3: use moto for multipart test
Browse files Browse the repository at this point in the history
Fixes iterative#1889

Signed-off-by: Ruslan Kuprieiev <[email protected]>
  • Loading branch information
efiop committed Jul 18, 2019
1 parent 36c07b8 commit 958a4fb
Show file tree
Hide file tree
Showing 3 changed files with 38 additions and 10 deletions.
4 changes: 4 additions & 0 deletions scripts/ci/install.sh
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,10 @@ function retry {
retry pip install --upgrade pip setuptools wheel
retry pip install .[all,tests]

# NOTE: waiting for https://github.com/spulec/moto/issues/2172
pip uninstall -y moto
retry pip install git+https://github.com/efiop/moto.git@move-env-mocking

git config --global user.email "[email protected]"
git config --global user.name "DVC Tester"

Expand Down
1 change: 1 addition & 0 deletions setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,7 @@ def run(self):
"pydocstyle<4.0",
"jaraco.windows==3.9.2",
"mock-ssh-server>=0.5.0",
"moto",
]

if (sys.version_info) >= (3, 6):
Expand Down
43 changes: 33 additions & 10 deletions tests/func/test_s3.py
Original file line number Diff line number Diff line change
@@ -1,22 +1,45 @@
import boto3
import pytest

from moto import mock_s3
from functools import wraps
import moto.s3.models as s3model

from dvc.remote.s3 import RemoteS3
from tests.func.test_data_cloud import _should_test_aws, get_aws_url
from tests.func.test_data_cloud import get_aws_url


# from https://github.com/spulec/moto/blob/v1.3.5/tests/test_s3/test_s3.py#L40
REDUCED_PART_SIZE = 256


def reduced_min_part_size(f):
""" speed up tests by temporarily making the multipart minimum part size
small
"""
orig_size = s3model.UPLOAD_PART_MIN_SIZE

@wraps(f)
def wrapped(*args, **kwargs):
try:
s3model.UPLOAD_PART_MIN_SIZE = REDUCED_PART_SIZE
return f(*args, **kwargs)
finally:
s3model.UPLOAD_PART_MIN_SIZE = orig_size

return wrapped


def _get_src_dst():
base_info = RemoteS3.path_cls(get_aws_url())
return base_info / "from", base_info / "to"


@mock_s3
def test_copy_singlepart_preserve_etag():
from_info, to_info = _get_src_dst()

if not _should_test_aws():
pytest.skip()

s3 = boto3.client("s3")
s3.create_bucket(Bucket=from_info.bucket)
s3.put_object(Bucket=from_info.bucket, Key=from_info.path, Body="data")

RemoteS3._copy(s3, from_info, to_info, {})
Expand All @@ -32,8 +55,8 @@ def _upload_multipart(s3, Bucket, Key):
# NOTE: Generation parts of variable size. Part size should be at
# least 5MB:
# https://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadComplete.html
part_size = 10 * 1024 * 1024 + 2 ** i
body = str(i) * part_size
part_size = REDUCED_PART_SIZE + i
body = b"1" * part_size
part = s3.upload_part(
Bucket=Bucket,
Key=Key,
Expand All @@ -52,12 +75,12 @@ def _upload_multipart(s3, Bucket, Key):
)


@mock_s3
@reduced_min_part_size
def test_copy_multipart_preserve_etag():
from_info, to_info = _get_src_dst()

if not _should_test_aws():
pytest.skip()

s3 = boto3.client("s3")
s3.create_bucket(Bucket=from_info.bucket)
_upload_multipart(s3, from_info.bucket, from_info.path)
RemoteS3._copy(s3, from_info, to_info, {})

0 comments on commit 958a4fb

Please sign in to comment.