Skip to content

Commit

Permalink
Core, AWS, Delta, Snowflake: Switch tests to JUnit5 (apache#7996)
Browse files Browse the repository at this point in the history
  • Loading branch information
@skytin1004
skytin1004 authored Jul 19, 2023
1 parent cf3e50b commit d3d9155
Show file tree
Hide file tree
Showing 21 changed files with 434 additions and 398 deletions.
76 changes: 39 additions & 37 deletions aws/src/test/java/org/apache/iceberg/aws/s3/TestS3FileIO.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,18 +18,14 @@
*/
package org.apache.iceberg.aws.s3;

import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.mockito.AdditionalAnswers.delegatesTo;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;

import com.adobe.testing.s3mock.junit4.S3MockRule;
import com.adobe.testing.s3mock.junit5.S3MockExtension;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
Expand Down Expand Up @@ -61,14 +57,12 @@
import org.apache.iceberg.types.Types;
import org.apache.iceberg.util.SerializableSupplier;
import org.assertj.core.api.Assertions;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.ClassRule;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.junit.MockitoJUnitRunner;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.junit.jupiter.api.extension.RegisterExtension;
import software.amazon.awssdk.core.sync.RequestBody;
import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient;
import software.amazon.awssdk.regions.Region;
Expand All @@ -79,10 +73,12 @@
import software.amazon.awssdk.services.s3.model.DeleteObjectsResponse;
import software.amazon.awssdk.services.s3.model.S3Error;

@RunWith(MockitoJUnitRunner.class)
@ExtendWith(S3MockExtension.class)
public class TestS3FileIO {
@ClassRule public static final S3MockRule S3_MOCK_RULE = S3MockRule.builder().silent().build();
public SerializableSupplier<S3Client> s3 = S3_MOCK_RULE::createS3ClientV2;
@RegisterExtension
public static final S3MockExtension S3_MOCK = S3MockExtension.builder().silent().build();

public SerializableSupplier<S3Client> s3 = S3_MOCK::createS3ClientV2;
private final S3Client s3mock = mock(S3Client.class, delegatesTo(s3.get()));
private final Random random = new Random(1);
private final int numBucketsForBatchDeletion = 3;
Expand All @@ -97,7 +93,7 @@ public class TestS3FileIO {
"s3.delete.batch-size",
Integer.toString(batchDeletionSize));

@Before
@BeforeEach
public void before() {
s3FileIO = new S3FileIO(() -> s3mock);
s3FileIO.initialize(properties);
Expand All @@ -108,7 +104,7 @@ public void before() {
StaticClientFactory.client = s3mock;
}

@After
@AfterEach
public void after() {
if (null != s3FileIO) {
s3FileIO.close();
Expand All @@ -122,25 +118,25 @@ public void testNewInputFile() throws IOException {
random.nextBytes(expected);

InputFile in = s3FileIO.newInputFile(location);
assertFalse(in.exists());
Assertions.assertThat(in.exists()).isFalse();

OutputFile out = s3FileIO.newOutputFile(location);
try (OutputStream os = out.createOrOverwrite()) {
IOUtils.write(expected, os);
}

assertTrue(in.exists());
Assertions.assertThat(in.exists()).isTrue();
byte[] actual;

try (InputStream is = in.newStream()) {
actual = IOUtils.readFully(is, expected.length);
}

assertArrayEquals(expected, actual);
Assertions.assertThat(actual).isEqualTo(expected);

s3FileIO.deleteFile(in);

assertFalse(s3FileIO.newInputFile(location).exists());
Assertions.assertThat(s3FileIO.newInputFile(location).exists()).isFalse();
}

@Test
Expand All @@ -162,17 +158,17 @@ public void testDeleteFilesSingleBatchWithRemainder() {
public void testDeleteEmptyList() throws IOException {
String location = "s3://bucket/path/to/file.txt";
InputFile in = s3FileIO.newInputFile(location);
assertFalse(in.exists());
Assertions.assertThat(in.exists()).isFalse();
OutputFile out = s3FileIO.newOutputFile(location);
try (OutputStream os = out.createOrOverwrite()) {
IOUtils.write(new byte[1024 * 1024], os);
}

s3FileIO.deleteFiles(Lists.newArrayList());

Assert.assertTrue(s3FileIO.newInputFile(location).exists());
Assertions.assertThat(s3FileIO.newInputFile(location).exists()).isTrue();
s3FileIO.deleteFile(in);
assertFalse(s3FileIO.newInputFile(location).exists());
Assertions.assertThat(s3FileIO.newInputFile(location).exists()).isFalse();
}

@Test
Expand Down Expand Up @@ -207,7 +203,7 @@ private void testBatchDelete(int numObjects) {
int expectedDeleteRequests = expectedNumberOfBatchesPerBucket * numBucketsForBatchDeletion;
verify(s3mock, times(expectedDeleteRequests)).deleteObjects((DeleteObjectsRequest) any());
for (String path : paths) {
Assert.assertFalse(s3FileIO.newInputFile(path).exists());
Assertions.assertThat(s3FileIO.newInputFile(path).exists()).isFalse();
}
}

Expand All @@ -223,7 +219,7 @@ public void testSerializeClient() {
byte[] data = SerializationUtils.serialize(pre);
SerializableSupplier<S3Client> post = SerializationUtils.deserialize(data);

assertEquals("s3", post.get().serviceName());
Assertions.assertThat(post.get().serviceName()).isEqualTo("s3");
}

@Test
Expand All @@ -239,19 +235,21 @@ public void testPrefixList() {
String scalePrefix = String.format("%s/%s/", prefix, scale);

createRandomObjects(scalePrefix, scale);
assertEquals((long) scale, Streams.stream(s3FileIO.listPrefix(scalePrefix)).count());
Assertions.assertThat(Streams.stream(s3FileIO.listPrefix(scalePrefix)).count())
.isEqualTo((long) scale);
});

long totalFiles = scaleSizes.stream().mapToLong(Integer::longValue).sum();
assertEquals(totalFiles, Streams.stream(s3FileIO.listPrefix(prefix)).count());
Assertions.assertThat(Streams.stream(s3FileIO.listPrefix(prefix)).count())
.isEqualTo(totalFiles);
}

/**
* Ignoring because the test is flaky, failing with 500s from S3Mock. Coverage of prefix delete
* exists through integration tests.
*/
@Test
@Ignore
@Disabled
public void testPrefixDelete() {
String prefix = "s3://bucket/path/to/delete";
List<Integer> scaleSizes = Lists.newArrayList(0, 5, 1001);
Expand All @@ -262,7 +260,8 @@ public void testPrefixDelete() {

createRandomObjects(scalePrefix, scale);
s3FileIO.deletePrefix(scalePrefix);
assertEquals(0L, Streams.stream(s3FileIO.listPrefix(scalePrefix)).count());
Assertions.assertThat(Streams.stream(s3FileIO.listPrefix(scalePrefix)).count())
.isEqualTo(0);
});
}

Expand Down Expand Up @@ -306,7 +305,7 @@ public void testMissingTableMetadata() {
.hasMessageStartingWith("Location does not exist");

long duration = System.currentTimeMillis() - start;
Assert.assertTrue("Should take less than 10 seconds", duration < 10_000);
Assertions.assertThat(duration < 10_000).as("Should take less than 10 seconds").isTrue();
}
}

Expand All @@ -321,8 +320,8 @@ public void testFileIOJsonSerialization() {

String json = FileIOParser.toJson(s3FileIO);
try (FileIO deserialized = FileIOParser.fromJson(json, conf)) {
Assert.assertTrue(deserialized instanceof S3FileIO);
Assert.assertEquals(s3FileIO.properties(), deserialized.properties());
Assertions.assertThat(deserialized).isInstanceOf(S3FileIO.class);
Assertions.assertThat(deserialized.properties()).isEqualTo(s3FileIO.properties());
}
}

Expand All @@ -334,7 +333,8 @@ public void testS3FileIOKryoSerialization() throws IOException {
testS3FileIO.initialize(ImmutableMap.of("k1", "v1"));
FileIO roundTripSerializedFileIO = TestHelpers.KryoHelpers.roundTripSerialize(testS3FileIO);

Assert.assertEquals(testS3FileIO.properties(), roundTripSerializedFileIO.properties());
Assertions.assertThat(roundTripSerializedFileIO.properties())
.isEqualTo(testS3FileIO.properties());
}

@Test
Expand All @@ -345,7 +345,8 @@ public void testS3FileIOWithEmptyPropsKryoSerialization() throws IOException {
testS3FileIO.initialize(ImmutableMap.of());
FileIO roundTripSerializedFileIO = TestHelpers.KryoHelpers.roundTripSerialize(testS3FileIO);

Assert.assertEquals(testS3FileIO.properties(), roundTripSerializedFileIO.properties());
Assertions.assertThat(roundTripSerializedFileIO.properties())
.isEqualTo(testS3FileIO.properties());
}

@Test
Expand All @@ -356,7 +357,8 @@ public void testS3FileIOJavaSerialization() throws IOException, ClassNotFoundExc
testS3FileIO.initialize(ImmutableMap.of("k1", "v1"));
FileIO roundTripSerializedFileIO = TestHelpers.roundTripSerialize(testS3FileIO);

Assert.assertEquals(testS3FileIO.properties(), roundTripSerializedFileIO.properties());
Assertions.assertThat(roundTripSerializedFileIO.properties())
.isEqualTo(testS3FileIO.properties());
}

private void createRandomObjects(String prefix, int count) {
Expand Down
41 changes: 22 additions & 19 deletions aws/src/test/java/org/apache/iceberg/aws/s3/TestS3InputStream.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,33 +18,33 @@
*/
package org.apache.iceberg.aws.s3;

import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThrows;

import com.adobe.testing.s3mock.junit4.S3MockRule;
import com.adobe.testing.s3mock.junit5.S3MockExtension;
import java.io.IOException;
import java.util.Arrays;
import java.util.Random;
import org.apache.commons.io.IOUtils;
import org.apache.iceberg.io.RangeReadable;
import org.apache.iceberg.io.SeekableInputStream;
import org.junit.Before;
import org.junit.ClassRule;
import org.junit.Test;
import org.assertj.core.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.junit.jupiter.api.extension.RegisterExtension;
import software.amazon.awssdk.core.sync.RequestBody;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.BucketAlreadyExistsException;
import software.amazon.awssdk.services.s3.model.CreateBucketRequest;
import software.amazon.awssdk.services.s3.model.PutObjectRequest;

@ExtendWith(S3MockExtension.class)
public class TestS3InputStream {
@ClassRule public static final S3MockRule S3_MOCK_RULE = S3MockRule.builder().silent().build();
@RegisterExtension
public static final S3MockExtension S3_MOCK = S3MockExtension.builder().silent().build();

private final S3Client s3 = S3_MOCK_RULE.createS3ClientV2();
private final S3Client s3 = S3_MOCK.createS3ClientV2();
private final Random random = new Random(1);

@Before
@BeforeEach
public void before() {
createBucket("bucket");
}
Expand Down Expand Up @@ -88,7 +88,7 @@ private void readAndCheck(
SeekableInputStream in, long rangeStart, int size, byte[] original, boolean buffered)
throws IOException {
in.seek(rangeStart);
assertEquals(rangeStart, in.getPos());
Assertions.assertThat(in.getPos()).isEqualTo(rangeStart);

long rangeEnd = rangeStart + size;
byte[] actual = new byte[size];
Expand All @@ -102,8 +102,9 @@ private void readAndCheck(
}
}

assertEquals(rangeEnd, in.getPos());
assertArrayEquals(Arrays.copyOfRange(original, (int) rangeStart, (int) rangeEnd), actual);
Assertions.assertThat(in.getPos()).isEqualTo(rangeEnd);
Assertions.assertThat(actual)
.isEqualTo(Arrays.copyOfRange(original, (int) rangeStart, (int) rangeEnd));
}

@Test
Expand Down Expand Up @@ -144,17 +145,18 @@ private void readAndCheckRanges(
throws IOException {
in.readFully(position, buffer, offset, length);

assertArrayEquals(
Arrays.copyOfRange(original, offset, offset + length),
Arrays.copyOfRange(buffer, offset, offset + length));
Assertions.assertThat(Arrays.copyOfRange(buffer, offset, offset + length))
.isEqualTo(Arrays.copyOfRange(original, offset, offset + length));
}

@Test
public void testClose() throws Exception {
S3URI uri = new S3URI("s3://bucket/path/to/closed.dat");
SeekableInputStream closed = new S3InputStream(s3, uri);
closed.close();
assertThrows(IllegalStateException.class, () -> closed.seek(0));
Assertions.assertThatThrownBy(() -> closed.seek(0))
.isInstanceOf(IllegalStateException.class)
.hasMessage("already closed");
}

@Test
Expand All @@ -167,7 +169,8 @@ public void testSeek() throws Exception {
try (SeekableInputStream in = new S3InputStream(s3, uri)) {
in.seek(expected.length / 2);
byte[] actual = IOUtils.readFully(in, expected.length / 2);
assertArrayEquals(Arrays.copyOfRange(expected, expected.length / 2, expected.length), actual);
Assertions.assertThat(actual)
.isEqualTo(Arrays.copyOfRange(expected, expected.length / 2, expected.length));
}
}

Expand Down
Loading

0 comments on commit d3d9155

Please sign in to comment.