Merged with Devin's change

deephaven · May 14, 2024 · 618bd6a · 618bd6a
1 parent 5e80889
commit 618bd6a
Show file tree

Hide file tree

Showing 9 changed files with 119 additions and 206 deletions.
diff --git a/extensions/parquet/table/gradle.properties b/extensions/parquet/table/gradle.properties
@@ -2,4 +2,4 @@ io.deephaven.project.ProjectType=JAVA_PUBLIC
 
 # TODO(deephaven-core#5115): EPIC: Dependency management
 testcontainers.localstack.image=localstack/localstack:3.1.0
-testcontainers.minio.image=minio/minio:RELEASE.2024-02-04T22-36-13Z
+testcontainers.minio.image=minio/minio:RELEASE.2024-02-04T22-36-13Z
diff --git a/...arquet/table/ParquetLocalStackS3Test.java → .../table/ParquetS3SimpleLocalStackTest.java b/...arquet/table/ParquetLocalStackS3Test.java → .../table/ParquetS3SimpleLocalStackTest.java
@@ -9,7 +9,7 @@
 import org.junit.BeforeClass;
 import software.amazon.awssdk.services.s3.S3AsyncClient;
 
-public class ParquetLocalStackS3Test extends ParquetS3TestBase {
+public class ParquetS3SimpleLocalStackTest extends ParquetS3SimpleTestBase {
 
     @BeforeClass
     public static void initContainer() {

diff --git a/...ven/parquet/table/ParquetMinIOS3Test.java → ...rquet/table/ParquetS3SimpleMinIOTest.java b/...ven/parquet/table/ParquetMinIOS3Test.java → ...rquet/table/ParquetS3SimpleMinIOTest.java
@@ -12,7 +12,7 @@
 
 import static org.junit.Assert.assertFalse;
 
-public class ParquetMinIOS3Test extends ParquetS3TestBase {
+public class ParquetS3SimpleMinIOTest extends ParquetS3SimpleTestBase {
 
     @BeforeClass
     public static void initContainer() {

diff --git a/...aven/parquet/table/ParquetS3TestBase.java → ...arquet/table/ParquetS3SimpleTestBase.java b/...aven/parquet/table/ParquetS3TestBase.java → ...arquet/table/ParquetS3SimpleTestBase.java
@@ -12,7 +12,7 @@
 import io.deephaven.engine.util.TableTools;
 import io.deephaven.extensions.s3.Credentials;
 import io.deephaven.extensions.s3.S3Instructions;
-import io.deephaven.extensions.s3.S3SeekableChannelTestBase;
+import io.deephaven.extensions.s3.testlib.S3SeekableChannelTestSetup;
 import io.deephaven.extensions.s3.testlib.S3Helper;
 import io.deephaven.test.types.OutOfBandTest;
 import org.junit.After;
@@ -37,9 +37,9 @@
 import static org.junit.Assert.assertTrue;
 
 @Category(OutOfBandTest.class)
-abstract class ParquetS3TestBase extends S3SeekableChannelTestBase {
+abstract class ParquetS3SimpleTestBase extends S3SeekableChannelTestSetup {
 
-    private static final File rootDir = new File(ParquetS3TestBase.class.getName() + "_root");
+    private static final File rootDir = new File(ParquetS3SimpleTestBase.class.getName() + "_root");
 
     // The following tests are disabled by default, as they are verifying against a remote system
     private static final boolean ENABLE_REMOTE_S3_TESTING = false;
@@ -49,7 +49,7 @@ abstract class ParquetS3TestBase extends S3SeekableChannelTestBase {
 
     @Before
     public void setUp() throws ExecutionException, InterruptedException, TimeoutException {
-        super.setUp();
+        super.doSetUp();
         if (rootDir.exists()) {
             FileUtils.deleteRecursively(rootDir);
         }
@@ -59,7 +59,7 @@ public void setUp() throws ExecutionException, InterruptedException, TimeoutExce
 
     @After
     public void tearDown() throws ExecutionException, InterruptedException, TimeoutException {
-        super.tearDown();
+        super.doTearDown();
         FileUtils.deleteRecursively(rootDir);
     }
 

diff --git a/...s/s3/S3SeekableChannelLocalStackTest.java → ...3SeekableChannelSimpleLocalStackTest.java b/...s/s3/S3SeekableChannelLocalStackTest.java → ...3SeekableChannelSimpleLocalStackTest.java
@@ -10,7 +10,7 @@
 import software.amazon.awssdk.services.s3.S3AsyncClient;
 
 @Tag("testcontainers")
-public class S3SeekableChannelLocalStackTest extends S3SeekableChannelCommonTests {
+public class S3SeekableChannelSimpleLocalStackTest extends S3SeekableChannelSimpleTestBase {
 
     @BeforeAll
     static void initContainer() {

diff --git a/...nsions/s3/S3SeekableChannelMinIOTest.java → .../s3/S3SeekableChannelSimpleMinIOTest.java b/...nsions/s3/S3SeekableChannelMinIOTest.java → .../s3/S3SeekableChannelSimpleMinIOTest.java
@@ -12,7 +12,7 @@
 import software.amazon.awssdk.services.s3.S3AsyncClient;
 
 @Tag("testcontainers")
-public class S3SeekableChannelMinIOTest extends S3SeekableChannelCommonTests {
+public class S3SeekableChannelSimpleMinIOTest extends S3SeekableChannelSimpleTestBase {
 
     @BeforeAll
     static void initContainer() {

diff --git a/...ions/s3/S3SeekableChannelCommonTests.java → ...s/s3/S3SeekableChannelSimpleTestBase.java b/...ions/s3/S3SeekableChannelCommonTests.java → ...s/s3/S3SeekableChannelSimpleTestBase.java
@@ -3,29 +3,37 @@
 //
 package io.deephaven.extensions.s3;
 
-import io.deephaven.extensions.s3.testlib.S3Helper;
+import io.deephaven.extensions.s3.testlib.S3SeekableChannelTestSetup;
 import io.deephaven.util.channel.CachedChannelProvider;
 import io.deephaven.util.channel.SeekableChannelContext;
 import io.deephaven.util.channel.SeekableChannelsProvider;
-import io.deephaven.util.channel.SeekableChannelsProviderPlugin;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import software.amazon.awssdk.core.async.AsyncRequestBody;
 import java.io.IOException;
 import java.io.InputStream;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.nio.ByteBuffer;
-import java.nio.channels.ReadableByteChannel;
 import java.nio.channels.SeekableByteChannel;
-import java.nio.file.Path;
 import java.nio.charset.StandardCharsets;
-import java.time.Duration;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeoutException;
 
 import static org.assertj.core.api.Assertions.assertThat;
 
-abstract class S3SeekableChannelCommonTests extends S3SeekableChannelTestBase {
+abstract class S3SeekableChannelSimpleTestBase extends S3SeekableChannelTestSetup {
+
+    @BeforeEach
+    void setUp() throws ExecutionException, InterruptedException, TimeoutException {
+        doSetUp();
+    }
+
+    @AfterEach
+    void tearDown() throws ExecutionException, InterruptedException, TimeoutException {
+        doTearDown();
+    }
 
     @Test
     void readSimpleFiles()
@@ -36,7 +44,7 @@ void readSimpleFiles()
             final ByteBuffer buffer = ByteBuffer.allocate(1);
             try (
                     final SeekableChannelsProvider providerImpl = providerImpl(uri);
-                    final SeekableChannelsProvider provider = new CachedChannelProvider(providerImpl, 32);
+                    final SeekableChannelsProvider provider = CachedChannelProvider.create(providerImpl, 32);
                     final SeekableChannelContext context = provider.makeContext();
                     final SeekableByteChannel readChannel = provider.getReadChannel(context, uri)) {
                 assertThat(readChannel.read(buffer)).isEqualTo(-1);
@@ -46,7 +54,7 @@ void readSimpleFiles()
             final URI uri = uri("hello/world.txt");
             try (
                     final SeekableChannelsProvider providerImpl = providerImpl(uri);
-                    final SeekableChannelsProvider provider = new CachedChannelProvider(providerImpl, 32);
+                    final SeekableChannelsProvider provider = CachedChannelProvider.create(providerImpl, 32);
                     final SeekableChannelContext context = provider.makeContext();
                     final SeekableByteChannel readChannel = provider.getReadChannel(context, uri)) {
                 final ByteBuffer bytes = readAll(readChannel, 32);
@@ -68,7 +76,7 @@ public int read() {
         final ByteBuffer buffer = ByteBuffer.allocate(1);
         try (
                 final SeekableChannelsProvider providerImpl = providerImpl(uri);
-                final SeekableChannelsProvider provider = new CachedChannelProvider(providerImpl, 32);
+                final SeekableChannelsProvider provider = CachedChannelProvider.create(providerImpl, 32);
                 final SeekableChannelContext context = provider.makeContext();
                 final SeekableByteChannel readChannel = provider.getReadChannel(context, uri)) {
             for (long p = 0; p < numBytes; ++p) {
@@ -79,34 +87,4 @@ public int read() {
             assertThat(readChannel.read(buffer)).isEqualTo(-1);
         }
     }
-
-    private void uploadDirectory(String resourceDir)
-            throws URISyntaxException, ExecutionException, InterruptedException, TimeoutException {
-        S3Helper.uploadDirectory(
-                asyncClient,
-                Path.of(io.deephaven.extensions.s3.S3SeekableChannelTestBase.class.getResource(resourceDir).toURI()),
-                bucket,
-                null,
-                Duration.ofSeconds(5));
-    }
-
-    private SeekableChannelsProvider providerImpl(URI uri) {
-        final SeekableChannelsProviderPlugin plugin = new S3SeekableChannelProviderPlugin();
-        final S3Instructions instructions = s3Instructions(S3Instructions.builder()).build();
-        return plugin.createProvider(uri, instructions);
-    }
-
-    private static ByteBuffer readAll(ReadableByteChannel channel, int maxBytes) throws IOException {
-        final ByteBuffer dst = ByteBuffer.allocate(maxBytes);
-        while (dst.remaining() > 0 && channel.read(dst) != -1) {
-            // continue
-        }
-        if (dst.remaining() == 0) {
-            if (channel.read(ByteBuffer.allocate(1)) != -1) {
-                throw new RuntimeException(String.format("channel has more than %d bytes", maxBytes));
-            }
-        }
-        dst.flip();
-        return dst;
-    }
 }
diff --git a/extensions/s3/src/test/java/io/deephaven/extensions/s3/S3SeekableChannelTestBase.java b/extensions/s3/src/test/java/io/deephaven/extensions/s3/S3SeekableChannelTestBase.java