[bug] Setup s3Client, hadoop opts earlier

(Still doesn't work)
treeverse · Nov 24, 2024 · f56b541 · f56b541
1 parent f9caad9
commit f56b541
Show file tree

Hide file tree

Showing 3 changed files with 17 additions and 16 deletions.
diff --git a/clients/hadoopfs/src/test/java/io/lakefs/FSTestBase.java b/clients/hadoopfs/src/test/java/io/lakefs/FSTestBase.java
@@ -121,20 +121,12 @@ protected String getS3Key(StagingLocation stagingLocation) {
         return removeStart(stagingLocation.getPhysicalAddress(), s3Base);
     }
 
-    /**
-     * Override to add to Hadoop configuration.
-     */
-    protected void addHadoopConfiguration(Configuration conf) {
-    }
-
     @Before
     public void hadoopSetup() throws IOException, URISyntaxException {
         s3Base = "s3a://UNUSED/"; // Overridden if S3 will be used!
 
         conf = new Configuration(false);
 
-        addHadoopConfiguration(conf);
-
         conf.set("fs.lakefs.impl", "io.lakefs.LakeFSFileSystem");
 
         conf.set("fs.lakefs.access.key", "unused-but-checked");

diff --git a/clients/hadoopfs/src/test/java/io/lakefs/LakeFSFileSystemServerS3Test.java b/clients/hadoopfs/src/test/java/io/lakefs/LakeFSFileSystemServerS3Test.java
@@ -15,6 +15,7 @@
 import com.amazonaws.services.s3.model.*;
 
 import org.junit.Assert;
+import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
@@ -45,12 +46,13 @@ public static Iterable<Object[]> data() {
     public String unusedAddressCreatorType;
 
     @Parameter(0)
-    public PhysicalAddressCreator pac;
+    public PhysicalAddressCreator pacParam;
 
     @Override
-    protected void moreHadoopSetup() {
-        super.moreHadoopSetup();
-        pac.initConfiguration(conf);
+    // pac is defined in a superclass but parametrized here.
+    protected void setPac() {
+        pac = pacParam;
+        LOG.info("Set pac!");
     }
 
     @Test

diff --git a/clients/hadoopfs/src/test/java/io/lakefs/S3FSTestBase.java b/clients/hadoopfs/src/test/java/io/lakefs/S3FSTestBase.java
@@ -49,10 +49,10 @@ public abstract class S3FSTestBase extends FSTestBase {
     protected String s3Endpoint;
     protected AmazonS3 s3Client;
 
-    private static final DockerImageName MINIO = DockerImageName.parse("minio/minio:RELEASE.2021-06-07T21-40-51Z");
+    private static final DockerImageName MINIO = DockerImageName.parse("minio/minio:RELEASE.2024-11-07T00-52-20Z");
 
     @Rule
-    public final GenericContainer s3 = new GenericContainer(MINIO.toString()).
+    public final GenericContainer s3 = new GenericContainer(MINIO).
         withCommand("minio", "server", "/data").
         withEnv("MINIO_ROOT_USER", S3_ACCESS_KEY_ID).
         withEnv("MINIO_ROOT_PASSWORD", S3_SECRET_ACCESS_KEY).
@@ -74,7 +74,7 @@ public void s3ClientSetup() {
 
         ClientConfiguration clientConfiguration = new ClientConfiguration()
                 .withSignerOverride("AWSS3V4SignerType");
-        s3Endpoint = String.format("http://s3.local.lakefs.io:%d", s3.getMappedPort(9000));
+        s3Endpoint = String.format("http://s3.local.lakefs.io:%d/", s3.getMappedPort(9000));
 
         s3Client = new AmazonS3Client(creds, clientConfiguration);
 
@@ -85,7 +85,8 @@ public void s3ClientSetup() {
 
         s3Bucket = makeS3BucketName();
         s3Base = String.format("s3://%s/", s3Bucket);
-        LOG.info("S3: bucket {} => base URL {}", s3Bucket, s3Base);
+        LOG.info("S3 [endpoint {}]: bucket {} => base URL {}",
+                 s3Endpoint, s3Bucket, s3Base);
 
         CreateBucketRequest cbr = new CreateBucketRequest(s3Bucket);
         s3Client.createBucket(cbr);
@@ -126,14 +127,20 @@ protected void assertS3Object(StagingLocation stagingLocation, String contents)
         }
     }
 
+    protected void setPac() {}
+
     protected void moreHadoopSetup() {
+        setPac();
         s3ClientSetup();
 
         conf.set("fs.s3a.impl", "org.apache.hadoop.fs.s3a.S3AFileSystem");
         conf.set(org.apache.hadoop.fs.s3a.Constants.ACCESS_KEY, S3_ACCESS_KEY_ID);
         conf.set(org.apache.hadoop.fs.s3a.Constants.SECRET_KEY, S3_SECRET_ACCESS_KEY);
         conf.set(org.apache.hadoop.fs.s3a.Constants.ENDPOINT, s3Endpoint);
         conf.set(org.apache.hadoop.fs.s3a.Constants.BUFFER_DIR, "/tmp/s3a");
+        pac.initConfiguration(conf);
+
+        LOG.info("Setup done!");
     }
 
     public static interface PhysicalAddressCreator {