Enable TAP_UPLOAD support for non GCS S3 buckets

lsst-sqre · Nov 1, 2024 · f01482f · f01482f
1 parent 345a6e2
commit f01482f
Show file tree

Hide file tree

Showing 3 changed files with 84 additions and 23 deletions.
diff --git a/changelog.d/20241101_002548_steliosvoutsinas_DM_47315.md b/changelog.d/20241101_002548_steliosvoutsinas_DM_47315.md
@@ -0,0 +1,5 @@
+<!-- Delete the sections that don't apply -->
+
+### Fixed
+
+- Enable non GCS S3 support for TAP_UPLOAD
diff --git a/tap/build.gradle b/tap/build.gradle
@@ -27,14 +27,20 @@ dependencies {
 
     implementation 'org.opencadc:cadc-adql:1.1.14'
     implementation 'org.opencadc:cadc-log:1.2.1'
+    implementation 'org.opencadc:cadc-gms:1.0.13'
+    implementation 'org.opencadc:cadc-tap:1.1.16'
+    implementation 'org.opencadc:cadc-tap:cadc-registry-1.7.6'
+    implementation 'org.opencadc:cadc-tap:cadc-tap-1.1.16'
+    implementation 'org.opencadc:cadc-tap:cadc-tap-schema-1.1.33'
     implementation 'org.opencadc:cadc-rest:1.3.20'
     implementation 'org.opencadc:cadc-tap-server:1.1.24'
-    implementation 'org.opencadc:cadc-tap-server-pg:[1.0.0,)'
-    implementation 'org.opencadc:cadc-util:1.11.2'
+    implementation 'org.opencadc:cadc-tap-server-pg:1.1.0'
+    implementation 'org.opencadc:cadc-util:1.11.3'
     implementation 'org.opencadc:cadc-uws:1.0.5'
     implementation 'org.opencadc:cadc-uws-server:1.2.21'
     implementation 'org.opencadc:cadc-vosi:1.4.6'
 
+
     // Switch out this to use any supported database instead of PostgreSQL.
     // ## START CUSTOM DATABASE ##
     implementation group: 'com.mysql', name: 'mysql-connector-j', version: '8.4.0'

diff --git a/tap/src/main/java/ca/nrc/cadc/sample/UploadManager.java b/tap/src/main/java/ca/nrc/cadc/sample/UploadManager.java
@@ -3,8 +3,13 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
-import java.nio.channels.Channels;
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.net.URL;
+import java.nio.channels.Channels;
+
+import org.apache.log4j.Logger;
+import org.apache.solr.s3.S3OutputStream;
 
 import com.google.cloud.storage.Blob;
 import com.google.cloud.storage.BlobId;
@@ -16,55 +21,100 @@
 import ca.nrc.cadc.uws.web.InlineContentException;
 import ca.nrc.cadc.uws.web.UWSInlineContentHandler;
 
-import org.apache.log4j.Logger;
-
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.S3Configuration;
 
 public class UploadManager implements UWSInlineContentHandler {
-
     private static Logger log = Logger.getLogger(UploadManager.class);
-
     private static final String bucket = System.getProperty("gcs_bucket");
     private static final String bucketURL = System.getProperty("gcs_bucket_url");
-    public UploadManager() {
+    private static final String bucketType = System.getProperty("gcs_bucket_type");
 
+    public UploadManager() {
     }
 
     @Override
     public Content accept(String name, String contentType, InputStream inputStream)
         throws InlineContentException, IOException {
-
-	log.debug("name: " + name);
+        log.debug("name: " + name);
         log.debug("Content-Type: " + contentType);
+
         if (inputStream == null) {
             throw new IOException("InputStream cannot be null");
         }
 
-	String filename = name + "-" + new RandomStringGenerator(16).getID();
-	OutputStream os = getOutputStream(filename, contentType);
+        String filename = name + "-" + new RandomStringGenerator(16).getID();
+        OutputStream os = getOutputStream(filename, contentType);
 
-	byte[] buf = new byte[16384];
-	int num = inputStream.read(buf);
-	while (num > 0) {
+        byte[] buf = new byte[16384];
+        int num = inputStream.read(buf);
+        while (num > 0) {
             os.write(buf, 0, num);
             num = inputStream.read(buf);
         }
-	os.flush();
-	os.close();
 
+        os.flush();
+        os.close();
 
-        URL retURL = new URL(bucketURL + "/" + filename);
+        // Construct return URL based on bucket type
+        URL retURL;
+        if (bucketType.equals("S3")) {
+            retURL = new URL(bucketURL + "/" + bucket + "/" + filename);
+        } else {
+            retURL = new URL(bucketURL + "/" + filename);
+        }
 
-	Content ret = new Content();
-	ret.name = UWSInlineContentHandler.CONTENT_PARAM_REPLACE;
-	ret.value = new UWSInlineContentHandler.ParameterReplacement("param:" + name, retURL.toExternalForm());
-	return ret;
+        Content ret = new Content();
+        ret.name = UWSInlineContentHandler.CONTENT_PARAM_REPLACE;
+        ret.value = new UWSInlineContentHandler.ParameterReplacement(
+            "param:" + name,
+            retURL.toExternalForm()
+        );
+        return ret;
     }
 
     private OutputStream getOutputStream(String filename, String contentType) {
+        if (bucketType.equals("S3")) {
+            return getOutputStreamS3(filename);
+        } else {
+            return getOutputStreamGCS(filename, contentType);
+        }
+    }
+
+    private OutputStream getOutputStreamS3(String filename) {
+        S3Configuration config = S3Configuration.builder()
+            .pathStyleAccessEnabled(true)
+            .useArnRegionEnabled(true)
+            .build();
+
+        S3Client s3Client = S3Client.builder()
+            .endpointOverride(getURI())
+            .serviceConfiguration(config)
+            .region(Region.US_WEST_2)
+            .build();
+
+        return new S3OutputStream(s3Client, filename, bucket);
+    }
+
+    private OutputStream getOutputStreamGCS(String filename, String contentType) {
         Storage storage = StorageOptions.getDefaultInstance().getService();
         BlobId blobId = BlobId.of(bucket, filename);
-        BlobInfo blobInfo = BlobInfo.newBuilder(blobId).setContentType("application/x-votable+xml").build();
+        BlobInfo blobInfo = BlobInfo.newBuilder(blobId)
+            .setContentType("application/x-votable+xml")
+            .build();
         Blob blob = storage.create(blobInfo);
         return Channels.newOutputStream(blob.writer());
     }
+
+    private URI getURI() {
+        try {
+            return new URI(bucketURL);
+        } catch (URISyntaxException e) {
+            throw new IllegalArgumentException(
+                "Invalid bucket URL in configuration: " + e.getMessage(),
+                e
+            );
+        }
+    }
 }