diff --git a/changelog.d/20241101_002548_steliosvoutsinas_DM_47315.md b/changelog.d/20241101_002548_steliosvoutsinas_DM_47315.md new file mode 100644 index 0000000..9d0319a --- /dev/null +++ b/changelog.d/20241101_002548_steliosvoutsinas_DM_47315.md @@ -0,0 +1,5 @@ + + +### Fixed + +- Enable non GCS S3 support for TAP_UPLOAD diff --git a/tap/build.gradle b/tap/build.gradle index 70b8ead..9c98c08 100644 --- a/tap/build.gradle +++ b/tap/build.gradle @@ -27,14 +27,20 @@ dependencies { implementation 'org.opencadc:cadc-adql:1.1.14' implementation 'org.opencadc:cadc-log:1.2.1' + implementation 'org.opencadc:cadc-gms:1.0.13' + implementation 'org.opencadc:cadc-tap:1.1.16' + implementation 'org.opencadc:cadc-tap:cadc-registry-1.7.6' + implementation 'org.opencadc:cadc-tap:cadc-tap-1.1.16' + implementation 'org.opencadc:cadc-tap:cadc-tap-schema-1.1.33' implementation 'org.opencadc:cadc-rest:1.3.20' implementation 'org.opencadc:cadc-tap-server:1.1.24' - implementation 'org.opencadc:cadc-tap-server-pg:[1.0.0,)' - implementation 'org.opencadc:cadc-util:1.11.2' + implementation 'org.opencadc:cadc-tap-server-pg:1.1.0' + implementation 'org.opencadc:cadc-util:1.11.3' implementation 'org.opencadc:cadc-uws:1.0.5' implementation 'org.opencadc:cadc-uws-server:1.2.21' implementation 'org.opencadc:cadc-vosi:1.4.6' + // Switch out this to use any supported database instead of PostgreSQL. // ## START CUSTOM DATABASE ## implementation group: 'com.mysql', name: 'mysql-connector-j', version: '8.4.0' diff --git a/tap/src/main/java/ca/nrc/cadc/sample/UploadManager.java b/tap/src/main/java/ca/nrc/cadc/sample/UploadManager.java index 1d75eb5..e764b5e 100644 --- a/tap/src/main/java/ca/nrc/cadc/sample/UploadManager.java +++ b/tap/src/main/java/ca/nrc/cadc/sample/UploadManager.java @@ -3,8 +3,13 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.nio.channels.Channels; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; +import java.nio.channels.Channels; + +import org.apache.log4j.Logger; +import org.apache.solr.s3.S3OutputStream; import com.google.cloud.storage.Blob; import com.google.cloud.storage.BlobId; @@ -16,55 +21,100 @@ import ca.nrc.cadc.uws.web.InlineContentException; import ca.nrc.cadc.uws.web.UWSInlineContentHandler; -import org.apache.log4j.Logger; - +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.S3Configuration; public class UploadManager implements UWSInlineContentHandler { - private static Logger log = Logger.getLogger(UploadManager.class); - private static final String bucket = System.getProperty("gcs_bucket"); private static final String bucketURL = System.getProperty("gcs_bucket_url"); - public UploadManager() { + private static final String bucketType = System.getProperty("gcs_bucket_type"); + public UploadManager() { } @Override public Content accept(String name, String contentType, InputStream inputStream) throws InlineContentException, IOException { - - log.debug("name: " + name); + log.debug("name: " + name); log.debug("Content-Type: " + contentType); + if (inputStream == null) { throw new IOException("InputStream cannot be null"); } - String filename = name + "-" + new RandomStringGenerator(16).getID(); - OutputStream os = getOutputStream(filename, contentType); + String filename = name + "-" + new RandomStringGenerator(16).getID(); + OutputStream os = getOutputStream(filename, contentType); - byte[] buf = new byte[16384]; - int num = inputStream.read(buf); - while (num > 0) { + byte[] buf = new byte[16384]; + int num = inputStream.read(buf); + while (num > 0) { os.write(buf, 0, num); num = inputStream.read(buf); } - os.flush(); - os.close(); + os.flush(); + os.close(); - URL retURL = new URL(bucketURL + "/" + filename); + // Construct return URL based on bucket type + URL retURL; + if (bucketType.equals("S3")) { + retURL = new URL(bucketURL + "/" + bucket + "/" + filename); + } else { + retURL = new URL(bucketURL + "/" + filename); + } - Content ret = new Content(); - ret.name = UWSInlineContentHandler.CONTENT_PARAM_REPLACE; - ret.value = new UWSInlineContentHandler.ParameterReplacement("param:" + name, retURL.toExternalForm()); - return ret; + Content ret = new Content(); + ret.name = UWSInlineContentHandler.CONTENT_PARAM_REPLACE; + ret.value = new UWSInlineContentHandler.ParameterReplacement( + "param:" + name, + retURL.toExternalForm() + ); + return ret; } private OutputStream getOutputStream(String filename, String contentType) { + if (bucketType.equals("S3")) { + return getOutputStreamS3(filename); + } else { + return getOutputStreamGCS(filename, contentType); + } + } + + private OutputStream getOutputStreamS3(String filename) { + S3Configuration config = S3Configuration.builder() + .pathStyleAccessEnabled(true) + .useArnRegionEnabled(true) + .build(); + + S3Client s3Client = S3Client.builder() + .endpointOverride(getURI()) + .serviceConfiguration(config) + .region(Region.US_WEST_2) + .build(); + + return new S3OutputStream(s3Client, filename, bucket); + } + + private OutputStream getOutputStreamGCS(String filename, String contentType) { Storage storage = StorageOptions.getDefaultInstance().getService(); BlobId blobId = BlobId.of(bucket, filename); - BlobInfo blobInfo = BlobInfo.newBuilder(blobId).setContentType("application/x-votable+xml").build(); + BlobInfo blobInfo = BlobInfo.newBuilder(blobId) + .setContentType("application/x-votable+xml") + .build(); Blob blob = storage.create(blobInfo); return Channels.newOutputStream(blob.writer()); } + + private URI getURI() { + try { + return new URI(bucketURL); + } catch (URISyntaxException e) { + throw new IllegalArgumentException( + "Invalid bucket URL in configuration: " + e.getMessage(), + e + ); + } + } }