From d12d27b322af89130c6810eb49b071e037557553 Mon Sep 17 00:00:00 2001 From: carlosthe19916 <2582866+carlosthe19916@users.noreply.github.com> Date: Wed, 21 Dec 2022 09:37:50 +0100 Subject: [PATCH] Add Google Cloud Storage support Signed-off-by: carlosthe19916 <2582866+carlosthe19916@users.noreply.github.com> --- application/pom.xml | 10 ++ ...S3FilesRoute.java => AwsS3FilesRoute.java} | 2 +- .../camel/GoogleCloudStorageFilesRoute.java | 125 ++++++++++++++++++ .../src/main/resources/application.properties | 7 +- .../resources/gcstorage_serviceaccountid.json | 0 .../openubl/ublhub/ProfileManager.java | 3 +- .../containers/GoogleCloudStorageServer.java | 50 +++++++ 7 files changed, 194 insertions(+), 3 deletions(-) rename application/src/main/java/io/github/project/openubl/ublhub/files/camel/{S3FilesRoute.java => AwsS3FilesRoute.java} (99%) create mode 100644 application/src/main/java/io/github/project/openubl/ublhub/files/camel/GoogleCloudStorageFilesRoute.java create mode 100644 application/src/main/resources/gcstorage_serviceaccountid.json create mode 100644 application/src/test/java/io/github/project/openubl/ublhub/containers/GoogleCloudStorageServer.java diff --git a/application/pom.xml b/application/pom.xml index b7fc478a..12b3ebbc 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -194,6 +194,10 @@ org.apache.camel.quarkus camel-quarkus-aws2-s3 + + org.apache.camel.quarkus + camel-quarkus-google-storage + org.apache.camel.quarkus camel-quarkus-json-validator @@ -239,6 +243,12 @@ ${testcontainers.version} test + + org.testcontainers + gcloud + ${testcontainers.version} + test + org.awaitility awaitility diff --git a/application/src/main/java/io/github/project/openubl/ublhub/files/camel/S3FilesRoute.java b/application/src/main/java/io/github/project/openubl/ublhub/files/camel/AwsS3FilesRoute.java similarity index 99% rename from application/src/main/java/io/github/project/openubl/ublhub/files/camel/S3FilesRoute.java rename to application/src/main/java/io/github/project/openubl/ublhub/files/camel/AwsS3FilesRoute.java index c0b8dd80..c846c6f7 100644 --- a/application/src/main/java/io/github/project/openubl/ublhub/files/camel/S3FilesRoute.java +++ b/application/src/main/java/io/github/project/openubl/ublhub/files/camel/AwsS3FilesRoute.java @@ -36,7 +36,7 @@ import java.util.UUID; @ApplicationScoped -public class S3FilesRoute extends RouteBuilder { +public class AwsS3FilesRoute extends RouteBuilder { @ConfigProperty(name = "openubl.storage.link-expiration", defaultValue = "5000") String linkExpiration; diff --git a/application/src/main/java/io/github/project/openubl/ublhub/files/camel/GoogleCloudStorageFilesRoute.java b/application/src/main/java/io/github/project/openubl/ublhub/files/camel/GoogleCloudStorageFilesRoute.java new file mode 100644 index 00000000..3dc814b2 --- /dev/null +++ b/application/src/main/java/io/github/project/openubl/ublhub/files/camel/GoogleCloudStorageFilesRoute.java @@ -0,0 +1,125 @@ +/* + * Copyright 2019 Project OpenUBL, Inc. and/or its affiliates + * and other contributors as indicated by the @author tags. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.project.openubl.ublhub.files.camel; + +import com.google.cloud.NoCredentials; +import com.google.cloud.ServiceOptions; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.StorageOptions; +import org.apache.camel.Exchange; +import org.apache.camel.builder.RouteBuilder; +import org.apache.camel.component.google.storage.GoogleCloudStorageConstants; +import org.eclipse.microprofile.config.inject.ConfigProperty; + +import javax.enterprise.context.ApplicationScoped; +import javax.enterprise.inject.Produces; +import javax.inject.Named; +import java.util.Iterator; +import java.util.Optional; +import java.util.UUID; + +@ApplicationScoped +public class GoogleCloudStorageFilesRoute extends RouteBuilder { + + @ConfigProperty(name = "openubl.storage.link-expiration", defaultValue = "5000") + String linkExpiration; + + @ConfigProperty(name = "openubl.storage.gcstorage.bucket") + String gcStorageBucket; + + @ConfigProperty(name = "openubl.storage.gcstorage.project_id") + String gcStorageProjectId; + + @ConfigProperty(name = "openubl.storage.gcstorage.service_account_id") + String gcStorageServiceAccountId; + + @ConfigProperty(name = "openubl.storage.gcstorage.host") + Optional gcStorageHost; + + @Named("gcStorageClient") + @Produces + public Storage gcStorageClient() { + if (gcStorageHost.isPresent()) { + StorageOptions options = StorageOptions.newBuilder() + .setHost(gcStorageHost.get()) + .setProjectId(gcStorageProjectId) +// .setCredentials(NoCredentials.getInstance()) +// .setRetrySettings(ServiceOptions.getNoRetrySettings()) + .build(); + return options.getService(); + } else { + return StorageOptions.getDefaultInstance() + .getService(); + } + } + + @Override + public void configure() throws Exception { + from("direct:gcstorage-save-file") + .id("gcstorage-save-file") + .choice() + .when(header("shouldZipFile").isEqualTo(true)) + .marshal().zipFile() + .endChoice() + .end() + .process(exchange -> { + exchange.getIn().setHeader(GoogleCloudStorageConstants.OBJECT_NAME, UUID.randomUUID().toString()); + exchange.getIn().setHeader(GoogleCloudStorageConstants.BUCKET_NAME, gcStorageBucket); + }) + .toD("google-storage://" + gcStorageBucket + "?autoCreateBucket=true&storageClient=#gcStorageClient") + .process(exchange -> { + String documentID = exchange.getIn().getHeader(GoogleCloudStorageConstants.OBJECT_NAME, String.class); + exchange.getIn().setBody(documentID); + }); + + from("direct:gcstorage-get-file") + .id("gcstorage-get-file") + .setHeader(GoogleCloudStorageConstants.OPERATION, constant("getObject")) + .setHeader(GoogleCloudStorageConstants.OBJECT_NAME, body()) + .choice() + .when(header("shouldUnzip").isEqualTo(true)) + .pollEnrich().simple("google-storage://" + gcStorageBucket + "?storageClient=#gcStorageClient&deleteAfterRead=false&serviceAccountKey=" + gcStorageServiceAccountId) + .setHeader(GoogleCloudStorageConstants.CONTENT_DISPOSITION, simple("$header.CamelAwsS3ContentDisposition")) + .setHeader(Exchange.CONTENT_TYPE, simple("$header.CamelAwsS3ContentType")) + .unmarshal(RouteUtils.getZipFileDataFormat()) + .split(bodyAs(Iterator.class), (oldExchange, newExchange) -> newExchange) + .streaming() + .convertBodyTo(byte[].class) + .end() + .endChoice() + .otherwise() + .pollEnrich().simple("google-storage://" + gcStorageBucket + "?storageClient=#gcStorageClient&deleteAfterRead=false&serviceAccountKey=" + gcStorageServiceAccountId) + .setHeader(GoogleCloudStorageConstants.CONTENT_DISPOSITION, simple("$header.CamelAwsS3ContentDisposition")) + .setHeader(Exchange.CONTENT_TYPE, simple("$header.CamelAwsS3ContentType")) + .endChoice() + .end(); + + from("direct:gcstorage-get-file-link") + .id("gcstorage-get-file-link") + .setHeader(GoogleCloudStorageConstants.OPERATION, constant("createDownloadLink")) + .setHeader(GoogleCloudStorageConstants.OBJECT_NAME, body()) + .setHeader(GoogleCloudStorageConstants.DOWNLOAD_LINK_EXPIRATION_TIME, constant(linkExpiration)) + .toD("google-storage://" + gcStorageBucket + "?storageClient=#gcStorageClient&serviceAccountKey=" + gcStorageServiceAccountId); + + from("direct:gcstorage-delete-file") + .id("gcstorage-delete-file") + .setHeader(GoogleCloudStorageConstants.OPERATION, constant("deleteObject")) + .setHeader(GoogleCloudStorageConstants.OBJECT_NAME, body()) + .toD("google-storage://" + gcStorageBucket + "?storageClient=#gcStorageClient&serviceAccountKey=" + gcStorageServiceAccountId); + } + +} diff --git a/application/src/main/resources/application.properties b/application/src/main/resources/application.properties index 0aeb1cdd..d8366c1e 100644 --- a/application/src/main/resources/application.properties +++ b/application/src/main/resources/application.properties @@ -53,10 +53,15 @@ openubl.storage.filesystem.folder=workspace openubl.storage.s3.health.url=http://localhost:9000/minio/health/live openubl.storage.s3.host=http://localhost:9000 openubl.storage.s3.region=us-east-1 -openubl.storage.s3.bucket=project-openubl +openubl.storage.s3.bucket=ublhub openubl.storage.s3.access_key_id=BQA2GEXO711FVBVXDWKM openubl.storage.s3.secret_access_key=uvgz3LCwWM3e400cDkQIH/y1Y4xgU4iV91CwFSPC +openubl.storage.gcstorage.host=http://localhost:9000 +openubl.storage.gcstorage.project_id=openubl-project +openubl.storage.gcstorage.bucket=ublhub +openubl.storage.gcstorage.service_account_id=classpath:gcstorage_serviceaccountid.json + # Basic HTTP Authentication quarkus.http.auth.basic=true diff --git a/application/src/main/resources/gcstorage_serviceaccountid.json b/application/src/main/resources/gcstorage_serviceaccountid.json new file mode 100644 index 00000000..e69de29b diff --git a/application/src/test/java/io/github/project/openubl/ublhub/ProfileManager.java b/application/src/test/java/io/github/project/openubl/ublhub/ProfileManager.java index 226daa94..fb408f77 100644 --- a/application/src/test/java/io/github/project/openubl/ublhub/ProfileManager.java +++ b/application/src/test/java/io/github/project/openubl/ublhub/ProfileManager.java @@ -16,6 +16,7 @@ */ package io.github.project.openubl.ublhub; +import io.github.project.openubl.ublhub.containers.GoogleCloudStorageServer; import io.github.project.openubl.ublhub.containers.MinioServer; import io.quarkus.test.junit.QuarkusTestProfile; @@ -27,7 +28,7 @@ public abstract class ProfileManager implements QuarkusTestProfile { List testResources = new ArrayList<>(); public ProfileManager() { - testResources.add(new TestResourceEntry(MinioServer.class)); + testResources.add(new TestResourceEntry(GoogleCloudStorageServer.class)); } public abstract String getProfile(); diff --git a/application/src/test/java/io/github/project/openubl/ublhub/containers/GoogleCloudStorageServer.java b/application/src/test/java/io/github/project/openubl/ublhub/containers/GoogleCloudStorageServer.java new file mode 100644 index 00000000..500f5b94 --- /dev/null +++ b/application/src/test/java/io/github/project/openubl/ublhub/containers/GoogleCloudStorageServer.java @@ -0,0 +1,50 @@ +/* + * Copyright 2019 Project OpenUBL, Inc. and/or its affiliates + * and other contributors as indicated by the @author tags. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.project.openubl.ublhub.containers; + +import io.quarkus.test.common.QuarkusTestResourceLifecycleManager; +import org.testcontainers.containers.DatastoreEmulatorContainer; +import org.testcontainers.utility.DockerImageName; + +import java.util.HashMap; +import java.util.Map; + +public class GoogleCloudStorageServer implements QuarkusTestResourceLifecycleManager { + + private DatastoreEmulatorContainer gcStorage; + + @Override + public Map start() { + gcStorage = new DatastoreEmulatorContainer( + DockerImageName.parse("gcr.io/google.com/cloudsdktool/cloud-sdk:412.0.0-emulators") + ); + gcStorage.start(); + + String host = gcStorage.getEmulatorEndpoint(); + + Map properties = new HashMap<>(); + properties.put("openubl.storage.type", "gcstorage"); + properties.put("openubl.storage.gcstorage.host", "http://" + host); + return properties; + } + + @Override + public void stop() { + gcStorage.stop(); + } + +}