From 89323865d9c80d291e172ca9fe97a08d49fd9cb0 Mon Sep 17 00:00:00 2001 From: Shun Fan Date: Thu, 25 Jun 2015 11:39:30 -0700 Subject: [PATCH] Add Storage Transfer Service samples for java --- cloud-storage/storage-transfer/README.md | 50 +++++++++++ cloud-storage/storage-transfer/pom.xml | 36 ++++++++ .../storagetransfer/samples/AwsRequester.java | 70 ++++++++++++++++ .../samples/NearlineRequester.java | 68 +++++++++++++++ .../samples/RequestChecker.java | 28 +++++++ .../samples/RetryHttpInitializerWrapper.java | 83 +++++++++++++++++++ .../samples/TransferClientCreator.java | 36 ++++++++ 7 files changed, 371 insertions(+) create mode 100644 cloud-storage/storage-transfer/README.md create mode 100644 cloud-storage/storage-transfer/pom.xml create mode 100644 cloud-storage/storage-transfer/src/main/java/com/google/storagetransfer/samples/AwsRequester.java create mode 100644 cloud-storage/storage-transfer/src/main/java/com/google/storagetransfer/samples/NearlineRequester.java create mode 100644 cloud-storage/storage-transfer/src/main/java/com/google/storagetransfer/samples/RequestChecker.java create mode 100644 cloud-storage/storage-transfer/src/main/java/com/google/storagetransfer/samples/RetryHttpInitializerWrapper.java create mode 100644 cloud-storage/storage-transfer/src/main/java/com/google/storagetransfer/samples/TransferClientCreator.java diff --git a/cloud-storage/storage-transfer/README.md b/cloud-storage/storage-transfer/README.md new file mode 100644 index 00000000000..025e94bb2b4 --- /dev/null +++ b/cloud-storage/storage-transfer/README.md @@ -0,0 +1,50 @@ +# Transfer Service sample using Java + +This app creates two types of transfers using the Transfer Service tool. + +## Prerequisites + +1. Set up a project on Cloud Console. + 1. Go to the [Cloud Console](https://cloud.google.com/console) and create or select your project. + You will need the project ID later. +1. Install jar + `mvn install:install-file -Dfile=libstoragetransfer-v1-java-public.jar \` + `-DgroupId=com.google.storagetransfer.samples -DartifactId=libstoragetransfer -Dversion=1 -Dpackaging=jar` +1. Set up gcloud for application default credentials. + 1. `gcloud components update` + 1. `gcloud auth login` + 1. `gcloud config set project PROJECT_ID` + +## Transfer from AWS S3 + +Creating a one-time transfer between Amazon AWS S3 and Google Cloud Storage. +1. Set up data sink. + 1. Go to the Cloud Console and create a bucket under Cloud Storage > Storage Browser. +1. Set up data source. + 1. Go to Amazon AWS S3 console and create a bucket. + 1. Under Security Credentials, create an IAM User with access to the bucket. + 1. Create an Access Key for the user. Note the Access Key ID and Secret Access Key. +1. In AwsRequester.java, fill in the user-provided constants. +1. Run with `mvn compile` and + `mvn exec:java -Dexec.mainClass="com.google.storagetransfer.samples.AwsRequester"` + 1. Note the job ID in the returned Transfer Job. + +## Transfer to Nearline + +Creating a daily transfer between a regular Cloud Storage bucket and a Nearline +Cloud Storage bucket for files untouched for 30 days. +1. Set up data sink. + 1. Go to the Cloud Console and create a bucket under Cloud Storage > Storage Browser. + 1. Select Nearline for Storage Class. +1. Set up data source. + 1. Go to the Cloud Console and create a bucket under Cloud Storage > Storage Browser. +1. In NearlineRequester.java, fill in the user-provided constants. +1. Run with `mvn compile` and + `mvn exec:java -Dexec.mainClass="com.google.storagetransfer.samples.NearlineRequester"` + 1. Note the job ID in the returned Transfer Job. + +## Checking the status of a transfer + +1. In RequestChecker.java, fill in the user-provided constants. Use the Job ID you recorded earlier. +1. Run with `mvn compile` and + `mvn exec:java -Dexec.mainClass="com.google.storagetransfer.samples.RequestChecker"` diff --git a/cloud-storage/storage-transfer/pom.xml b/cloud-storage/storage-transfer/pom.xml new file mode 100644 index 00000000000..eba63fff0db --- /dev/null +++ b/cloud-storage/storage-transfer/pom.xml @@ -0,0 +1,36 @@ + + 4.0.0 + + com.google.storagetransfer.samples + transfersample + 0.1 + jar + + transfersample + http://maven.apache.org + + + UTF-8 + + + + + junit + junit + 3.8.1 + test + + + com.google.apis + google-api-services-storage + v1-rev35-1.20.0 + + + com.google.storagetransfer.samples + libstoragetransfer + 1 + + + + \ No newline at end of file diff --git a/cloud-storage/storage-transfer/src/main/java/com/google/storagetransfer/samples/AwsRequester.java b/cloud-storage/storage-transfer/src/main/java/com/google/storagetransfer/samples/AwsRequester.java new file mode 100644 index 00000000000..501bc4e5362 --- /dev/null +++ b/cloud-storage/storage-transfer/src/main/java/com/google/storagetransfer/samples/AwsRequester.java @@ -0,0 +1,70 @@ +package com.google.storagetransfer.samples; + +import java.io.IOException; +import java.util.logging.Logger; + +import com.google.api.services.storagetransfer_googleapis.StoragetransferGoogleapis; +import com.google.api.services.storagetransfer_googleapis.model.AwsAccessKey; +import com.google.api.services.storagetransfer_googleapis.model.AwsS3Data; +import com.google.api.services.storagetransfer_googleapis.model.Date; +import com.google.api.services.storagetransfer_googleapis.model.GcsData; +import com.google.api.services.storagetransfer_googleapis.model.Schedule; +import com.google.api.services.storagetransfer_googleapis.model.TimeOfDay; +import com.google.api.services.storagetransfer_googleapis.model.TransferJob; +import com.google.api.services.storagetransfer_googleapis.model.TransferSpec; + +public class AwsRequester { + + // User-provided constants + private static final String projectId = ""; + private static final String projectDesc = ""; + private static final String awsSourceName = ""; + private static final String awsAccessKeyId = ""; + private static final String awsSecretAccessKey = ""; + private static final String gcsSinkName = ""; + // US Pacific Time Zone + private static final String startDate = "YYYY-MM-DD"; + private static final String startTime = "HH:MM:SS"; + + private static final Logger log = Logger.getLogger(AwsRequester.class.getName()); + + public static void createAwsRequest(StoragetransferGoogleapis client, String projectId, + String desc, Date date, TimeOfDay time, String awsSourceName, String awsAccessKeyId, + String awsSecretAccessKey, String gcsSinkName) + throws InstantiationException, IllegalAccessException, IOException { + TransferJob t = TransferJob.class + .newInstance() + .setProjectId(projectId) + .setSchedule( + Schedule.class.newInstance() + .setScheduleStartDate(date) + .setScheduleEndDate(date) + .setStartTimeOfDay(time)) + .setTransferSpec( + TransferSpec.class.newInstance() + .setAwsS3DataSource( + AwsS3Data.class.newInstance() + .setBucketName(awsSourceName) + .setAwsAccessKey(AwsAccessKey.class.newInstance().setAccessKeyId(awsAccessKeyId) + .setSecretAccessKey(awsSecretAccessKey))) + .setGcsDataSink( + GcsData.class.newInstance().setBucketName(gcsSinkName))) + .setJobDescription(desc).setJobState("ENABLED"); + + TransferJob responseT = client.transferJobs().create(t).execute(); + log.info("Return transferJob: " + responseT.toPrettyString()); + } + + public static void main(String[] args) throws IOException, NumberFormatException, + InstantiationException, IllegalAccessException { + Date date = Date.class.newInstance().setYear(Integer.decode(startDate.split("-")[0])) + .setMonth(Integer.decode(startDate.split("-")[1])) + .setDay(Integer.decode(startDate.split("-")[2])); + TimeOfDay time = TimeOfDay.class.newInstance() + .setHours(Integer.decode(startTime.split(":")[0])) + .setMinutes(Integer.decode(startTime.split(":")[1])) + .setSeconds(Integer.decode(startTime.split(":")[2])); + createAwsRequest(TransferClientCreator.createStorageTransferClient(), projectId, projectDesc, + date, time, awsSourceName, awsAccessKeyId, awsSecretAccessKey, gcsSinkName); + } +} diff --git a/cloud-storage/storage-transfer/src/main/java/com/google/storagetransfer/samples/NearlineRequester.java b/cloud-storage/storage-transfer/src/main/java/com/google/storagetransfer/samples/NearlineRequester.java new file mode 100644 index 00000000000..73589a642bc --- /dev/null +++ b/cloud-storage/storage-transfer/src/main/java/com/google/storagetransfer/samples/NearlineRequester.java @@ -0,0 +1,68 @@ +package com.google.storagetransfer.samples; + +import java.io.IOException; +import java.util.logging.Logger; + +import com.google.api.services.storagetransfer_googleapis.StoragetransferGoogleapis; +import com.google.api.services.storagetransfer_googleapis.model.Date; +import com.google.api.services.storagetransfer_googleapis.model.GcsData; +import com.google.api.services.storagetransfer_googleapis.model.ObjectConditions; +import com.google.api.services.storagetransfer_googleapis.model.Schedule; +import com.google.api.services.storagetransfer_googleapis.model.TimeOfDay; +import com.google.api.services.storagetransfer_googleapis.model.TransferJob; +import com.google.api.services.storagetransfer_googleapis.model.TransferOptions; +import com.google.api.services.storagetransfer_googleapis.model.TransferSpec; + +public class NearlineRequester { + + // User-provided constants + private static final String projectId = ""; + private static final String projectDesc = ""; + private static final String gcsSourceName = ""; + private static final String gcsSinkName = ""; + // US Pacific Time Zone + private static final String startDate = "YYYY-MM-DD"; + private static final String startTime = "HH:MM:SS"; + + private static final Logger log = Logger.getLogger(NearlineRequester.class.getName()); + + public static void createNearlineRequest(StoragetransferGoogleapis client, String projectId, + String desc, Date date, TimeOfDay time, String gcsSourceName, String gcsNearlineName) + throws InstantiationException, + IllegalAccessException, IOException { + TransferJob t = TransferJob.class + .newInstance() + .setProjectId(projectId) + .setSchedule( + Schedule.class.newInstance() + .setScheduleStartDate(date) + .setStartTimeOfDay(time)) + .setTransferSpec( + TransferSpec.class.newInstance() + .setGcsDataSource( + GcsData.class.newInstance().setBucketName(gcsSourceName)) + .setGcsDataSink( + GcsData.class.newInstance().setBucketName(gcsNearlineName)) + .setObjectConditions( + ObjectConditions.class.newInstance().setMinTimeElapsedSinceLastModification("2592000s")) + .setTransferOptions( + TransferOptions.class.newInstance().setDeleteObjectsFromSourceAfterTransfer(true))) + .setJobDescription(desc).setJobState("ENABLED"); + + TransferJob responseT = client.transferJobs().create(t).execute(); + log.info("Return transferJob: " + responseT.toPrettyString()); + } + + public static void main(String[] args) throws NumberFormatException, InstantiationException, + IllegalAccessException, IOException { + Date date = Date.class.newInstance().setYear(Integer.decode(startDate.split("-")[0])) + .setMonth(Integer.decode(startDate.split("-")[1])) + .setDay(Integer.decode(startDate.split("-")[2])); + TimeOfDay time = TimeOfDay.class.newInstance() + .setHours(Integer.decode(startTime.split(":")[0])) + .setMinutes(Integer.decode(startTime.split(":")[1])) + .setSeconds(Integer.decode(startTime.split(":")[2])); + createNearlineRequest(TransferClientCreator.createStorageTransferClient(), projectId, + projectDesc, date, time, gcsSourceName, gcsSinkName); + } +} diff --git a/cloud-storage/storage-transfer/src/main/java/com/google/storagetransfer/samples/RequestChecker.java b/cloud-storage/storage-transfer/src/main/java/com/google/storagetransfer/samples/RequestChecker.java new file mode 100644 index 00000000000..14d3c5ffbd0 --- /dev/null +++ b/cloud-storage/storage-transfer/src/main/java/com/google/storagetransfer/samples/RequestChecker.java @@ -0,0 +1,28 @@ +package com.google.storagetransfer.samples; + +import java.io.IOException; +import java.util.logging.Logger; + +import com.google.api.services.storagetransfer_googleapis.StoragetransferGoogleapis; +import com.google.api.services.storagetransfer_googleapis.model.ListOperationsResponse; + +public class RequestChecker { + + // User-provided constants + private static final String projectId = ""; + private static final String jobId = ""; + private static final Logger log = Logger.getLogger(RequestChecker.class.getName()); + + public static void checkTransfer(StoragetransferGoogleapis client, String projectId, String jobId) + throws InstantiationException, IllegalAccessException, IOException { + ListOperationsResponse r = client.transferOperations().list("transferOperations") + .setFilter("{\"project_id\": \"" + projectId + "\", \"job_id\": [\"" + jobId + "\"] }") + .execute(); + log.info("Result of transferOperations/list: " + r.toPrettyString()); + } + + public static void main(String[] args) throws InstantiationException, IllegalAccessException, + IOException { + checkTransfer(TransferClientCreator.createStorageTransferClient(), projectId, jobId); + } +} \ No newline at end of file diff --git a/cloud-storage/storage-transfer/src/main/java/com/google/storagetransfer/samples/RetryHttpInitializerWrapper.java b/cloud-storage/storage-transfer/src/main/java/com/google/storagetransfer/samples/RetryHttpInitializerWrapper.java new file mode 100644 index 00000000000..5939b98c6d5 --- /dev/null +++ b/cloud-storage/storage-transfer/src/main/java/com/google/storagetransfer/samples/RetryHttpInitializerWrapper.java @@ -0,0 +1,83 @@ +package com.google.storagetransfer.samples; + +import com.google.api.client.auth.oauth2.Credential; +import com.google.api.client.http.HttpBackOffIOExceptionHandler; +import com.google.api.client.http.HttpBackOffUnsuccessfulResponseHandler; +import com.google.api.client.http.HttpRequest; +import com.google.api.client.http.HttpRequestInitializer; +import com.google.api.client.http.HttpResponse; +import com.google.api.client.http.HttpUnsuccessfulResponseHandler; +import com.google.api.client.util.ExponentialBackOff; +import com.google.api.client.util.Sleeper; +import com.google.common.base.Preconditions; + +import java.io.IOException; +import java.util.logging.Logger; + +/** + * RetryHttpInitializerWrapper will automatically retry upon RPC + * failures, preserving the auto-refresh behavior of the Google + * Credentials. + */ +public class RetryHttpInitializerWrapper implements HttpRequestInitializer { + + private static final Logger LOG = + Logger.getLogger(RetryHttpInitializerWrapper.class.getName()); + + // Intercepts the request for filling in the "Authorization" + // header field, as well as recovering from certain unsuccessful + // error codes wherein the Credential must refresh its token for a + // retry. + private final Credential wrappedCredential; + + // A sleeper; you can replace it with a mock in your test. + private final Sleeper sleeper; + + public RetryHttpInitializerWrapper(Credential wrappedCredential) { + this(wrappedCredential, Sleeper.DEFAULT); + } + + // Use only for testing. + RetryHttpInitializerWrapper( + Credential wrappedCredential, Sleeper sleeper) { + this.wrappedCredential = Preconditions.checkNotNull(wrappedCredential); + this.sleeper = sleeper; + } + + public void initialize(HttpRequest request) { + request.setReadTimeout(2 * 60000); // 2 minutes read timeout + final HttpUnsuccessfulResponseHandler backoffHandler = + new HttpBackOffUnsuccessfulResponseHandler( + new ExponentialBackOff()) + .setSleeper(sleeper); + request.setInterceptor(wrappedCredential); + request.setUnsuccessfulResponseHandler( + new HttpUnsuccessfulResponseHandler() { + public boolean handleResponse( + HttpRequest request, + HttpResponse response, + boolean supportsRetry) throws IOException { + if (wrappedCredential.handleResponse( + request, response, supportsRetry)) { + // If credential decides it can handle it, + // the return code or message indicated + // something specific to authentication, + // and no backoff is desired. + return true; + } else if (backoffHandler.handleResponse( + request, response, supportsRetry)) { + // Otherwise, we defer to the judgement of + // our internal backoff handler. + LOG.info("Retrying " + + request.getUrl().toString()); + return true; + } else { + return false; + } + } + }); + request.setIOExceptionHandler( + new HttpBackOffIOExceptionHandler(new ExponentialBackOff()) + .setSleeper(sleeper)); + } +} diff --git a/cloud-storage/storage-transfer/src/main/java/com/google/storagetransfer/samples/TransferClientCreator.java b/cloud-storage/storage-transfer/src/main/java/com/google/storagetransfer/samples/TransferClientCreator.java new file mode 100644 index 00000000000..27ba8ff81fd --- /dev/null +++ b/cloud-storage/storage-transfer/src/main/java/com/google/storagetransfer/samples/TransferClientCreator.java @@ -0,0 +1,36 @@ +package com.google.storagetransfer.samples; + +import java.io.IOException; + +import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; +import com.google.api.client.googleapis.util.Utils; +import com.google.api.client.http.HttpRequestInitializer; +import com.google.api.client.http.HttpTransport; +import com.google.api.client.json.JsonFactory; +import com.google.api.services.storagetransfer_googleapis.StoragetransferGoogleapis; +import com.google.api.services.storagetransfer_googleapis.StoragetransferGoogleapisScopes; +import com.google.common.base.Preconditions; + +public class TransferClientCreator { + public static StoragetransferGoogleapis createStorageTransferClient() throws IOException { + return createStorageTransferClient(Utils.getDefaultTransport(), Utils.getDefaultJsonFactory()); + } + + public static StoragetransferGoogleapis createStorageTransferClient(HttpTransport httpTransport, + JsonFactory jsonFactory) throws IOException { + Preconditions.checkNotNull(httpTransport); + Preconditions.checkNotNull(jsonFactory); + GoogleCredential credential = GoogleCredential + .getApplicationDefault(httpTransport, jsonFactory); + // In some cases, you need to add the scope explicitly. + if (credential.createScopedRequired()) { + credential = credential.createScoped(StoragetransferGoogleapisScopes.all()); + } + // Please use custom HttpRequestInitializer for automatic + // retry upon failures. We provide a simple reference + // implementation in the "Retry Handling" section. + HttpRequestInitializer initializer = new RetryHttpInitializerWrapper(credential); + return new StoragetransferGoogleapis.Builder(httpTransport, jsonFactory, initializer) + .setApplicationName("storagetransfer-sample").build(); + } +}