diff --git a/google-cloud-clients/google-cloud-contrib/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/testing/FakeStorageRpc.java b/google-cloud-clients/google-cloud-contrib/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/testing/FakeStorageRpc.java index 515235cc5176..bee18dda5290 100644 --- a/google-cloud-clients/google-cloud-contrib/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/testing/FakeStorageRpc.java +++ b/google-cloud-clients/google-cloud-contrib/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/testing/FakeStorageRpc.java @@ -312,6 +312,11 @@ public String open(StorageObject object, Map options) throws StorageE return fullname(object); } + @Override + public String open(String signedURL) { + return null; + } + @Override public void write( String uploadId, byte[] toWrite, int toWriteOffset, long destOffset, int length, boolean last) diff --git a/google-cloud-clients/google-cloud-storage/.attach_pid8524 b/google-cloud-clients/google-cloud-storage/.attach_pid8524 new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobWriteChannel.java b/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobWriteChannel.java index eade1d245d83..ec5376697ce3 100644 --- a/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobWriteChannel.java +++ b/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobWriteChannel.java @@ -24,6 +24,7 @@ import com.google.cloud.RetryHelper; import com.google.cloud.WriteChannel; import com.google.cloud.storage.spi.v1.StorageRpc; +import java.net.URL; import java.util.Map; import java.util.concurrent.Callable; @@ -34,10 +35,18 @@ class BlobWriteChannel extends BaseWriteChannel { this(options, blob, open(options, blob, optionsMap)); } + BlobWriteChannel(StorageOptions options, URL signedURL) { + this(options, open(signedURL, options)); + } + BlobWriteChannel(StorageOptions options, BlobInfo blobInfo, String uploadId) { super(options, blobInfo, uploadId); } + BlobWriteChannel(StorageOptions options, String uploadId) { + super(options, null, uploadId); + } + @Override protected void flushBuffer(final int length, final boolean last) { try { @@ -83,6 +92,46 @@ public String call() { } } + private static String open(final URL signedURL, final StorageOptions options) { + try { + return runWithRetries( + new Callable() { + @Override + public String call() { + if (!isValidSignedURL(signedURL.getQuery())) { + throw new StorageException(2, "invalid signedURL"); + } + return options.getStorageRpcV1().open(signedURL.toString()); + } + }, + options.getRetrySettings(), + StorageImpl.EXCEPTION_HANDLER, + options.getClock()); + } catch (RetryHelper.RetryHelperException e) { + throw StorageException.translateAndThrow(e); + } + } + + private static boolean isValidSignedURL(String signedURLQuery) { + boolean isValid = true; + if (signedURLQuery.startsWith("X-Goog-Algorithm=")) { + if (!signedURLQuery.contains("&X-Goog-Credential=") + || !signedURLQuery.contains("&X-Goog-Date=") + || !signedURLQuery.contains("&X-Goog-Expires=") + || !signedURLQuery.contains("&X-Goog-SignedHeaders=") + || !signedURLQuery.contains("&X-Goog-Signature=")) { + isValid = false; + } + } else if (signedURLQuery.startsWith("GoogleAccessId=")) { + if (!signedURLQuery.contains("&Expires=") || !signedURLQuery.contains("&Signature=")) { + isValid = false; + } + } else { + isValid = false; + } + return isValid; + } + static class StateImpl extends BaseWriteChannel.BaseState { private static final long serialVersionUID = -9028324143780151286L; diff --git a/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java b/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java index 720664e40a57..82dc0775ad32 100644 --- a/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java +++ b/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java @@ -2091,6 +2091,27 @@ Blob create( */ WriteChannel writer(BlobInfo blobInfo, BlobWriteOption... options); + /** + * Accepts signed URL and return a channel for writing content. + * + *

Example of writing content through a writer using signed URL. + * + *

{@code
+   * String bucketName = "my_unique_bucket";
+   * String blobName = "my_blob_name";
+   * BlobId blobId = BlobId.of(bucketName, blobName);
+   * byte[] content = "Hello, World!".getBytes(UTF_8);
+   * BlobInfo blobInfo = BlobInfo.newBuilder(blobId).setContentType("text/plain").build();
+   * URL signedURL = storage.signUrl(blobInfo, 1, TimeUnit.HOURS, Storage.SignUrlOption.httpMethod(HttpMethod.POST));
+   * try (WriteChannel writer = storage.writer(signedURL)) {
+   *    writer.write(ByteBuffer.wrap(content, 0, content.length));
+   * }
+   * }
+ * + * @throws StorageException upon failure + */ + WriteChannel writer(URL signedURL); + /** * Generates a signed URL for a blob. If you have a blob that you want to allow access to for a * fixed amount of time, you can use this method to generate a URL that is only valid within a diff --git a/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java b/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java index 6611613f45a3..6f127ac5fb0c 100644 --- a/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java +++ b/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java @@ -599,6 +599,11 @@ public BlobWriteChannel writer(BlobInfo blobInfo, BlobWriteOption... options) { return writer(targetOptions.x(), targetOptions.y()); } + @Override + public BlobWriteChannel writer(URL signedURL) { + return new BlobWriteChannel(getOptions(), signedURL); + } + private BlobWriteChannel writer(BlobInfo blobInfo, BlobTargetOption... options) { final Map optionsMap = optionMap(blobInfo, options); return new BlobWriteChannel(getOptions(), blobInfo, optionsMap); diff --git a/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java b/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java index b736600fcf69..d162b33c5c7f 100644 --- a/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java +++ b/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java @@ -822,6 +822,39 @@ public String open(StorageObject object, Map options) { } } + @Override + public String open(String signedURL) { + Span span = startSpan(HttpStorageRpcSpans.SPAN_NAME_OPEN); + Scope scope = tracer.withSpan(span); + try { + GenericUrl url = new GenericUrl(signedURL); + url.set("uploadType", "resumable"); + String bytesArrayParameters = ""; + byte[] bytesArray = new byte[bytesArrayParameters.length()]; + HttpRequestFactory requestFactory = storage.getRequestFactory(); + HttpRequest httpRequest = + requestFactory.buildPostRequest( + url, new ByteArrayContent("", bytesArray, 0, bytesArray.length)); + HttpHeaders requestHeaders = httpRequest.getHeaders(); + requestHeaders.set("X-Upload-Content-Type", ""); + requestHeaders.set("x-goog-resumable", "start"); + HttpResponse response = httpRequest.execute(); + if (response.getStatusCode() != 201) { + GoogleJsonError error = new GoogleJsonError(); + error.setCode(response.getStatusCode()); + error.setMessage(response.getStatusMessage()); + throw translate(error); + } + return response.getHeaders().getLocation(); + } catch (IOException ex) { + span.setStatus(Status.UNKNOWN.withDescription(ex.getMessage())); + throw translate(ex); + } finally { + scope.close(); + span.end(); + } + } + @Override public RewriteResponse openRewrite(RewriteRequest rewriteRequest) { Span span = startSpan(HttpStorageRpcSpans.SPAN_NAME_OPEN_REWRITE); diff --git a/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java b/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java index eecf311a7033..c85958176d69 100644 --- a/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java +++ b/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java @@ -289,6 +289,13 @@ StorageObject compose( */ String open(StorageObject object, Map options); + /** + * Opens a resumable upload channel for a given signedURL. + * + * @throws StorageException upon failure + */ + String open(String signedURL); + /** * Writes the provided bytes to a storage object at the provided location. * diff --git a/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobWriteChannelTest.java b/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobWriteChannelTest.java index 8e2d7d14bdb5..d3cd439d203e 100644 --- a/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobWriteChannelTest.java +++ b/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobWriteChannelTest.java @@ -36,7 +36,9 @@ import com.google.cloud.storage.spi.v1.StorageRpc; import com.google.common.collect.ImmutableMap; import java.io.IOException; +import java.net.MalformedURLException; import java.net.SocketException; +import java.net.URL; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Map; @@ -60,6 +62,8 @@ public class BlobWriteChannelTest { private static final int DEFAULT_CHUNK_SIZE = 8 * MIN_CHUNK_SIZE; private static final int CUSTOM_CHUNK_SIZE = 4 * MIN_CHUNK_SIZE; private static final Random RANDOM = new Random(); + private static final String SIGNED_URL = + "http://www.test.com/test-bucket/test1.txt?GoogleAccessId=testClient-test@test.com&Expires=1553839761&Signature=MJUBXAZ7"; @Rule public ExpectedException thrown = ExpectedException.none(); @@ -265,6 +269,133 @@ public void testStateEquals() { assertEquals(state.toString(), state2.toString()); } + @Test + public void testWriteWithSignedURLAndWithoutFlush() throws IOException { + expect(storageRpcMock.open(SIGNED_URL)).andReturn(UPLOAD_ID); + replay(storageRpcMock); + writer = new BlobWriteChannel(options, new URL(SIGNED_URL)); + assertEquals(MIN_CHUNK_SIZE, writer.write(ByteBuffer.allocate(MIN_CHUNK_SIZE))); + } + + @Test + public void testWriteWithSignedURLAndWithFlush() throws IOException { + expect(storageRpcMock.open(SIGNED_URL)).andReturn(UPLOAD_ID); + Capture capturedBuffer = Capture.newInstance(); + storageRpcMock.write( + eq(UPLOAD_ID), capture(capturedBuffer), eq(0), eq(0L), eq(CUSTOM_CHUNK_SIZE), eq(false)); + replay(storageRpcMock); + writer = new BlobWriteChannel(options, new URL(SIGNED_URL)); + writer.setChunkSize(CUSTOM_CHUNK_SIZE); + ByteBuffer buffer = randomBuffer(CUSTOM_CHUNK_SIZE); + assertEquals(CUSTOM_CHUNK_SIZE, writer.write(buffer)); + assertArrayEquals(buffer.array(), capturedBuffer.getValue()); + } + + @Test + public void testWriteWithSignedURLAndFlush() throws IOException { + expect(storageRpcMock.open(SIGNED_URL)).andReturn(UPLOAD_ID); + Capture capturedBuffer = Capture.newInstance(); + storageRpcMock.write( + eq(UPLOAD_ID), capture(capturedBuffer), eq(0), eq(0L), eq(DEFAULT_CHUNK_SIZE), eq(false)); + replay(storageRpcMock); + writer = new BlobWriteChannel(options, new URL(SIGNED_URL)); + ByteBuffer[] buffers = new ByteBuffer[DEFAULT_CHUNK_SIZE / MIN_CHUNK_SIZE]; + for (int i = 0; i < buffers.length; i++) { + buffers[i] = randomBuffer(MIN_CHUNK_SIZE); + assertEquals(MIN_CHUNK_SIZE, writer.write(buffers[i])); + } + for (int i = 0; i < buffers.length; i++) { + assertArrayEquals( + buffers[i].array(), + Arrays.copyOfRange( + capturedBuffer.getValue(), MIN_CHUNK_SIZE * i, MIN_CHUNK_SIZE * (i + 1))); + } + } + + @Test + public void testCloseWithSignedURLWithoutFlush() throws IOException { + expect(storageRpcMock.open(SIGNED_URL)).andReturn(UPLOAD_ID); + Capture capturedBuffer = Capture.newInstance(); + storageRpcMock.write(eq(UPLOAD_ID), capture(capturedBuffer), eq(0), eq(0L), eq(0), eq(true)); + replay(storageRpcMock); + writer = new BlobWriteChannel(options, new URL(SIGNED_URL)); + assertTrue(writer.isOpen()); + writer.close(); + assertArrayEquals(new byte[0], capturedBuffer.getValue()); + assertTrue(!writer.isOpen()); + } + + @Test + public void testCloseWithSignedURLWithFlush() throws IOException { + expect(storageRpcMock.open(SIGNED_URL)).andReturn(UPLOAD_ID); + Capture capturedBuffer = Capture.newInstance(); + ByteBuffer buffer = randomBuffer(MIN_CHUNK_SIZE); + storageRpcMock.write( + eq(UPLOAD_ID), capture(capturedBuffer), eq(0), eq(0L), eq(MIN_CHUNK_SIZE), eq(true)); + replay(storageRpcMock); + writer = new BlobWriteChannel(options, new URL(SIGNED_URL)); + assertTrue(writer.isOpen()); + writer.write(buffer); + writer.close(); + assertEquals(DEFAULT_CHUNK_SIZE, capturedBuffer.getValue().length); + assertArrayEquals(buffer.array(), Arrays.copyOf(capturedBuffer.getValue(), MIN_CHUNK_SIZE)); + assertTrue(!writer.isOpen()); + } + + @Test + public void testWriteWithSignedURLClosed() throws IOException { + expect(storageRpcMock.open(SIGNED_URL)).andReturn(UPLOAD_ID); + Capture capturedBuffer = Capture.newInstance(); + storageRpcMock.write(eq(UPLOAD_ID), capture(capturedBuffer), eq(0), eq(0L), eq(0), eq(true)); + replay(storageRpcMock); + writer = new BlobWriteChannel(options, new URL(SIGNED_URL)); + writer.close(); + try { + writer.write(ByteBuffer.allocate(MIN_CHUNK_SIZE)); + fail("Expected BlobWriteChannel write to throw IOException"); + } catch (IOException ex) { + // expected + } + } + + @Test + public void testSaveAndRestoreWithSignedURL() throws IOException { + expect(storageRpcMock.open(SIGNED_URL)).andReturn(UPLOAD_ID); + Capture capturedBuffer = Capture.newInstance(CaptureType.ALL); + Capture capturedPosition = Capture.newInstance(CaptureType.ALL); + storageRpcMock.write( + eq(UPLOAD_ID), + capture(capturedBuffer), + eq(0), + captureLong(capturedPosition), + eq(DEFAULT_CHUNK_SIZE), + eq(false)); + expectLastCall().times(2); + replay(storageRpcMock); + ByteBuffer buffer1 = randomBuffer(DEFAULT_CHUNK_SIZE); + ByteBuffer buffer2 = randomBuffer(DEFAULT_CHUNK_SIZE); + writer = new BlobWriteChannel(options, new URL(SIGNED_URL)); + assertEquals(DEFAULT_CHUNK_SIZE, writer.write(buffer1)); + assertArrayEquals(buffer1.array(), capturedBuffer.getValues().get(0)); + assertEquals(new Long(0L), capturedPosition.getValues().get(0)); + RestorableState writerState = writer.capture(); + WriteChannel restoredWriter = writerState.restore(); + assertEquals(DEFAULT_CHUNK_SIZE, restoredWriter.write(buffer2)); + assertArrayEquals(buffer2.array(), capturedBuffer.getValues().get(1)); + assertEquals(new Long(DEFAULT_CHUNK_SIZE), capturedPosition.getValues().get(1)); + } + + @Test + public void testRuntimeExceptionWithSignedURL() throws MalformedURLException { + String exceptionMessage = "invalid signedURL"; + expect(new BlobWriteChannel(options, new URL(SIGNED_URL))) + .andThrow(new RuntimeException(exceptionMessage)); + replay(storageRpcMock); + thrown.expect(StorageException.class); + thrown.expectMessage(exceptionMessage); + writer = new BlobWriteChannel(options, new URL(SIGNED_URL)); + } + private static ByteBuffer randomBuffer(int size) { byte[] byteArray = new byte[size]; RANDOM.nextBytes(byteArray); diff --git a/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java b/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java index 9a9db310e99d..db003a92a7c6 100644 --- a/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java +++ b/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java @@ -59,6 +59,7 @@ import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.UnsupportedEncodingException; +import java.net.MalformedURLException; import java.net.URL; import java.net.URLDecoder; import java.nio.ByteBuffer; @@ -319,6 +320,9 @@ public class StorageImplTest { + "EkPPhszldvQTY486uPxyD/D7HdfnGW/Nbw5JUhfvecAdudDEhNAQ3PNabyDMI+TpiHy4NTWOrgdcWrzj6VXcdc" + "+uuABnPwRCdcyJ1xl2kOrPksRnp1auNGMLOe4IpEBjGY7baX9UG8+A45MbG0aHmkR59Op/aR9XowIDAQAB"; + private static final String SIGNED_URL = + "http://www.test.com/test-bucket/test1.txt?GoogleAccessId=testClient-test@test.com&Expires=1553839761&Signature=MJUBXAZ7"; + private static final ApiClock TIME_SOURCE = new ApiClock() { @Override @@ -2835,4 +2839,14 @@ public void testRuntimeException() { thrown.expectMessage(exceptionMessage); storage.get(blob); } + + @Test + public void testWriterWithSignedURL() throws MalformedURLException { + EasyMock.expect(storageRpcMock.open(SIGNED_URL)).andReturn("upload-id"); + EasyMock.replay(storageRpcMock); + initializeService(); + WriteChannel writer = new BlobWriteChannel(options, new URL(SIGNED_URL)); + assertNotNull(writer); + assertTrue(writer.isOpen()); + } } diff --git a/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java b/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java index f5bbe7218b3e..8b2d627b23db 100644 --- a/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java +++ b/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java @@ -2560,4 +2560,28 @@ public void testEnableAndDisableBucketPolicyOnlyOnExistingBucket() throws Except RemoteStorageHelper.forceDelete(storage, bpoBucket, 1, TimeUnit.MINUTES); } } + + @Test + public void testUploadUsingSignedURL() throws Exception { + String blobName = "test-signed-url-upload"; + BlobInfo blob = BlobInfo.newBuilder(BUCKET, blobName).build(); + assertNotNull(storage.create(blob)); + URL signUrl = + storage.signUrl(blob, 1, TimeUnit.HOURS, Storage.SignUrlOption.httpMethod(HttpMethod.POST)); + byte[] bytesArrayToUpload = BLOB_STRING_CONTENT.getBytes(); + try (WriteChannel writer = storage.writer(signUrl)) { + writer.write(ByteBuffer.wrap(bytesArrayToUpload, 0, bytesArrayToUpload.length)); + } + + int lengthOfDownLoadBytes = -1; + BlobId blobId = BlobId.of(BUCKET, blobName); + Blob blobToRead = storage.get(blobId); + try (ReadChannel reader = blobToRead.reader()) { + ByteBuffer bytes = ByteBuffer.allocate(64 * 1024); + lengthOfDownLoadBytes = reader.read(bytes); + } + + assertEquals(bytesArrayToUpload.length, lengthOfDownLoadBytes); + assertTrue(storage.delete(BUCKET, blobName)); + } }