diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileAttributeView.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileAttributeView.java index fe5a0b17f798..11b92116b4c3 100644 --- a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileAttributeView.java +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileAttributeView.java @@ -18,9 +18,9 @@ import static com.google.common.base.Preconditions.checkNotNull; -import com.google.common.base.MoreObjects; import com.google.cloud.storage.BlobInfo; import com.google.cloud.storage.Storage; +import com.google.common.base.MoreObjects; import java.io.IOException; import java.nio.file.NoSuchFileException; diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileAttributes.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileAttributes.java index 04a7c9509ebc..ff77f0b96ab6 100644 --- a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileAttributes.java +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileAttributes.java @@ -16,9 +16,9 @@ package com.google.cloud.storage.contrib.nio; +import com.google.cloud.storage.Acl; import com.google.common.base.Optional; import com.google.common.collect.ImmutableMap; -import com.google.cloud.storage.Acl; import java.nio.file.attribute.BasicFileAttributes; import java.util.List; diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystem.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystem.java index e67dd79cc205..80004792d2f8 100644 --- a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystem.java +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystem.java @@ -19,8 +19,8 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; -import com.google.common.collect.ImmutableSet; import com.google.cloud.storage.StorageOptions; +import com.google.common.collect.ImmutableSet; import java.io.IOException; import java.net.URI; diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystemProvider.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystemProvider.java index 81fea40ea690..cc1832686fbb 100644 --- a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystemProvider.java +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystemProvider.java @@ -21,11 +21,6 @@ import static com.google.common.base.Strings.isNullOrEmpty; import com.google.auto.service.AutoService; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.MoreObjects; -import com.google.common.base.Throwables; -import com.google.common.collect.AbstractIterator; -import com.google.common.primitives.Ints; import com.google.cloud.storage.Acl; import com.google.cloud.storage.Blob; import com.google.cloud.storage.BlobId; @@ -34,6 +29,11 @@ import com.google.cloud.storage.Storage; import com.google.cloud.storage.StorageException; import com.google.cloud.storage.StorageOptions; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.MoreObjects; +import com.google.common.base.Throwables; +import com.google.common.collect.AbstractIterator; +import com.google.common.primitives.Ints; import java.io.BufferedInputStream; import java.io.IOException; @@ -87,7 +87,8 @@ private static class LazyPathIterator extends AbstractIterator { private final Filter filter; private final CloudStorageFileSystem fileSystem; - LazyPathIterator(CloudStorageFileSystem fileSystem, Iterator blobIterator, Filter filter) { + LazyPathIterator(CloudStorageFileSystem fileSystem, Iterator blobIterator, + Filter filter) { this.blobIterator = blobIterator; this.filter = filter; this.fileSystem = fileSystem; @@ -164,7 +165,8 @@ public CloudStorageFileSystem newFileSystem(URI uri, Map env) { CloudStorageFileSystem.URI_SCHEME, uri); checkArgument( - !isNullOrEmpty(uri.getHost()), "%s:// URIs must have a host: %s", CloudStorageFileSystem.URI_SCHEME, uri); + !isNullOrEmpty(uri.getHost()), "%s:// URIs must have a host: %s", + CloudStorageFileSystem.URI_SCHEME, uri); checkArgument( uri.getPort() == -1 && isNullOrEmpty(uri.getPath()) @@ -179,7 +181,8 @@ && isNullOrEmpty(uri.getUserInfo()), @Override public CloudStoragePath getPath(URI uri) { - return CloudStoragePath.getPath(getFileSystem(CloudStorageUtil.stripPathFromUri(uri)), uri.getPath()); + return CloudStoragePath.getPath( + getFileSystem(CloudStorageUtil.stripPathFromUri(uri)), uri.getPath()); } @Override @@ -562,7 +565,9 @@ public DirectoryStream newDirectoryStream(Path dir, final Filter blobIterator = storage.list(cloudPath.bucket(), Storage.BlobListOption.prefix(prefix), Storage.BlobListOption.fields()).iterateAll(); + final Iterator blobIterator = storage.list(cloudPath.bucket(), + Storage.BlobListOption.prefix(prefix), Storage.BlobListOption.currentDirectory(), + Storage.BlobListOption.fields()).iterateAll(); return new DirectoryStream() { @Override public Iterator iterator() { diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageObjectAttributes.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageObjectAttributes.java index 2f98e1bf8aed..93814b99243a 100644 --- a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageObjectAttributes.java +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageObjectAttributes.java @@ -16,14 +16,14 @@ package com.google.cloud.storage.contrib.nio; -import static com.google.common.base.Preconditions.checkNotNull; import static com.google.cloud.storage.contrib.nio.CloudStorageFileSystem.FILE_TIME_UNKNOWN; +import static com.google.common.base.Preconditions.checkNotNull; +import com.google.cloud.storage.Acl; +import com.google.cloud.storage.BlobInfo; import com.google.common.base.MoreObjects; import com.google.common.base.Optional; import com.google.common.collect.ImmutableMap; -import com.google.cloud.storage.Acl; -import com.google.cloud.storage.BlobInfo; import java.nio.file.attribute.FileTime; import java.util.List; diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStoragePath.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStoragePath.java index 778aa53baf83..fdb407ba1852 100644 --- a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStoragePath.java +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStoragePath.java @@ -19,8 +19,8 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; -import com.google.common.collect.UnmodifiableIterator; import com.google.cloud.storage.BlobId; +import com.google.common.collect.UnmodifiableIterator; import java.io.File; import java.net.URI; @@ -308,7 +308,8 @@ public String toString() { @Override public URI toUri() { try { - return new URI(CloudStorageFileSystem.URI_SCHEME, bucket(), path.toAbsolutePath().toString(), null); + return new URI( + CloudStorageFileSystem.URI_SCHEME, bucket(), path.toAbsolutePath().toString(), null); } catch (URISyntaxException e) { throw new AssertionError(e); } diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStoragePseudoDirectoryAttributes.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStoragePseudoDirectoryAttributes.java index c4aa2dc5afb3..a7a8c7293688 100644 --- a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStoragePseudoDirectoryAttributes.java +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStoragePseudoDirectoryAttributes.java @@ -16,9 +16,9 @@ package com.google.cloud.storage.contrib.nio; +import com.google.cloud.storage.Acl; import com.google.common.base.Optional; import com.google.common.collect.ImmutableMap; -import com.google.cloud.storage.Acl; import java.nio.file.attribute.FileTime; import java.util.List; diff --git a/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/cloud/storage/contrib/nio/it/ITGcsNio.java b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/cloud/storage/contrib/nio/it/ITGcsNio.java index 534fffdca94e..cd2c897f609a 100644 --- a/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/cloud/storage/contrib/nio/it/ITGcsNio.java +++ b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/cloud/storage/contrib/nio/it/ITGcsNio.java @@ -27,10 +27,12 @@ import java.nio.ByteBuffer; import java.nio.channels.ReadableByteChannel; import java.nio.channels.SeekableByteChannel; +import java.nio.file.FileSystem; import java.nio.file.Files; import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.nio.file.StandardOpenOption; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Random; @@ -317,6 +319,29 @@ public void testCopy() throws IOException { } } + @Test + public void testListFiles() throws IOException { + try (FileSystem fs = getTestBucket()) { + List goodPaths = new ArrayList<>(); + List paths = new ArrayList<>(); + goodPaths.add(fs.getPath("dir/angel")); + goodPaths.add(fs.getPath("dir/alone")); + paths.add(fs.getPath("dir/dir2/another_angel")); + paths.add(fs.getPath("atroot")); + paths.addAll(goodPaths); + goodPaths.add(fs.getPath("dir/dir2/")); + for (Path path : paths) { + fillFile(storage, path.toString(), SML_SIZE); + } + + List got = new ArrayList<>(); + for (Path path : Files.newDirectoryStream(fs.getPath("dir/"))) { + got.add(path); + } + assertThat(got).containsExactlyElementsIn(goodPaths); + } + } + private int readFully(ReadableByteChannel chan, byte[] outputBuf) throws IOException { ByteBuffer buf = ByteBuffer.wrap(outputBuf); int sofar = 0; diff --git a/gcloud-java-storage/src/main/java/com/google/cloud/storage/testing/FakeStorageRpc.java b/gcloud-java-storage/src/main/java/com/google/cloud/storage/testing/FakeStorageRpc.java index f98ee25aec14..52e300405e64 100644 --- a/gcloud-java-storage/src/main/java/com/google/cloud/storage/testing/FakeStorageRpc.java +++ b/gcloud-java-storage/src/main/java/com/google/cloud/storage/testing/FakeStorageRpc.java @@ -93,6 +93,7 @@ public Tuple> list(Map options) throws Stora @Override public Tuple> list(String bucket, Map options) throws StorageException { + String delimiter = null; String preprefix = ""; for (Map.Entry e : options.entrySet()) { switch (e.getKey()) { @@ -102,6 +103,9 @@ public Tuple> list(String bucket, Map preprefix = preprefix.substring(1); } break; + case DELIMITER: + delimiter = (String) e.getValue(); + break; case FIELDS: // ignore and return all the fields break; @@ -117,17 +121,7 @@ public Tuple> list(String bucket, Map if (!so.getName().startsWith(prefix)) { continue; } - int nextSlash = so.getName().indexOf("/", prefix.length()); - if (nextSlash >= 0) { - String folderName = so.getName().substring(0, nextSlash + 1); - if (folders.containsKey(folderName)) { - continue; - } - StorageObject fakeFolder = new StorageObject(); - fakeFolder.setName(folderName); - fakeFolder.setBucket(so.getBucket()); - fakeFolder.setGeneration(so.getGeneration()); - folders.put(folderName, fakeFolder); + if (processedAsFolder(so, delimiter, prefix, folders)) { continue; } values.add(so); @@ -333,4 +327,25 @@ private void potentiallyThrow(Map options) throws UnsupportedOperatio throw new UnsupportedOperationException(); } } + + // Returns true if this is a folder. Adds it to folders if it isn't already there. + private static boolean processedAsFolder(StorageObject so, String delimiter, String prefix, /* inout */ Map folders) { + if (delimiter == null) { + return false; + } + int nextSlash = so.getName().indexOf(delimiter, prefix.length()); + if (nextSlash < 0) { + return false; + } + String folderName = so.getName().substring(0, nextSlash + 1); + if (folders.containsKey(folderName)) { + return true; + } + StorageObject fakeFolder = new StorageObject(); + fakeFolder.setName(folderName); + fakeFolder.setBucket(so.getBucket()); + fakeFolder.setGeneration(so.getGeneration()); + folders.put(folderName, fakeFolder); + return true; + } }