Skip to content

Commit

Permalink
Merge pull request #884 from jean-philippe-martin/fake_list
Browse files Browse the repository at this point in the history
Implement bare-bones blob listing
  • Loading branch information
jean-philippe-martin committed Apr 14, 2016
2 parents 9e91dc5 + 9c9dc0d commit 9a8c51a
Show file tree
Hide file tree
Showing 4 changed files with 119 additions and 12 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,10 @@
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.MoreObjects;
import com.google.common.base.Throwables;
import com.google.common.collect.AbstractIterator;
import com.google.common.primitives.Ints;
import com.google.gcloud.storage.Acl;
import com.google.gcloud.storage.Blob;
import com.google.gcloud.storage.BlobId;
import com.google.gcloud.storage.BlobInfo;
import com.google.gcloud.storage.CopyWriter;
Expand All @@ -46,6 +48,7 @@
import java.nio.file.AccessMode;
import java.nio.file.AtomicMoveNotSupportedException;
import java.nio.file.CopyOption;
import java.nio.file.DirectoryIteratorException;
import java.nio.file.DirectoryStream;
import java.nio.file.DirectoryStream.Filter;
import java.nio.file.FileAlreadyExistsException;
Expand All @@ -64,11 +67,11 @@
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;

import javax.annotation.Nullable;
import javax.annotation.concurrent.ThreadSafe;

Expand All @@ -84,6 +87,33 @@ public final class CloudStorageFileSystemProvider extends FileSystemProvider {
// used only when we create a new instance of CloudStorageFileSystemProvider.
private static StorageOptions defaultStorageOptions;

private static class LazyPathIterator extends AbstractIterator<Path> {
private final Iterator<Blob> blobIterator;
private final Filter<? super Path> filter;
private final CloudStorageFileSystem fileSystem;

LazyPathIterator(CloudStorageFileSystem fileSystem, Iterator<Blob> blobIterator, Filter<? super Path> filter) {
this.blobIterator = blobIterator;
this.filter = filter;
this.fileSystem = fileSystem;
}

@Override
protected Path computeNext() {
while (blobIterator.hasNext()) {
Path path = fileSystem.getPath(blobIterator.next().name());
try {
if (filter.accept(path)) {
return path;
}
} catch (IOException ex) {
throw new DirectoryIteratorException(ex);
}
}
return endOfData();
}
}

/**
* Sets default options that are only used by the constructor.
*/
Expand Down Expand Up @@ -532,13 +562,23 @@ public void createDirectory(Path dir, FileAttribute<?>... attrs) {
checkNotNullArray(attrs);
}

/**
* Throws {@link UnsupportedOperationException} because this feature hasn't been implemented yet.
*/
@Override
public DirectoryStream<Path> newDirectoryStream(Path dir, Filter<? super Path> filter) {
// TODO: Implement me.
throw new UnsupportedOperationException();
public DirectoryStream<Path> newDirectoryStream(Path dir, final Filter<? super Path> filter) {
final CloudStoragePath cloudPath = checkPath(dir);
checkNotNull(filter);
String prefix = cloudPath.toString();
final Iterator<Blob> blobIterator = storage.list(cloudPath.bucket(), Storage.BlobListOption.prefix(prefix), Storage.BlobListOption.fields()).iterateAll();
return new DirectoryStream<Path>() {
@Override
public Iterator<Path> iterator() {
return new LazyPathIterator(cloudPath.getFileSystem(), blobIterator, filter);
}

@Override
public void close() throws IOException {
// Does nothing since there's nothing to close. Commenting this method to quiet codacy.
}
};
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,10 @@
import java.nio.file.FileSystem;
import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;

/**
* Unit tests for {@link CloudStorageFileSystem}.
Expand Down Expand Up @@ -134,4 +137,27 @@ public void testNullness() throws IOException, NoSuchMethodException, SecurityEx
tester.testAllPublicInstanceMethods(fs);
}
}

@Test
public void testListFiles() throws IOException {
try (FileSystem fs = CloudStorageFileSystem.forBucket("bucket")) {
List<Path> goodPaths = new ArrayList<>();
List<Path> paths = new ArrayList<>();
goodPaths.add(fs.getPath("dir/angel"));
goodPaths.add(fs.getPath("dir/alone"));
paths.add(fs.getPath("dir/dir2/another_angel"));
paths.add(fs.getPath("atroot"));
paths.addAll(goodPaths);
goodPaths.add(fs.getPath("dir/dir2/"));
for (Path path : paths) {
Files.write(path, ALONE.getBytes(UTF_8));
}

List<Path> got = new ArrayList<>();
for (Path path : Files.newDirectoryStream(fs.getPath("/dir/"))) {
got.add(path);
}
assertThat(got).containsExactlyElementsIn(goodPaths);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -22,14 +22,12 @@
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoMoreInteractions;
import static org.mockito.Mockito.verifyZeroInteractions;
import static org.mockito.Mockito.when;

import com.google.gcloud.ReadChannel;
import com.google.gcloud.storage.Blob;
import com.google.gcloud.storage.BlobId;
import com.google.gcloud.storage.BlobInfo;
import com.google.gcloud.storage.Storage;

import org.junit.Before;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,10 @@
import java.io.InputStream;
import java.math.BigInteger;
import java.nio.file.FileAlreadyExistsException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import javax.annotation.concurrent.NotThreadSafe;
Expand Down Expand Up @@ -91,8 +93,49 @@ public Tuple<String, Iterable<Bucket>> list(Map<Option, ?> options) throws Stora
@Override
public Tuple<String, Iterable<StorageObject>> list(String bucket, Map<Option, ?> options)
throws StorageException {
potentiallyThrow(options);
return null;
String preprefix = "";
for (Map.Entry<Option, ?> e : options.entrySet()) {
switch (e.getKey()) {
case PREFIX:
preprefix = (String) e.getValue();
if (preprefix.startsWith("/")) {
preprefix = preprefix.substring(1);
}
break;
case FIELDS:
// ignore and return all the fields
break;
default:
throw new UnsupportedOperationException("Unknown option: " + e.getKey());
}
}
final String prefix = preprefix;

List<StorageObject> values = new ArrayList<>();
Map<String, StorageObject> folders = new HashMap<>();
for (StorageObject so : stuff.values()) {
if (!so.getName().startsWith(prefix)) {
continue;
}
int nextSlash = so.getName().indexOf("/", prefix.length());
if (nextSlash >= 0) {
String folderName = so.getName().substring(0, nextSlash + 1);
if (folders.containsKey(folderName)) {
continue;
}
StorageObject fakeFolder = new StorageObject();
fakeFolder.setName(folderName);
fakeFolder.setBucket(so.getBucket());
fakeFolder.setGeneration(so.getGeneration());
folders.put(folderName, fakeFolder);
continue;
}
values.add(so);
}
values.addAll(folders.values());
// null cursor to indicate there is no more data (empty string would cause us to be called again).
// The type cast seems to be necessary to help Java's typesystem remember that collections are iterable.
return Tuple.of(null, (Iterable<StorageObject>) values);
}

/**
Expand All @@ -111,7 +154,7 @@ public Bucket get(Bucket bucket, Map<Option, ?> options) throws StorageException
public StorageObject get(StorageObject object, Map<Option, ?> options) throws StorageException {
// we allow the "ID" option because we need to, but then we give a whole answer anyways
// because the caller won't mind the extra fields.
if (throwIfOption && !options.isEmpty() && options.size()>1
if (throwIfOption && !options.isEmpty() && options.size() > 1
&& options.keySet().toArray()[0] != Storage.BlobGetOption.fields(Storage.BlobField.ID)) {
throw new UnsupportedOperationException();
}
Expand Down

0 comments on commit 9a8c51a

Please sign in to comment.