Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

walkFiles consistently relative or absolute #3773

Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -102,15 +102,19 @@ private static class LazyPathIterator extends AbstractIterator<Path> {
private final Filter<? super Path> filter;
private final CloudStorageFileSystem fileSystem;
private final String prefix;
// whether to make the paths absolute before returning them.
private final boolean absolutePaths;

LazyPathIterator(CloudStorageFileSystem fileSystem,
String prefix,
Iterator<Blob> blobIterator,
Filter<? super Path> filter) {
Filter<? super Path> filter,
boolean absolutePaths) {
this.prefix = prefix;
this.blobIterator = blobIterator;
this.filter = filter;
this.fileSystem = fileSystem;
this.absolutePaths = absolutePaths;
}

@Override
Expand All @@ -123,6 +127,9 @@ protected Path computeNext() {
continue;
}
if (filter.accept(path)) {
if (absolutePaths) {
return path.toAbsolutePath();
}
return path;
}
} catch (IOException ex) {
Expand Down Expand Up @@ -769,7 +776,7 @@ public void createDirectory(Path dir, FileAttribute<?>... attrs) {
}

@Override
public DirectoryStream<Path> newDirectoryStream(Path dir, final Filter<? super Path> filter) {
public DirectoryStream<Path> newDirectoryStream(final Path dir, final Filter<? super Path> filter) {
final CloudStoragePath cloudPath = CloudStorageUtil.checkPath(dir);
checkNotNull(filter);
initStorage();
Expand All @@ -793,7 +800,7 @@ public DirectoryStream<Path> newDirectoryStream(Path dir, final Filter<? super P
return new DirectoryStream<Path>() {
@Override
public Iterator<Path> iterator() {
return new LazyPathIterator(cloudPath.getFileSystem(), prefix, blobIterator, filter);
return new LazyPathIterator(cloudPath.getFileSystem(), prefix, blobIterator, filter, dir.isAbsolute());
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
package com.google.cloud.storage.contrib.nio.it;

import static com.google.common.truth.Truth.assertThat;
import static com.google.common.truth.Truth.assertWithMessage;
import static java.nio.charset.StandardCharsets.UTF_8;

import com.google.api.client.http.HttpResponseException;
Expand Down Expand Up @@ -538,6 +539,52 @@ public void testListFiles() throws IOException {
}
}

@Test
public void testRelativityOfResolve() throws IOException {
try (FileSystem fs = getTestBucket()) {
Path abs1 = fs.getPath("/dir");
Path abs2 = abs1.resolve("subdir/");
Path rel1 = fs.getPath("dir");
Path rel2 = rel1.resolve("subdir/");
// children of absolute paths are absolute,
// children of relative paths are relative.
assertThat(abs1.isAbsolute()).isTrue();
assertThat(abs2.isAbsolute()).isTrue();
assertThat(rel1.isAbsolute()).isFalse();
assertThat(rel2.isAbsolute()).isFalse();
}
}

@Test
public void testWalkFiles() throws IOException {
try (FileSystem fs = getTestBucket()) {
List<Path> goodPaths = new ArrayList<>();
List<Path> paths = new ArrayList<>();
goodPaths.add(fs.getPath("dir/angel"));
goodPaths.add(fs.getPath("dir/alone"));
paths.add(fs.getPath("dir/dir2/another_angel"));
paths.add(fs.getPath("atroot"));
paths.addAll(goodPaths);
goodPaths.add(fs.getPath("dir/dir2/"));
jean-philippe-martin marked this conversation as resolved.
Show resolved Hide resolved
for (Path path : paths) {
fillFile(storage, BUCKET, path.toString(), SML_SIZE);
}
// Given a relative path as starting point, walkFileTree must return only relative paths.
List<Path> relativePaths = postTraversalWalker.walkFileTree(fs.getPath("dir/"));
for (Path p : relativePaths) {
assertWithMessage("Should have been relative: " + p.toString()).that(p.isAbsolute()).isFalse();
}
assertThat(relativePaths.size()).isEqualTo(5);
jean-philippe-martin marked this conversation as resolved.
Show resolved Hide resolved

// Given an absolute path as starting point, walkFileTree must return only relative paths.
List<Path> absolutePaths = postTraversalWalker.walkFileTree(fs.getPath("/dir/"));
for (Path p : absolutePaths) {
assertWithMessage("Should have been absolute: " + p.toString()).that(p.isAbsolute()).isTrue();
}
assertThat(absolutePaths.size()).isEqualTo(5);
}
}


@Test
public void testDeleteRecursive() throws IOException {
Expand Down Expand Up @@ -620,6 +667,32 @@ private String randomSuffix() {
return "-" + rnd.nextInt(99999);
}

private static class postTraversalWalker extends SimpleFileVisitor<Path> {
jean-philippe-martin marked this conversation as resolved.
Show resolved Hide resolved
private final List<Path> paths = new ArrayList<>();

// Traverse the tree, return the list of files and folders.
static public ImmutableList<Path> walkFileTree(Path start) throws IOException {
postTraversalWalker walker = new postTraversalWalker();
Files.walkFileTree(start, walker);
return walker.getPaths();
}

@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
paths.add(file);
return FileVisitResult.CONTINUE;
}

@Override
public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
paths.add(dir);
return FileVisitResult.CONTINUE;
}

public ImmutableList<Path> getPaths() {
return ImmutableList.copyOf(paths);
}
}

private CloudStorageFileSystem getTestBucket() throws IOException {
// in typical usage we use the single-argument version of forBucket
Expand Down