Skip to content

Commit

Permalink
consistently urlencode paths correctly
Browse files Browse the repository at this point in the history 
When Manta paths appear as part of the of a url (which they do on most
api requests), reserved characters must be encoded just like in any
other url.

Paths were encoded inconsistently (in that they were sometimes
encoded, sometimes not, and sometimes encoded twice).  When paths were
encoded, spaces were not encoded correctly (turned to `+`) because
despite the tantalizing name `java.net.URLEncoder` does does not
encode strings for inclusion in a url.

The contract for `MantaObject.getPath` has been clarified. It should
always return the "real" (un-encoded) path.

ref TritonDataCenter#229 TritonDataCenter#230 TritonDataCenter#231
  • Loading branch information
@cburroughs
cburroughs committed Apr 21, 2017
1 parent 60a3037 commit 6357b03
Show file tree
Hide file tree
Showing 14 changed files with 278 additions and 49 deletions.
21 changes: 21 additions & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,27 @@
All notable changes to this project will be documented in this file.
This project aims to adhere to [Semantic Versioning](http://semver.org/).

## [x.x.x] - YYYY-MM-DD

Several related URL encoding bugs have been fixed. Objects with
non-alphanumeric characters created by java-manta may have been
created with unexpected encoding characters. Only the object names
were affected, not the content.

### Changed
- Paths with URL unsafe characters are now encoded correctly.
Previously the
[space character was being transformed](https://github.com/joyent/java-manta/issues/229)
into a plus (`+`) character. So a PUT to the Manta object
`/user/stor/Hello World.txt`, would instead create
`/user/stor/Hello+World.txt`.
- All `MantaClient` operations now
[consistently encode](https://github.com/joyent/java-manta/issues/230),
and `MantaObject.getPath` always returns the original (not encoded)
path.
- Recursive directory creation will no longer
[encode the path twice](https://github.com/joyent/java-manta/issues/231).

## [3.0.0] - 2017-04-06
### Changed
- Upgraded HTTP Signatures library to 4.0.1.
Expand Down
36 changes: 34 additions & 2 deletions java-manta-cli/src/main/java/com/joyent/manta/cli/MantaCLI.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
package com.joyent.manta.cli;

import com.joyent.manta.client.MantaClient;
import com.joyent.manta.client.MantaObject;
import com.joyent.manta.client.MantaObjectResponse;
import com.joyent.manta.client.crypto.SecretKeyUtils;
import com.joyent.manta.config.ChainedConfigContext;
Expand All @@ -20,14 +21,15 @@
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.exception.ExceptionUtils;

import javax.crypto.SecretKey;
import javax.crypto.spec.SecretKeySpec;
import java.io.File;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.security.NoSuchAlgorithmException;
import java.util.stream.Stream;
import javax.crypto.SecretKey;
import javax.crypto.spec.SecretKeySpec;

/**
* Class providing a CLI interface to the Java Manta SDK.
Expand Down Expand Up @@ -83,6 +85,15 @@ public static void main(final String[] argv) {
System.out.println(generateKey(argv[1].trim(), Integer.valueOf(argv[2].trim()),
Paths.get(argv[3].trim())));
break;
case "ls":
if (argv.length < 2) {
System.err.println(help());
System.err.println();
System.err.println("ls requires one parameter: dir");
break;
}
System.out.println(listDir(argv[1].trim()));
break;
case "get-file":
if (argv.length < 2) {
System.err.println(help());
Expand Down Expand Up @@ -230,6 +241,27 @@ protected static String generateKey(final String cipher, final int bits,
return b.toString();
}


/**
* ls.
*
* @param dirPath dir to ls
* @return String containing the output of the operation
* @throws IOException thrown when we are unable to connect to Manta
*/
protected static String listDir(final String dirPath) throws IOException {
final StringBuilder b = new StringBuilder();
ConfigContext config = buildConfig();
try (MantaClient client = new MantaClient(config)) {
final Stream<MantaObject> objs = client.listObjects(dirPath);
objs.forEach(obj -> {
b.append(INDENT).append(obj.getPath()).append(BR);
});
}

return b.toString();
}

/**
* Performs a download of file in Manta.
* @param filePath in Manta to download
Expand Down
11 changes: 11 additions & 0 deletions java-manta-client/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,13 @@
</exclusions>
</dependency>

<!-- url encoding utility -->
<dependency>
<groupId>io.mikael</groupId>
<artifactId>urlbuilder</artifactId>
<version>${dependency.urlbuilder.version}</version>
</dependency>

<!-- These dependencies are declared at the module level because we can not
inherit exclusions from the parent. -->
<dependency>
Expand Down Expand Up @@ -227,6 +234,10 @@
<pattern>com.fasterxml</pattern>
<shadedPattern>com.joyent.manta.com.fasterxml</shadedPattern>
</relocation>
<relocation>
<pattern>io.mikael.urlbuilder</pattern>
<shadedPattern>com.joyent.manta.io.mikael.urlbuilder</shadedPattern>
</relocation>
</relocations>
<filters>
<!-- explicitly remove class that causes security concerns -->
Expand Down
37 changes: 20 additions & 17 deletions java-manta-client/src/main/java/com/joyent/manta/client/MantaClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -751,7 +751,7 @@ public Stream<MantaObject> listObjects(final String path) throws IOException {
}
}

return new MantaObjectResponse(objPath, headers);
return new MantaObjectResponse(formatPath(objPath), headers);
});

danglingStreams.add(stream);
Expand Down Expand Up @@ -798,7 +798,7 @@ public boolean isDirectoryEmpty(final String path) throws IOException {
*/
public boolean existsAndIsAccessible(final String path) {
try {
httpHelper.httpHead(path);
head(path);
} catch (IOException e) {
return false;
}
Expand Down Expand Up @@ -860,7 +860,7 @@ public MantaObjectResponse put(final String path,
/**
* Puts an object into Manta.
*
* @param path The path to the Manta object.
* @param rawPath The path to the Manta object.
* @param source {@link InputStream} to copy object data from
* @param contentLength the total length of the stream (-1 if unknown)
* @param headers optional HTTP headers to include when copying the object
Expand All @@ -869,13 +869,14 @@ public MantaObjectResponse put(final String path,
* @throws IOException If an IO exception has occurred.
* @throws MantaClientHttpResponseException If a http status code {@literal > 300} is returned.
*/
public MantaObjectResponse put(final String path,
public MantaObjectResponse put(final String rawPath,
final InputStream source,
final long contentLength,
final MantaHttpHeaders headers,
final MantaMetadata metadata) throws IOException {
Validate.notNull(path, "Path must not be null");
Validate.notNull(rawPath, "rawPath must not be null");
Validate.notNull(source, "Input stream must not be null");
final String path = formatPath(rawPath);

final ContentType contentType = ContentTypeLookup.findOrDefaultContentType(headers,
ContentType.APPLICATION_OCTET_STREAM);
Expand Down Expand Up @@ -1020,15 +1021,16 @@ public MantaObjectOutputStream putAsOutputStream(final String path,
* to upload using an {@link java.io.OutputStream}. Additionally, if you do not close()
* the stream, the data will not be uploaded.
*
* @param path The fully qualified path of the object. i.e. /user/stor/foo/bar/baz
* @param rawPath The fully qualified path of the object. i.e. /user/stor/foo/bar/baz
* @param metadata optional user-supplied metadata for object
* @param headers optional HTTP headers to include when copying the object
* @return A OutputStream that allows for directly uploading to Manta
*/
public MantaObjectOutputStream putAsOutputStream(final String path,
public MantaObjectOutputStream putAsOutputStream(final String rawPath,
final MantaHttpHeaders headers,
final MantaMetadata metadata) {
Validate.notNull(path, "Path must not be null");
Validate.notNull(rawPath, "rawPath must not be null");
final String path = formatPath(rawPath);

final ContentType contentType = ContentTypeLookup.findOrDefaultContentType(headers,
path, ContentType.APPLICATION_OCTET_STREAM);
Expand Down Expand Up @@ -1182,19 +1184,20 @@ public MantaObjectResponse put(final String path,
* Copies the supplied {@link File} to a remote Manta object at the specified
* path using the default JVM character encoding as a binary representation.
*
* @param path The fully qualified path of the object. i.e. /user/stor/foo/bar/baz
* @param rawPath The fully qualified path of the object. i.e. /user/stor/foo/bar/baz
* @param file file to upload
* @param headers optional HTTP headers to include when copying the object
* @param metadata optional user-supplied metadata for object
* @return Manta response object
* @throws IOException when there is a problem sending the object over the network
*/
public MantaObjectResponse put(final String path,
public MantaObjectResponse put(final String rawPath,
final File file,
final MantaHttpHeaders headers,
final MantaMetadata metadata) throws IOException {
Validate.notNull(path, "Path must not be null");
Validate.notNull(rawPath, "rawPath must not be null");
Validate.notNull(file, "File must not be null");
final String path = formatPath(rawPath);

if (!file.exists()) {
String msg = String.format("File doesn't exist: %s",
Expand Down Expand Up @@ -1266,19 +1269,20 @@ public MantaObjectResponse put(final String path,
* Copies the supplied byte array to a remote Manta object at the specified
* path using the default JVM character encoding as a binary representation.
*
* @param path The fully qualified path of the object. i.e. /user/stor/foo/bar/baz
* @param rawPath The fully qualified path of the object. i.e. /user/stor/foo/bar/baz
* @param bytes byte array to upload
* @param headers optional HTTP headers to include when copying the object
* @param metadata optional user-supplied metadata for object
* @return Manta response object
* @throws IOException when there is a problem sending the object over the network
*/
public MantaObjectResponse put(final String path,
public MantaObjectResponse put(final String rawPath,
final byte[] bytes,
final MantaHttpHeaders headers,
final MantaMetadata metadata) throws IOException {
Validate.notNull(path, "Path must not be null");
Validate.notNull(rawPath, "rawPath must not be null");
Validate.notNull(bytes, "Byte array must not be null");
final String path = formatPath(rawPath);

final ContentType contentType = ContentTypeLookup.findOrDefaultContentType(
headers, path, ContentType.APPLICATION_OCTET_STREAM);
Expand Down Expand Up @@ -1421,14 +1425,13 @@ public void putDirectory(final String path, final boolean recursive)
public void putDirectory(final String rawPath, final boolean recursive,
final MantaHttpHeaders headers)
throws IOException {
String path = formatPath(rawPath);

if (!recursive) {
putDirectory(path, headers);
putDirectory(rawPath, headers);
return;
}

final String[] parts = path.split(SEPARATOR);
final String[] parts = rawPath.split(SEPARATOR);
final Iterator<Path> itr = Paths.get("", parts).iterator();
final StringBuilder sb = new StringBuilder(SEPARATOR);

Expand Down
3 changes: 2 additions & 1 deletion java-manta-client/src/main/java/com/joyent/manta/client/MantaObject.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,8 @@ public interface MantaObject extends Serializable {
String MANTA_OBJECT_TYPE_DIRECTORY = "directory";

/**
* Returns the path value.
* Returns the (decoded) path value. In other words, the path as
* given by the user.
*
* @return the path
*/
Expand Down
12 changes: 8 additions & 4 deletions java-manta-client/src/main/java/com/joyent/manta/client/MantaObjectResponse.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
package com.joyent.manta.client;

import com.joyent.manta.http.MantaHttpHeaders;
import com.joyent.manta.util.MantaUtils;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.Validate;
Expand Down Expand Up @@ -113,7 +114,8 @@ public MantaObjectResponse() {
/**
* Creates a MantaObject.
*
* @param path The fully qualified path of the object in Manta. i.e. "/user/stor/path/to/some/file/or/dir".
* @param path The (encoded) fully qualified path of the object in
* Manta. i.e. "/user/stor/path/to/some/file/or/dir".
*/
public MantaObjectResponse(final String path) {
Validate.notNull(path, "Path must not be null");
Expand All @@ -125,7 +127,8 @@ public MantaObjectResponse(final String path) {
/**
* Creates a MantaObject.
*
* @param path The fully qualified path of the object in Manta. i.e. "/user/stor/path/to/some/file/or/dir".
* @param path The (encoded) fully qualified path of the object in
* Manta. i.e. "/user/stor/path/to/some/file/or/dir".
* @param headers Optional {@link MantaHttpHeaders}. Use this to set any additional headers on the Manta object.
* For the full list of Manta headers see the
* <a href="http://apidocs.joyent.com/manta/manta/">Manta API</a>.
Expand All @@ -137,7 +140,8 @@ public MantaObjectResponse(final String path, final MantaHttpHeaders headers) {
/**
* Creates a MantaObject.
*
* @param path The fully qualified path of the object in Manta. i.e. "/user/stor/path/to/some/file/or/dir".
* @param path The (encoded) fully qualified path of the object in
* Manta. i.e. "/user/stor/path/to/some/file/or/dir".
* @param headers Optional {@link MantaHttpHeaders}. Use this to set any additional headers on the Manta object.
* For the full list of Manta headers see the
* <a href="http://apidocs.joyent.com/manta/manta/">Manta API</a>.
Expand Down Expand Up @@ -179,7 +183,7 @@ public MantaObjectResponse(final String path, final MantaHttpHeaders headers,

@Override
public final String getPath() {
return this.path;
return MantaUtils.decodePath(this.path);
}

/**
Expand Down
6 changes: 5 additions & 1 deletion java-manta-client/src/main/java/com/joyent/manta/http/StandardHttpHelper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -281,7 +281,11 @@ public MantaObjectInputStream httpRequestAsInputStream(final HttpUriRequest requ

final MantaHttpHeaders responseHeaders = new MantaHttpHeaders(response.getAllHeaders());
final String path = request.getURI().getPath();
final MantaObjectResponse metadata = new MantaObjectResponse(path, responseHeaders);
// MantaObjectResponse expects to be constructed with the
// encoded path, which it then decodes when a caller does
// getPath. However, here the HttpUriRequest has already
// decoded.
final MantaObjectResponse metadata = new MantaObjectResponse(MantaUtils.formatPath(path), responseHeaders);

if (metadata.isDirectory()) {
final String msg = "Directories do not have data, so data streams "
Expand Down
35 changes: 20 additions & 15 deletions java-manta-client/src/main/java/com/joyent/manta/util/MantaUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@
*/
package com.joyent.manta.util;

import io.mikael.urlbuilder.util.Encoder;
import io.mikael.urlbuilder.util.Decoder;
import org.apache.commons.codec.binary.Hex;
import org.apache.commons.lang3.BooleanUtils;
import org.apache.commons.lang3.StringUtils;
Expand All @@ -17,10 +19,9 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.UnsupportedEncodingException;
import java.lang.reflect.Field;
import java.net.InetAddress;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
Expand Down Expand Up @@ -203,23 +204,27 @@ public static <T extends Enum<T>> T parseEnumOrNull(final Object value,
}

/**
* Format the path according to RFC3986.
* Format the manta path for inclusion in a URI path according to
* RFC3986. Note that the encoding rules vary for each part of
* the url
*
* @param path the raw path string.
* @return the URI formatted string with the exception of '/' which is special in manta.
* @throws UnsupportedEncodingException If UTF-8 is not supported on this system.
*/
public static String formatPath(final String path) throws UnsupportedEncodingException {
// first split the path by slashes.
final String[] elements = path.split("/");
final StringBuilder encodedPath = new StringBuilder();
for (final String string : elements) {
if (string.equals("")) {
continue;
}
encodedPath.append("/").append(URLEncoder.encode(string, "UTF-8"));
}
return encodedPath.toString();
public static String formatPath(final String path) {
final Encoder encoder = new Encoder(StandardCharsets.UTF_8);
return encoder.encodePath(path);
}

/**
* Decodes a percent encoded manta path.
*
* @param encodedPath The percent encoded path
* @return The percent decoded path
*/
public static String decodePath(final String encodedPath) {
final Decoder decoder = new Decoder(StandardCharsets.UTF_8);
return decoder.decodePath(encodedPath);
}

/**
Expand Down
Loading

0 comments on commit 6357b03

Please sign in to comment.