url: resource and query encoding should support unicode

python2.7 lacks proper unicode support, so we need
a proper wrapper function for url encoding. With
this the url and query encodings works for all
unicode characters.

Fixes #529

minio/api.py

@@ -45,7 +45,8 @@
 
 # Internal imports
 from . import __title__, __version__
-from .compat import urlsplit, range, urlencode, basestring
+from .compat import (urlsplit, queryencode,
+                     range, basestring)
 from .error import (KnownResponseError, ResponseError, NoSuchBucket,
                     InvalidArgumentError, InvalidSizeError, NoSuchBucketPolicy)
 from .definitions import Object, UploadPart
@@ -841,7 +842,7 @@ def copy_object(self, bucket_name, object_name, object_source,
         if conditions:
             headers = {k: v for k, v in conditions.items()}
 
-        headers['X-Amz-Copy-Source'] = urlencode(object_source)
+        headers['X-Amz-Copy-Source'] = queryencode(object_source)
         response = self._url_open('PUT',
                                   bucket_name=bucket_name,
                                   object_name=object_name,

minio/compat.py

@@ -35,7 +35,7 @@
 
 if _is_py2:
     from urllib import quote
-    urlencode = quote
+    _urlencode = quote
 
     from urllib import unquote
     urldecode = unquote
@@ -57,7 +57,7 @@
     basestring = basestring
 elif _is_py3:
     from urllib.request import quote
-    urlencode = quote
+    _urlencode = quote
 
     from urllib.request import unquote
     urldecode = unquote
@@ -80,3 +80,20 @@
     str = str
 
 numeric_types = (int, long, float)
+
+def urlencode(resource):
+    """
+    This implementation of urlencode supports all unicode characters
+
+    :param: resource: Resource value to be url encoded.
+    """
+    resource_raw = '{0}'.format(resource)
+    return _urlencode(resource_raw)
+
+def queryencode(query):
+    """
+    This implementation of queryencode supports all unicode characters
+
+    :param: query: Query value to be url encoded.
+    """
+    return urlencode(query).replace('/', '%2F')

minio/helpers.py

@@ -35,7 +35,8 @@
 import errno
 import math
 
-from .compat import urlsplit, urlencode, str, bytes, basestring
+from .compat import (urlsplit, urlencode, queryencode,
+                     str, bytes, basestring)
 from .error import (InvalidBucketError, InvalidEndpointError,
                     InvalidArgumentError)
 
@@ -226,13 +227,15 @@ def get_target_url(endpoint_url, bucket_name=None, object_name=None,
             if ordered_query[component_key] is not None:
                 if isinstance(ordered_query[component_key], list):
                     for value in ordered_query[component_key]:
-                        encoded_query = urlencode(
-                            str(value)).replace('/', '%2F')
-                        query_components.append(component_key+'='+encoded_query)
+                        query_components.append(component_key+'='+
+                                                queryencode(value))
                 else:
-                    encoded_query = urlencode(
-                        str(ordered_query[component_key])).replace('/', '%2F')
-                    query_components.append(component_key+'='+encoded_query)
+                    query_components.append(
+                        component_key+'='+
+                        queryencode(
+                            ordered_query[component_key]
+                        )
+                    )
             else:
                 query_components.append(component_key)
 

minio/signer.py

@@ -31,7 +31,7 @@
 
 from datetime import datetime
 from .error import InvalidArgumentError
-from .compat import urlsplit, urlencode
+from .compat import urlsplit, queryencode
 from .helpers import get_sha256_hexdigest
 from .fold_case_dict import FoldCaseDict
 
@@ -117,10 +117,8 @@ def presign_v4(method, url, access_key, secret_key, region=None,
             if ordered_query[component_key] is not None:
                 single_component.append('=')
                 single_component.append(
-                    urlencode(
-                        str(ordered_query[component_key])
-                    ).replace('/',
-                              '%2F'))
+                    queryencode(ordered_query[component_key])
+                )
             query_components.append(''.join(single_component))
 
         query_string = '&'.join(query_components)

tests/unit/sign_test.py

@@ -24,7 +24,7 @@
                           generate_signing_key, generate_authorization_header,
                           presign_v4)
 from minio.error import InvalidArgumentError
-from minio.compat import urlsplit
+from minio.compat import urlsplit, urlencode, queryencode
 from minio.fold_case_dict import FoldCaseDict
 
 empty_hash = 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855'
@@ -115,3 +115,10 @@ def test_presigned_no_access_key(self):
     @raises(InvalidArgumentError)
     def test_presigned_invalid_expires(self):
         presign_v4('GET', 'http://localhost:9000/hello', None, None, region=None, headers={}, expires=0)
+
+class UnicodeEncodeTest(TestCase):
+    def test_unicode_urlencode(self):
+        eq_(urlencode('/test/123/汉字'), '/test/123/%E6%B1%89%E5%AD%97')
+
+    def test_unicode_queryencode(self):
+        eq_(queryencode('/test/123/汉字'), '%2Ftest%2F123%2F%E6%B1%89%E5%AD%97')