Skip to content

Commit

Permalink
Rearranged the location of the sort function.
Browse files Browse the repository at this point in the history
  • Loading branch information
kyleknap committed Aug 6, 2014
1 parent 607c579 commit 4eb7773
Show file tree
Hide file tree
Showing 4 changed files with 32 additions and 33 deletions.
12 changes: 10 additions & 2 deletions awscli/customizations/s3/filegenerator.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
from awscli.customizations.s3.fileinfo import FileInfo
from awscli.customizations.s3.utils import find_bucket_key, get_file_stat
from awscli.customizations.s3.utils import BucketLister
from awscli.customizations.s3.utils import normalize_sort
from awscli.errorhandler import ClientError


Expand Down Expand Up @@ -125,7 +124,7 @@ def list_files(self, path, dir_op):
file_path = join(path, name)
if isdir(file_path):
names[i] = name + os.path.sep
normalize_sort(names, os.sep, '/')
self.normalize_sort(names, os.sep, '/')
for name in names:
file_path = join(path, name)
if not self.should_ignore_file(file_path):
Expand All @@ -142,6 +141,15 @@ def list_files(self, path, dir_op):
size, last_update = get_file_stat(file_path)
yield file_path, size, last_update

def normalize_sort(self, names, os_sep, character):
"""
The purpose of this function is to ensure that the same path seperator
is used when sorting. In windows, the path operator is a backslash as
opposed to a forward slash which can lead to differences in sorting
between s3 and a windows machine.
"""
names.sort(key=lambda item: item.replace(os_sep, character))

def _check_paths_decoded(self, path, names):
# We can get a UnicodeDecodeError if we try to listdir(<unicode>) and
# can't decode the contents with sys.getfilesystemencoding(). In this
Expand Down
10 changes: 0 additions & 10 deletions awscli/customizations/s3/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -231,16 +231,6 @@ def relative_path(filename, start=os.path.curdir):
return os.path.abspath(filename)


def normalize_sort(names, os_sep, character):
"""
The purpose of this function is to ensure that the same path seperator
is used when sorting. In windows, the path operator is a backslash as
opposed to a forward slash which can lead to differences in sorting
between s3 and a windows machine.
"""
names.sort(key=lambda item: item.replace(os_sep, character))


class ReadFileChunk(object):
def __init__(self, filename, start_byte, size):
self._filename = filename
Expand Down
22 changes: 22 additions & 0 deletions tests/unit/customizations/s3/test_filegenerator.py
Original file line number Diff line number Diff line change
Expand Up @@ -344,6 +344,28 @@ def test_list_local_files_with_unicode_chars(self):
self.assertEqual(values, expected_order)


class TestNormalizeSort(unittest.TestCase):
def test_normalize_sort(self):
names = ['xyz123456789',
'xyz1' + os.path.sep + 'test',
'xyz' + os.path.sep + 'test']
ref_names = [names[2], names[1], names[0]]
filegenerator = FileGenerator(None, None, None)
filegenerator.normalize_sort(names, os.path.sep, '/')
for i in range(len(ref_names)):
self.assertEqual(ref_names[i], names[i])

def test_normalize_sort_backslash(self):
names = ['xyz123456789',
'xyz1\\test',
'xyz\\test']
ref_names = [names[2], names[1], names[0]]
filegenerator = FileGenerator(None, None, None)
filegenerator.normalize_sort(names, '\\', '/')
for i in range(len(ref_names)):
self.assertEqual(ref_names[i], names[i])


class S3FileGeneratorTest(unittest.TestCase):
def setUp(self):
self.session = FakeSession()
Expand Down
21 changes: 0 additions & 21 deletions tests/unit/customizations/s3/test_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
from awscli.customizations.s3.utils import BucketLister
from awscli.customizations.s3.utils import ScopedEventHandler
from awscli.customizations.s3.utils import get_file_stat
from awscli.customizations.s3.utils import normalize_sort
from awscli.customizations.s3.constants import MAX_SINGLE_UPLOAD_SIZE


Expand Down Expand Up @@ -265,26 +264,6 @@ def test_urlencoded_with_unicode_keys(self):
# And note how it's been converted to '\r'.
self.assertEqual(objects, [(u'foo/\u2713', 1, now)])


class TestNormalizeSort(unittest.TestCase):
def test_normalize_sort(self):
names = ['xyz123456789',
'xyz1' + os.path.sep + 'test',
'xyz' + os.path.sep + 'test']
ref_names = [names[2], names[1], names[0]]
normalize_sort(names, os.path.sep, '/')
for i in range(len(ref_names)):
self.assertEqual(ref_names[i], names[i])

def test_normalize_sort_backslash(self):
names = ['xyz123456789',
'xyz1\\test',
'xyz\\test']
ref_names = [names[2], names[1], names[0]]
normalize_sort(names, '\\', '/')
for i in range(len(ref_names)):
self.assertEqual(ref_names[i], names[i])


class TestScopedEventHandler(unittest.TestCase):
def test_scoped_session_handler(self):
Expand Down

0 comments on commit 4eb7773

Please sign in to comment.