From 1e68f0abed7b0d05c52fe7969e16957c2ef5e6f8 Mon Sep 17 00:00:00 2001 From: kyleknap Date: Mon, 11 Aug 2014 16:18:28 -0700 Subject: [PATCH 1/3] Added support for warning about and skipping files. Files that do not exist or user does not have read access to are skipped and users are warned that the file is skipped. --- awscli/customizations/s3/executor.py | 18 +++- awscli/customizations/s3/filegenerator.py | 29 +++++- awscli/customizations/s3/s3handler.py | 10 ++- awscli/customizations/s3/subcommands.py | 15 +++- .../customizations/s3/test_plugin.py | 30 ++++++- .../customizations/s3/test_filegenerator.py | 90 +++++++++++++++---- 6 files changed, 164 insertions(+), 28 deletions(-) diff --git a/awscli/customizations/s3/executor.py b/awscli/customizations/s3/executor.py index 9cc5f7f153ab..dbfbe24b9ab5 100644 --- a/awscli/customizations/s3/executor.py +++ b/awscli/customizations/s3/executor.py @@ -62,6 +62,13 @@ def num_tasks_failed(self): tasks_failed = self.print_thread.num_errors_seen return tasks_failed + @property + def num_tasks_warned(self): + tasks_warned = 0 + if self.print_thread is not None: + tasks_warned = self.print_thread.num_warnings_seen + return tasks_warned + def start(self): self.io_thread.start() # Note that we're *not* adding the IO thread to the threads_list. @@ -233,6 +240,7 @@ def __init__(self, result_queue, quiet): # This is a public attribute that clients can inspect to determine # whether or not we saw any results indicating that an error occurred. self.num_errors_seen = 0 + self.num_warnings_seen = 0 def set_total_parts(self, total_parts): with self._lock: @@ -265,8 +273,16 @@ def _process_print_task(self, print_task): print_str = print_task['message'] if print_task['error']: self.num_errors_seen += 1 + warning = False + if 'warning' in print_task: + if print_task['warning']: + warning = True + self.num_warnings_seen += 1 final_str = '' - if 'total_parts' in print_task: + if warning: + final_str += print_str.ljust(self._progress_length, ' ') + final_str += '\n' + elif 'total_parts' in print_task: # Normalize keys so failures and sucess # look the same. op_list = print_str.split(':') diff --git a/awscli/customizations/s3/filegenerator.py b/awscli/customizations/s3/filegenerator.py index 7984a8a31e5a..4e37f91625be 100644 --- a/awscli/customizations/s3/filegenerator.py +++ b/awscli/customizations/s3/filegenerator.py @@ -14,6 +14,7 @@ import sys import six +from six.moves import queue from dateutil.parser import parse from dateutil.tz import tzlocal @@ -59,6 +60,16 @@ def __init__(self, src, dest=None, compare_key=None, size=None, self.operation_name = operation_name +def create_warning(path, error_message): + print_string = "WARNING: " + print_string = print_string + "Skipping file " + path + ". " + print_string = print_string + error_message + warning_message = {'message': print_string, + 'error': False, + 'warning': True} + return warning_message + + class FileGenerator(object): """ This is a class the creates a generator to yield files based on information @@ -68,11 +79,14 @@ class FileGenerator(object): ``FileInfo`` objects to send to a ``Comparator`` or ``S3Handler``. """ def __init__(self, service, endpoint, operation_name, - follow_symlinks=True): + follow_symlinks=True, result_queue=None): self._service = service self._endpoint = endpoint self.operation_name = operation_name self.follow_symlinks = follow_symlinks + self.result_queue = result_queue + if not result_queue: + self.result_queue = queue.Queue() def call(self, files): """ @@ -186,6 +200,19 @@ def should_ignore_file(self, path): path = path[:-1] if os.path.islink(path): return True + if self.throws_warning(path): + return True + return False + + def throws_warning(self, path): + if not os.path.exists(path): + warning = create_warning(path, "File does not exist.") + self.result_queue.put(warning) + return True + if not os.access(path, os.R_OK): + warning = create_warning(path, "File read access is denied.") + self.result_queue.put(warning) + return True return False def list_objects(self, s3_path, dir_op): diff --git a/awscli/customizations/s3/s3handler.py b/awscli/customizations/s3/s3handler.py index d17ab1359690..fa549681cdae 100644 --- a/awscli/customizations/s3/s3handler.py +++ b/awscli/customizations/s3/s3handler.py @@ -33,14 +33,16 @@ class pull tasks from to complete. """ MAX_IO_QUEUE_SIZE = 20 - def __init__(self, session, params, multi_threshold=MULTI_THRESHOLD, - chunksize=CHUNKSIZE): + def __init__(self, session, params, result_queue=None, + multi_threshold=MULTI_THRESHOLD, chunksize=CHUNKSIZE): self.session = session - self.result_queue = queue.Queue() # The write_queue has potential for optimizations, so the constant # for maxsize is scoped to this class (as opposed to constants.py) # so we have the ability to change this value later. self.write_queue = queue.Queue(maxsize=self.MAX_IO_QUEUE_SIZE) + self.result_queue = result_queue + if not self.result_queue: + self.result_queue = queue.Queue() self.params = {'dryrun': False, 'quiet': False, 'acl': None, 'guess_mime_type': True, 'sse': False, 'storage_class': None, 'website_redirect': None, @@ -94,7 +96,7 @@ def call(self, files): priority=self.executor.IMMEDIATE_PRIORITY) self._shutdown() self.executor.wait_until_shutdown() - return self.executor.num_tasks_failed + return [self.executor.num_tasks_failed, self.executor.num_tasks_warned] def _shutdown(self): # And finally we need to make a pass through all the existing diff --git a/awscli/customizations/s3/subcommands.py b/awscli/customizations/s3/subcommands.py index a4f9586aa9dc..9a379c2cab3f 100644 --- a/awscli/customizations/s3/subcommands.py +++ b/awscli/customizations/s3/subcommands.py @@ -12,6 +12,7 @@ # language governing permissions and limitations under the License. import os import six +from six.moves import queue import sys from dateutil.parser import parse @@ -554,13 +555,16 @@ def run(self): 'mb': 'make_bucket', 'rb': 'remove_bucket' } + result_queue = queue.Queue() operation_name = cmd_translation[paths_type][self.cmd] file_generator = FileGenerator(self._service, self._source_endpoint, operation_name, - self.parameters['follow_symlinks']) + self.parameters['follow_symlinks'], + result_queue=result_queue) rev_generator = FileGenerator(self._service, self._endpoint, '', - self.parameters['follow_symlinks']) + self.parameters['follow_symlinks'], + result_queue=result_queue) taskinfo = [TaskInfo(src=files['src']['path'], src_type='s3', operation_name=operation_name, @@ -568,7 +572,8 @@ def run(self): endpoint=self._endpoint)] file_info_builder = FileInfoBuilder(self._service, self._endpoint, self._source_endpoint, self.parameters) - s3handler = S3Handler(self.session, self.parameters) + s3handler = S3Handler(self.session, self.parameters, + result_queue=result_queue) command_dict = {} if self.cmd == 'sync': @@ -625,8 +630,10 @@ def run(self): # keeping it simple and saying that > 0 failed tasks # will give a 1 RC. rc = 0 - if files[0] > 0: + if files[0][0] > 0: rc = 1 + if files[0][1] > 0: + rc = 2 return rc diff --git a/tests/integration/customizations/s3/test_plugin.py b/tests/integration/customizations/s3/test_plugin.py index 56801f0a9970..3cca1a90b4d9 100644 --- a/tests/integration/customizations/s3/test_plugin.py +++ b/tests/integration/customizations/s3/test_plugin.py @@ -20,6 +20,7 @@ import platform import contextlib import time +import stat import signal import string @@ -673,6 +674,29 @@ def testMvRegion(self): self.assertFalse( self.key_exists(bucket_name=self.src_bucket, key_name='foo.txt')) +class TestWarnings(BaseS3CLICommand): + def extra_setup(self): + self.bucket_name = self.create_bucket() + + def test_no_exist(self): + filename = os.path.join(self.files.rootdir, "no-exists-file") + p = aws('s3 cp %s s3://%s/' % (filename, self.bucket_name)) + self.assertEqual(p.rc, 2, p.stdout) + self.assertIn('WARNING: Skipping file %s. File does not exist.' % + filename, p.stdout) + + def test_no_read_access(self): + self.files.create_file('foo.txt', 'foo') + filename = os.path.join(self.files.rootdir, 'foo.txt') + permissions = stat.S_IMODE(os.stat(filename).st_mode) + # Remove read permissions + permissions = permissions ^ stat.S_IRUSR + os.chmod(filename, permissions) + p = aws('s3 cp %s s3://%s/' % (filename, self.bucket_name)) + self.assertEqual(p.rc, 2, p.stdout) + self.assertIn('WARNING: Skipping file %s. File read access' + ' is denied.' % filename, p.stdout) + @unittest.skipIf(platform.system() not in ['Darwin', 'Linux'], 'Symlink tests only supported on mac/linux') @@ -751,8 +775,10 @@ def test_follow_symlinks_default(self): def test_bad_symlink(self): p = aws('s3 sync %s s3://%s/' % (self.files.rootdir, self.bucket_name)) - self.assertEqual(p.rc, 1, p.stdout) - self.assertIn('[Errno 2] No such file or directory', p.stdout) + self.assertEqual(p.rc, 2, p.stdout) + self.assertIn('WARNING: Skipping file %s. File does not exist.' % + os.path.join(self.files.rootdir, 'b-badsymlink'), + p.stdout) class TestUnicode(BaseS3CLICommand): diff --git a/tests/unit/customizations/s3/test_filegenerator.py b/tests/unit/customizations/s3/test_filegenerator.py index 96b2a2739c2b..082fb665d5bd 100644 --- a/tests/unit/customizations/s3/test_filegenerator.py +++ b/tests/unit/customizations/s3/test_filegenerator.py @@ -13,6 +13,7 @@ import os import platform from awscli.testutils import unittest, FileCreator +import stat import tempfile import shutil @@ -20,7 +21,7 @@ import mock from awscli.customizations.s3.filegenerator import FileGenerator, \ - FileDecodingError, FileStat + FileDecodingError, FileStat, create_warning from awscli.customizations.s3.utils import get_file_stat import botocore.session from tests.unit.customizations.s3 import make_loc_files, clean_loc_files, \ @@ -28,6 +29,17 @@ from tests.unit.customizations.s3.fake_session import FakeSession +class TestCreateWarning(unittest.TestCase): + def test_create_warning(self): + path = '/foo/' + error_message = 'There was an error' + warning_message = create_warning(path, error_message) + self.assertEqual(warning_message['message'], + 'WARNING: Skipping file /foo/. There was an error') + self.assertFalse(warning_message['error']) + self.assertTrue(warning_message['warning']) + + class LocalFileGeneratorTest(unittest.TestCase): def setUp(self): self.local_file = six.text_type(os.path.abspath('.') + @@ -120,12 +132,12 @@ def setUp(self): def tearDown(self): self.files.remove_all() - def test_bad_symlink(self): + def test_warning(self): path = os.path.join(self.files.rootdir, 'badsymlink') os.symlink('non-existent-file', path) filegenerator = FileGenerator(self.service, self.endpoint, '', True) - self.assertFalse(filegenerator.should_ignore_file(path)) + self.assertTrue(filegenerator.should_ignore_file(path)) def test_skip_symlink(self): filename = 'foo.txt' @@ -162,6 +174,52 @@ def test_no_skip_symlink_dir(self): self.assertFalse(filegenerator.should_ignore_file(path)) +class TestThrowsWarning(unittest.TestCase): + def setUp(self): + self.files = FileCreator() + self.root = self.files.rootdir + self.session = FakeSession() + self.service = self.session.get_service('s3') + self.endpoint = self.service.get_endpoint('us-east-1') + + def tearDown(self): + self.files.remove_all() + + def test_no_warning(self): + file_gen = FileGenerator(self.service, self.endpoint, '', False) + self.files.create_file("foo.txt", contents="foo") + full_path = os.path.join(self.root, "foo.txt") + return_val = file_gen.throws_warning(full_path) + self.assertFalse(return_val) + self.assertTrue(file_gen.result_queue.empty()) + + def test_no_exists(self): + file_gen = FileGenerator(self.service, self.endpoint, '', False) + symlink = os.path.join(self.root, 'symlink') + os.symlink('non-existent-file', symlink) + return_val = file_gen.throws_warning(symlink) + self.assertTrue(return_val) + warning_message = file_gen.result_queue.get() + self.assertEqual(warning_message['message'], + ("WARNING: Skipping file %s. File does not exist." % + symlink)) + + def test_no_read_access(self): + file_gen = FileGenerator(self.service, self.endpoint, '', False) + self.files.create_file("foo.txt", contents="foo") + full_path = os.path.join(self.root, "foo.txt") + permissions = stat.S_IMODE(os.stat(full_path).st_mode) + # Remove read permissions + permissions = permissions ^ stat.S_IRUSR + os.chmod(full_path, permissions) + return_val = file_gen.throws_warning(full_path) + self.assertTrue(return_val) + warning_message = file_gen.result_queue.get() + self.assertEqual(warning_message['message'], + ("WARNING: Skipping file %s. File read access is " + "denied." % full_path)) + + @unittest.skipIf(platform.system() not in ['Darwin', 'Linux'], 'Symlink tests only supported on mac/linux') class TestSymlinksIgnoreFiles(unittest.TestCase): @@ -229,7 +287,7 @@ def test_no_follow_symlink(self): filename = six.text_type(os.path.abspath(self.filenames[i])) self.assertEqual(result_list[i], filename) - def test_follow_bad_symlink(self): + def test_warn_bad_symlink(self): """ This tests to make sure it fails when following bad symlinks. """ @@ -241,19 +299,19 @@ def test_follow_bad_symlink(self): 'dir_op': True, 'use_src_name': True} file_stats = FileGenerator(self.service, self.endpoint, '', True).call(input_local_dir) + file_gen = FileGenerator(self.service, self.endpoint, '', True) + file_stats = file_gen.call(input_local_dir) + all_filenames = self.filenames + self.symlink_files + all_filenames.sort() result_list = [] - rc = 0 - try: - for file_stat in file_stats: - result_list.append(getattr(file_stat, 'src')) - rc = 1 - except OSError as e: - pass - # Error shows up as ValueError in Python 3. - except ValueError as e: - pass - self.assertEquals(0, rc) - + for file_stat in file_stats: + result_list.append(getattr(file_stat, 'src')) + self.assertEqual(len(result_list), len(all_filenames)) + # Just check to make sure the right local files are generated. + for i in range(len(result_list)): + filename = six.text_type(os.path.abspath(all_filenames[i])) + self.assertEqual(result_list[i], filename) + self.assertFalse(file_gen.result_queue.empty()) def test_follow_symlink(self): # First remove the bad symlink. From 348302dbb70a7f64ace0f98d41242ec912f543f5 Mon Sep 17 00:00:00 2001 From: kyleknap Date: Thu, 14 Aug 2014 20:44:33 -0700 Subject: [PATCH 2/3] Enabled skipping and warning special files. Files include character special devices, block special devices, FIFO, and sockets. --- awscli/compat.py | 25 ++++++++ awscli/customizations/s3/executor.py | 17 ++--- awscli/customizations/s3/filegenerator.py | 39 +++++++----- awscli/customizations/s3/s3handler.py | 25 +++++--- awscli/customizations/s3/subcommands.py | 14 +++-- awscli/customizations/s3/tasks.py | 21 ++++--- awscli/customizations/s3/utils.py | 26 ++++++++ .../customizations/s3/test_plugin.py | 24 +++++-- tests/unit/customizations/s3/test_executor.py | 19 +++++- .../customizations/s3/test_filegenerator.py | 45 ++++++++------ tests/unit/customizations/s3/test_utils.py | 12 ++++ tests/unit/test_compat.py | 62 +++++++++++++++++++ 12 files changed, 253 insertions(+), 76 deletions(-) create mode 100644 tests/unit/test_compat.py diff --git a/awscli/compat.py b/awscli/compat.py index a664d8ea7891..e79c078f9b62 100644 --- a/awscli/compat.py +++ b/awscli/compat.py @@ -10,8 +10,33 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. +import os import sys import six +import stat + + +def is_special_file(path): + """ + This function checks to see if a special file. It checks if the + file is a character special device, block special device, FIFO, or + socket. + """ + mode = os.stat(path).st_mode + # Character special device. + if stat.S_ISCHR(mode): + return True + # Block special device + if stat.S_ISBLK(mode): + return True + # FIFO. + if stat.S_ISFIFO(mode): + return True + # Socket. + if stat.S_ISSOCK(mode): + return True + return False + if six.PY3: import locale diff --git a/awscli/customizations/s3/executor.py b/awscli/customizations/s3/executor.py index dbfbe24b9ab5..872f181ef055 100644 --- a/awscli/customizations/s3/executor.py +++ b/awscli/customizations/s3/executor.py @@ -213,7 +213,8 @@ class PrintThread(threading.Thread): Result Queue ------------ - Result queue items are dictionaries that have the following keys: + Result queue items are PrintTask objects that have the following + attributes: * message: An arbitrary string associated with the entry. This can be used to communicate the result of the task. @@ -221,6 +222,8 @@ class PrintThread(threading.Thread): successfully. * total_parts: The total number of parts for multipart transfers ( deprecated, will be removed in the future). + * warning: Boolean indicating whether or not a file generated a + warning. """ def __init__(self, result_queue, quiet): @@ -270,24 +273,24 @@ def run(self): pass def _process_print_task(self, print_task): - print_str = print_task['message'] - if print_task['error']: + print_str = print_task.message + if print_task.error: self.num_errors_seen += 1 warning = False - if 'warning' in print_task: - if print_task['warning']: + if print_task.warning: + if print_task.warning: warning = True self.num_warnings_seen += 1 final_str = '' if warning: final_str += print_str.ljust(self._progress_length, ' ') final_str += '\n' - elif 'total_parts' in print_task: + elif print_task.total_parts: # Normalize keys so failures and sucess # look the same. op_list = print_str.split(':') print_str = ':'.join(op_list[1:]) - total_part = print_task['total_parts'] + total_part = print_task.total_parts self._num_parts += 1 if print_str in self._progress_dict: self._progress_dict[print_str]['parts'] += 1 diff --git a/awscli/customizations/s3/filegenerator.py b/awscli/customizations/s3/filegenerator.py index 4e37f91625be..86fed796b4b2 100644 --- a/awscli/customizations/s3/filegenerator.py +++ b/awscli/customizations/s3/filegenerator.py @@ -19,9 +19,9 @@ from dateutil.tz import tzlocal from awscli.customizations.s3.utils import find_bucket_key, get_file_stat -from awscli.customizations.s3.utils import BucketLister +from awscli.customizations.s3.utils import BucketLister, create_warning from awscli.errorhandler import ClientError - +from awscli.compat import is_special_file # This class is provided primarily to provide a detailed error message. @@ -60,16 +60,6 @@ def __init__(self, src, dest=None, compare_key=None, size=None, self.operation_name = operation_name -def create_warning(path, error_message): - print_string = "WARNING: " - print_string = print_string + "Skipping file " + path + ". " - print_string = print_string + error_message - warning_message = {'message': print_string, - 'error': False, - 'warning': True} - return warning_message - - class FileGenerator(object): """ This is a class the creates a generator to yield files based on information @@ -192,7 +182,7 @@ def should_ignore_file(self, path): """ This function checks whether a file should be ignored in the file generation process. This includes symlinks that are not to be - followed. + followed and files that generate warnings. """ if not self.follow_symlinks: if os.path.isdir(path) and path.endswith(os.sep): @@ -200,17 +190,34 @@ def should_ignore_file(self, path): path = path[:-1] if os.path.islink(path): return True - if self.throws_warning(path): + warning_triggered = self.triggers_warning(path) + if warning_triggered: return True return False - def throws_warning(self, path): + def triggers_warning(self, path): + """ + This function checks the specific types and properties of a file. + If the file would cause trouble, the function adds a + warning to the result queue to be printed out and returns a boolean + value notify whether the file caused a warning to be generated. + Files that generate warnings are skipped. Currently, this function + checks for files that do not exist and files that the user does + not have read access. + """ if not os.path.exists(path): warning = create_warning(path, "File does not exist.") self.result_queue.put(warning) return True if not os.access(path, os.R_OK): - warning = create_warning(path, "File read access is denied.") + warning = create_warning(path, "Read access is denied.") + self.result_queue.put(warning) + return True + if is_special_file(path): + warning = create_warning(path, + ("File is character special device, " + "block special device, FIFO, or " + "socket.")) self.result_queue.put(warning) return True return False diff --git a/awscli/customizations/s3/s3handler.py b/awscli/customizations/s3/s3handler.py index fa549681cdae..91f701bbd83d 100644 --- a/awscli/customizations/s3/s3handler.py +++ b/awscli/customizations/s3/s3handler.py @@ -10,6 +10,7 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. +from collections import namedtuple import logging import math import os @@ -18,12 +19,15 @@ from awscli.customizations.s3.constants import MULTI_THRESHOLD, CHUNKSIZE, \ NUM_THREADS, MAX_UPLOAD_SIZE, MAX_QUEUE_SIZE from awscli.customizations.s3.utils import find_chunksize, \ - operate, find_bucket_key, relative_path + operate, find_bucket_key, relative_path, PrintTask, create_warning from awscli.customizations.s3.executor import Executor from awscli.customizations.s3 import tasks LOGGER = logging.getLogger(__name__) +CommandResult = namedtuple('CommandResult', + ['num_tasks_failed', 'num_tasks_warned']) + class S3Handler(object): """ @@ -84,19 +88,22 @@ def call(self, files): except Exception as e: LOGGER.debug('Exception caught during task execution: %s', str(e), exc_info=True) - self.result_queue.put({'message': str(e), 'error': True}) + self.result_queue.put(PrintTask(message=str(e), error=True)) self.executor.initiate_shutdown( priority=self.executor.IMMEDIATE_PRIORITY) self._shutdown() self.executor.wait_until_shutdown() except KeyboardInterrupt: - self.result_queue.put({'message': "Cleaning up. Please wait...", - 'error': True}) + self.result_queue.put(PrintTask(message=("Cleaning up. " + "Please wait..."), + error=True)) self.executor.initiate_shutdown( priority=self.executor.IMMEDIATE_PRIORITY) self._shutdown() self.executor.wait_until_shutdown() - return [self.executor.num_tasks_failed, self.executor.num_tasks_warned] + + return CommandResult(self.executor.num_tasks_failed, + self.executor.num_tasks_warned) def _shutdown(self): # And finally we need to make a pass through all the existing @@ -160,9 +167,10 @@ def _enqueue_tasks(self, files): if hasattr(filename, 'size'): too_large = filename.size > MAX_UPLOAD_SIZE if too_large and filename.operation_name == 'upload': - warning = "Warning %s exceeds 5 TB and upload is " \ - "being skipped" % relative_path(filename.src) - self.result_queue.put({'message': warning, 'error': True}) + warning_message = "File exceeds s3 upload limit of 5 TB." + warning = create_warning(relative_path(filename.src), + message=warning_message) + self.result_queue.put(warning) elif is_multipart_task and not self.params['dryrun']: # If we're in dryrun mode, then we don't need the # real multipart tasks. We can just use a BasicTask @@ -302,3 +310,4 @@ def _enqueue_upload_end_task(self, filename, upload_context): result_queue=self.result_queue, upload_context=upload_context) self.executor.submit(complete_multipart_upload_task) self._multipart_uploads.append((upload_context, filename)) + diff --git a/awscli/customizations/s3/subcommands.py b/awscli/customizations/s3/subcommands.py index 9a379c2cab3f..43185f0108b9 100644 --- a/awscli/customizations/s3/subcommands.py +++ b/awscli/customizations/s3/subcommands.py @@ -625,14 +625,16 @@ def run(self): # will replaces the files attr with the return value of the # file_list. The very last call is a single list of # [s3_handler], and the s3_handler returns the number of - # tasks failed. This means that files[0] now contains - # the number of failed tasks. In terms of the RC, we're - # keeping it simple and saying that > 0 failed tasks - # will give a 1 RC. + # tasks failed and the number of tasks warned. + # This means that files[0] now contains a namedtuple with + # the number of failed tasks and the number of warned tasks. + # In terms of the RC, we're keeping it simple and saying + # that > 0 failed tasks will give a 1 RC and > 0 warned + # tasks will give a 2 RC. Otherwise a RC of zero is returned. rc = 0 - if files[0][0] > 0: + if files[0].num_tasks_failed > 0: rc = 1 - if files[0][1] > 0: + if files[0].num_tasks_warned > 0: rc = 2 return rc diff --git a/awscli/customizations/s3/tasks.py b/awscli/customizations/s3/tasks.py index 430004ca3ae8..37089c42a2b9 100644 --- a/awscli/customizations/s3/tasks.py +++ b/awscli/customizations/s3/tasks.py @@ -9,7 +9,8 @@ from botocore.exceptions import IncompleteReadError from awscli.customizations.s3.utils import find_bucket_key, MD5Error, \ - operate, ReadFileChunk, relative_path, IORequest, IOCloseRequest + operate, ReadFileChunk, relative_path, IORequest, IOCloseRequest, \ + PrintTask LOGGER = logging.getLogger(__name__) @@ -113,7 +114,7 @@ def _queue_print_message(self, filename, failed, dryrun, if error_message is not None: message += ' ' + error_message result = {'message': message, 'error': failed} - self.result_queue.put(result) + self.result_queue.put(PrintTask(**result)) except Exception as e: LOGGER.debug('%s' % str(e)) @@ -165,7 +166,7 @@ def __call__(self): message = print_operation(self._filename, 0) result = {'message': message, 'total_parts': self._total_parts(), 'error': False} - self._result_queue.put(result) + self._result_queue.put(PrintTask(**result)) except UploadCancelledError as e: # We don't need to do anything in this case. The task # has been cancelled, and the task that cancelled the @@ -178,7 +179,7 @@ def __call__(self): dryrun=False) message += '\n' + str(e) result = {'message': message, 'error': True} - self._result_queue.put(result) + self._result_queue.put(PrintTask(**result)) self._upload_context.cancel_upload() else: LOGGER.debug("Copy part number %s completed for filename: %s", @@ -235,7 +236,7 @@ def __call__(self): message = print_operation(self._filename, 0) result = {'message': message, 'total_parts': total, 'error': False} - self._result_queue.put(result) + self._result_queue.put(PrintTask(**result)) except UploadCancelledError as e: # We don't need to do anything in this case. The task # has been cancelled, and the task that cancelled the @@ -248,7 +249,7 @@ def __call__(self): dryrun=False) message += '\n' + str(e) result = {'message': message, 'error': True} - self._result_queue.put(result) + self._result_queue.put(PrintTask(**result)) self._upload_context.cancel_upload() else: LOGGER.debug("Part number %s completed for filename: %s", @@ -303,7 +304,7 @@ def __call__(self): message = print_operation(self._filename, False, self._parameters['dryrun']) print_task = {'message': message, 'error': False} - self._result_queue.put(print_task) + self._result_queue.put(PrintTask(**print_task)) self._io_queue.put(IOCloseRequest(self._filename.dest)) @@ -369,7 +370,7 @@ def _download_part(self): total_parts = int(self._filename.size / self._chunk_size) result = {'message': message, 'error': False, 'total_parts': total_parts} - self._result_queue.put(result) + self._result_queue.put(PrintTask(**result)) LOGGER.debug("Task complete: %s", self) return except (socket.timeout, socket.error) as e: @@ -426,7 +427,7 @@ def __call__(self): self.parameters['dryrun']) message += '\n' + str(e) result = {'message': message, 'error': True} - self.result_queue.put(result) + self.result_queue.put(PrintTask(**result)) raise e @@ -485,7 +486,7 @@ def __call__(self): self.parameters['dryrun']) result = {'message': message, 'error': False} self._upload_context.announce_completed() - self.result_queue.put(result) + self.result_queue.put(PrintTask(**result)) class RemoveFileTask(BasicTask): diff --git a/awscli/customizations/s3/utils.py b/awscli/customizations/s3/utils.py index c96c47fb94c2..c02b4c572ea9 100644 --- a/awscli/customizations/s3/utils.py +++ b/awscli/customizations/s3/utils.py @@ -171,6 +171,18 @@ def check_error(response_data): raise Exception("Error: %s\n" % error['Message']) +def create_warning(path, error_message): + """ + This creates a ``PrintTask`` for whenever a warning is to be thrown. + """ + print_string = "warning: " + print_string = print_string + "Skipping file " + path + ". " + print_string = print_string + error_message + warning_message = PrintTask(message=print_string, error=False, + warning=True) + return warning_message + + def operate(service, cmd, kwargs): """ A helper function that universally calls any command by taking in the @@ -369,6 +381,20 @@ def __exit__(self, exc_type, exc_value, traceback): self._unique_id) +class PrintTask(namedtuple('PrintTask', + ['message', 'error', 'total_parts', 'warning'])): + def __new__(cls, message, error=False, total_parts=None, warning=None): + """ + :var message: An arbitrary string associated with the entry. This + can be used to communicate the result of the task. + :var error: Boolean indicating a failure. + :var total_parts: The total number of parts for multipart transfers. + :var warning: Boolean indicating a warning + """ + return super(PrintTask, cls).__new__(cls, message, error, total_parts, + warning) + + IORequest = namedtuple('IORequest', ['filename', 'offset', 'data']) # Used to signal that IO for the filename is finished, and that # any associated resources may be cleaned up. diff --git a/tests/integration/customizations/s3/test_plugin.py b/tests/integration/customizations/s3/test_plugin.py index 3cca1a90b4d9..e41047da5a81 100644 --- a/tests/integration/customizations/s3/test_plugin.py +++ b/tests/integration/customizations/s3/test_plugin.py @@ -23,6 +23,7 @@ import stat import signal import string +import socket import botocore.session import six @@ -682,20 +683,31 @@ def test_no_exist(self): filename = os.path.join(self.files.rootdir, "no-exists-file") p = aws('s3 cp %s s3://%s/' % (filename, self.bucket_name)) self.assertEqual(p.rc, 2, p.stdout) - self.assertIn('WARNING: Skipping file %s. File does not exist.' % + self.assertIn('warning: Skipping file %s. File does not exist.' % filename, p.stdout) def test_no_read_access(self): self.files.create_file('foo.txt', 'foo') filename = os.path.join(self.files.rootdir, 'foo.txt') permissions = stat.S_IMODE(os.stat(filename).st_mode) - # Remove read permissions - permissions = permissions ^ stat.S_IRUSR + # Remove read permissions + permissions = permissions ^ stat.S_IREAD os.chmod(filename, permissions) p = aws('s3 cp %s s3://%s/' % (filename, self.bucket_name)) self.assertEqual(p.rc, 2, p.stdout) - self.assertIn('WARNING: Skipping file %s. File read access' - ' is denied.' % filename, p.stdout) + self.assertIn('warning: Skipping file %s. Read access' + ' is denied.' % filename, p.stdout) + + def test_is_special_file(self): + file_path = os.path.join(self.files.rootdir, 'foo') + # Use socket for special file. + sock=socket.socket(socket.AF_UNIX,socket.SOCK_STREAM) + sock.bind(file_path) + p = aws('s3 cp %s s3://%s/' % (file_path, self.bucket_name)) + self.assertEqual(p.rc, 2, p.stdout) + self.assertIn(("warning: Skipping file %s. File is character " + "special device, block special device, FIFO, or " + "socket." % file_path), p.stdout) @unittest.skipIf(platform.system() not in ['Darwin', 'Linux'], @@ -776,7 +788,7 @@ def test_follow_symlinks_default(self): def test_bad_symlink(self): p = aws('s3 sync %s s3://%s/' % (self.files.rootdir, self.bucket_name)) self.assertEqual(p.rc, 2, p.stdout) - self.assertIn('WARNING: Skipping file %s. File does not exist.' % + self.assertIn('warning: Skipping file %s. File does not exist.' % os.path.join(self.files.rootdir, 'b-badsymlink'), p.stdout) diff --git a/tests/unit/customizations/s3/test_executor.py b/tests/unit/customizations/s3/test_executor.py index 0b3b6c8a8739..9afaacd3ba22 100644 --- a/tests/unit/customizations/s3/test_executor.py +++ b/tests/unit/customizations/s3/test_executor.py @@ -13,14 +13,17 @@ import os import tempfile import shutil -import mock +import six from six.moves import queue +import mock + from awscli.testutils import unittest, temporary_file from awscli.customizations.s3.executor import IOWriterThread from awscli.customizations.s3.executor import ShutdownThreadRequest -from awscli.customizations.s3.executor import Executor -from awscli.customizations.s3.utils import IORequest, IOCloseRequest +from awscli.customizations.s3.executor import Executor, PrintThread +from awscli.customizations.s3.utils import IORequest, IOCloseRequest, \ + PrintTask class TestIOWriterThread(unittest.TestCase): @@ -86,3 +89,13 @@ def __call__(self): executor.initiate_shutdown() executor.wait_until_shutdown() self.assertEqual(open(f.name, 'rb').read(), b'foobar') + +class TestPrintThread(unittest.TestCase): + def test_print_warning(self): + result_queue = queue.Queue() + print_task = PrintTask(message="Bad File.", warning=True) + thread = PrintThread(result_queue, False) + with mock.patch('sys.stdout', new=six.StringIO()) as mock_stdout: + thread._process_print_task(print_task) + self.assertIn("Bad File.", mock_stdout.getvalue()) + diff --git a/tests/unit/customizations/s3/test_filegenerator.py b/tests/unit/customizations/s3/test_filegenerator.py index 082fb665d5bd..5febc10fbbcd 100644 --- a/tests/unit/customizations/s3/test_filegenerator.py +++ b/tests/unit/customizations/s3/test_filegenerator.py @@ -16,12 +16,13 @@ import stat import tempfile import shutil +import socket import six import mock from awscli.customizations.s3.filegenerator import FileGenerator, \ - FileDecodingError, FileStat, create_warning + FileDecodingError, FileStat from awscli.customizations.s3.utils import get_file_stat import botocore.session from tests.unit.customizations.s3 import make_loc_files, clean_loc_files, \ @@ -29,17 +30,6 @@ from tests.unit.customizations.s3.fake_session import FakeSession -class TestCreateWarning(unittest.TestCase): - def test_create_warning(self): - path = '/foo/' - error_message = 'There was an error' - warning_message = create_warning(path, error_message) - self.assertEqual(warning_message['message'], - 'WARNING: Skipping file /foo/. There was an error') - self.assertFalse(warning_message['error']) - self.assertTrue(warning_message['warning']) - - class LocalFileGeneratorTest(unittest.TestCase): def setUp(self): self.local_file = six.text_type(os.path.abspath('.') + @@ -189,7 +179,7 @@ def test_no_warning(self): file_gen = FileGenerator(self.service, self.endpoint, '', False) self.files.create_file("foo.txt", contents="foo") full_path = os.path.join(self.root, "foo.txt") - return_val = file_gen.throws_warning(full_path) + return_val = file_gen.triggers_warning(full_path) self.assertFalse(return_val) self.assertTrue(file_gen.result_queue.empty()) @@ -197,11 +187,11 @@ def test_no_exists(self): file_gen = FileGenerator(self.service, self.endpoint, '', False) symlink = os.path.join(self.root, 'symlink') os.symlink('non-existent-file', symlink) - return_val = file_gen.throws_warning(symlink) + return_val = file_gen.triggers_warning(symlink) self.assertTrue(return_val) warning_message = file_gen.result_queue.get() - self.assertEqual(warning_message['message'], - ("WARNING: Skipping file %s. File does not exist." % + self.assertEqual(warning_message.message, + ("warning: Skipping file %s. File does not exist." % symlink)) def test_no_read_access(self): @@ -210,15 +200,30 @@ def test_no_read_access(self): full_path = os.path.join(self.root, "foo.txt") permissions = stat.S_IMODE(os.stat(full_path).st_mode) # Remove read permissions - permissions = permissions ^ stat.S_IRUSR + permissions = permissions ^ stat.S_IREAD os.chmod(full_path, permissions) - return_val = file_gen.throws_warning(full_path) + return_val = file_gen.triggers_warning(full_path) self.assertTrue(return_val) warning_message = file_gen.result_queue.get() - self.assertEqual(warning_message['message'], - ("WARNING: Skipping file %s. File read access is " + self.assertEqual(warning_message.message, + ("warning: Skipping file %s. Read access is " "denied." % full_path)) + def test_is_special_file_warning(self): + file_gen = FileGenerator(self.service, self.endpoint, '', False) + file_path = os.path.join(self.files.rootdir, 'foo') + # Use socket for special file. + sock=socket.socket(socket.AF_UNIX,socket.SOCK_STREAM) + sock.bind(file_path) + return_val = file_gen.triggers_warning(file_path) + self.assertTrue(return_val) + warning_message = file_gen.result_queue.get() + self.assertEqual(warning_message.message, + ("warning: Skipping file %s. File is character " + "special device, block special device, FIFO, or " + "socket." % file_path)) + + @unittest.skipIf(platform.system() not in ['Darwin', 'Linux'], 'Symlink tests only supported on mac/linux') diff --git a/tests/unit/customizations/s3/test_utils.py b/tests/unit/customizations/s3/test_utils.py index 9da4a1434764..fad31afc2c66 100644 --- a/tests/unit/customizations/s3/test_utils.py +++ b/tests/unit/customizations/s3/test_utils.py @@ -19,6 +19,7 @@ from awscli.customizations.s3.utils import ScopedEventHandler from awscli.customizations.s3.utils import get_file_stat from awscli.customizations.s3.utils import AppendFilter +from awscli.customizations.s3.utils import create_warning from awscli.customizations.s3.constants import MAX_SINGLE_UPLOAD_SIZE @@ -47,6 +48,17 @@ def test_unicode(self): self.assertEqual(key, '\u5678') +class TestCreateWarning(unittest.TestCase): + def test_create_warning(self): + path = '/foo/' + error_message = 'There was an error' + warning_message = create_warning(path, error_message) + self.assertEqual(warning_message.message, + 'warning: Skipping file /foo/. There was an error') + self.assertFalse(warning_message.error) + self.assertTrue(warning_message.warning) + + class FindChunksizeTest(unittest.TestCase): """ This test ensures that the ``find_chunksize`` function works diff --git a/tests/unit/test_compat.py b/tests/unit/test_compat.py new file mode 100644 index 000000000000..3e5bbd78c0fb --- /dev/null +++ b/tests/unit/test_compat.py @@ -0,0 +1,62 @@ +# Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +import os +import stat +import socket + +from mock import Mock, patch + +from awscli.compat import is_special_file +from awscli.testutils import unittest, FileCreator + + +class TestIsSpecialFile(unittest.TestCase): + def setUp(self): + self.files = FileCreator() + self.filename = 'foo' + + def tearDown(self): + self.files.remove_all() + + def test_is_character_device(self): + mock_class = Mock() + mock_class.return_value = True + file_path = os.path.join(self.files.rootdir, self.filename) + self.files.create_file(self.filename, contents='') + with patch('stat.S_ISCHR') as mock_class: + self.assertTrue(is_special_file(file_path)) + + def test_is_block_device(self): + mock_class = Mock() + mock_class.return_value = True + file_path = os.path.join(self.files.rootdir, self.filename) + self.files.create_file(self.filename, contents='') + with patch('stat.S_ISBLK') as mock_class: + self.assertTrue(is_special_file(file_path)) + + def test_is_fifo(self): + file_path = os.path.join(self.files.rootdir, self.filename) + mode = 0o600 | stat.S_IFIFO + os.mknod(file_path, mode) + self.assertTrue(is_special_file(file_path)) + + + def test_is_socket(self): + file_path = os.path.join(self.files.rootdir, self.filename) + sock=socket.socket(socket.AF_UNIX,socket.SOCK_STREAM) + sock.bind(file_path) + self.assertTrue(is_special_file(file_path)) + + +if __name__ == "__main__": + unittest.main() From 8b1320ebc5dca4c774cbd89d3177de685f6bb23d Mon Sep 17 00:00:00 2001 From: kyleknap Date: Fri, 15 Aug 2014 14:53:23 -0700 Subject: [PATCH 3/3] Added Windows compatibility to warning about files. --- CHANGELOG.rst | 5 ++ awscli/compat.py | 25 ------ awscli/customizations/s3/filegenerator.py | 53 +++++++++-- awscli/customizations/s3/utils.py | 8 +- .../customizations/s3/test_plugin.py | 11 ++- .../customizations/s3/test_filegenerator.py | 89 +++++++++++++++++-- tests/unit/test_compat.py | 62 ------------- 7 files changed, 145 insertions(+), 108 deletions(-) delete mode 100644 tests/unit/test_compat.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 6269473dc25e..ac223359cad9 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -5,6 +5,11 @@ CHANGELOG Next Release (TBD) ================== +* bugfix:``aws s3``: Added support for ignoring and warning + about files that do not exist, user does not have read + permissions, or are special files (i.e. sockets, FIFOs, + character special devices, and block special devices) + (`issue 881 `__) * feature:Parameter Shorthand: Added support for ``structure(list-scalar, scalar)`` parameter shorthand. (`issue 882 `__) diff --git a/awscli/compat.py b/awscli/compat.py index e79c078f9b62..a664d8ea7891 100644 --- a/awscli/compat.py +++ b/awscli/compat.py @@ -10,33 +10,8 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. -import os import sys import six -import stat - - -def is_special_file(path): - """ - This function checks to see if a special file. It checks if the - file is a character special device, block special device, FIFO, or - socket. - """ - mode = os.stat(path).st_mode - # Character special device. - if stat.S_ISCHR(mode): - return True - # Block special device - if stat.S_ISBLK(mode): - return True - # FIFO. - if stat.S_ISFIFO(mode): - return True - # Socket. - if stat.S_ISSOCK(mode): - return True - return False - if six.PY3: import locale diff --git a/awscli/customizations/s3/filegenerator.py b/awscli/customizations/s3/filegenerator.py index 86fed796b4b2..19e593c793de 100644 --- a/awscli/customizations/s3/filegenerator.py +++ b/awscli/customizations/s3/filegenerator.py @@ -12,6 +12,7 @@ # language governing permissions and limitations under the License. import os import sys +import stat import six from six.moves import queue @@ -21,7 +22,49 @@ from awscli.customizations.s3.utils import find_bucket_key, get_file_stat from awscli.customizations.s3.utils import BucketLister, create_warning from awscli.errorhandler import ClientError -from awscli.compat import is_special_file + + +def is_special_file(path): + """ + This function checks to see if a special file. It checks if the + file is a character special device, block special device, FIFO, or + socket. + """ + mode = os.stat(path).st_mode + # Character special device. + if stat.S_ISCHR(mode): + return True + # Block special device + if stat.S_ISBLK(mode): + return True + # FIFO. + if stat.S_ISFIFO(mode): + return True + # Socket. + if stat.S_ISSOCK(mode): + return True + return False + + +def is_readable(path): + """ + This function checks to see if a file or a directory can be read. + This is tested by performing an operation that requires read access + on the file or the directory. + """ + if os.path.isdir(path): + try: + os.listdir(path) + except (OSError, IOError): + return False + else: + try: + with open(path, 'r') as fd: + pass + except (OSError, IOError): + return False + return True + # This class is provided primarily to provide a detailed error message. @@ -209,10 +252,6 @@ def triggers_warning(self, path): warning = create_warning(path, "File does not exist.") self.result_queue.put(warning) return True - if not os.access(path, os.R_OK): - warning = create_warning(path, "Read access is denied.") - self.result_queue.put(warning) - return True if is_special_file(path): warning = create_warning(path, ("File is character special device, " @@ -220,6 +259,10 @@ def triggers_warning(self, path): "socket.")) self.result_queue.put(warning) return True + if not is_readable(path): + warning = create_warning(path, "File/Directory is not readable.") + self.result_queue.put(warning) + return True return False def list_objects(self, s3_path, dir_op): diff --git a/awscli/customizations/s3/utils.py b/awscli/customizations/s3/utils.py index c02b4c572ea9..8613481a5fbc 100644 --- a/awscli/customizations/s3/utils.py +++ b/awscli/customizations/s3/utils.py @@ -385,11 +385,11 @@ class PrintTask(namedtuple('PrintTask', ['message', 'error', 'total_parts', 'warning'])): def __new__(cls, message, error=False, total_parts=None, warning=None): """ - :var message: An arbitrary string associated with the entry. This + :param message: An arbitrary string associated with the entry. This can be used to communicate the result of the task. - :var error: Boolean indicating a failure. - :var total_parts: The total number of parts for multipart transfers. - :var warning: Boolean indicating a warning + :param error: Boolean indicating a failure. + :param total_parts: The total number of parts for multipart transfers. + :param warning: Boolean indicating a warning """ return super(PrintTask, cls).__new__(cls, message, error, total_parts, warning) diff --git a/tests/integration/customizations/s3/test_plugin.py b/tests/integration/customizations/s3/test_plugin.py index e41047da5a81..a4e6d6d5919b 100644 --- a/tests/integration/customizations/s3/test_plugin.py +++ b/tests/integration/customizations/s3/test_plugin.py @@ -675,6 +675,7 @@ def testMvRegion(self): self.assertFalse( self.key_exists(bucket_name=self.src_bucket, key_name='foo.txt')) + class TestWarnings(BaseS3CLICommand): def extra_setup(self): self.bucket_name = self.create_bucket() @@ -686,6 +687,8 @@ def test_no_exist(self): self.assertIn('warning: Skipping file %s. File does not exist.' % filename, p.stdout) + @unittest.skipIf(platform.system() not in ['Darwin', 'Linux'], + 'Read permissions tests only supported on mac/linux') def test_no_read_access(self): self.files.create_file('foo.txt', 'foo') filename = os.path.join(self.files.rootdir, 'foo.txt') @@ -695,13 +698,15 @@ def test_no_read_access(self): os.chmod(filename, permissions) p = aws('s3 cp %s s3://%s/' % (filename, self.bucket_name)) self.assertEqual(p.rc, 2, p.stdout) - self.assertIn('warning: Skipping file %s. Read access' - ' is denied.' % filename, p.stdout) + self.assertIn('warning: Skipping file %s. File/Directory is ' + 'not readable.' % filename, p.stdout) + @unittest.skipIf(platform.system() not in ['Darwin', 'Linux'], + 'Special files only supported on mac/linux') def test_is_special_file(self): file_path = os.path.join(self.files.rootdir, 'foo') # Use socket for special file. - sock=socket.socket(socket.AF_UNIX,socket.SOCK_STREAM) + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) sock.bind(file_path) p = aws('s3 cp %s s3://%s/' % (file_path, self.bucket_name)) self.assertEqual(p.rc, 2, p.stdout) diff --git a/tests/unit/customizations/s3/test_filegenerator.py b/tests/unit/customizations/s3/test_filegenerator.py index 5febc10fbbcd..844db5e53567 100644 --- a/tests/unit/customizations/s3/test_filegenerator.py +++ b/tests/unit/customizations/s3/test_filegenerator.py @@ -22,7 +22,7 @@ import mock from awscli.customizations.s3.filegenerator import FileGenerator, \ - FileDecodingError, FileStat + FileDecodingError, FileStat, is_special_file, is_readable from awscli.customizations.s3.utils import get_file_stat import botocore.session from tests.unit.customizations.s3 import make_loc_files, clean_loc_files, \ @@ -30,6 +30,75 @@ from tests.unit.customizations.s3.fake_session import FakeSession +@unittest.skipIf(platform.system() not in ['Darwin', 'Linux'], + 'Special files only supported on mac/linux') +class TestIsSpecialFile(unittest.TestCase): + def setUp(self): + self.files = FileCreator() + self.filename = 'foo' + + def tearDown(self): + self.files.remove_all() + + def test_is_character_device(self): + file_path = os.path.join(self.files.rootdir, self.filename) + self.files.create_file(self.filename, contents='') + with mock.patch('stat.S_ISCHR') as mock_class: + mock_class.return_value = True + self.assertTrue(is_special_file(file_path)) + + def test_is_block_device(self): + file_path = os.path.join(self.files.rootdir, self.filename) + self.files.create_file(self.filename, contents='') + with mock.patch('stat.S_ISBLK') as mock_class: + mock_class.return_value = True + self.assertTrue(is_special_file(file_path)) + + def test_is_fifo(self): + file_path = os.path.join(self.files.rootdir, self.filename) + mode = 0o600 | stat.S_IFIFO + os.mknod(file_path, mode) + self.assertTrue(is_special_file(file_path)) + + def test_is_socket(self): + file_path = os.path.join(self.files.rootdir, self.filename) + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + sock.bind(file_path) + self.assertTrue(is_special_file(file_path)) + + +@unittest.skipIf(platform.system() not in ['Darwin', 'Linux'], + 'Read permissions tests only supported on mac/linux') +class TestIsReadable(unittest.TestCase): + def setUp(self): + self.files = FileCreator() + self.filename = 'foo' + self.full_path = os.path.join(self.files.rootdir, self.filename) + + def tearDown(self): + permissions = stat.S_IMODE(os.stat(self.full_path).st_mode) + # Reinstate read permissions + permissions = permissions | stat.S_IREAD + os.chmod(self.full_path, permissions) + self.files.remove_all() + + def test_unreadable_file(self): + self.files.create_file(self.filename, contents="foo") + permissions = stat.S_IMODE(os.stat(self.full_path).st_mode) + # Remove read permissions + permissions = permissions ^ stat.S_IREAD + os.chmod(self.full_path, permissions) + self.assertFalse(is_readable(self.full_path)) + + def test_unreadable_directory(self): + os.mkdir(self.full_path) + permissions = stat.S_IMODE(os.stat(self.full_path).st_mode) + # Remove read permissions + permissions = permissions ^ stat.S_IREAD + os.chmod(self.full_path, permissions) + self.assertFalse(is_readable(self.full_path)) + + class LocalFileGeneratorTest(unittest.TestCase): def setUp(self): self.local_file = six.text_type(os.path.abspath('.') + @@ -185,15 +254,16 @@ def test_no_warning(self): def test_no_exists(self): file_gen = FileGenerator(self.service, self.endpoint, '', False) - symlink = os.path.join(self.root, 'symlink') - os.symlink('non-existent-file', symlink) - return_val = file_gen.triggers_warning(symlink) + filename = os.path.join(self.root, 'file') + return_val = file_gen.triggers_warning(filename) self.assertTrue(return_val) warning_message = file_gen.result_queue.get() self.assertEqual(warning_message.message, ("warning: Skipping file %s. File does not exist." % - symlink)) + filename)) + @unittest.skipIf(platform.system() not in ['Darwin', 'Linux'], + 'Read permissions tests only supported on mac/linux') def test_no_read_access(self): file_gen = FileGenerator(self.service, self.endpoint, '', False) self.files.create_file("foo.txt", contents="foo") @@ -206,14 +276,16 @@ def test_no_read_access(self): self.assertTrue(return_val) warning_message = file_gen.result_queue.get() self.assertEqual(warning_message.message, - ("warning: Skipping file %s. Read access is " - "denied." % full_path)) + ("warning: Skipping file %s. File/Directory is " + "not readable." % full_path)) + @unittest.skipIf(platform.system() not in ['Darwin', 'Linux'], + 'Special files only supported on mac/linux') def test_is_special_file_warning(self): file_gen = FileGenerator(self.service, self.endpoint, '', False) file_path = os.path.join(self.files.rootdir, 'foo') # Use socket for special file. - sock=socket.socket(socket.AF_UNIX,socket.SOCK_STREAM) + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) sock.bind(file_path) return_val = file_gen.triggers_warning(file_path) self.assertTrue(return_val) @@ -224,7 +296,6 @@ def test_is_special_file_warning(self): "socket." % file_path)) - @unittest.skipIf(platform.system() not in ['Darwin', 'Linux'], 'Symlink tests only supported on mac/linux') class TestSymlinksIgnoreFiles(unittest.TestCase): diff --git a/tests/unit/test_compat.py b/tests/unit/test_compat.py deleted file mode 100644 index 3e5bbd78c0fb..000000000000 --- a/tests/unit/test_compat.py +++ /dev/null @@ -1,62 +0,0 @@ -# Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). You -# may not use this file except in compliance with the License. A copy of -# the License is located at -# -# http://aws.amazon.com/apache2.0/ -# -# or in the "license" file accompanying this file. This file is -# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF -# ANY KIND, either express or implied. See the License for the specific -# language governing permissions and limitations under the License. -import os -import stat -import socket - -from mock import Mock, patch - -from awscli.compat import is_special_file -from awscli.testutils import unittest, FileCreator - - -class TestIsSpecialFile(unittest.TestCase): - def setUp(self): - self.files = FileCreator() - self.filename = 'foo' - - def tearDown(self): - self.files.remove_all() - - def test_is_character_device(self): - mock_class = Mock() - mock_class.return_value = True - file_path = os.path.join(self.files.rootdir, self.filename) - self.files.create_file(self.filename, contents='') - with patch('stat.S_ISCHR') as mock_class: - self.assertTrue(is_special_file(file_path)) - - def test_is_block_device(self): - mock_class = Mock() - mock_class.return_value = True - file_path = os.path.join(self.files.rootdir, self.filename) - self.files.create_file(self.filename, contents='') - with patch('stat.S_ISBLK') as mock_class: - self.assertTrue(is_special_file(file_path)) - - def test_is_fifo(self): - file_path = os.path.join(self.files.rootdir, self.filename) - mode = 0o600 | stat.S_IFIFO - os.mknod(file_path, mode) - self.assertTrue(is_special_file(file_path)) - - - def test_is_socket(self): - file_path = os.path.join(self.files.rootdir, self.filename) - sock=socket.socket(socket.AF_UNIX,socket.SOCK_STREAM) - sock.bind(file_path) - self.assertTrue(is_special_file(file_path)) - - -if __name__ == "__main__": - unittest.main()