Merge pull request #921 from kyleknap/show-error

Added a ``--only-show-errors`` argument.

CHANGELOG.rst

@@ -12,6 +12,8 @@ Next Release (TBD)
 * bugfix:``aws s3``: Write warnings and errors to standard error as
   opposed to standard output.
   (`issue 919 <https://github.com/aws/aws-cli/pull/919>`__)
+* feature:``aws s3``: Add ``--only-show-errors`` option that displays
+  errors and warnings but suppresses all other output.
 
 
 1.4.4

awscli/customizations/s3/executor.py

@@ -40,18 +40,19 @@ class Executor(object):
     STANDARD_PRIORITY = 11
     IMMEDIATE_PRIORITY= 1
 
-    def __init__(self, num_threads, result_queue,
-                 quiet, max_queue_size, write_queue):
+    def __init__(self, num_threads, result_queue, quiet,
+                 only_show_errors, max_queue_size, write_queue):
         self._max_queue_size = max_queue_size
         self.queue = StablePriorityQueue(maxsize=self._max_queue_size,
                                          max_priority=20)
         self.num_threads = num_threads
         self.result_queue = result_queue
         self.quiet = quiet
+        self.only_show_errors = only_show_errors
         self.threads_list = []
         self.write_queue = write_queue
-        self.print_thread = PrintThread(self.result_queue,
-                                        self.quiet)
+        self.print_thread = PrintThread(self.result_queue, self.quiet,
+                                        self.only_show_errors)
         self.print_thread.daemon = True
         self.io_thread = IOWriterThread(self.write_queue)
 
@@ -226,11 +227,12 @@ class PrintThread(threading.Thread):
             warning.
 
     """
-    def __init__(self, result_queue, quiet):
+    def __init__(self, result_queue, quiet, only_show_errors):
         threading.Thread.__init__(self)
         self._progress_dict = {}
         self._result_queue = result_queue
         self._quiet = quiet
+        self._only_show_errors = only_show_errors
         self._progress_length = 0
         self._num_parts = 0
         self._file_count = 0
@@ -317,7 +319,7 @@ def _process_print_task(self, print_task):
         is_done = self._total_files == self._file_count
         if not is_done:
             final_str += self._make_progress_bar()
-        if not self._quiet:
+        if not (self._quiet or self._only_show_errors):
             uni_print(final_str)
             self._needs_newline = not final_str.endswith('\n')
 

awscli/customizations/s3/s3handler.py

@@ -53,7 +53,7 @@ def __init__(self, session, params, result_queue=None,
                        'content_type': None, 'cache_control': None,
                        'content_disposition': None, 'content_encoding': None,
                        'content_language': None, 'expires': None,
-                       'grants': None}
+                       'grants': None, 'only_show_errors': False}
         self.params['region'] = params['region']
         for key in self.params.keys():
             if key in params:
@@ -62,8 +62,9 @@ def __init__(self, session, params, result_queue=None,
         self.chunksize = chunksize
         self.executor = Executor(
             num_threads=NUM_THREADS, result_queue=self.result_queue,
-            quiet=self.params['quiet'], max_queue_size=MAX_QUEUE_SIZE,
-            write_queue=self.write_queue
+            quiet=self.params['quiet'],
+            only_show_errors=self.params['only_show_errors'],
+            max_queue_size=MAX_QUEUE_SIZE, write_queue=self.write_queue
         )
         self._multipart_uploads = []
         self._multipart_downloads = []

awscli/customizations/s3/subcommands.py

@@ -206,11 +206,16 @@
                       'The object key name to use when '
                       'a 4XX class error occurs.')}
 
+ONLY_SHOW_ERRORS = {'name': 'only-show-errors', 'action': 'store_true',
+                    'help_text': (
+                        'Only errors and warnings are displayed. All other '
+                        'output is suppressed.')}
+
 TRANSFER_ARGS = [DRYRUN, QUIET, RECURSIVE, INCLUDE, EXCLUDE, ACL,
                  FOLLOW_SYMLINKS, NO_FOLLOW_SYMLINKS, NO_GUESS_MIME_TYPE,
                  SSE, STORAGE_CLASS, GRANTS, WEBSITE_REDIRECT, CONTENT_TYPE,
                  CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING,
-                 CONTENT_LANGUAGE, EXPIRES, SOURCE_REGION]
+                 CONTENT_LANGUAGE, EXPIRES, SOURCE_REGION, ONLY_SHOW_ERRORS]
 
 SYNC_ARGS = [DELETE, EXACT_TIMESTAMPS, SIZE_ONLY] + TRANSFER_ARGS
 
@@ -431,7 +436,7 @@ class RmCommand(S3TransferCommand):
     USAGE = "<S3Path>"
     ARG_TABLE = [{'name': 'paths', 'nargs': 1, 'positional_arg': True,
                   'synopsis': USAGE}, DRYRUN, QUIET, RECURSIVE, INCLUDE,
-                 EXCLUDE]
+                 EXCLUDE, ONLY_SHOW_ERRORS]
     EXAMPLES = BasicCommand.FROM_FILE('s3/rm.rst')
 
 

tests/integration/customizations/s3/test_plugin.py

@@ -949,6 +949,106 @@ def test_fail_mb_rb(self):
         self.assertEqual(p.rc, 1)
 
 
+class TestOutput(BaseS3CLICommand):
+    """
+    This ensures that arguments that affect output i.e. ``--quiet`` and
+    ``--only-show-errors`` behave as expected.
+    """
+    def test_normal_output(self):
+        # Make a bucket.
+        bucket_name = self.create_bucket()
+        foo_txt = self.files.create_file('foo.txt', 'foo contents')
+
+        # Copy file into bucket.
+        p = aws('s3 cp %s s3://%s/' % (foo_txt, bucket_name))
+        self.assertEqual(p.rc, 0)
+        # Check that there were no errors and that parts of the expected
+        # progress message are written to stdout.
+        self.assert_no_errors(p)
+        self.assertIn('upload', p.stdout)
+        self.assertIn('s3://%s/foo.txt' % bucket_name, p.stdout)
+
+    def test_normal_output_quiet(self):
+        # Make a bucket.
+        bucket_name = self.create_bucket()
+        foo_txt = self.files.create_file('foo.txt', 'foo contents')
+
+        # Copy file into bucket.
+        p = aws('s3 cp %s s3://%s/ --quiet' % (foo_txt, bucket_name))
+        self.assertEqual(p.rc, 0)
+        # Check that nothing was printed to stdout.
+        self.assertEqual('', p.stdout)
+
+    def test_normal_output_only_show_errors(self):
+        # Make a bucket.
+        bucket_name = self.create_bucket()
+        foo_txt = self.files.create_file('foo.txt', 'foo contents')
+
+        # Copy file into bucket.
+        p = aws('s3 cp %s s3://%s/ --only-show-errors' % (foo_txt, bucket_name))
+        self.assertEqual(p.rc, 0)
+        # Check that nothing was printed to stdout.
+        self.assertEqual('', p.stdout)
+
+    def test_error_output(self):
+        foo_txt = self.files.create_file('foo.txt', 'foo contents')
+
+        # Copy file into bucket.
+        p = aws('s3 cp %s s3://non-existant-bucket/' % foo_txt)
+        # Check that there were errors and that the error was print to stderr.
+        self.assertEqual(p.rc, 1)
+        self.assertIn('upload failed', p.stderr)
+
+    def test_error_ouput_quiet(self):
+        foo_txt = self.files.create_file('foo.txt', 'foo contents')
+
+        # Copy file into bucket.
+        p = aws('s3 cp %s s3://non-existant-bucket/ --quiet' % foo_txt)
+        # Check that there were errors and that the error was not
+        # print to stderr.
+        self.assertEqual(p.rc, 1)
+        self.assertEqual('', p.stderr)
+
+    def test_error_ouput_only_show_errors(self):
+        foo_txt = self.files.create_file('foo.txt', 'foo contents')
+
+        # Copy file into bucket.
+        p = aws('s3 cp %s s3://non-existant-bucket/ --only-show-errors'
+                % foo_txt)
+        # Check that there were errors and that the error was print to stderr.
+        self.assertEqual(p.rc, 1)
+        self.assertIn('upload failed', p.stderr)
+
+    def test_error_and_success_output_only_show_errors(self):
+        # Make a bucket.
+        bucket_name = self.create_bucket()
+
+        # Create one file.
+        self.files.create_file('f', 'foo contents')
+
+        # Create another file that has a slightly longer name than the first.
+        self.files.create_file('bar.txt', 'bar contents')
+
+        # Create a prefix that will cause the second created file to have a key
+        # longer than 1024 bytes which is not allowed in s3.
+        long_prefix = 'd' * 1022
+
+        p = aws('s3 cp %s s3://%s/%s/ --only-show-errors --recursive'
+                % (self.files.rootdir, bucket_name, long_prefix))
+
+        # Check that there was at least one error.
+        self.assertEqual(p.rc, 1)
+
+        # Check that there was nothing written to stdout for successful upload.
+        self.assertEqual('', p.stdout)
+
+        # Check that the failed message showed up in stderr.
+        self.assertIn('upload failed', p.stderr)
+
+        # Ensure the expected successful key exists in the bucket.
+        self.assertTrue(self.key_exists(bucket_name, long_prefix + '/f'))
+
+
 class TestDryrun(BaseS3CLICommand):
     """
     This ensures that dryrun works.

tests/unit/customizations/s3/test_executor.py

@@ -75,7 +75,7 @@ def test_multiple_files_in_queue(self):
 
 class TestExecutor(unittest.TestCase):
     def test_shutdown_does_not_hang(self):
-        executor = Executor(2, queue.Queue(), False,
+        executor = Executor(2, queue.Queue(), False, False,
                             10, queue.Queue(maxsize=1))
         with temporary_file('rb+') as f:
             executor.start()
@@ -94,20 +94,97 @@ def __call__(self):
 class TestPrintThread(unittest.TestCase):
     def setUp(self):
         self.result_queue = queue.Queue()
-        self.thread = PrintThread(result_queue=self.result_queue, quiet=False)
 
-    def test_print_error(self):
-        print_task = PrintTask(message="Fail File.", error=True)
-        with mock.patch('sys.stderr', new=six.StringIO()) as mock_stderr:
+    def assert_expected_output(self, print_task, expected_output, thread,
+                               out_file):
+        with mock.patch(out_file, new=six.StringIO()) as mock_out:
             self.result_queue.put(print_task)
             self.result_queue.put(ShutdownThreadRequest())
-            self.thread.run()
-            self.assertIn("Fail File.", mock_stderr.getvalue())
+            thread.run()
+            self.assertIn(expected_output, mock_out.getvalue())
+
+    def test_print(self):
+        print_task = PrintTask(message="Success", error=False)
+
+        # Ensure a successful task is printed to stdout when
+        # ``quiet`` and ``only_show_errors`` is False.
+        thread = PrintThread(result_queue=self.result_queue,
+                             quiet=False, only_show_errors=False)
+        self.assert_expected_output(print_task, 'Success', thread,
+                                    'sys.stdout')
+
+    def test_print_quiet(self):
+        print_task = PrintTask(message="Success", error=False)
+
+        # Ensure a succesful task is not printed to stdout when
+        # ``quiet`` is True.
+        thread = PrintThread(result_queue=self.result_queue,
+                             quiet=True, only_show_errors=False)
+        self.assert_expected_output(print_task, '', thread, 'sys.stdout')
+
+    def test_print_only_show_errors(self):
+        print_task = PrintTask(message="Success", error=False)
+
+        # Ensure a succesful task is not printed to stdout when
+        # ``only_show_errors`` is True.
+        thread = PrintThread(result_queue=self.result_queue,
+                             quiet=False, only_show_errors=True)
+        self.assert_expected_output(print_task, '', thread, 'sys.stdout')
+
+    def test_print_error(self):
+        print_task = PrintTask(message="Fail File.", error=True)
+
+        # Ensure a failed task is printed to stderr when
+        # ``quiet`` and ``only_show_errors`` is False.
+        thread = PrintThread(result_queue=self.result_queue,
+                             quiet=False, only_show_errors=False)
+        self.assert_expected_output(print_task, 'Fail File.', thread,
+                                    'sys.stderr')
+
+    def test_print_error_quiet(self):
+        print_task = PrintTask(message="Fail File.", error=True)
+
+        # Ensure a failed task is not printed to stderr when
+        # ``quiet`` is True.
+        thread = PrintThread(result_queue=self.result_queue,
+                             quiet=True, only_show_errors=False)
+        self.assert_expected_output(print_task, '', thread, 'sys.stderr')
+
+    def test_print_error_only_show_errors(self):
+        print_task = PrintTask(message="Fail File.", error=True)
+
+        # Ensure a failed task is printed to stderr when
+        # ``only_show_errors`` is True.
+        thread = PrintThread(result_queue=self.result_queue,
+                             quiet=False, only_show_errors=True)
+        self.assert_expected_output(print_task, 'Fail File.', thread,
+                                    'sys.stderr')
 
     def test_print_warning(self):
         print_task = PrintTask(message="Bad File.", warning=True)
-        with mock.patch('sys.stderr', new=six.StringIO()) as mock_stderr:
-            self.result_queue.put(print_task)
-            self.result_queue.put(ShutdownThreadRequest())
-            self.thread.run()
-            self.assertIn("Bad File.", mock_stderr.getvalue())
+
+        # Ensure a warned task is printed to stderr when
+        # ``quiet`` and ``only_show_errors`` is False.
+        thread = PrintThread(result_queue=self.result_queue,
+                             quiet=False, only_show_errors=False)
+        self.assert_expected_output(print_task, 'Bad File.', thread,
+                                    'sys.stderr')
+
+    def test_print_warning_quiet(self):
+        print_task = PrintTask(message="Bad File.", warning=True)
+
+        # Ensure a warned task is not printed to stderr when
+        # ``quiet`` is True.
+        thread = PrintThread(result_queue=self.result_queue,
+                             quiet=True, only_show_errors=False)
+        self.assert_expected_output(print_task, '', thread, 'sys.stderr')
+
+    def test_print_warning_only_show_errors(self):
+        print_task = PrintTask(message="Bad File.", warning=True)
+
+        # Ensure a warned task is printed to stderr when
+        # ``only_show_errors`` is True.
+        thread = PrintThread(result_queue=self.result_queue,
+                             quiet=False, only_show_errors=True)
+        self.assert_expected_output(print_task, 'Bad File.', thread,
+                                    'sys.stderr')

tests/unit/test_completer.py

@@ -73,7 +73,8 @@
                              '--cache-control', '--content-type',
                              '--content-disposition', '--source-region',
                              '--content-encoding', '--content-language',
-                             '--expires', '--grants'] + GLOBALOPTS)),
+                             '--expires', '--grants', '--only-show-errors']
+                            + GLOBALOPTS)),
     ('aws s3 cp --quiet -', -1, set(['--no-guess-mime-type', '--dryrun',
                                      '--recursive', '--content-type',
                                      '--follow-symlinks', '--no-follow-symlinks',
@@ -83,7 +84,8 @@
                                      '--storage-class', '--sse',
                                      '--exclude', '--include',
                                      '--source-region',
-                                     '--grants'] + GLOBALOPTS)),
+                                     '--grants', '--only-show-errors']
+                                    + GLOBALOPTS)),
     ('aws emr ', -1, set(['add-instance-groups', 'add-steps', 'add-tags',
                           'create-cluster', 'create-default-roles',
                           'create-hbase-backup', 'describe-cluster',