Integrate plugin system with cli driver

* Created main() function in clidriver

This gives room for the plugins to be loaded, rather than
have CLIDriver know anything about the plugin loading process.

As a result, I've updated aws and aws.cmd to use clidriver.main()
instead of creating the CLIDriver instance themselves.

* Load plugins from config file

* Emit events in clidriver

* Also removed the nargs=1 for non lists.  This simplified the unpack
  arg, and it will make it easier for plugins to manipulate the args.

* Add first_non_none_response function.

awscli/clidriver.py

@@ -22,6 +22,15 @@
 from .help import get_provider_help, get_service_help, get_operation_help
 from .formatter import get_formatter
 from .paramfile import get_paramfile
+from .plugin import load_plugins, first_non_none_response
+from .hooks import BaseEventHooks
+
+
+def main():
+    session = botocore.session.get_session(EnvironmentVariables)
+    emitter = load_plugins(session.full_config.get('plugins', {}))
+    driver = CLIDriver(session=session, emitter=emitter)
+    return driver.main()
 
 
 class CLIDriver(object):
@@ -41,13 +50,17 @@ class CLIDriver(object):
         'double': float,
         'blob': str}
 
-    def __init__(self, session=None):
+    def __init__(self, session=None, emitter=None):
         if session is None:
             self.session = botocore.session.get_session(EnvironmentVariables)
             self.session.user_agent_name = 'aws-cli'
             self.session.user_agent_version = __version__
         else:
             self.session = session
+        if emitter is None:
+            self._emitter = BaseEventHooks()
+        else:
+            self._emitter = emitter
         self.args = None
         self.service = None
         self.region = None
@@ -89,6 +102,7 @@ def create_main_parser(self):
             parser.add_argument(option_name, **option_data)
         parser.add_argument('--version', action="version",
                             version=self.session.user_agent())
+        self._emitter.emit('parser-created.main', parser=parser)
         return parser
 
     def create_service_parser(self, remaining, main_parser):
@@ -115,6 +129,8 @@ def create_service_parser(self, remaining, main_parser):
         parser.add_argument('operation', help='The operation',
                             metavar='operation',
                             choices=operations)
+        self._emitter.emit('parser-created.%s' % self.service.cli_name,
+                           parser=parser)
         return parser
 
     def _create_operation_parser(self, remaining, main_parser):
@@ -165,7 +181,6 @@ def _create_operation_parser(self, remaining, main_parser):
             else:
                 parser.add_argument(param.cli_name,
                                     help=param.documentation,
-                                    nargs=1,
                                     type=self.type_map[param.type],
                                     required=param.required,
                                     dest=param.py_name)
@@ -175,6 +190,8 @@ def _create_operation_parser(self, remaining, main_parser):
         if 'help' in remaining:
             get_operation_help(self.operation)
             return 0
+        self._emitter.emit('parser-created.%s-%s' % (self.service.cli_name,
+                                                     self.operation.cli_name))
         return parser
 
     def _unpack_cli_arg(self, param, s):
@@ -184,17 +201,11 @@ def _unpack_cli_arg(self, param, s):
         to the Operation.
         """
         if param.type == 'integer':
-            if isinstance(s, list):
-                s = s[0]
             return int(s)
         elif param.type == 'float' or param.type == 'double':
             # TODO: losing precision on double types
-            if isinstance(s, list):
-                s = s[0]
             return float(s)
         elif param.type == 'structure' or param.type == 'map':
-            if isinstance(s, list) and len(s) == 1:
-                s = s[0]
             if s[0] == '{':
                 d = json.loads(s)
             else:
@@ -210,38 +221,53 @@ def _unpack_cli_arg(self, param, s):
                     return json.loads(s[0])
             return [self._unpack_cli_arg(param.members, v) for v in s]
         elif param.type == 'blob' and param.payload and param.streaming:
-            if isinstance(s, list) and len(s) == 1:
-                file_path = s[0]
-            file_path = os.path.expandvars(file_path)
+            file_path = os.path.expandvars(s)
             file_path = os.path.expanduser(file_path)
             if not os.path.isfile(file_path):
                 msg = 'Blob values must be a path to a file.'
                 raise ValueError(msg)
             return open(file_path, 'rb')
         else:
-            if isinstance(s, list):
-                s = s[0]
             return str(s)
 
     def _build_call_parameters(self, args, param_dict):
+        service_name = self.service.cli_name
+        operation_name = self.operation.cli_name
         for param in self.operation.params:
             value = getattr(args, param.py_name)
             if value is not None:
-                # Don't include non-required boolean params whose
-                # values are False
+                # Plugins can override the cli -> python conversion
+                # process for CLI args.
+                responses = self._emitter.emit('process-cli-arg.%s.%s' % (
+                    service_name, operation_name), param=param, value=value,
+                    service=self.service, operation=self.operation)
+                override = first_non_none_response(responses)
+                if override is not None:
+                    # A plugin supplied an alternate conversion,
+                    # use it instead.
+                    param_dict[param.py_name] = override
+                    continue
+                # Otherwise fall back to our normal built in cli -> python
+                # conversion process.
                 if param.type == 'boolean' and not param.required and \
                         value is False:
+                    # Don't include non-required boolean params whose
+                    # values are False
                     continue
                 if not hasattr(param, 'no_paramfile'):
-                    if isinstance(value, list) and len(value) == 1:
-                        temp = value[0]
-                    else:
-                        temp = value
-                    temp = get_paramfile(self.session, temp)
-                    if temp:
-                        value = temp
+                    value = self._handle_param_file(value)
                 param_dict[param.py_name] = self._unpack_cli_arg(param, value)
 
+    def _handle_param_file(self, value):
+        if isinstance(value, list) and len(value) == 1:
+            temp = value[0]
+        else:
+            temp = value
+        temp = get_paramfile(self.session, temp)
+        if temp:
+            value = temp
+        return value
+
     def display_error_and_exit(self, ex):
         if self.args.debug:
             traceback.print_exc()
@@ -273,20 +299,31 @@ def save_output(self, body_name, response_data, path):
                 data = response_data[body_name].read(buffsize)
         del response_data[body_name]
 
-    def call(self, args):
+    def _call(self, args):
         try:
             params = {}
             self._build_call_parameters(args, params)
             self.endpoint = self.service.get_endpoint(
                 self.args.region, endpoint_url=self.args.endpoint_url)
             self.endpoint.verify = not self.args.no_verify_ssl
+            self._emitter.emit(
+                'before-operation.%s.%s' % (self.service.cli_name,
+                                            self.operation.cli_name),
+                service=self.service, operation=self.operation,
+                endpoint=self.endpoint, params=params)
             if self.operation.can_paginate:
                 pages = self.operation.paginate(self.endpoint, **params)
+                self._emitter.emit(
+                    'after-operation.%s.%s' % (self.service.cli_name,
+                                               self.operation.cli_name),
+                    service=self.service, operation=self.operation,
+                    endpoint=self.endpoint, params=params)
                 self._display_response(self.operation, pages)
                 # TODO: need to handle http error responses.  I believe
                 # this will be addressed with the plugin refactoring,
                 # but the other alternative is going to be that we'll need
                 # to cache the fully buffered response.
+                return 0
             else:
                 http_response, response_data = self.operation.call(
                     self.endpoint, **params)
@@ -378,7 +415,16 @@ def _parse_args(self, main_parser, args):
                 return -1
             return args
 
-    def main(self):
+    def main(self, args=None):
+        """
+
+        :param args: List of arguments, with the 'aws' removed.  For example,
+            the command "aws s3 list-objects --bucket foo" will have an
+            args list of ``['s3', 'list-objects', '--bucket', 'foo']``.
+
+        """
+        if args is None:
+            args = sys.argv[1:]
         main_parser = self.create_main_parser()
-        args = self._parse_args(main_parser, sys.argv[1:])
-        return self.call(args)
+        remaining_args = self._parse_args(main_parser, args)
+        return self._call(remaining_args)

awscli/hooks.py

@@ -15,7 +15,18 @@
 from collections import defaultdict
 
 
-class EventHooks(object):
+class BaseEventHooks(object):
+    def emit(self, event_name, **kwargs):
+        return []
+
+    def register(self, event_name, handler):
+        pass
+
+    def unregister(self, event_name, handler):
+        pass
+
+
+class EventHooks(BaseEventHooks):
     def __init__(self):
         # event_name -> [handler, ...]
         self._handlers = defaultdict(list)
@@ -75,3 +86,26 @@ def _verify_accept_kwargs(self, func):
                 raise ValueError("Event handler %s must accept keyword "
                                  "arguments (**kwargs)" % func)
 
+
+class HierarchicalEmitter(BaseEventHooks):
+    def __init__(self, event_hooks):
+        self._event_hooks = event_hooks
+
+    def emit(self, event, **kwargs):
+        responses = []
+        # Invoke the event handlers from most specific
+        # to least specific, each time stripping off a dot.
+        while event:
+            responses.extend(self._event_hooks.emit(event, **kwargs))
+            next_event = event.rsplit('.', 1)
+            if len(next_event) == 2:
+                event = next_event[0]
+            else:
+                event = None
+        return responses
+
+    def register(self, event_name, handler):
+        return self._event_hooks.register(event_name, handler)
+
+    def unregister(self, event_name, handler):
+        return self._event_hooks.unregister(event_name, handler)

awscli/plugin.py

@@ -10,70 +10,77 @@
 # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
 # ANY KIND, either express or implied. See the License for the specific
 # language governing permissions and limitations under the License.
+import logging
 
-from awscli.hooks import EventHooks
+from awscli.hooks import EventHooks, HierarchicalEmitter
 
+log = logging.getLogger('awscli.plugin')
 
-def load_plugins(plugin_names, event_hooks=None):
-    modules = _import_plugins(plugin_names)
+
+def load_plugins(plugin_mapping, event_hooks=None):
+    """
+
+    :type plugin_mapping: dict
+    :param plugin_mapping: A dict of plugin name to import path,
+        e.g. ``{"plugingName": "package.modulefoo"}``.
+
+    :type event_hooks: ``EventHooks``
+    :param event_hooks: Event hook emitter.
+
+    :rtype: HierarchicalEmitter
+    :return: An event emitter object.
+
+    """
+    modules = _import_plugins(plugin_mapping)
     if event_hooks is None:
         event_hooks = EventHooks()
     cli = CLI(event_hooks)
-    for plugin in modules:
+    for name, plugin in zip(plugin_mapping.keys(), modules):
+        log.debug("Initializing plugin %s: %s", name, plugin)
         plugin.awscli_initialize(cli)
     return HierarchicalEmitter(event_hooks)
 
 
 def _import_plugins(plugin_names):
     plugins = []
-    for name in plugin_names:
+    for name, path in plugin_names.items():
+        log.debug("Importing plugin %s: %s", name, path)
         if '.' not in name:
-            plugins.append(__import__(name))
+            plugins.append(__import__(path))
     return plugins
 
 
-class HierarchicalEmitter(object):
-    def __init__(self, event_hooks):
-        self._event_hooks = event_hooks
+def first_non_none_response(responses, default=None):
+    """Find first non None response in a list of tuples.
+
+    This function can be used to find the first non None response from
+    handlers connected to an event.  This is useful if you are interested
+    in the returned responses from event handlers. Example usage::
 
-    def emit(self, event):
-        responses = []
-        # Invoke the event handlers from most specific
-        # to least specific, each time stripping off a dot.
-        while event:
-            responses.extend(self._event_hooks.emit(event))
-            next_event = event.rsplit('.', 1)
-            if len(next_event) == 2:
-                event = next_event[0]
-            else:
-                event = None
-        return responses
+        print(first_non_none_response([(func1, None), (func2, 'foo'),
+                                       (func3, 'bar')]))
+        # This will print 'foo'
+
+    :type responses: list of tuples
+    :param responses: The responses from the ``EventHooks.emit`` method.
+        This is a list of tuples, and each tuple is
+        (handler, handler_response).
+
+    :param default: If no non-None responses are found, then this default
+        value will be returned.
+
+    :return: The first non-None response in the list of tuples.
+
+    """
+    for response in responses:
+        if response[1] is not None:
+            return response[1]
+    return default
 
 
 class CLI(object):
     def __init__(self, event_hooks):
         self._event_hooks = event_hooks
 
-    def before_call(self, handler, service_name=None, operation_name=None):
-        op_event_name = self._get_event_name(service_name, operation_name)
-        if op_event_name:
-            event_name = 'before_call.%s' % op_event_name
-        else:
-            event_name = 'before_call'
+    def register(self, event_name, handler):
         self._event_hooks.register(event_name, handler)
-
-    def after_call(self, handler, service_name=None, operation_name=None):
-        op_event_name = self._get_event_name(service_name, operation_name)
-        if op_event_name:
-            event_name = 'after_call.%s' % op_event_name
-        else:
-            event_name = 'after_call'
-        self._event_hooks.register(event_name, handler)
-
-    def _get_event_name(self, service_name, operation_name):
-        if service_name is None:
-            return ''
-        if service_name is not None and operation_name is None:
-            return service_name
-        elif service_name is not None and operation_name is not None:
-            return '%s.%s' % (service_name, operation_name)

bin/aws

@@ -16,8 +16,7 @@ import awscli.clidriver
 
 
 def main():
-    driver = awscli.clidriver.CLIDriver()
-    return driver.main()
+    return awscli.clidriver.main()
 
 
 if __name__ == '__main__':

bin/aws.cmd

@@ -50,9 +50,8 @@ import awscli.clidriver
 
 
 def main():
-    driver = awscli.clidriver.CLIDriver()
-    driver.main()
+    return awscli.clidriver.main()
 
 
 if __name__ == '__main__':
-    main()
+    sys.exit(main())