RPCv2 and CBOR support (#3006)

.github/workflows/changelog.yml

@@ -19,4 +19,4 @@ jobs:
         uses: thollander/actions-comment-pull-request@main
         with:
           message: 'You have made a change to core without a corresponding change to the CHANGELOG.md. This change will not result in a new version and will not published unless an entry is added to CHANGELOG.md.'
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

Gemfile

@@ -9,13 +9,13 @@ gem 'aws-crt' if ENV['CRT']
 gem 'http-2'
 gem 'jmespath'
 
-# json and xml parsers
+# protocol parsers
 gem 'json'
 gem 'nokogiri', '>= 1.6.8.1'
 gem 'oga'
 gem 'rexml', '= 3.2.6' # Temporary Workaround (https://github.com/ruby/rexml/issues/131)
 
-# These json and xml parsers do not have java gems
+# These protocol parsers do not have java gems
 unless defined?(JRUBY_VERSION)
   gem 'libxml-ruby'
   gem 'oj'

build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/client_operation_documentation.rb

@@ -16,6 +16,7 @@ def initialize(options)
       @method_name = options.fetch(:method_name)
       @operation = options.fetch(:operation)
       @api = options.fetch(:api)
+      @protocol = options.fetch(:protocol)
       @client_examples = options.fetch(:client_examples, [])
       @examples = options.fetch(:examples)
       @module_name = options.fetch(:module_name)
@@ -279,7 +280,7 @@ def waiters_tag(waiters)
 
     def see_also_tag(operation, api)
       uid = api['metadata']['uid']
-      if api['metadata']['protocol'] != 'api-gateway' && Crosslink.taggable?(uid)
+      if @protocol != 'api-gateway' && Crosslink.taggable?(uid)
         "# " + Crosslink.tag_string(uid, operation)
       end
     end

build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/client_operation_list.rb

@@ -9,6 +9,7 @@ def initialize(options)
       api = options.fetch(:api)
       examples = options.fetch(:examples, {})
       module_name = options.fetch(:module_name)
+      protocol = options.fetch(:protocol)
       protocol_settings = options.fetch(:protocol_settings, {})
       client_examples = options.fetch(:client_examples, {})
       paginators = options.fetch(:paginators, {})
@@ -41,6 +42,7 @@ def initialize(options)
                   method_name: method_name,
                   operation: operation,
                   api: api,
+                  protocol: protocol,
                   examples: examples,
                   client_examples: client_examples[method_name] || [],
                   async_client: true
@@ -60,6 +62,7 @@ def initialize(options)
                 method_name: method_name,
                 operation: operation,
                 api: api,
+                protocol: protocol,
                 examples: examples,
                 client_examples: client_examples[method_name] || [],
                 async_client: false
@@ -80,6 +83,7 @@ def initialize(options)
               method_name: method_name,
               operation: operation,
               api: api,
+              protocol: protocol,
               examples: examples,
               client_examples: client_examples[method_name] || [],
               async_client: false,

build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/code_builder.rb

@@ -185,13 +185,13 @@ def client_class(codegenerated_plugins)
       Views::ClientClass.new(
         service_identifier: @service.identifier,
         service_name: @service.name,
+        protocol: @service.protocol,
         protocol_settings: @service.protocol_settings,
         module_name: @service.module_name,
         gem_name: @service.gem_name,
         gem_version: @service.gem_version,
         aws_sdk_core_lib_path: @aws_sdk_core_lib_path,
         client_examples: @client_examples,
-        protocol: @service.protocol,
         signature_version: @service.signature_version,
         require_endpoint_discovery: @service.require_endpoint_discovery,
         add_plugins: @service.add_plugins,
@@ -210,12 +210,12 @@ def async_client_class(codegenerated_plugins)
       Views::AsyncClientClass.new(
         service_identifier: @service.identifier,
         service_name: @service.name,
+        protocol: @service.protocol,
         protocol_settings: @service.protocol_settings,
         module_name: @service.module_name,
         gem_name: @service.gem_name,
         gem_version: @service.gem_version,
         aws_sdk_core_lib_path: @aws_sdk_core_lib_path,
-        protocol: @service.protocol,
         signature_version: @service.signature_version,
         add_plugins: @service.add_plugins,
         remove_plugins: @service.remove_plugins,

build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/plugin_list.rb

@@ -86,6 +86,7 @@ def protocol_plugins(protocol)
         'rest-xml'  => { 'Aws::Plugins::Protocols::RestXml' => "#{core_plugins}/protocols/rest_xml.rb" },
         'query'     => { 'Aws::Plugins::Protocols::Query' => "#{core_plugins}/protocols/query.rb" },
         'ec2'       => { 'Aws::Plugins::Protocols::EC2' => "#{core_plugins}/protocols/ec2.rb" },
+        'smithy-rpc-v2-cbor' => { 'Aws::Plugins::Protocols::RpcV2' => "#{core_plugins}/protocols/rpc_v2.rb" },
         'api-gateway' => {
           'Aws::Plugins::Protocols::ApiGateway' => "#{core_plugins}/protocols/api_gateway.rb",
           'Aws::Plugins::ApiKey' => "#{core_plugins}/api_key.rb",

build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/service.rb

@@ -2,12 +2,26 @@
 
 module AwsSdkCodeGenerator
   class Service
+    # Ordered priority list of supported protocols
+    # api-gateway is a special case and is always first.
+    SUPPORTED_PROTOCOLS = %w[
+      api-gateway
+      smithy-rpc-v2-cbor
+      json_1.0
+      json_1.1
+      rest-json
+      rest-xml
+      query
+      ec2
+    ]
 
     # @param [Hash] options
-    # @option options [required, String] :gem_version Gem version, e.g. "1.0.0".
     # @option options [required, String] :name The service name, e.g. "S3"
     # @option options [String] :module_name The service module name, defaults
     #   to "Aws::#{name}", e.g. "Aws::S3".
+    # @option options [String] :gem_name The gem name, defaults to
+    #  "aws-sdk-#{name}", e.g. "aws-sdk-s3".
+    # @option options [required, String] :gem_version Gem version, e.g. "1.0.0".
     # @option options [required, Hash, String] :api
     # @option options [Hash, String] :docs
     # @option options [Hash, String] :paginators
@@ -21,44 +35,48 @@ class Service
     # @option options [Hash] :add_plugins ({})
     # @option options [Hash] :remove_plugins ([])
     # @option options [Boolean] :deprecated (false)
+    # @option options [String] :default_endpoint (nil)
+    # @option options [String] :endpoints_key (nil)
     def initialize(options)
       @name = options.fetch(:name)
       @identifier = name.downcase
       @module_name = options[:module_name] || "Aws::#{name}"
       @gem_name = options[:gem_name] || "aws-sdk-#{identifier}"
       @gem_version = options.fetch(:gem_version)
+
       @api = load_json(options.fetch(:api))
-      unless @api['metadata']['protocol'] == 'api-gateway'
-        # Dont reply on API Gateway doc.json
+
+      # computed attributes
+      metadata = @api.fetch('metadata')
+      @protocol = select_protocol(metadata)
+      @protocol_settings = metadata['protocolSettings'] || {}
+      @api_version = metadata['apiVersion']
+      @signature_version = metadata['signatureVersion']
+      @full_name = metadata['serviceFullName']
+      @short_name = metadata['serviceAbbreviation'] || @full_name
+
+      # Dont reply on API Gateway doc.json
+      unless @protocol == 'api-gateway'
         ApplyDocs.new(@api).apply(load_json(options[:docs]))
       end
       @paginators = load_json(options[:paginators])
       @waiters = load_json(options[:waiters])
       @resources = load_json(options[:resources])
       @examples = load_json(options[:examples])
-      @smoke_tests = load_json(options[:smoke_tests])
       unless options[:legacy_endpoints]
         @endpoint_rules = load_json(options[:endpoint_rules])
         @endpoint_tests = load_json(options[:endpoint_tests])
       end
+      @smoke_tests = load_json(options[:smoke_tests])
+
       @gem_dependencies = options[:gem_dependencies] || {}
       @add_plugins = options[:add_plugins] || {}
       @remove_plugins = options[:remove_plugins] || []
+      @deprecated = options[:deprecated] || false
+      @default_endpoint = options[:default_endpoint] # APIG custom service only
       @endpoints_key = options.fetch(:endpoints_key, nil)
-      # APIG custom service only
-      @default_endpoint = options[:default_endpoint]
 
-      # computed attributes
-      @protocol = api.fetch('metadata').fetch('protocol')
-      @protocol_settings = api.fetch('metadata')['protocolSettings'] || {}
-      @api_version = api.fetch('metadata')['apiVersion']
-      @signature_version = api.fetch('metadata')['signatureVersion']
-      @full_name = api.fetch('metadata')['serviceFullName']
-      @short_name = api.fetch('metadata')['serviceAbbreviation'] || @full_name
-      @require_endpoint_discovery = api.fetch('operations', []).any? do |_, o|
-        o['endpointdiscovery'] && o['endpointdiscovery']['required']
-      end
-      @deprecated = options[:deprecated] || false
+      @require_endpoint_discovery = endpoint_discovery_required?
     end
 
     # @return [String] The service name, e.g. "S3"
@@ -129,6 +147,9 @@ def included_in_core?
     # @return [String] The service protocol, e.g. "json", "query", etc.
     attr_reader :protocol
 
+    # @return [Array<String>] The list of supported protocols
+    attr_reader :protocols
+
     # @return [Hash] The service protocol settings
     attr_reader :protocol_settings
 
@@ -174,6 +195,29 @@ def inspect
 
     private
 
+    def select_protocol(metadata)
+      protocols = metadata.fetch('protocols', [metadata['protocol']])
+      protocol = SUPPORTED_PROTOCOLS.find do |supported_protocol|
+        if %w[json_1.0 json_1.1].include?(supported_protocol)
+          supported_protocol, version = supported_protocol.split('_')
+        end
+
+        if protocols.include?(supported_protocol) &&
+           (version.nil? || version == metadata['jsonVersion'])
+          return supported_protocol
+        end
+      end
+      return protocol if protocol
+
+      raise "unsupported protocols `#{protocols.join(', ')}'"
+    end
+
+    def endpoint_discovery_required?
+      @api.fetch('operations', []).any? do |_, o|
+        o['endpointdiscovery'] && o['endpointdiscovery']['required']
+      end
+    end
+
     def load_json(value)
       case value
       when nil then nil

build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/views/client_api_module.rb

@@ -27,25 +27,26 @@ class ClientApiModule < View
       SHAPE_KEYS = {
         # keep
         'flattened' => true,
-        'timestampFormat' => true, # glacier api customization
+        'timestampFormat' => true,
         'xmlNamespace' => true,
-        'streaming' => true, # transfer-encoding
-        'requiresLength' => true, # transfer-encoding
-        'union' => false, # should remain false
+        'streaming' => true,
+        'requiresLength' => true,
+        'union' => false, # handled separately - should remain false
         'document' => true,
         'jsonvalue' => true,
-        'error' => true, # parsing customized error code in query protocol
-        'locationName' => true, # to recognize xmlName defined on shape
+        'error' => true,
+        'locationName' => true,
+        # ignore
         # event stream modeling
         'event' => false,
         'eventstream' => false,
         'eventheader' => false,
         'eventpayload' => false,
-        # ignore
+        'exceptionEvent' => false, # internal, exceptions cannot be events
+        # other
         'synthetic' => false,
         'box' => false,
         'fault' => false,
-        'exception_event' => false, # internal, exceptions cannot be events
         'deprecated' => false,
         'deprecatedMessage' => false,
         'type' => false,
@@ -68,7 +69,7 @@ class ClientApiModule < View
       }
 
       METADATA_KEYS = {
-        # keep all
+        # keep
         'endpointPrefix' => true,
         'signatureVersion' => true,
         'auth' => true,
@@ -79,18 +80,18 @@ class ClientApiModule < View
         'targetPrefix' => true,
         'jsonVersion' => true,
         'errorPrefix' => true,
-        'timestampFormat' => true, # glacier api customization
+        'timestampFormat' => true,
         'xmlNamespace' => true,
-        'protocolSettings' => {}, # current unused unless for h2 exclude
+        'protocolSettings' => {},
         'serviceId' => true,
         'apiVersion' => true,
         'checksumFormat' => true,
         'globalEndpoint' => true,
         'serviceAbbreviation' => true,
         'uid' => true,
-        'awsQueryCompatible' => true, # AwsQuery migration
+        'awsQueryCompatible' => true,
         # ignore
-        'ripServiceName' => true
+        'ripServiceName' => false
       }
 
       # @option options [required, Service] :service

build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/views/rbs/types_module.rb

@@ -31,7 +31,7 @@ def structures
               # exceptions will not have the event trait.
               shape['members'].each do |name, ref|
                 if !!@service.api['shapes'][ref['shape']]['exception']
-                  @service.api['shapes'][ref['shape']]['exception_event'] = true
+                  @service.api['shapes'][ref['shape']]['exceptionEvent'] = true
                 end
               end
             end
@@ -90,7 +90,7 @@ def struct_members(shape)
               returns: AwsSdkCodeGenerator::RBS.to_type(member_ref, @api)
             )
           end
-          if shape['event'] || shape['exception_event']
+          if shape['event'] || shape['exceptionEvent']
             members << StructMember.new(
               member_name: 'event_type',
               returns: 'untyped'

build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/views/types_module.rb

@@ -34,7 +34,7 @@ def structures
             # exceptions will not have the event trait.
             shape['members'].each do |name, ref|
               if !!@service.api['shapes'][ref['shape']]['exception']
-                @service.api['shapes'][ref['shape']]['exception_event'] = true
+                @service.api['shapes'][ref['shape']]['exceptionEvent'] = true
               end
             end
           end
@@ -92,7 +92,7 @@ def struct_members(shape)
             sensitive: sensitive
           )
         end
-        if shape['event'] || shape['exception_event']
+        if shape['event'] || shape['exceptionEvent']
           members << StructMember.new(member_name: 'event_type')
         end
         members
@@ -180,7 +180,7 @@ def attribute_macros_docs(shape_name)
 
       def see_also_tag(shape_name)
         uid = @api['metadata']['uid']
-        if @api['metadata']['protocol'] != 'api-gateway' && Crosslink.taggable?(uid)
+        if @service.protocol != 'api-gateway' && Crosslink.taggable?(uid)
           Crosslink.tag_string(uid, shape_name)
         end
       end

build_tools/aws-sdk-code-generator/spec/fixtures/interfaces/endpoints_built_ins/api.json

@@ -1,6 +1,6 @@
 {
   "metadata": {
-    "endpointPrefix": "svcname",
+    "endpointPrefix": "svc",
     "serviceId": "sample_svc",
     "protocol": "rest-json"
   },

build_tools/aws-sdk-code-generator/spec/fixtures/interfaces/endpoints_plugin/api.json

@@ -1,6 +1,6 @@
 {
   "metadata": {
-    "endpointPrefix": "svcname",
+    "endpointPrefix": "svc",
     "serviceId": "sample_svc",
     "protocol": "rest-json"
   },

build_tools/aws-sdk-code-generator/spec/fixtures/interfaces/endpoints_precedence/api.json

@@ -1,6 +1,6 @@
 {
   "metadata": {
-    "endpointPrefix": "svcname",
+    "endpointPrefix": "svc",
     "serviceId": "sample_svc",
     "protocol": "rest-json"
   },

build_tools/aws-sdk-code-generator/spec/fixtures/interfaces/idempotency_token/api.json

@@ -3,6 +3,7 @@
   "metadata":{
     "endpointPrefix":"svc",
     "protocol":"json",
+    "jsonVersion":"1.1",
     "signatureVersion":"v4"
   },
   "operations":{