Add GetMetadata RPC to provider protocol (#33739)

Reference: #33486

This is a followup to the new provider server capability to make the `GetProviderSchema` RPC optional. While this server capability would perform its intended function when directly talking to a single provider server SDK implementation, provider servers using terraform-plugin-mux need a methodology for the mux server to determine the available resource types of each underlying provider server to properly route resource-specific RPCs. Since the only methodology available to the mux server would be calling the `GetProviderSchema` RPC to each of underlying provider servers, any memory optimization of core caching would be lost.

The choice of adding a new RPC instead of adjusting the existing `GetProviderSchema` RPC with additional request information, such as "only list the type names and not the schema information in the response", is two-fold:

- Prevents the introduction of conditional logic for the existing RPC.
- Clearly delineates the purpose of the RPC and can be documented easier.

The choice of adding this to the existing provider service is two-fold:

- Implementing a separate protocol and/or service only on the provider side of the protocol would be a novel design change. This small of a change does not warrant the potential research and testing effort that would be associated with that implementation.
- While the core implementation will not use the new RPC immediately, there is no reason why it should be restricted from doing so in the future if a valid use case surfaces. Other ecosystem tools, beyond terraform-plugin-mux, can also potentially benefit from the lightweight RPC now.

This is changing the 5.4 and 6.4 protocol versions following the guidance of this comment in the definition files, since it directly relates to the prior intention of the new minor versions:

```protobuf
// Note that only the proto files included in a release tag of Terraform are
// official protocol releases. Proto files taken from other commits may include
// incomplete changes or features that did not make it into a final release.
// In all reasonable cases, plugin developers should take the proto file from
// the tag of the most recent release of Terraform, and not from the main
// branch or any other development branch.
```

As with any Protocol Buffers definition update, protocol compatibility is guaranteed within a major version, however generated protocol source code compatibility is not guaranteed. In this case, implementing the new RPC method in protocol wrapper types and the moving of the `ServerCapabilities` message to the top namespace are considered acceptable changes.

docs/plugin-protocol/tfplugin5.4.proto

@@ -128,8 +128,34 @@ message Schema {
     Block block = 2;
 }
 
+// ServerCapabilities allows providers to communicate extra information
+// regarding supported protocol features. This is used to indicate
+// availability of certain forward-compatible changes which may be optional
+// in a major protocol version, but cannot be tested for directly.
+message ServerCapabilities {
+    // The plan_destroy capability signals that a provider expects a call
+    // to PlanResourceChange when a resource is going to be destroyed.
+    bool plan_destroy = 1;
+
+    // The get_provider_schema_optional capability indicates that this
+    // provider does not require calling GetProviderSchema to operate
+    // normally, and the caller can used a cached copy of the provider's
+    // schema.
+    bool get_provider_schema_optional = 2;
+}
+
 service Provider {
     //////// Information about what a provider supports/expects
+
+    // GetMetadata returns upfront information about server capabilities and
+    // supported resource types without requiring the server to instantiate all
+    // schema information, which may be memory intensive. This RPC is optional,
+    // where clients may receive an unimplemented RPC error. Clients should
+    // ignore the error and call the GetSchema RPC as a fallback.
+    rpc GetMetadata(GetMetadata.Request) returns (GetMetadata.Response);
+
+    // GetSchema returns schema information for the provider, data resources,
+    // and managed resources.
     rpc GetSchema(GetProviderSchema.Request) returns (GetProviderSchema.Response);
     rpc PrepareProviderConfig(PrepareProviderConfig.Request) returns (PrepareProviderConfig.Response);
     rpc ValidateResourceTypeConfig(ValidateResourceTypeConfig.Request) returns (ValidateResourceTypeConfig.Response);
@@ -151,6 +177,26 @@ service Provider {
     rpc Stop(Stop.Request) returns (Stop.Response);
 }
 
+message GetMetadata {
+    message Request {
+    }
+
+    message Response {
+        ServerCapabilities server_capabilities = 1;
+        repeated Diagnostic diagnostics = 2;
+        repeated DataSourceMetadata data_sources = 3;
+        repeated ResourceMetadata resources = 4;
+    }
+
+    message DataSourceMetadata {
+        string type_name = 1;
+    }
+
+    message ResourceMetadata {
+        string type_name = 1;
+    }
+}
+
 message GetProviderSchema {
     message Request {
     }
@@ -162,23 +208,6 @@ message GetProviderSchema {
         Schema provider_meta = 5;
         ServerCapabilities server_capabilities = 6;
     }
-
-
-    // ServerCapabilities allows providers to communicate extra information
-    // regarding supported protocol features. This is used to indicate
-    // availability of certain forward-compatible changes which may be optional
-    // in a major protocol version, but cannot be tested for directly.
-    message ServerCapabilities {
-        // The plan_destroy capability signals that a provider expects a call
-        // to PlanResourceChange when a resource is going to be destroyed.
-        bool plan_destroy = 1;
-
-        // The get_provider_schema_optional capability indicates that this
-        // provider does not require calling GetProviderSchema to operate
-        // normally, and the caller can used a cached copy of the provider's
-        // schema.
-        bool get_provider_schema_optional = 2;
-    }
 }
 
 message PrepareProviderConfig {

docs/plugin-protocol/tfplugin6.4.proto

@@ -147,8 +147,34 @@ message Schema {
     Block block = 2;
 }
 
+// ServerCapabilities allows providers to communicate extra information
+// regarding supported protocol features. This is used to indicate
+// availability of certain forward-compatible changes which may be optional
+// in a major protocol version, but cannot be tested for directly.
+message ServerCapabilities {
+    // The plan_destroy capability signals that a provider expects a call
+    // to PlanResourceChange when a resource is going to be destroyed.
+    bool plan_destroy = 1;
+
+    // The get_provider_schema_optional capability indicates that this
+    // provider does not require calling GetProviderSchema to operate
+    // normally, and the caller can used a cached copy of the provider's
+    // schema.
+    bool get_provider_schema_optional = 2;
+}
+
 service Provider {
     //////// Information about what a provider supports/expects
+
+    // GetMetadata returns upfront information about server capabilities and
+    // supported resource types without requiring the server to instantiate all
+    // schema information, which may be memory intensive. This RPC is optional,
+    // where clients may receive an unimplemented RPC error. Clients should
+    // ignore the error and call the GetProviderSchema RPC as a fallback.
+    rpc GetMetadata(GetMetadata.Request) returns (GetMetadata.Response);
+
+    // GetSchema returns schema information for the provider, data resources,
+    // and managed resources.
     rpc GetProviderSchema(GetProviderSchema.Request) returns (GetProviderSchema.Response);
     rpc ValidateProviderConfig(ValidateProviderConfig.Request) returns (ValidateProviderConfig.Response);
     rpc ValidateResourceConfig(ValidateResourceConfig.Request) returns (ValidateResourceConfig.Response);
@@ -170,6 +196,26 @@ service Provider {
     rpc StopProvider(StopProvider.Request) returns (StopProvider.Response);
 }
 
+message GetMetadata {
+    message Request {
+    }
+
+    message Response {
+        ServerCapabilities server_capabilities = 1;
+        repeated Diagnostic diagnostics = 2;
+        repeated DataSourceMetadata data_sources = 3;
+        repeated ResourceMetadata resources = 4;
+    }
+
+    message DataSourceMetadata {
+        string type_name = 1;
+    }
+
+    message ResourceMetadata {
+        string type_name = 1;
+    }
+}
+
 message GetProviderSchema {
     message Request {
     }
@@ -181,23 +227,6 @@ message GetProviderSchema {
         Schema provider_meta = 5;
         ServerCapabilities server_capabilities = 6;
     }
-
-
-    // ServerCapabilities allows providers to communicate extra information
-    // regarding supported protocol features. This is used to indicate
-    // availability of certain forward-compatible changes which may be optional
-    // in a major protocol version, but cannot be tested for directly.
-    message ServerCapabilities {
-        // The plan_destroy capability signals that a provider expects a call
-        // to PlanResourceChange when a resource is going to be destroyed.
-        bool plan_destroy = 1;
-
-        // The get_provider_schema_optional capability indicates that this
-        // provider does not require calling GetProviderSchema to operate
-        // normally, and the caller can used a cached copy of the provider's
-        // schema.
-        bool get_provider_schema_optional = 2;
-    }
 }
 
 message ValidateProviderConfig {

internal/grpcwrap/provider.go

@@ -12,6 +12,8 @@ import (
 	"github.com/zclconf/go-cty/cty"
 	ctyjson "github.com/zclconf/go-cty/cty/json"
 	"github.com/zclconf/go-cty/cty/msgpack"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
 )
 
 // New wraps a providers.Interface to implement a grpc ProviderServer.
@@ -29,6 +31,10 @@ type provider struct {
 	schema   providers.GetProviderSchemaResponse
 }
 
+func (p *provider) GetMetadata(_ context.Context, req *tfplugin5.GetMetadata_Request) (*tfplugin5.GetMetadata_Response, error) {
+	return nil, status.Error(codes.Unimplemented, "GetMetadata is not implemented by core")
+}
+
 func (p *provider) GetSchema(_ context.Context, req *tfplugin5.GetProviderSchema_Request) (*tfplugin5.GetProviderSchema_Response, error) {
 	resp := &tfplugin5.GetProviderSchema_Response{
 		ResourceSchemas:   make(map[string]*tfplugin5.Schema),
@@ -62,7 +68,7 @@ func (p *provider) GetSchema(_ context.Context, req *tfplugin5.GetProviderSchema
 		}
 	}
 
-	resp.ServerCapabilities = &tfplugin5.GetProviderSchema_ServerCapabilities{
+	resp.ServerCapabilities = &tfplugin5.ServerCapabilities{
 		PlanDestroy: p.schema.ServerCapabilities.PlanDestroy,
 	}
 

internal/grpcwrap/provider6.go

@@ -12,6 +12,8 @@ import (
 	"github.com/zclconf/go-cty/cty"
 	ctyjson "github.com/zclconf/go-cty/cty/json"
 	"github.com/zclconf/go-cty/cty/msgpack"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
 )
 
 // New wraps a providers.Interface to implement a grpc ProviderServer using
@@ -29,6 +31,10 @@ type provider6 struct {
 	schema   providers.GetProviderSchemaResponse
 }
 
+func (p *provider6) GetMetadata(_ context.Context, req *tfplugin6.GetMetadata_Request) (*tfplugin6.GetMetadata_Response, error) {
+	return nil, status.Error(codes.Unimplemented, "GetMetadata is not implemented by core")
+}
+
 func (p *provider6) GetProviderSchema(_ context.Context, req *tfplugin6.GetProviderSchema_Request) (*tfplugin6.GetProviderSchema_Response, error) {
 	resp := &tfplugin6.GetProviderSchema_Response{
 		ResourceSchemas:   make(map[string]*tfplugin6.Schema),
@@ -62,8 +68,9 @@ func (p *provider6) GetProviderSchema(_ context.Context, req *tfplugin6.GetProvi
 		}
 	}
 
-	resp.ServerCapabilities = &tfplugin6.GetProviderSchema_ServerCapabilities{
-		PlanDestroy: p.schema.ServerCapabilities.PlanDestroy,
+	resp.ServerCapabilities = &tfplugin6.ServerCapabilities{
+		GetProviderSchemaOptional: p.schema.ServerCapabilities.GetProviderSchemaOptional,
+		PlanDestroy:               p.schema.ServerCapabilities.PlanDestroy,
 	}
 
 	// include any diagnostics from the original GetSchema call