Implement GetSchema. (#1181)

These changes implement the resource provider's GetSchema method in order to facilitate code generation (e.g. for docs and for the proposed import command). As part of these changes, I refactored the Makefile s.t. each language has its own target, and I updated the generator s.t. the schema is read from disk unless it is the target language. The only exception is the .NET code generator, which still uses the Swagger spec for YAML support. Fixes #1180.
pulumi · Jun 30, 2020 · da35c89 · da35c89
1 parent 6345139
commit da35c89
Show file tree

Hide file tree

Showing 10 changed files with 164 additions and 60 deletions.
diff --git a/.gitignore b/.gitignore
@@ -8,4 +8,6 @@
 .pulumi
 Pulumi.*.yaml
 /provider/pkg/gen/openapi-specs
+/provider/cmd/pulumi-resource-kubernetes/schema.go
+/provider/cmd/pulumi-resource-kubernetes/schema.json
 yarn.lock
diff --git a/.golangci.yml b/.golangci.yml
@@ -18,3 +18,6 @@ linters:
     - lll
     - megacheck # Disabled due to OOM errors in golangci-lint@v1.18.0
     - staticcheck # Disabled due to OOM errors in golangci-lint@v1.18.0
+run:
+  skip-files:
+  - schema.go
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -9,6 +9,7 @@
 -   Fix prometheus-operator test to wait for the CRD to be ready before use (https://github.com/pulumi/pulumi-kubernetes/pull/1172)
 -   Set supported environment variables in SDK Provider classes (https://github.com/pulumi/pulumi-kubernetes/pull/1166)
 -   Python SDK updated to align with other Pulumi Python SDKs. (https://github.com/pulumi/pulumi-kubernetes/pull/1160)
+-   Implement GetSchema to enable example and import code generation. (https://github.com/pulumi/pulumi-kubernetes/pull/1181)
 
 ## 2.3.1 (June 17, 2020)
 

diff --git a/Makefile b/Makefile
@@ -15,6 +15,7 @@ KUBE_VERSION    ?= v1.18.0
 SWAGGER_URL     ?= https://github.com/kubernetes/kubernetes/raw/${KUBE_VERSION}/api/openapi-spec/swagger.json
 OPENAPI_DIR     := provider/pkg/gen/openapi-specs
 OPENAPI_FILE    := ${OPENAPI_DIR}/swagger-${KUBE_VERSION}.json
+SCHEMA_FILE     := provider/cmd/pulumi-resource-kubernetes/schema.json
 
 VERSION_FLAGS   := -ldflags "-X github.com/pulumi/pulumi-kubernetes/provider/v2/pkg/version.Version=${VERSION}"
 
@@ -37,29 +38,51 @@ $(OPENAPI_FILE)::
 	@mkdir -p $(OPENAPI_DIR)
 	test -f $(OPENAPI_FILE) || $(CURL) -s -L $(SWAGGER_URL) > $(OPENAPI_FILE)
 
-build:: $(OPENAPI_FILE)
-	cd provider && $(GO) install $(VERSION_FLAGS) $(PROJECT)/provider/v2/cmd/$(PROVIDER)
+k8sgen::
 	cd provider && $(GO) install $(VERSION_FLAGS) $(PROJECT)/provider/v2/cmd/$(CODEGEN)
-	# Delete only files and folders that are generated.
-	rm -r sdk/python/pulumi_kubernetes/*/ sdk/python/pulumi_kubernetes/__init__.py
-	for LANGUAGE in "dotnet" "go" "nodejs" "python"; do \
-		$(CODEGEN) $$LANGUAGE $(OPENAPI_FILE) $(CURDIR) || exit 3 ; \
-	done
+
+$(SCHEMA_FILE):: k8sgen $(OPENAPI_FILE)
+	$(call STEP_MESSAGE)
+	@echo "Generating Pulumi schema..."
+	$(CODEGEN) schema $(OPENAPI_FILE) $(CURDIR)
+	@echo "Finished generating schema."
+
+k8sprovider:: $(SCHEMA_FILE)
+	$(CODEGEN) -version=${VERSION} kinds $(SCHEMA_FILE) $(CURDIR)
+	cd provider && $(GO) generate cmd/${PROVIDER}/main.go
+	cd provider && $(GO) install $(VERSION_FLAGS) $(PROJECT)/provider/v2/cmd/$(PROVIDER)
+
+dotnet_sdk:: k8sgen $(OPENAPI_FILE)
+	$(CODEGEN) -version=${VERSION} dotnet $(OPENAPI_FILE) $(CURDIR)
+	cd ${PACKDIR}/dotnet/&& \
+		echo "${VERSION:v%=%}" >version.txt && \
+		dotnet build /p:Version=${DOTNET_VERSION}
+
+go_sdk:: k8sgen $(SCHEMA_FILE)
+	$(CODEGEN) -version=${VERSION} go $(SCHEMA_FILE) $(CURDIR)
+
+nodejs_sdk:: k8sgen $(SCHEMA_FILE)
+	$(CODEGEN) -version=${VERSION} nodejs $(SCHEMA_FILE) $(CURDIR)
 	cd ${PACKDIR}/nodejs/ && \
 		yarn install && \
 		yarn run tsc
 	cp README.md LICENSE ${PACKDIR}/nodejs/package.json ${PACKDIR}/nodejs/yarn.lock ${PACKDIR}/nodejs/bin/
-	cp README.md ${PACKDIR}/python/
 	sed -i.bak 's/$${VERSION}/$(VERSION)/g' ${PACKDIR}/nodejs/bin/package.json
+
+python_sdk:: k8sgen $(SCHEMA_FILE)
+	# Delete only files and folders that are generated.
+	rm -r sdk/python/pulumi_kubernetes/*/ sdk/python/pulumi_kubernetes/__init__.py
+	$(CODEGEN) -version=${VERSION} python $(SCHEMA_FILE) $(CURDIR)
+	cp README.md ${PACKDIR}/python/
 	cd ${PACKDIR}/python/ && \
 		$(PYTHON) setup.py clean --all 2>/dev/null && \
 		rm -rf ./bin/ ../python.bin/ && cp -R . ../python.bin && mv ../python.bin ./bin && \
 		sed -i.bak -e "s/\$${VERSION}/$(PYPI_VERSION)/g" -e "s/\$${PLUGIN_VERSION}/$(VERSION)/g" ./bin/setup.py && \
 		rm ./bin/setup.py.bak && \
 		cd ./bin && $(PYTHON) setup.py build sdist
-	cd ${PACKDIR}/dotnet/&& \
-		echo "${VERSION:v%=%}" >version.txt && \
-		dotnet build /p:Version=${DOTNET_VERSION}
+
+.PHONY: build
+build:: k8sgen k8sprovider dotnet_sdk go_sdk nodejs_sdk python_sdk
 
 lint::
 	for DIR in "provider" "sdk" "tests" ; do \
@@ -91,12 +114,8 @@ test_all::
 	cd provider/pkg && $(GO_TEST_FAST) ./...
 	cd tests && $(GO_TEST) ./...
 
-generate_schema:: $(OPENAPI_FILE)
-	$(call STEP_MESSAGE)
-	cd provider && $(GO) install $(VERSION_FLAGS) $(PROJECT)/provider/v2/cmd/$(CODEGEN)
-	echo "Generating Pulumi schema..."
-	$(CODEGEN) schema $(OPENAPI_FILE) "" $(PACKDIR)
-	echo "Finished generating schema."
+generate_schema:: $(SCHEMA_FILE)
+	cp $(SCHEMA_FILE) $(PACKDIR)/schema/schema.json
 
 .PHONY: publish_tgz
 publish_tgz:

diff --git a/provider/cmd/pulumi-gen-kubernetes/main.go b/provider/cmd/pulumi-gen-kubernetes/main.go
@@ -18,10 +18,10 @@ package main
 import (
 	"bytes"
 	"encoding/json"
+	"flag"
 	"fmt"
 	"go/format"
 	"io/ioutil"
-	"log"
 	"os"
 	"path"
 	"path/filepath"
@@ -31,6 +31,7 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/pulumi/pulumi-kubernetes/provider/v2/pkg/gen"
+	providerVersion "github.com/pulumi/pulumi-kubernetes/provider/v2/pkg/version"
 	"github.com/pulumi/pulumi/pkg/v2/codegen"
 	dotnetgen "github.com/pulumi/pulumi/pkg/v2/codegen/dotnet"
 	gogen "github.com/pulumi/pulumi/pkg/v2/codegen/go"
@@ -63,69 +64,103 @@ const (
 	DotNet Language = "dotnet"
 	Go     Language = "go"
 	NodeJS Language = "nodejs"
+	Kinds  Language = "kinds"
 	Python Language = "python"
 	Schema Language = "schema"
 )
 
 func main() {
-	if len(os.Args) < 4 {
-		log.Fatal("Usage: gen <language> <swagger-file> <root-pulumi-kubernetes-dir>")
+	flag.Usage = func() {
+		const usageFormat = "Usage: %s <language> <swagger-or-schema-file> <root-pulumi-kubernetes-dir>"
+		_, err := fmt.Fprintf(flag.CommandLine.Output(), usageFormat, os.Args[0])
+		contract.IgnoreError(err)
+		flag.PrintDefaults()
 	}
 
-	language := Language(os.Args[1])
+	var version string
+	flag.StringVar(&version, "version", providerVersion.Version, "the provider version to record in the generated code")
 
-	swagger, err := ioutil.ReadFile(os.Args[2])
-	if err != nil {
-		panic(err)
+	flag.Parse()
+	args := flag.Args()
+	if len(args) < 3 {
+		flag.Usage()
+		return
 	}
 
-	swaggerDir := filepath.Dir(os.Args[2])
-
-	legacySwaggerPath := filepath.Join(swaggerDir, Swagger117FileName)
-	err = DownloadFile(legacySwaggerPath, Swagger117Url)
-	if err != nil {
-		panic(err)
-	}
-	legacySwagger, err := ioutil.ReadFile(legacySwaggerPath)
-	if err != nil {
-		panic(err)
-	}
-	mergedSwagger := mergeSwaggerSpecs(legacySwagger, swagger)
-	data := mergedSwagger.(map[string]interface{})
+	language, inputFile := Language(args[0]), args[1]
 
-	BaseDir = os.Args[3]
+	BaseDir = args[2]
 	TemplateDir = path.Join(BaseDir, "provider", "pkg", "gen")
 	outdir := path.Join(BaseDir, "sdk", string(language))
 
-	// Generate schema
-	pkgSpec := gen.PulumiSchema(data)
-
-	// Generate package from schema
-	pkg := genPulumiSchemaPackage(pkgSpec)
-
-	// Generate provider code
-	genK8sResourceTypes(pkg)
-
 	switch language {
 	case NodeJS:
 		templateDir := path.Join(TemplateDir, "nodejs-templates")
-		writeNodeJSClient(pkg, outdir, templateDir)
+		writeNodeJSClient(readSchema(inputFile), outdir, templateDir)
 	case Python:
 		templateDir := path.Join(TemplateDir, "python-templates")
-		writePythonClient(pkg, outdir, templateDir)
+		writePythonClient(readSchema(inputFile), outdir, templateDir)
 	case DotNet:
 		templateDir := path.Join(TemplateDir, "dotnet-templates")
-		writeDotnetClient(pkg, data, outdir, templateDir)
+		data, pkgSpec := generateSchema(inputFile, version)
+		writeDotnetClient(genPulumiSchemaPackage(pkgSpec), data, outdir, templateDir)
 	case Go:
 		templateDir := path.Join(TemplateDir, "go-templates")
-		writeGoClient(pkg, outdir, templateDir)
+		writeGoClient(readSchema(inputFile), outdir, templateDir)
+	case Kinds:
+		pkg := readSchema(inputFile)
+		genK8sResourceTypes(pkg)
 	case Schema:
+		_, pkgSpec := generateSchema(inputFile, version)
 		mustWritePulumiSchema(pkgSpec, outdir)
 	default:
 		panic(fmt.Sprintf("Unrecognized language '%s'", language))
 	}
 }
 
+func readSchema(schemaPath string) *schema.Package {
+	// Read in, decode, and import the schema.
+	schemaBytes, err := ioutil.ReadFile(schemaPath)
+	if err != nil {
+		panic(err)
+	}
+
+	var pkgSpec schema.PackageSpec
+	if err = json.Unmarshal(schemaBytes, &pkgSpec); err != nil {
+		panic(err)
+	}
+
+	pkg, err := schema.ImportSpec(pkgSpec, nil)
+	if err != nil {
+		panic(err)
+	}
+	return pkg
+}
+
+func generateSchema(swaggerPath, version string) (map[string]interface{}, schema.PackageSpec) {
+	swagger, err := ioutil.ReadFile(swaggerPath)
+	if err != nil {
+		panic(err)
+	}
+
+	swaggerDir := filepath.Dir(swaggerPath)
+
+	legacySwaggerPath := filepath.Join(swaggerDir, Swagger117FileName)
+	err = DownloadFile(legacySwaggerPath, Swagger117Url)
+	if err != nil {
+		panic(err)
+	}
+	legacySwagger, err := ioutil.ReadFile(legacySwaggerPath)
+	if err != nil {
+		panic(err)
+	}
+	mergedSwagger := mergeSwaggerSpecs(legacySwagger, swagger)
+	data := mergedSwagger.(map[string]interface{})
+
+	// Generate schema
+	return data, gen.PulumiSchema(data, version)
+}
+
 func writeNodeJSClient(pkg *schema.Package, outdir, templateDir string) {
 	resources, err := nodejsgen.LanguageResources(pkg)
 	if err != nil {
@@ -361,5 +396,5 @@ func mustWritePulumiSchema(pkgSpec schema.PackageSpec, outDir string) {
 		panic(errors.Wrap(err, "marshaling Pulumi schema"))
 	}
 
-	mustWriteFile(outDir, "schema.json", schemaJSON)
+	mustWriteFile(BaseDir, "provider/cmd/pulumi-resource-kubernetes/schema.json", schemaJSON)
 }
diff --git a/provider/cmd/pulumi-resource-kubernetes/generate.go b/provider/cmd/pulumi-resource-kubernetes/generate.go
@@ -0,0 +1,37 @@
+// Copyright 2016-2020, Pulumi Corporation.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build ignore
+
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log"
+)
+
+func main() {
+	schemaContents, err := ioutil.ReadFile("./schema.json")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	err = ioutil.WriteFile("./schema.go", []byte(fmt.Sprintf(`package main
+var pulumiSchema = %#v
+`, schemaContents)), 0600)
+	if err != nil {
+		log.Fatal(err)
+	}
+}
diff --git a/provider/cmd/pulumi-resource-kubernetes/main.go b/provider/cmd/pulumi-resource-kubernetes/main.go
@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:generate go run ./generate.go
+
 package main
 
 import (
@@ -22,5 +24,5 @@ import (
 var providerName = "kubernetes"
 
 func main() {
-	provider.Serve(providerName, version.Version)
+	provider.Serve(providerName, version.Version, pulumiSchema)
 }
diff --git a/provider/pkg/gen/schema.go b/provider/pkg/gen/schema.go
@@ -20,16 +20,15 @@ import (
 	"fmt"
 	"strings"
 
-	providerVersion "github.com/pulumi/pulumi-kubernetes/provider/v2/pkg/version"
 	pschema "github.com/pulumi/pulumi/pkg/v2/codegen/schema"
 	"github.com/pulumi/pulumi/sdk/v2/go/common/util/contract"
 )
 
 // PulumiSchema will generate a Pulumi schema for the given k8s schema.
-func PulumiSchema(swagger map[string]interface{}) pschema.PackageSpec {
+func PulumiSchema(swagger map[string]interface{}, version string) pschema.PackageSpec {
 	pkg := pschema.PackageSpec{
 		Name:        "kubernetes",
-		Version:     providerVersion.Version,
+		Version:     version,
 		Description: "A Pulumi package for creating and managing Kubernetes resources.",
 		License:     "Apache-2.0",
 		Keywords:    []string{"pulumi", "kubernetes"},

diff --git a/provider/pkg/provider/provider.go b/provider/pkg/provider/provider.go
@@ -102,6 +102,7 @@ type kubeProvider struct {
 	canceler         *cancellationContext
 	name             string
 	version          string
+	pulumiSchema     []byte
 	providerPackage  string
 	opts             kubeOpts
 	defaultNamespace string
@@ -129,13 +130,14 @@ type kubeProvider struct {
 var _ pulumirpc.ResourceProviderServer = (*kubeProvider)(nil)
 
 func makeKubeProvider(
-	host *provider.HostClient, name, version string,
+	host *provider.HostClient, name, version string, pulumiSchema []byte,
 ) (pulumirpc.ResourceProviderServer, error) {
 	return &kubeProvider{
 		host:                        host,
 		canceler:                    makeCancellationContext(),
 		name:                        name,
 		version:                     version,
+		pulumiSchema:                pulumiSchema,
 		providerPackage:             name,
 		enableDryRun:                false,
 		enableSecrets:               false,
@@ -171,7 +173,11 @@ func (k *kubeProvider) invalidateResources() {
 }
 
 func (k *kubeProvider) GetSchema(ctx context.Context, req *pulumirpc.GetSchemaRequest) (*pulumirpc.GetSchemaResponse, error) {
-	return nil, rpcerror.New(codes.Unimplemented, "GetSchema is unimplemented")
+	if v := req.GetVersion(); v != 0 {
+		return nil, fmt.Errorf("unsupported schema version %d", v)
+	}
+
+	return &pulumirpc.GetSchemaResponse{Schema: string(k.pulumiSchema)}, nil
 }
 
 // CheckConfig validates the configuration for this provider.

diff --git a/provider/pkg/provider/serve.go b/provider/pkg/provider/serve.go
@@ -24,11 +24,11 @@ import (
 )
 
 // Serve launches the gRPC server for the Pulumi Kubernetes resource provider.
-func Serve(providerName, version string) {
+func Serve(providerName, version string, pulumiSchema []byte) {
 	// Start gRPC service.
 	err := provider.Main(
 		providerName, func(host *provider.HostClient) (lumirpc.ResourceProviderServer, error) {
-			return makeKubeProvider(host, providerName, version)
+			return makeKubeProvider(host, providerName, version, pulumiSchema)
 		})
 
 	if err != nil {