From 334b3b75e18af5015b820fd82bc712c209f5f381 Mon Sep 17 00:00:00 2001 From: Sean Sullivan Date: Mon, 12 Oct 2020 13:24:53 -0700 Subject: [PATCH] StreamManifestReader for ResourceGroup inventory --- internal/live/rgstream.go | 92 ++++++++++++++ internal/live/rgstream_test.go | 224 +++++++++++++++++++++++++++++++++ 2 files changed, 316 insertions(+) create mode 100644 internal/live/rgstream.go create mode 100644 internal/live/rgstream_test.go diff --git a/internal/live/rgstream.go b/internal/live/rgstream.go new file mode 100644 index 0000000000..5d7fa8f0e6 --- /dev/null +++ b/internal/live/rgstream.go @@ -0,0 +1,92 @@ +// Copyright 2020 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package live + +import ( + "bytes" + "fmt" + "io" + + "github.com/GoogleContainerTools/kpt/pkg/kptfile" + "k8s.io/cli-runtime/pkg/resource" + "k8s.io/klog" + "sigs.k8s.io/cli-utils/pkg/manifestreader" + "sigs.k8s.io/kustomize/kyaml/yaml" +) + +// ResourceGroupStreamManifestReader encapsulates the default stream +// manifest reader. +type ResourceGroupStreamManifestReader struct { + streamReader *manifestreader.StreamManifestReader +} + +var ResourceSeparator = []byte("\n---\n") + +// Read reads the manifests and returns them as Info objects. +// Transforms the Kptfile into the ResourceGroup inventory object, +// and appends it to the rest of the standard StreamManifestReader +// generated objects. Returns an error if one occurs. If the +// ResourceGroup inventory object does not exist, it is NOT an error. +func (p *ResourceGroupStreamManifestReader) Read() ([]*resource.Info, error) { + var resourceBytes bytes.Buffer + _, err := io.Copy(&resourceBytes, p.streamReader.Reader) + if err != nil { + return []*resource.Info{}, err + } + // Split the bytes into resource configs, and if the resource + // config is a Kptfile, transform it into a ResourceGroup object. + var rgInfo *resource.Info + var filteredBytes bytes.Buffer + resources := bytes.Split(resourceBytes.Bytes(), ResourceSeparator) + for _, r := range resources { + if !isKptfile(r) { + r = append(r, ResourceSeparator...) + _, err := filteredBytes.Write(r) + if err != nil { + return []*resource.Info{}, err + } + } else { + rgInfo, err = transformKptfile(r) + if err != nil { + return []*resource.Info{}, err + } + } + } + // Reset the stream reader, and generate the infos. Append the + // ResourceGroup inventory info if it exists. + p.streamReader.Reader = bytes.NewReader(filteredBytes.Bytes()) + infos, err := p.streamReader.Read() + if rgInfo != nil { + infos = append(infos, rgInfo) + } + return infos, err +} + +var kptFileTemplate = kptfile.KptFile{ResourceMeta: kptfile.TypeMeta} + +// isKptfile returns true if the passed resource config is a Kptfile; false otherwise +func isKptfile(resource []byte) bool { + d := yaml.NewDecoder(bytes.NewReader(resource)) + d.KnownFields(true) + if err := d.Decode(&kptFileTemplate); err == nil { + return kptFileTemplate.ResourceMeta.TypeMeta == kptfile.TypeMeta.TypeMeta + } + return false +} + +// transformKptfile transforms the passed kptfile resource config +// into the ResourceGroup inventory object, or an error. +func transformKptfile(resource []byte) (*resource.Info, error) { + d := yaml.NewDecoder(bytes.NewReader(resource)) + d.KnownFields(true) + if err := d.Decode(&kptFileTemplate); err != nil { + return nil, err + } + if kptFileTemplate.ResourceMeta.TypeMeta != kptfile.TypeMeta.TypeMeta { + return nil, fmt.Errorf("invalid kptfile type: %s", kptFileTemplate.ResourceMeta.TypeMeta) + } + inv := kptFileTemplate.Inventory + klog.V(4).Infof("generating ResourceGroup inventory object %s/%s/%s", inv.Namespace, inv.Name, inv.InventoryID) + return generateInventoryObj(inv.Name, inv.Namespace, inv.InventoryID) +} diff --git a/internal/live/rgstream_test.go b/internal/live/rgstream_test.go new file mode 100644 index 0000000000..ea1e821a00 --- /dev/null +++ b/internal/live/rgstream_test.go @@ -0,0 +1,224 @@ +// Copyright 2020 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package live + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "k8s.io/cli-runtime/pkg/resource" + cmdtesting "k8s.io/kubectl/pkg/cmd/testing" + "sigs.k8s.io/cli-utils/pkg/manifestreader" +) + +func TestResourceStreamManifestReader_Read(t *testing.T) { + testCases := map[string]struct { + manifests map[string]string + numInfos int + }{ + "Kptfile only is valid": { + manifests: map[string]string{ + "Kptfile": kptFile, + }, + numInfos: 1, + }, + "Only a pod is valid": { + manifests: map[string]string{ + "pod-a.yaml": podA, + }, + numInfos: 1, + }, + "Multiple pods are valid": { + manifests: map[string]string{ + "pod-a.yaml": podA, + "deployment-a.yaml": deploymentA, + }, + numInfos: 2, + }, + "Basic ResourceGroup inventory object created": { + manifests: map[string]string{ + "Kptfile": kptFile, + "pod-a.yaml": podA, + }, + numInfos: 2, + }, + "ResourceGroup inventory object created, multiple objects": { + manifests: map[string]string{ + "Kptfile": kptFile, + "pod-a.yaml": podA, + "deployment-a.yaml": deploymentA, + }, + numInfos: 3, + }, + "ResourceGroup inventory object created, Kptfile last": { + manifests: map[string]string{ + "deployment-a.yaml": deploymentA, + "Kptfile": kptFile, + }, + numInfos: 2, + }, + } + + for tn, tc := range testCases { + t.Run(tn, func(t *testing.T) { + tf := cmdtesting.NewTestFactory().WithNamespace("test-ns") + defer tf.Cleanup() + + streamStr := "" + for _, manifestStr := range tc.manifests { + streamStr = streamStr + "\n---\n" + manifestStr + } + streamStr += "\n---\n" + streamReader := &manifestreader.StreamManifestReader{ + ReaderName: "rgstream", + Reader: strings.NewReader(streamStr), + ReaderOptions: manifestreader.ReaderOptions{ + Factory: tf, + Namespace: inventoryNamespace, + EnforceNamespace: false, + }, + } + rgStreamReader := &ResourceGroupStreamManifestReader{ + streamReader: streamReader, + } + readInfos, err := rgStreamReader.Read() + assert.NoError(t, err) + assert.Equal(t, tc.numInfos, len(readInfos)) + for _, info := range readInfos { + assert.Equal(t, inventoryNamespace, info.Namespace) + } + invInfo := findResourceGroupInventory(readInfos) + if invInfo != nil { + assert.Equal(t, inventoryName, invInfo.Name) + actualID, err := getInventoryLabel(invInfo) + assert.NoError(t, err) + assert.Equal(t, inventoryID, actualID) + } + }) + } +} + +func TestResourceStreamManifestReader_isKptfile(t *testing.T) { + testCases := map[string]struct { + kptfile string + expected bool + }{ + "Empty kptfile is invalid": { + kptfile: "", + expected: false, + }, + "Kptfile with foo/bar GVK is invalid": { + kptfile: ` +apiVersion: foo/v1 +kind: FooBar +metadata: + name: test1 +`, + expected: false, + }, + "Kptfile with bad apiVersion is invalid": { + kptfile: ` +apiVersion: foo/v1 +kind: Kptfile +metadata: + name: test1 +`, + expected: false, + }, + "Kptfile with wrong kind is invalid": { + kptfile: ` +apiVersion: kpt.dev/v1alpha1 +kind: foo +metadata: + name: test1 +`, + expected: false, + }, + "Kptfile with different GVK is invalid": { + kptfile: ` +kind: Deployment +apiVersion: apps/v1 +metadata: + name: test-deployment +spec: + replicas: 1 +`, + expected: false, + }, + "Wrong fields (foo/bar) in kptfile is invalid": { + kptfile: ` +apiVersion: kpt.dev/v1alpha1 +kind: Kptfile +foo: bar +`, + expected: false, + }, + "Kptfile with deployment/replicas fields is invalid": { + kptfile: ` +apiVersion: kpt.dev/v1alpha1 +kind: Kptfile +metadata: + name: test-deployment +spec: + replicas: 1 +`, + expected: false, + }, + "Wrong fields (foo/bar) in kptfile inventory is invalid": { + kptfile: ` +apiVersion: kpt.dev/v1alpha1 +kind: Kptfile +metadata: + name: test1 +inventory: + namespace: test-namespace + name: inventory-obj-name + foo: bar +`, + expected: false, + }, + "Full, regular kptfile is valid": { + kptfile: kptFile, + expected: true, + }, + "Kptfile with only GVK is valid": { + kptfile: ` +apiVersion: kpt.dev/v1alpha1 +kind: Kptfile +`, + expected: true, + }, + "Kptfile missing optional inventory is still valid": { + kptfile: ` +apiVersion: kpt.dev/v1alpha1 +kind: Kptfile +metadata: + name: test1 +`, + expected: true, + }, + } + + for tn, tc := range testCases { + t.Run(tn, func(t *testing.T) { + actual := isKptfile([]byte(tc.kptfile)) + if tc.expected != actual { + t.Errorf("expected isKptfile (%t), got (%t)", tc.expected, actual) + } + }) + } +} + +// Returns the ResourceGroup inventory object from a slice +// of objects, or nil if it does not exist. +func findResourceGroupInventory(infos []*resource.Info) *resource.Info { + for _, info := range infos { + invLabel, _ := getInventoryLabel(info) + if len(invLabel) != 0 { + return info + } + } + return nil +}