From 41914ce07dab077fbd9c0ab849661489039c3157 Mon Sep 17 00:00:00 2001 From: Mikalai Radchuk Date: Thu, 7 Sep 2023 15:14:33 +0100 Subject: [PATCH] Adds client to fetch bundles Signed-off-by: Mikalai Radchuk --- internal/catalogmetadata/client/client.go | 92 +++++ .../catalogmetadata/client/client_test.go | 321 ++++++++++++++++++ internal/catalogmetadata/types.go | 3 +- 3 files changed, 415 insertions(+), 1 deletion(-) create mode 100644 internal/catalogmetadata/client/client.go create mode 100644 internal/catalogmetadata/client/client_test.go diff --git a/internal/catalogmetadata/client/client.go b/internal/catalogmetadata/client/client.go new file mode 100644 index 000000000..b472d14ff --- /dev/null +++ b/internal/catalogmetadata/client/client.go @@ -0,0 +1,92 @@ +package client + +import ( + "context" + "fmt" + + "sigs.k8s.io/controller-runtime/pkg/client" + + catalogd "github.com/operator-framework/catalogd/api/core/v1alpha1" + "github.com/operator-framework/operator-registry/alpha/declcfg" + + "github.com/operator-framework/operator-controller/internal/catalogmetadata" +) + +func NewClient(cl client.Client) *Client { + return &Client{cl: cl} +} + +// Client is reading catalog metadata +type Client struct { + // Note that eventually we will be reading from catalogd http API + // instead of kube API server. We will need to swap this implementation. + cl client.Client +} + +func (c *Client) Bundles(ctx context.Context) ([]*catalogmetadata.Bundle, error) { + var allBundles []*catalogmetadata.Bundle + + var catalogList catalogd.CatalogList + if err := c.cl.List(ctx, &catalogList); err != nil { + return nil, err + } + for _, catalog := range catalogList.Items { + channels, err := fetchCatalogMetadata[catalogmetadata.Channel](ctx, c.cl, catalog.Name, declcfg.SchemaChannel) + if err != nil { + return nil, err + } + + bundles, err := fetchCatalogMetadata[catalogmetadata.Bundle](ctx, c.cl, catalog.Name, declcfg.SchemaBundle) + if err != nil { + return nil, err + } + + bundles, err = populateExtraFields(catalog.Name, channels, bundles) + if err != nil { + return nil, err + } + + allBundles = append(allBundles, bundles...) + } + + return allBundles, nil +} + +func fetchCatalogMetadata[T catalogmetadata.Schemas](ctx context.Context, cl client.Client, catalogName, schema string) ([]*T, error) { + var cmList catalogd.CatalogMetadataList + err := cl.List(ctx, &cmList, client.MatchingLabels{"catalog": catalogName, "schema": schema}) + if err != nil { + return nil, err + } + + content, err := catalogmetadata.Unmarshal[T](cmList.Items) + if err != nil { + return nil, fmt.Errorf("error unmarshalling catalog metadata: %s", err) + } + + return content, nil +} + +func populateExtraFields(catalogName string, channels []*catalogmetadata.Channel, bundles []*catalogmetadata.Bundle) ([]*catalogmetadata.Bundle, error) { + bundlesMap := map[string]*catalogmetadata.Bundle{} + for i := range bundles { + bundleKey := fmt.Sprintf("%s-%s", bundles[i].Package, bundles[i].Name) + bundlesMap[bundleKey] = bundles[i] + + bundles[i].CatalogName = catalogName + } + + for _, ch := range channels { + for _, chEntry := range ch.Entries { + bundleKey := fmt.Sprintf("%s-%s", ch.Package, chEntry.Name) + bundle, ok := bundlesMap[bundleKey] + if !ok { + return nil, fmt.Errorf("bundle %q not found in catalog %q (package %q, channel %q)", chEntry.Name, catalogName, ch.Package, ch.Name) + } + + bundle.InChannels = append(bundle.InChannels, ch) + } + } + + return bundles, nil +} diff --git a/internal/catalogmetadata/client/client_test.go b/internal/catalogmetadata/client/client_test.go new file mode 100644 index 000000000..4b28dadd9 --- /dev/null +++ b/internal/catalogmetadata/client/client_test.go @@ -0,0 +1,321 @@ +package client_test + +import ( + "context" + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + catalogd "github.com/operator-framework/catalogd/api/core/v1alpha1" + "github.com/operator-framework/operator-registry/alpha/declcfg" + "github.com/operator-framework/operator-registry/alpha/property" + + "github.com/operator-framework/operator-controller/internal/catalogmetadata" + catalogClient "github.com/operator-framework/operator-controller/internal/catalogmetadata/client" +) + +var ( + scheme *runtime.Scheme +) + +func init() { + scheme = runtime.NewScheme() + utilruntime.Must(catalogd.AddToScheme(scheme)) +} + +func TestClient(t *testing.T) { + t.Run("Bundles", func(t *testing.T) { + for _, tt := range []struct { + name string + fakeCatalog func() ([]client.Object, []*catalogmetadata.Bundle) + wantErr string + }{ + { + name: "valid catalog", + fakeCatalog: defaultFakeCatalog, + }, + { + name: "channel has a ref to a missing bundle", + fakeCatalog: func() ([]client.Object, []*catalogmetadata.Bundle) { + objs, _ := defaultFakeCatalog() + + objs = append(objs, &catalogd.CatalogMetadata{ + ObjectMeta: metav1.ObjectMeta{ + Name: "catalog-1-fake1-channel-with-missing-bundle", + Labels: map[string]string{"schema": declcfg.SchemaChannel, "catalog": "catalog-1"}, + }, + Spec: catalogd.CatalogMetadataSpec{ + Content: json.RawMessage(`{ + "schema": "olm.channel", + "name": "channel-with-missing-bundle", + "package": "fake1", + "entries": [ + { + "name": "fake1.v9.9.9" + } + ] + }`), + }, + }) + + return objs, nil + }, + wantErr: `bundle "fake1.v9.9.9" not found in catalog "catalog-1" (package "fake1", channel "channel-with-missing-bundle")`, + }, + { + name: "invalid bundle", + fakeCatalog: func() ([]client.Object, []*catalogmetadata.Bundle) { + objs, _ := defaultFakeCatalog() + + objs = append(objs, &catalogd.CatalogMetadata{ + ObjectMeta: metav1.ObjectMeta{ + Name: "catalog-1-broken-bundle", + Labels: map[string]string{"schema": declcfg.SchemaBundle, "catalog": "catalog-1"}, + }, + Spec: catalogd.CatalogMetadataSpec{ + Content: json.RawMessage(`{"name":123123123}`), + }, + }) + + return objs, nil + }, + wantErr: "error unmarshalling catalog metadata: json: cannot unmarshal number into Go struct field Bundle.name of type string", + }, + { + name: "invalid channel", + fakeCatalog: func() ([]client.Object, []*catalogmetadata.Bundle) { + objs, _ := defaultFakeCatalog() + + objs = append(objs, &catalogd.CatalogMetadata{ + ObjectMeta: metav1.ObjectMeta{ + Name: "catalog-1-fake1-broken-channel", + Labels: map[string]string{"schema": declcfg.SchemaChannel, "catalog": "catalog-1"}, + }, + Spec: catalogd.CatalogMetadataSpec{ + Content: json.RawMessage(`{"name":123123123}`), + }, + }) + + return objs, nil + }, + wantErr: "error unmarshalling catalog metadata: json: cannot unmarshal number into Go struct field Channel.name of type string", + }, + } { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + objs, expectedBundles := tt.fakeCatalog() + + fakeCatalogClient := catalogClient.NewClient( + fake.NewClientBuilder().WithScheme(scheme).WithObjects(objs...).Build(), + ) + + bundles, err := fakeCatalogClient.Bundles(ctx) + if tt.wantErr == "" { + assert.NoError(t, err) + } else { + assert.EqualError(t, err, tt.wantErr) + } + assert.Equal(t, expectedBundles, bundles) + }) + } + }) +} + +func defaultFakeCatalog() ([]client.Object, []*catalogmetadata.Bundle) { + package1 := `{ + "schema": "olm.bundle", + "name": "fake1" + }` + + bundle1 := `{ + "schema": "olm.bundle", + "name": "fake1.v1.0.0", + "package": "fake1", + "image": "fake-image", + "properties": [ + { + "type": "olm.package", + "value": {"packageName":"fake1","version":"1.0.0"} + } + ] + }` + + stableChannel := `{ + "schema": "olm.channel", + "name": "stable", + "package": "fake1", + "entries": [ + { + "name": "fake1.v1.0.0" + } + ] + }` + + betaChannel := `{ + "schema": "olm.channel", + "name": "beta", + "package": "fake1", + "entries": [ + { + "name": "fake1.v1.0.0" + } + ] + }` + + objs := []client.Object{ + &catalogd.Catalog{ + ObjectMeta: metav1.ObjectMeta{ + Name: "catalog-1", + }, + }, + &catalogd.Catalog{ + ObjectMeta: metav1.ObjectMeta{ + Name: "catalog-2", + }, + }, + &catalogd.CatalogMetadata{ + ObjectMeta: metav1.ObjectMeta{ + Name: "catalog-1-fake1", + Labels: map[string]string{"schema": declcfg.SchemaPackage, "catalog": "catalog-1"}, + }, + Spec: catalogd.CatalogMetadataSpec{ + Content: json.RawMessage(package1), + }, + }, + &catalogd.CatalogMetadata{ + ObjectMeta: metav1.ObjectMeta{ + Name: "catalog-1-fake1-channel-stable", + Labels: map[string]string{"schema": declcfg.SchemaChannel, "catalog": "catalog-1"}, + }, + Spec: catalogd.CatalogMetadataSpec{ + Content: json.RawMessage(stableChannel), + }, + }, + &catalogd.CatalogMetadata{ + ObjectMeta: metav1.ObjectMeta{ + Name: "catalog-1-fake1-channel-beta", + Labels: map[string]string{"schema": declcfg.SchemaChannel, "catalog": "catalog-1"}, + }, + Spec: catalogd.CatalogMetadataSpec{ + Content: json.RawMessage(betaChannel), + }, + }, + &catalogd.CatalogMetadata{ + ObjectMeta: metav1.ObjectMeta{ + Name: "catalog-1-fake1-bundle-1", + Labels: map[string]string{"schema": declcfg.SchemaBundle, "catalog": "catalog-1"}, + }, + Spec: catalogd.CatalogMetadataSpec{ + Content: json.RawMessage(bundle1), + }, + }, + &catalogd.CatalogMetadata{ + ObjectMeta: metav1.ObjectMeta{ + Name: "catalog-2-fake1", + Labels: map[string]string{"schema": declcfg.SchemaPackage, "catalog": "catalog-2"}, + }, + Spec: catalogd.CatalogMetadataSpec{ + Content: json.RawMessage(package1), + }, + }, + &catalogd.CatalogMetadata{ + ObjectMeta: metav1.ObjectMeta{ + Name: "catalog-2-fake1-channel-stable", + Labels: map[string]string{"schema": declcfg.SchemaChannel, "catalog": "catalog-2"}, + }, + Spec: catalogd.CatalogMetadataSpec{ + Content: json.RawMessage(stableChannel), + }, + }, + &catalogd.CatalogMetadata{ + ObjectMeta: metav1.ObjectMeta{ + Name: "catalog-2-fake1-bundle-1", + Labels: map[string]string{"schema": declcfg.SchemaBundle, "catalog": "catalog-2"}, + }, + Spec: catalogd.CatalogMetadataSpec{ + Content: json.RawMessage(bundle1), + }, + }, + } + + expectedBundles := []*catalogmetadata.Bundle{ + { + CatalogName: "catalog-1", + Bundle: declcfg.Bundle{ + Schema: declcfg.SchemaBundle, + Name: "fake1.v1.0.0", + Package: "fake1", + Image: "fake-image", + Properties: []property.Property{ + { + Type: property.TypePackage, + Value: json.RawMessage(`{"packageName":"fake1","version":"1.0.0"}`), + }, + }, + }, + InChannels: []*catalogmetadata.Channel{ + { + Channel: declcfg.Channel{ + Schema: declcfg.SchemaChannel, + Name: "beta", + Package: "fake1", + Entries: []declcfg.ChannelEntry{ + { + Name: "fake1.v1.0.0", + }, + }, + }, + }, + { + Channel: declcfg.Channel{ + Schema: declcfg.SchemaChannel, + Name: "stable", + Package: "fake1", + Entries: []declcfg.ChannelEntry{ + { + Name: "fake1.v1.0.0", + }, + }, + }, + }, + }, + }, + { + CatalogName: "catalog-2", + Bundle: declcfg.Bundle{ + Schema: declcfg.SchemaBundle, + Name: "fake1.v1.0.0", + Package: "fake1", + Image: "fake-image", + Properties: []property.Property{ + { + Type: property.TypePackage, + Value: json.RawMessage(`{"packageName":"fake1","version":"1.0.0"}`), + }, + }, + }, + InChannels: []*catalogmetadata.Channel{ + { + Channel: declcfg.Channel{ + Schema: declcfg.SchemaChannel, + Name: "stable", + Package: "fake1", + Entries: []declcfg.ChannelEntry{ + { + Name: "fake1.v1.0.0", + }, + }, + }, + }, + }, + }, + } + + return objs, expectedBundles +} diff --git a/internal/catalogmetadata/types.go b/internal/catalogmetadata/types.go index 52d55248c..624ab4eec 100644 --- a/internal/catalogmetadata/types.go +++ b/internal/catalogmetadata/types.go @@ -41,7 +41,8 @@ func (g GVKRequired) AsGVK() GVK { type Bundle struct { declcfg.Bundle - InChannels []*Channel + CatalogName string + InChannels []*Channel mu sync.RWMutex // these properties are lazy loaded as they are requested