WIP: Proof of concept: introduce experimental OCI artifact source behind feature gate

api/core/v1alpha1/catalog_types.go

@@ -26,7 +26,8 @@ import (
 type SourceType string
 
 const (
-	SourceTypeImage SourceType = "image"
+	SourceTypeImage       SourceType = "image"
+	SourceTypeOCIArtifact SourceType = "ociArtifact"
 
 	TypeUnpacked = "Unpacked"
 
@@ -86,6 +87,8 @@ type CatalogSource struct {
 	Type SourceType `json:"type"`
 	// Image is the catalog image that backs the content of this catalog.
 	Image *ImageSource `json:"image,omitempty"`
+	// OCIArtifact is the catalog OCI artifact that backs the content of this catalog.
+	OCIArtifact *ImageSource `json:"ociArtifact,omitempty"`
 }
 
 // ImageSource contains information required for sourcing a Catalog from an OCI image

cmd/manager/main.go

@@ -21,26 +21,22 @@ import (
 	"fmt"
 	"os"
 
-	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
-	// to ensure that exec-entrypoint and run can make use of them.
-	_ "k8s.io/client-go/plugin/pkg/client/auth"
-
+	"github.com/spf13/pflag"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	_ "k8s.io/client-go/plugin/pkg/client/auth"
+	"oras.land/oras-go/v2/content/oci"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 
+	"github.com/operator-framework/catalogd/api/core/v1alpha1"
 	"github.com/operator-framework/catalogd/internal/source"
 	"github.com/operator-framework/catalogd/internal/version"
 	corecontrollers "github.com/operator-framework/catalogd/pkg/controllers/core"
 	"github.com/operator-framework/catalogd/pkg/features"
 	"github.com/operator-framework/catalogd/pkg/profile"
-	"github.com/spf13/pflag"
-
-	//+kubebuilder:scaffold:imports
-	"github.com/operator-framework/catalogd/api/core/v1alpha1"
 )
 
 var (
@@ -92,6 +88,13 @@ func main() {
 
 	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
 
+	for f, spec := range features.CatalogdFeatureGate.GetAll() {
+		enabled := features.CatalogdFeatureGate.Enabled(f)
+		if spec.Default != enabled {
+			setupLog.Info("non-default feature gate in use", "name", f, "enabled", enabled)
+		}
+	}
+
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
 		Scheme:                 scheme,
 		MetricsBindAddress:     metricsAddr,
@@ -104,8 +107,16 @@ func main() {
 		setupLog.Error(err, "unable to start manager")
 		os.Exit(1)
 	}
+	ctx := ctrl.SetupSignalHandler()
+
+	ociStore, err := oci.NewWithContext(ctx, "/var/cache/oci")
+	if err != nil {
+		setupLog.Error(err, "unable to create oci store")
+		os.Exit(1)
+	}
+	fbcRootDir := "/var/cache/fbc"
 
-	unpacker, err := source.NewDefaultUnpacker(mgr, sysNs, unpackImage)
+	unpacker, err := source.NewDefaultUnpacker(mgr, sysNs, unpackImage, ociStore, fbcRootDir)
 	if err != nil {
 		setupLog.Error(err, "unable to create unpacker")
 		os.Exit(1)
@@ -138,7 +149,7 @@ func main() {
 	}
 
 	setupLog.Info("starting manager")
-	if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
+	if err := mgr.Start(ctx); err != nil {
 		setupLog.Error(err, "problem running manager")
 		os.Exit(1)
 	}

config/crd/bases/catalogd.operatorframework.io_catalogs.yaml

@@ -53,6 +53,21 @@ spec:
                     required:
                     - ref
                     type: object
+                  ociArtifact:
+                    description: OCIArtifact is the catalog OCI artifact that backs
+                      the content of this catalog.
+                    properties:
+                      pullSecret:
+                        description: PullSecret contains the name of the image pull
+                          secret in the namespace that catalogd is deployed.
+                        type: string
+                      ref:
+                        description: Ref contains the reference to a container image
+                          containing Catalog contents.
+                        type: string
+                    required:
+                    - ref
+                    type: object
                   type:
                     description: Type defines the kind of Catalog content being sourced.
                     type: string
@@ -156,6 +171,21 @@ spec:
                     required:
                     - ref
                     type: object
+                  ociArtifact:
+                    description: OCIArtifact is the catalog OCI artifact that backs
+                      the content of this catalog.
+                    properties:
+                      pullSecret:
+                        description: PullSecret contains the name of the image pull
+                          secret in the namespace that catalogd is deployed.
+                        type: string
+                      ref:
+                        description: Ref contains the reference to a container image
+                          containing Catalog contents.
+                        type: string
+                    required:
+                    - ref
+                    type: object
                   type:
                     description: Type defines the kind of Catalog content being sourced.
                     type: string

config/crd/patches/catalog_validation.yaml

@@ -3,4 +3,6 @@
   path: /spec/versions/0/schema/openAPIV3Schema/properties/spec/properties/source/oneOf
   value:
   - required:
-    - image
+    - image
+  - required:
+    - ociArtifact

config/default/manager_auth_proxy_patch.yaml

@@ -53,3 +53,4 @@ spec:
         - "--health-probe-bind-address=:8081"
         - "--metrics-bind-address=127.0.0.1:8080"
         - "--leader-elect"
+        - "--feature-gates=AllAlpha=true"

config/manager/manager.yaml

@@ -40,6 +40,11 @@ spec:
         runAsNonRoot: true
         seccompProfile:
           type: RuntimeDefault
+      volumes:
+        - name: oci
+          emptyDir: {}
+        - name: fbc
+          emptyDir: {}
       containers:
       - command:
         - "./manager"
@@ -72,5 +77,10 @@ spec:
             cpu: 1000m
             memory: 200Mi
         imagePullPolicy: IfNotPresent
+        volumeMounts:
+          - mountPath: /var/cache/oci
+            name: oci
+          - mountPath: /var/cache/fbc
+            name: fbc
       serviceAccountName: controller-manager
       terminationGracePeriodSeconds: 10

config/samples/core_v1alpha1_catalog.yaml

@@ -1,9 +1,9 @@
 apiVersion: catalogd.operatorframework.io/v1alpha1
 kind: Catalog
 metadata:
-  name: operatorhubio
+  name: oci-artifacts
 spec:
   source:
-    type: image
-    image:
-      ref: quay.io/operatorhubio/catalog:latest
+    type: ociArtifact
+    ociArtifact:
+      ref: docker.io/joelanford/catalog:catalog

go.mod

@@ -4,23 +4,35 @@ go 1.19
 
 require (
 	github.com/blang/semver/v4 v4.0.0
+	github.com/docker/distribution v2.8.2+incompatible
+	github.com/joelanford/olm-oci v0.0.0-20230616171019-74f773d44a80
 	github.com/nlepage/go-tarfs v1.1.0
 	github.com/onsi/ginkgo/v2 v2.9.7
 	github.com/onsi/gomega v1.27.7
-	github.com/operator-framework/operator-registry v1.26.3
+	github.com/operator-framework/operator-registry v1.28.0
 	github.com/spf13/pflag v1.0.5
-	k8s.io/api v0.26.0
-	k8s.io/apimachinery v0.26.0
-	k8s.io/client-go v0.26.0
-	k8s.io/component-base v0.26.0
+	k8s.io/api v0.26.1
+	k8s.io/apimachinery v0.26.1
+	k8s.io/client-go v0.26.1
+	k8s.io/component-base v0.26.1
 	k8s.io/utils v0.0.0-20221128185143-99ec85e7a448
-	sigs.k8s.io/controller-runtime v0.14.0
+	oras.land/oras-go/v2 v2.2.0
+	sigs.k8s.io/controller-runtime v0.14.4
 )
 
 require (
+	github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect
+	github.com/BurntSushi/toml v1.2.0 // indirect
+	github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
+	github.com/containers/image/v5 v5.23.1 // indirect
+	github.com/containers/storage v1.43.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/docker/cli v24.0.2+incompatible // indirect
+	github.com/docker/docker v24.0.2+incompatible // indirect
+	github.com/docker/docker-credential-helpers v0.7.0 // indirect
+	github.com/docker/go-units v0.5.0 // indirect
 	github.com/emicklei/go-restful/v3 v3.9.0 // indirect
 	github.com/evanphx/json-patch/v5 v5.6.0 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
@@ -36,47 +48,64 @@ require (
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/google/cel-go v0.12.6 // indirect
 	github.com/google/gnostic v0.5.7-v3refs // indirect
 	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/google/gofuzz v1.1.0 // indirect
 	github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 // indirect
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/h2non/filetype v1.1.1 // indirect
 	github.com/h2non/go-is-svg v0.0.0-20160927212452-35e8c4b0612c // indirect
-	github.com/imdario/mergo v0.3.12 // indirect
+	github.com/hashicorp/errwrap v1.1.0 // indirect
+	github.com/hashicorp/go-multierror v1.1.1 // indirect
+	github.com/imdario/mergo v0.3.13 // indirect
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
 	github.com/joelanford/ignore v0.0.0-20210607151042-0d25dc18b62d // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/mailru/easyjson v0.7.6 // indirect
-	github.com/matttproud/golang_protobuf_extensions v1.0.2 // indirect
+	github.com/mattn/go-isatty v0.0.12 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
+	github.com/moby/sys/mountinfo v0.6.2 // indirect
+	github.com/moby/term v0.5.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
-	github.com/operator-framework/api v0.17.2-0.20220915200120-ff2dbc53d381 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.1.0-rc2 // indirect
+	github.com/opencontainers/runc v1.1.4 // indirect
+	github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 // indirect
+	github.com/operator-framework/api v0.17.4-0.20230223191600-0131a6301e42 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/prometheus/client_golang v1.14.0 // indirect
 	github.com/prometheus/client_model v0.3.0 // indirect
 	github.com/prometheus/common v0.37.0 // indirect
 	github.com/prometheus/procfs v0.8.0 // indirect
+	github.com/sirupsen/logrus v1.9.2 // indirect
+	github.com/stoewer/go-strcase v1.2.0 // indirect
+	github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect
 	go.uber.org/atomic v1.7.0 // indirect
 	go.uber.org/multierr v1.6.0 // indirect
 	go.uber.org/zap v1.24.0 // indirect
 	golang.org/x/net v0.10.0 // indirect
-	golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 // indirect
+	golang.org/x/oauth2 v0.1.0 // indirect
+	golang.org/x/sync v0.2.0 // indirect
 	golang.org/x/sys v0.8.0 // indirect
 	golang.org/x/term v0.8.0 // indirect
 	golang.org/x/text v0.9.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
 	golang.org/x/tools v0.9.1 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect
+	google.golang.org/grpc v1.51.0 // indirect
 	google.golang.org/protobuf v1.28.1 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	k8s.io/apiextensions-apiserver v0.26.0 // indirect
+	k8s.io/apiextensions-apiserver v0.26.1 // indirect
 	k8s.io/klog/v2 v2.80.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 // indirect
 	sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect