(feat): direct image registry client unpacker

Signed-off-by: Bryce Palmer <bpalmer@redhat.com>
operator-framework · Oct 3, 2023 · 3c8584e · 3c8584e
1 parent 8a90d8d
commit 3c8584e
Show file tree

Hide file tree

Showing 19 changed files with 1,863 additions and 320 deletions.
diff --git a/Makefile b/Makefile
@@ -20,6 +20,7 @@ CATALOGD_NAMESPACE      ?= catalogd-system
 
 # E2E configuration
 TESTDATA_DIR            ?= testdata
+REGISTRY_FORWARD_PORT ?= 5000
 CONTAINER_RUNTIME ?= docker
 
 ##@ General
@@ -67,7 +68,10 @@ test-e2e: $(GINKGO) ## Run the e2e tests
 	$(GINKGO) --tags $(GO_BUILD_TAGS) $(E2E_FLAGS) -trace -progress $(FOCUS) test/e2e
 
 e2e: KIND_CLUSTER_NAME=catalogd-e2e
-e2e: run kind-load-test-artifacts test-e2e kind-cluster-cleanup ## Run e2e test suite on local kind cluster
+e2e: run image-registry test-e2e kind-cluster-cleanup ## Run e2e test suite on local kind cluster
+
+image-registry: ## Setup in-cluster image registry
+	./test/tools/imageregistry/registry.sh ${CONTAINER_RUNTIME} ${REGISTRY_FORWARD_PORT}
 
 .PHONY: tidy
 tidy: ## Update dependencies
@@ -147,10 +151,6 @@ kind-load: $(KIND) ## Load the built images onto the local cluster
 	$(KIND) export kubeconfig --name $(KIND_CLUSTER_NAME)
 	$(KIND) load docker-image $(IMAGE) --name $(KIND_CLUSTER_NAME)
 
-kind-load-test-artifacts: $(KIND) ## Load the e2e testdata container images into a kind cluster
-	$(CONTAINER_RUNTIME) build $(TESTDATA_DIR)/catalogs -f $(TESTDATA_DIR)/catalogs/test-catalog.Dockerfile -t localhost/testdata/catalogs/test-catalog:e2e
-	$(KIND) load docker-image localhost/testdata/catalogs/test-catalog:e2e --name $(KIND_CLUSTER_NAME)
-
 .PHONY: install
 install: build-container kind-load deploy wait ## Install local catalogd
 

diff --git a/cmd/manager/main.go b/cmd/manager/main.go
@@ -17,25 +17,31 @@ limitations under the License.
 package main
 
 import (
+	"context"
 	"flag"
 	"fmt"
 	"net/http"
 	"net/url"
 	"os"
+	"path/filepath"
 	"time"
 
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
 	// to ensure that exec-entrypoint and run can make use of them.
+	"k8s.io/client-go/metadata"
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
 
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/sets"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 	"sigs.k8s.io/controller-runtime/pkg/metrics"
 
+	"github.com/go-logr/logr"
 	"github.com/spf13/pflag"
 
 	"github.com/operator-framework/catalogd/internal/source"
@@ -56,6 +62,8 @@ var (
 	setupLog = ctrl.Log.WithName("setup")
 )
 
+const storageDir = "catalogs"
+
 func init() {
 	utilruntime.Must(clientgoscheme.AddToScheme(scheme))
 
@@ -68,25 +76,22 @@ func main() {
 		metricsAddr          string
 		enableLeaderElection bool
 		probeAddr            string
-		unpackImage          string
 		profiling            bool
 		catalogdVersion      bool
 		systemNamespace      string
-		storageDir           string
 		catalogServerAddr    string
 		httpExternalAddr     string
+		cacheDir             string
 	)
 	flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
 	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
 	flag.BoolVar(&enableLeaderElection, "leader-elect", false,
 		"Enable leader election for controller manager. "+
 			"Enabling this will ensure there is only one active controller manager.")
-	// TODO: should we move the unpacker to some common place? Or... hear me out... should catalogd just be a rukpak provisioner?
-	flag.StringVar(&unpackImage, "unpack-image", "quay.io/operator-framework/rukpak:v0.12.0", "The unpack image to use when unpacking catalog images")
 	flag.StringVar(&systemNamespace, "system-namespace", "", "The namespace catalogd uses for internal state, configuration, and workloads")
-	flag.StringVar(&storageDir, "catalogs-storage-dir", "/var/cache/catalogs", "The directory in the filesystem where unpacked catalog content will be stored and served from")
 	flag.StringVar(&catalogServerAddr, "catalogs-server-addr", ":8083", "The address where the unpacked catalogs' content will be accessible")
 	flag.StringVar(&httpExternalAddr, "http-external-address", "http://catalogd-catalogserver.catalogd-system.svc", "The external address at which the http server is reachable.")
+	flag.StringVar(&cacheDir, "cache-dir", "/var/cache/", "The directory in the filesystem that catalogd will use for file based caching")
 	flag.BoolVar(&profiling, "profiling", false, "enable profiling endpoints to allow for using pprof")
 	flag.BoolVar(&catalogdVersion, "version", false, "print the catalogd version and exit")
 	opts := zap.Options{
@@ -105,8 +110,8 @@ func main() {
 	}
 
 	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
-
-	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
+	cfg := ctrl.GetConfigOrDie()
+	mgr, err := ctrl.NewManager(cfg, ctrl.Options{
 		Scheme:                 scheme,
 		MetricsBindAddress:     metricsAddr,
 		Port:                   9443,
@@ -123,7 +128,12 @@ func main() {
 		systemNamespace = podNamespace()
 	}
 
-	unpacker, err := source.NewDefaultUnpacker(mgr, systemNamespace, unpackImage)
+	if err := os.MkdirAll(cacheDir, 0700); err != nil {
+		setupLog.Error(err, "unable to create cache directory")
+		os.Exit(1)
+	}
+
+	unpacker, err := source.NewDefaultUnpacker(mgr, systemNamespace, cacheDir)
 	if err != nil {
 		setupLog.Error(err, "unable to create unpacker")
 		os.Exit(1)
@@ -133,7 +143,7 @@ func main() {
 	if features.CatalogdFeatureGate.Enabled(features.HTTPServer) {
 		metrics.Registry.MustRegister(catalogdmetrics.RequestDurationMetric)
 
-		if err := os.MkdirAll(storageDir, 0700); err != nil {
+		if err := os.MkdirAll(filepath.Join(cacheDir, storageDir), 0700); err != nil {
 			setupLog.Error(err, "unable to create storage directory for catalogs")
 			os.Exit(1)
 		}
@@ -143,7 +153,7 @@ func main() {
 			setupLog.Error(err, "unable to create base storage URL")
 			os.Exit(1)
 		}
-		localStorage = storage.LocalDir{RootDir: storageDir, BaseURL: baseStorageURL}
+		localStorage = storage.LocalDir{RootDir: filepath.Join(cacheDir, storageDir), BaseURL: baseStorageURL}
 		shutdownTimeout := 30 * time.Second
 		catalogServer := server.Server{
 			Kind: "catalogs",
@@ -189,6 +199,14 @@ func main() {
 		}
 	}
 
+	metaClient, err := metadata.NewForConfig(cfg)
+	if err != nil {
+		setupLog.Error(err, "unable to setup client for garbage collection")
+		os.Exit(1)
+	}
+
+	unpackStartupGarbageCollection(context.Background(), filepath.Join(cacheDir, source.UnpackCacheDir), setupLog, metaClient)
+
 	setupLog.Info("starting manager")
 	if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
 		setupLog.Error(err, "problem running manager")
@@ -203,3 +221,32 @@ func podNamespace() string {
 	}
 	return string(namespace)
 }
+
+func unpackStartupGarbageCollection(ctx context.Context, cachePath string, log logr.Logger, metaClient metadata.Interface) error {
+	getter := metaClient.Resource(v1alpha1.GroupVersion.WithResource("catalogs"))
+	metaList, err := getter.List(ctx, v1.ListOptions{})
+	if err != nil {
+		return fmt.Errorf("error listing catalogs: %w", err)
+	}
+
+	expectedCatalogs := sets.New[string]()
+	for _, meta := range metaList.Items {
+		expectedCatalogs.Insert(meta.GetName())
+	}
+
+	cacheDirEntries, err := os.ReadDir(cachePath)
+	if err != nil {
+		return fmt.Errorf("error reading cache directory: %w", err)
+	}
+	for _, cacheDirEntry := range cacheDirEntries {
+		if cacheDirEntry.IsDir() && expectedCatalogs.Has(cacheDirEntry.Name()) {
+			continue
+		}
+		if err := os.RemoveAll(filepath.Join(cachePath, cacheDirEntry.Name())); err != nil {
+			log.Error(err, "error removing cache directory entry", "path", cacheDirEntry.Name(), "isDir", cacheDirEntry.IsDir())
+		} else {
+			log.Info("deleted unexpected cache directory entry", "path", cacheDirEntry.Name(), "isDir", cacheDirEntry.IsDir())
+		}
+	}
+	return nil
+}
diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml
@@ -75,14 +75,14 @@ spec:
         - --leader-elect
         - --health-probe-bind-address=:8081
         - --metrics-bind-address=127.0.0.1:8080
-        - --catalogs-storage-dir=/var/cache/catalogs
+        - --cache-dir=/var/cache/
         - --feature-gates=HTTPServer=true
         - --http-external-address=http://catalogd-catalogserver.catalogd-system.svc
         image: controller:latest
         name: manager
         volumeMounts:
-            - name: catalog-cache
-              mountPath: /var/cache/catalogs
+            - name: cache
+              mountPath: /var/cache/
         securityContext:
           allowPrivilegeEscalation: false
           capabilities:
@@ -108,5 +108,5 @@ spec:
       serviceAccountName: controller-manager
       terminationGracePeriodSeconds: 10
       volumes:
-        - name: catalog-cache
+        - name: cache
           emptyDir: {}
diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml
@@ -50,3 +50,16 @@ rules:
   - get
   - list
   - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: manager-role
+  namespace: system
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - get
diff --git a/config/rbac/role_binding.yaml b/config/rbac/role_binding.yaml
@@ -13,3 +13,24 @@ subjects:
 - kind: ServiceAccount
   name: controller-manager
   namespace: system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  labels:
+    app.kubernetes.io/name: rolebinding
+    app.kubernetes.io/instance: manager-rolebinding
+    app.kubernetes.io/component: rbac
+    app.kubernetes.io/created-by: catalogd
+    app.kubernetes.io/part-of: catalogd
+    app.kubernetes.io/managed-by: kustomize
+  name: manager-rolebinding
+  namespace: system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: manager-role
+subjects:
+- kind: ServiceAccount
+  name: controller-manager
+  namespace: system
diff --git a/go.mod b/go.mod
@@ -4,9 +4,12 @@ go 1.20
 
 require (
 	github.com/blang/semver/v4 v4.0.0
+	github.com/containerd/containerd v1.6.22
 	github.com/go-logr/logr v1.2.4
 	github.com/google/go-cmp v0.5.9
-	github.com/nlepage/go-tarfs v1.1.0
+	github.com/google/go-containerregistry v0.14.0
+	github.com/google/go-containerregistry/pkg/authn/k8schain v0.0.0-20230310164735-e94d40893b2d
+	github.com/google/go-containerregistry/pkg/authn/kubernetes v0.0.0-20230310164735-e94d40893b2d
 	github.com/onsi/ginkgo/v2 v2.9.7
 	github.com/onsi/gomega v1.27.7
 	github.com/operator-framework/operator-registry v1.29.0
@@ -21,10 +24,47 @@ require (
 )
 
 require (
+	cloud.google.com/go/compute v1.18.0 // indirect
+	cloud.google.com/go/compute/metadata v0.2.3 // indirect
+	github.com/Azure/azure-sdk-for-go v68.0.0+incompatible // indirect
+	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
+	github.com/Azure/go-autorest/autorest v0.11.28 // indirect
+	github.com/Azure/go-autorest/autorest/adal v0.9.22 // indirect
+	github.com/Azure/go-autorest/autorest/azure/auth v0.5.12 // indirect
+	github.com/Azure/go-autorest/autorest/azure/cli v0.4.6 // indirect
+	github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
+	github.com/Azure/go-autorest/logger v0.2.1 // indirect
+	github.com/Azure/go-autorest/tracing v0.6.0 // indirect
+	github.com/Microsoft/go-winio v0.6.1 // indirect
+	github.com/Microsoft/hcsshim v0.9.8 // indirect
 	github.com/acomagu/bufpipe v1.0.3 // indirect
+	github.com/aws/aws-sdk-go-v2 v1.17.5 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.18.15 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.13.15 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.23 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.29 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.23 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.3.30 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ecr v1.18.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.15.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.23 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.12.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.18.5 // indirect
+	github.com/aws/smithy-go v1.13.5 // indirect
+	github.com/awslabs/amazon-ecr-credential-helper/ecr-login v0.0.0-20230228174139-39c3d18f0af1 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/chrismellard/docker-credential-acr-env v0.0.0-20230304212654-82a0ddb27589 // indirect
+	github.com/containerd/cgroups v1.0.4 // indirect
+	github.com/containerd/continuity v0.3.0 // indirect
+	github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/dimchansky/utfbom v1.1.1 // indirect
+	github.com/docker/cli v23.0.1+incompatible // indirect
+	github.com/docker/distribution v2.8.2+incompatible // indirect
+	github.com/docker/docker v23.0.1+incompatible // indirect
+	github.com/docker/docker-credential-helpers v0.7.0 // indirect
 	github.com/emicklei/go-restful/v3 v3.10.2 // indirect
 	github.com/evanphx/json-patch v5.6.0+incompatible // indirect
 	github.com/evanphx/json-patch/v5 v5.6.0 // indirect
@@ -34,40 +74,50 @@ require (
 	github.com/go-git/go-git/v5 v5.4.2 // indirect
 	github.com/go-logr/zapr v1.2.4 // indirect
 	github.com/go-openapi/jsonpointer v0.19.6 // indirect
-	github.com/go-openapi/jsonreference v0.20.1 // indirect
+	github.com/go-openapi/jsonreference v0.20.2 // indirect
 	github.com/go-openapi/swag v0.22.3 // indirect
 	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
-	github.com/google/gnostic v0.5.7-v3refs // indirect
+	github.com/google/gnostic v0.6.9 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/h2non/filetype v1.1.1 // indirect
 	github.com/h2non/go-is-svg v0.0.0-20160927212452-35e8c4b0612c // indirect
 	github.com/imdario/mergo v0.3.13 // indirect
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/joelanford/ignore v0.0.0-20210607151042-0d25dc18b62d // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/klauspost/compress v1.16.0 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b // indirect
 	github.com/operator-framework/api v0.17.7 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_model v0.4.0 // indirect
 	github.com/prometheus/common v0.42.0 // indirect
 	github.com/prometheus/procfs v0.9.0 // indirect
 	github.com/sirupsen/logrus v1.9.2 // indirect
+	github.com/vbatts/tar-split v0.11.2 // indirect
+	go.opencensus.io v0.24.0 // indirect
 	go.uber.org/atomic v1.7.0 // indirect
 	go.uber.org/multierr v1.6.0 // indirect
 	go.uber.org/zap v1.24.0 // indirect
+	golang.org/x/crypto v0.7.0 // indirect
+	golang.org/x/mod v0.10.0 // indirect
 	golang.org/x/net v0.10.0 // indirect
-	golang.org/x/oauth2 v0.5.0 // indirect
+	golang.org/x/oauth2 v0.6.0 // indirect
 	golang.org/x/sync v0.2.0 // indirect
 	golang.org/x/sys v0.8.0 // indirect
 	golang.org/x/term v0.8.0 // indirect
@@ -84,7 +134,7 @@ require (
 	k8s.io/apiextensions-apiserver v0.27.2 // indirect
 	k8s.io/klog/v2 v2.90.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect
-	k8s.io/utils v0.0.0-20230209194617-a36077c30491 // indirect
+	k8s.io/utils v0.0.0-20230308161112-d77c459e9343 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
 	sigs.k8s.io/yaml v1.3.0 // indirect