Skip to content
This repository has been archived by the owner on Aug 12, 2024. It is now read-only.

update image source unpacking to use a direct image registry client #874

Merged
merged 1 commit into from
May 13, 2024
Merged

update image source unpacking to use a direct image registry client #874

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
22 changes: 18 additions & 4 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,7 @@ test-e2e: $(GINKGO) ## Run the e2e tests
$(GINKGO) $(E2E_FLAGS) --trace $(FOCUS) test/e2e

e2e: KIND_CLUSTER_NAME := rukpak-e2e
e2e: run image-registry local-git kind-load-bundles registry-load-bundles test-e2e kind-cluster-cleanup ## Run e2e tests against an ephemeral kind cluster
e2e: run image-registry secure-image-registry local-git kind-load-bundles registry-load-bundles secure-registry-load-bundles test-e2e kind-cluster-cleanup ## Run e2e tests against an ephemeral kind cluster

kind-cluster: $(KIND) kind-cluster-cleanup ## Standup a kind cluster
$(KIND) create cluster --name ${KIND_CLUSTER_NAME} ${KIND_CLUSTER_CONFIG}
Expand All @@ -127,7 +127,10 @@ kind-cluster-cleanup: $(KIND) ## Delete the kind cluster
$(KIND) delete cluster --name ${KIND_CLUSTER_NAME}

image-registry: ## Setup in-cluster image registry
./test/tools/imageregistry/setup_imageregistry.sh ${KIND_CLUSTER_NAME}
./test/tools/imageregistry/image-registry.sh ${REGISTRY_NAMESPACE} ${REGISTRY_NAME}

secure-image-registry: ## Setup a private in-cluster image registry
./test/tools/imageregistry/image-registry-secure.sh ${REGISTRY_NAMESPACE} ${REGISTRY_NAME}

local-git: ## Setup in-cluster git repository
./test/tools/git/setup_git.sh ${KIND_CLUSTER_NAME}
Expand Down Expand Up @@ -218,8 +221,19 @@ kind-load: $(KIND) ## Loads the currently constructed image onto the cluster
$(KIND) load docker-image $(IMAGE) --name $(KIND_CLUSTER_NAME)

registry-load-bundles: ## Load selected e2e testdata container images created in kind-load-bundles into registry
$(CONTAINER_RUNTIME) tag localhost/testdata/bundles/plain-v0:valid $(DNS_NAME):5000/bundles/plain-v0:valid
./test/tools/imageregistry/load_test_image.sh $(KIND) $(KIND_CLUSTER_NAME)
testdata/bundles/plain-v0/valid/build-push-e2e-bundle.sh ${REGISTRY_NAMESPACE} $(DNS_NAME):5000/bundles/plain-v0:valid
testdata/bundles/plain-v0/dependent/build-push-e2e-bundle.sh ${REGISTRY_NAMESPACE} $(DNS_NAME):5000/bundles/plain-v0:dependent
testdata/bundles/plain-v0/provides/build-push-e2e-bundle.sh ${REGISTRY_NAMESPACE} $(DNS_NAME):5000/bundles/plain-v0:provides
testdata/bundles/plain-v0/empty/build-push-e2e-bundle.sh ${REGISTRY_NAMESPACE} $(DNS_NAME):5000/bundles/plain-v0:empty
testdata/bundles/plain-v0/no-manifests/build-push-e2e-bundle.sh ${REGISTRY_NAMESPACE} $(DNS_NAME):5000/bundles/plain-v0:no-manifests
testdata/bundles/plain-v0/invalid-missing-crds/build-push-e2e-bundle.sh ${REGISTRY_NAMESPACE} $(DNS_NAME):5000/bundles/plain-v0:invalid-missing-crds
testdata/bundles/plain-v0/invalid-crds-and-crs/build-push-e2e-bundle.sh ${REGISTRY_NAMESPACE} $(DNS_NAME):5000/bundles/plain-v0:invalid-crds-and-crs
testdata/bundles/plain-v0/subdir/build-push-e2e-bundle.sh ${REGISTRY_NAMESPACE} $(DNS_NAME):5000/bundles/plain-v0:subdir
testdata/bundles/registry/valid/build-push-e2e-bundle.sh ${REGISTRY_NAMESPACE} $(DNS_NAME):5000/bundles/registry:valid
testdata/bundles/registry/invalid/build-push-e2e-bundle.sh ${REGISTRY_NAMESPACE} $(DNS_NAME):5000/bundles/registry:invalid

secure-registry-load-bundles: ## Load selected e2e testdata container images created in kind-load-bundles into private registry
testdata/bundles/plain-v0/valid/build-push-e2e-bundle-secure.sh ${REGISTRY_NAMESPACE} docker-registry-secure.rukpak-e2e.svc.cluster.local:5000/bundles/plain-v0:valid

###########
# Release #
Expand Down
6 changes: 6 additions & 0 deletions api/v1alpha2/bundle_types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,12 @@ type ImageSource struct {
Ref string `json:"ref"`
// ImagePullSecretName contains the name of the image pull secret in the namespace that the provisioner is deployed.
ImagePullSecretName string `json:"pullSecret,omitempty"`
// InsecureSkipTLSVerify indicates that TLS certificate validation should be skipped.
// If this option is specified, the HTTPS protocol will still be used to
// fetch the specified image reference.
// This should not be used in a production environment.
// +optional
InsecureSkipTLSVerify bool `json:"insecureSkipTLSVerify,omitempty"`
}

type GitSource struct {
Expand Down
16 changes: 11 additions & 5 deletions cmd/core/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ func main() {
enableLeaderElection bool
probeAddr string
systemNamespace string
unpackImage string
unpackCacheDir string
rukpakVersion bool
provisionerStorageDirectory string
)
Expand All @@ -86,7 +86,7 @@ func main() {
flag.StringVar(&bundleCAFile, "bundle-ca-file", "", "The file containing the certificate authority for connecting to bundle content servers.")
flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
flag.StringVar(&systemNamespace, "system-namespace", "", "Configures the namespace that gets used to deploy system resources.")
flag.StringVar(&unpackImage, "unpack-image", util.DefaultUnpackImage, "Configures the container image that gets used to unpack Bundle contents.")
flag.StringVar(&unpackCacheDir, "unpack-cache-dir", "/var/cache/unpack", "Configures the directory that gets used to unpack and cache Bundle contents.")
flag.BoolVar(&enableLeaderElection, "leader-elect", false,
"Enable leader election for controller manager. "+
"Enabling this will ensure there is only one active controller manager.")
Expand All @@ -107,7 +107,7 @@ func main() {
}

ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
setupLog.Info("starting up the core controllers and servers", "git commit", version.String(), "unpacker image", unpackImage)
setupLog.Info("starting up the core controllers and servers", "git commit", version.String(), "unpacker cache", unpackCacheDir)

dependentRequirement, err := labels.NewRequirement(util.CoreOwnerKindKey, selection.In, []string{rukpakv1alpha2.BundleDeploymentKind})
if err != nil {
Expand Down Expand Up @@ -196,12 +196,17 @@ func main() {
os.Exit(1)
}

unpacker, err := source.NewDefaultUnpacker(mgr, systemNamespace, unpackImage, rootCAs)
unpacker, err := source.NewDefaultUnpacker(mgr, systemNamespace, unpackCacheDir, rootCAs)
if err != nil {
setupLog.Error(err, "unable to setup bundle unpacker")
os.Exit(1)
}

if err := bundleFinalizers.Register(finalizer.CleanupUnpackCacheKey, &finalizer.CleanupUnpackCache{Unpacker: unpacker}); err != nil {
setupLog.Error(err, "unable to register finalizer", "finalizerKey", finalizer.CleanupUnpackCacheKey)
os.Exit(1)
}

bdNamespaceMapper := func(obj client.Object) (string, error) {
bd, ok := obj.(*rukpakv1alpha2.BundleDeployment)
if !ok {
Expand Down Expand Up @@ -261,8 +266,9 @@ func main() {
os.Exit(1)
}

ctx := ctrl.SetupSignalHandler()
setupLog.Info("starting manager")
if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
if err := mgr.Start(ctx); err != nil {
setupLog.Error(err, "problem running manager")
os.Exit(1)
}
Expand Down
14 changes: 10 additions & 4 deletions cmd/helm/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,15 +70,15 @@ func main() {
enableLeaderElection bool
probeAddr string
systemNamespace string
unpackImage string
unpackCacheDir string
rukpakVersion bool
storageDirectory string
)
flag.StringVar(&httpBindAddr, "http-bind-address", ":8080", "The address the http server binds to.")
flag.StringVar(&httpExternalAddr, "http-external-address", "http://localhost:8080", "The external address at which the http server is reachable.")
flag.StringVar(&bundleCAFile, "bundle-ca-file", "", "The file containing the certificate authority for connecting to bundle content servers.")
flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
flag.StringVar(&unpackImage, "unpack-image", util.DefaultUnpackImage, "Configures the container image that gets used to unpack Bundle contents.")
flag.StringVar(&unpackCacheDir, "unpack-cache-dir", "/var/cache/unpack", "Configures the directory that gets used to unpack and cache Bundle contents.")
flag.StringVar(&systemNamespace, "system-namespace", "", "Configures the namespace that gets used to deploy system resources.")
flag.BoolVar(&enableLeaderElection, "leader-elect", false,
"Enable leader election for controller manager. "+
Expand Down Expand Up @@ -188,12 +188,17 @@ func main() {
os.Exit(1)
}

unpacker, err := source.NewDefaultUnpacker(mgr, systemNamespace, unpackImage, rootCAs)
unpacker, err := source.NewDefaultUnpacker(mgr, systemNamespace, unpackCacheDir, rootCAs)
if err != nil {
setupLog.Error(err, "unable to setup bundle unpacker")
os.Exit(1)
}

if err := bundleFinalizers.Register(finalizer.CleanupUnpackCacheKey, &finalizer.CleanupUnpackCache{Unpacker: unpacker}); err != nil {
setupLog.Error(err, "unable to register finalizer", "finalizerKey", finalizer.CleanupUnpackCacheKey)
os.Exit(1)
}

bdNamespaceMapper := func(obj client.Object) (string, error) {
bd, ok := obj.(*rukpakv1alpha2.BundleDeployment)
if !ok {
Expand Down Expand Up @@ -243,8 +248,9 @@ func main() {
os.Exit(1)
}

ctx := ctrl.SetupSignalHandler()
setupLog.Info("starting manager")
if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
if err := mgr.Start(ctx); err != nil {
setupLog.Error(err, "problem running manager")
os.Exit(1)
}
Expand Down
37 changes: 36 additions & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,13 @@ go 1.21
toolchain go1.21.0

require (
github.com/containerd/containerd v1.7.15
github.com/go-git/go-billy/v5 v5.5.0
github.com/go-git/go-git/v5 v5.12.0
github.com/go-logr/logr v1.4.1
github.com/google/go-containerregistry v0.19.1
github.com/google/go-containerregistry/pkg/authn/k8schain v0.0.0-20240418155129-98dd3e91704f
github.com/google/go-containerregistry/pkg/authn/kubernetes v0.0.0-20240418155129-98dd3e91704f
everettraven marked this conversation as resolved.
Show resolved Hide resolved
github.com/gorilla/handlers v1.5.2
github.com/nlepage/go-tarfs v1.2.1
github.com/onsi/ginkgo/v2 v2.17.2
Expand Down Expand Up @@ -35,9 +39,20 @@ require (
)

require (
cloud.google.com/go/compute v1.24.0 // indirect
cloud.google.com/go/compute/metadata v0.2.3 // indirect
dario.cat/mergo v1.0.0 // indirect
github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect
github.com/Azure/azure-sdk-for-go v68.0.0+incompatible // indirect
github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect
github.com/Azure/go-autorest v14.2.0+incompatible // indirect
github.com/Azure/go-autorest/autorest v0.11.29 // indirect
github.com/Azure/go-autorest/autorest/adal v0.9.23 // indirect
github.com/Azure/go-autorest/autorest/azure/auth v0.5.12 // indirect
github.com/Azure/go-autorest/autorest/azure/cli v0.4.6 // indirect
github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
github.com/Azure/go-autorest/logger v0.2.1 // indirect
github.com/Azure/go-autorest/tracing v0.6.0 // indirect
github.com/BurntSushi/toml v1.3.2 // indirect
github.com/MakeNowJust/heredoc v1.0.0 // indirect
github.com/Masterminds/goutils v1.1.1 // indirect
Expand All @@ -49,17 +64,33 @@ require (
github.com/ProtonMail/go-crypto v1.0.0 // indirect
github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df // indirect
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
github.com/aws/aws-sdk-go-v2 v1.18.0 // indirect
github.com/aws/aws-sdk-go-v2/config v1.18.25 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.13.24 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.3 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.33 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.27 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.34 // indirect
github.com/aws/aws-sdk-go-v2/service/ecr v1.18.11 // indirect
github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.16.2 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.27 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.12.10 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.10 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.19.0 // indirect
github.com/aws/smithy-go v1.13.5 // indirect
github.com/awslabs/amazon-ecr-credential-helper/ecr-login v0.0.0-20230510185313-f5e39e5f34c7 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/blang/semver/v4 v4.0.0 // indirect
github.com/cenkalti/backoff/v4 v4.2.1 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/chai2010/gettext-go v1.0.2 // indirect
github.com/chrismellard/docker-credential-acr-env v0.0.0-20230304212654-82a0ddb27589 // indirect
github.com/cloudflare/circl v1.3.7 // indirect
github.com/containerd/cgroups/v3 v3.0.2 // indirect
github.com/containerd/containerd v1.7.15 // indirect
github.com/containerd/continuity v0.4.2 // indirect
github.com/containerd/errdefs v0.1.0 // indirect
github.com/containerd/log v0.1.0 // indirect
github.com/containerd/stargz-snapshotter/estargz v0.15.1 // indirect
github.com/containerd/ttrpc v1.2.3 // indirect
github.com/containerd/typeurl/v2 v2.1.1 // indirect
github.com/containers/common v0.58.2 // indirect
Expand All @@ -69,6 +100,7 @@ require (
github.com/containers/storage v1.53.0 // indirect
github.com/cyphar/filepath-securejoin v0.2.4 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dimchansky/utfbom v1.1.1 // indirect
github.com/distribution/reference v0.6.0 // indirect
github.com/docker/cli v26.0.1+incompatible // indirect
github.com/docker/distribution v2.8.3+incompatible // indirect
Expand Down Expand Up @@ -96,6 +128,7 @@ require (
github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
github.com/gobwas/glob v0.2.3 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/google/btree v1.1.2 // indirect
Expand All @@ -119,6 +152,7 @@ require (
github.com/imdario/mergo v0.3.16 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/jmoiron/sqlx v1.3.5 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
Expand All @@ -134,6 +168,7 @@ require (
github.com/mattn/go-isatty v0.0.17 // indirect
github.com/mattn/go-runewidth v0.0.15 // indirect
github.com/mitchellh/copystructure v1.2.0 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/mitchellh/go-wordwrap v1.0.1 // indirect
github.com/mitchellh/reflectwalk v1.0.2 // indirect
github.com/moby/locker v1.0.1 // indirect
Expand Down