refactor: create the minio bucket by the job (#620)

Create the minio bucket by the job in the target namespace of the harbor to avoid problem when network is not accessable between operator and harbor namespaces. Signed-off-by: He Weiwei <hweiwei@vmware.com>
goharbor · Apr 28, 2021 · 6dcbdc9 · 6dcbdc9
1 parent ba4e811
commit 6dcbdc9
Show file tree

Hide file tree

Showing 14 changed files with 424 additions and 72 deletions.
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -298,7 +298,7 @@ jobs:
         cd ..
         kubectl -n cluster-sample-ns patch deploy harborcluster-sample-harbor-harbor-core -p '{"spec":{"template":{"spec":{"containers":[{"name":"core","env":[{"name":"GC_TIME_WINDOW_HOURS","value":"0"}]}]}}}}'
         sleep 5
-        kubectl -n cluster-sample-ns wait --for=condition=Ready pod --all --timeout 600s
+        kubectl -n cluster-sample-ns wait --for=condition=Ready -l job-type!=minio-init pod --all --timeout 600s
         docker run --rm -t --privileged -v `pwd`/harbor:/drone -v /etc/docker/certs.d/$CORE_HOST/:/ca -v /etc/docker/certs.d/$CORE_HOST/:/root/.docker/tls/$NOTARY_HOST/ -e NOTARY_URL=https://$NOTARY_HOST -w /drone goharbor/harbor-e2e-engine:2.6.3 \
           python -u -m robot -v DOCKER_USER:${{ secrets.DOCKER_USER }} -v DOCKER_PWD:${{ secrets.DOCKER_TOKEN }} -v ip:$CORE_HOST -v ip1: -v HARBOR_PASSWORD:Harbor12345 -X \
           --exclude gc --exclude metrics --exclude singularity --exclude proxy_cache \
@@ -446,7 +446,7 @@ jobs:
             echo $i
             kubectl get all
           done
-          if ! time kubectl wait --for=condition=Ready pod --all --timeout 600s && ! time kubectl wait --for=condition=Ready pod --all --timeout 60s; then
+          if ! time kubectl wait --for=condition=Ready -l job-type!=minio-init pod --all --timeout 600s && ! time kubectl wait --for=condition=Ready -l job-type!=minio-init pod --all --timeout 60s; then
             echo install harbor failed
             kubectl get all
 
@@ -487,7 +487,7 @@ jobs:
           git clone https://github.com/goharbor/harbor -b v2.2.1
           kubectl patch deploy harborcluster-sample-harbor-harbor-core -p '{"spec":{"template":{"spec":{"containers":[{"name":"core","env":[{"name":"GC_TIME_WINDOW_HOURS","value":"0"}]}]}}}}'
           sleep 5
-          kubectl wait --for=condition=Ready pod --all --timeout 600s
+          kubectl wait --for=condition=Ready -l job-type!=minio-init pod --all --timeout 600s
           docker run --rm -t --privileged -v `pwd`/harbor:/drone -v /etc/docker/certs.d/$CORE_HOST/:/ca -v /etc/docker/certs.d/$CORE_HOST/:/root/.docker/tls/$NOTARY_HOST/ -e NOTARY_URL=https://$NOTARY_HOST -w /drone goharbor/harbor-e2e-engine:2.6.3 \
           python -u -m robot -v DOCKER_USER:${{ secrets.DOCKER_USER }} -v DOCKER_PWD:${{ secrets.DOCKER_TOKEN }} -v ip:$CORE_HOST -v ip1: -v HARBOR_PASSWORD:Harbor12345 -X \
           --exclude gc --exclude metrics --exclude singularity --exclude proxy_cache \

diff --git a/apis/goharbor.io/v1alpha3/harborcluster_types.go b/apis/goharbor.io/v1alpha3/harborcluster_types.go
@@ -153,6 +153,14 @@ type MinIOSpec struct {
 	// If provided, use these requests and limit for cpu/memory resource allocation
 	// +kubebuilder:validation:Optional
 	Resources corev1.ResourceRequirements `json:"resources,omitempty"`
+
+	// MinIOClientSpec the spec for the mc
+	// +kubebuilder:validation:Optional
+	MinIOClientSpec *MinIOClientSpec `json:"mc,omitempty"`
+}
+
+type MinIOClientSpec struct {
+	harbormetav1.ImageSpec `json:",inline"`
 }
 
 // HarborClusterStatus defines the observed state of HarborCluster.

diff --git a/apis/goharbor.io/v1alpha3/zz_generated.deepcopy.go b/apis/goharbor.io/v1alpha3/zz_generated.deepcopy.go
diff --git a/charts/harbor-operator/templates/clusterrole.yaml b/charts/harbor-operator/templates/clusterrole.yaml
@@ -112,6 +112,18 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - batch
+  resources:
+  - jobs
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - cert-manager.io
   resources:

diff --git a/controllers/goharbor/harborcluster/ctrl_setup.go b/controllers/goharbor/harborcluster/ctrl_setup.go
@@ -22,6 +22,7 @@ import (
 	"github.com/pkg/errors"
 	redisOp "github.com/spotahome/redis-operator/api/redisfailover/v1"
 	postgresv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1"
+	batchv1 "k8s.io/api/batch/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -64,6 +65,7 @@ type Reconciler struct {
 // +kubebuilder:rbac:groups=apps,resources=statefulsets;deployments,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=goharbor.io,resources=harbors,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=apiextensions.k8s.io,resources=customresourcedefinitions,verbs=get;list;watch
+// +kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;watch;create;update;patch;delete
 
 func (r *Reconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager) error {
 	if err := r.ctrl.SetupWithManager(ctx, mgr); err != nil {
@@ -112,6 +114,7 @@ func (r *Reconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager) err
 
 	return builder.ControllerManagedBy(mgr).
 		For(&goharborv1.HarborCluster{}).
+		Owns(&batchv1.Job{}).
 		Owns(&goharborv1.Harbor{}).
 		TryOwns(&minio.Tenant{}, minioCRD).
 		TryOwns(&postgresv1.Postgresql{}, postgresCRD).

diff --git a/manifests/cluster/deployment.yaml b/manifests/cluster/deployment.yaml
@@ -3049,6 +3049,29 @@ spec:
                               type: string
                           type: object
                         type: array
+                      mc:
+                        description: MinIOClientSpec the spec for the mc
+                        properties:
+                          image:
+                            description: Image name for the component.
+                            type: string
+                          imagePullPolicy:
+                            description: 'Image pull policy. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images'
+                            enum:
+                            - Always
+                            - Never
+                            - IfNotPresent
+                            type: string
+                          imagePullSecrets:
+                            items:
+                              description: LocalObjectReference contains enough information to let you locate the referenced object inside the same namespace.
+                              properties:
+                                name:
+                                  description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'
+                                  type: string
+                              type: object
+                            type: array
+                        type: object
                       redirect:
                         description: Determine if the redirection of minio storage is disabled.
                         properties:
@@ -19987,6 +20010,18 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - batch
+  resources:
+  - jobs
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - cert-manager.io
   resources:

diff --git a/manifests/cluster/patch/images.yaml b/manifests/cluster/patch/images.yaml
@@ -32,3 +32,5 @@ spec:
             value: ghcr.io/goharbor/redis:5.0-alpine
           - name: HARBORCLUSTER_POSTGRESQL_DOCKER_IMAGE_2_2_1
             value: ghcr.io/goharbor/spilo-12:1.6-p3
+          - name: HARBORCLUSTER_MINIO_INIT_DOCKER_IMAGE_2_2_1
+            value: ghcr.io/goharbor/mc:RELEASE.2021-03-23T05-46-11Z
diff --git a/manifests/harbor/deployment.yaml b/manifests/harbor/deployment.yaml
@@ -3049,6 +3049,29 @@ spec:
                               type: string
                           type: object
                         type: array
+                      mc:
+                        description: MinIOClientSpec the spec for the mc
+                        properties:
+                          image:
+                            description: Image name for the component.
+                            type: string
+                          imagePullPolicy:
+                            description: 'Image pull policy. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images'
+                            enum:
+                            - Always
+                            - Never
+                            - IfNotPresent
+                            type: string
+                          imagePullSecrets:
+                            items:
+                              description: LocalObjectReference contains enough information to let you locate the referenced object inside the same namespace.
+                              properties:
+                                name:
+                                  description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'
+                                  type: string
+                              type: object
+                            type: array
+                        type: object
                       redirect:
                         description: Determine if the redirection of minio storage is disabled.
                         properties:
@@ -12991,6 +13014,18 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - batch
+  resources:
+  - jobs
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - cert-manager.io
   resources:

diff --git a/pkg/cluster/controllers/storage/constants.go b/pkg/cluster/controllers/storage/constants.go
@@ -12,4 +12,8 @@ const (
 	GetMinIOProperties       = "get MinIO Properties error"
 	UpdateIngressError       = "update minIO ingress error"
 	GetMinIOIngressError     = "get minIO ingress error"
+	CreateInitJobError       = "create minIO init job error"
+	DeleteInitJobError       = "delete minIO init job error"
+	GetInitJobError          = "get minIO init job error"
+	UpdateInitJobError       = "update minIO init job error"
 )
diff --git a/pkg/cluster/controllers/storage/image.go b/pkg/cluster/controllers/storage/image.go
@@ -18,16 +18,21 @@ import (
 	"context"
 
 	goharborv1 "github.com/goharbor/harbor-operator/apis/goharbor.io/v1alpha3"
+	"github.com/goharbor/harbor-operator/pkg/config"
 	"github.com/goharbor/harbor-operator/pkg/image"
+	corev1 "k8s.io/api/core/v1"
 )
 
 const (
 	ComponentName  = "cluster-minio"
 	ConfigImageKey = "minio-docker-image"
+
+	MinIOClientComponentName  = "cluster-minio-init"
+	MinIOClientConfigImageKey = "minio-init-docker-image"
 )
 
-// GetImage returns the configured image via configstore or default one.
-func (m *MinIOController) GetImage(ctx context.Context, harborcluster *goharborv1.HarborCluster) (string, error) {
+// getImage returns the configured image via configstore or default one.
+func (m *MinIOController) getImage(ctx context.Context, harborcluster *goharborv1.HarborCluster) (string, error) {
 	if harborcluster.Spec.InClusterStorage.MinIOSpec.Image != "" {
 		return harborcluster.Spec.InClusterStorage.MinIOSpec.Image, nil
 	}
@@ -52,3 +57,80 @@ func (m *MinIOController) GetImage(ctx context.Context, harborcluster *goharborv
 
 	return image, nil
 }
+
+func (m *MinIOController) getImagePullPolicy(_ context.Context, harborcluster *goharborv1.HarborCluster) corev1.PullPolicy {
+	if harborcluster.Spec.InClusterStorage.MinIOSpec.ImagePullPolicy != nil {
+		return *harborcluster.Spec.InClusterStorage.MinIOSpec.ImagePullPolicy
+	}
+
+	if harborcluster.Spec.ImageSource != nil && harborcluster.Spec.ImageSource.ImagePullPolicy != nil {
+		return *harborcluster.Spec.ImageSource.ImagePullPolicy
+	}
+
+	return config.DefaultImagePullPolicy
+}
+
+func (m *MinIOController) getImagePullSecret(_ context.Context, harborcluster *goharborv1.HarborCluster) corev1.LocalObjectReference {
+	if len(harborcluster.Spec.InClusterStorage.MinIOSpec.ImagePullSecrets) > 0 {
+		return harborcluster.Spec.InClusterStorage.MinIOSpec.ImagePullSecrets[0]
+	}
+
+	if harborcluster.Spec.ImageSource != nil && len(harborcluster.Spec.ImageSource.ImagePullSecrets) > 0 {
+		return harborcluster.Spec.ImageSource.ImagePullSecrets[0]
+	}
+
+	return corev1.LocalObjectReference{Name: ""} // empty name means not using pull secret in minio
+}
+
+func (m *MinIOController) getMinIOClientImage(ctx context.Context, harborcluster *goharborv1.HarborCluster) (string, error) {
+	spec := harborcluster.Spec.InClusterStorage.MinIOSpec.MinIOClientSpec
+	if spec != nil && spec.Image != "" {
+		return spec.Image, nil
+	}
+
+	options := []image.Option{image.WithHarborVersion(harborcluster.Spec.Version)}
+	if harborcluster.Spec.ImageSource != nil && (harborcluster.Spec.ImageSource.Repository != "" || harborcluster.Spec.ImageSource.TagSuffix != "") {
+		options = append(options,
+			image.WithRepository(harborcluster.Spec.ImageSource.Repository),
+			image.WithTagSuffix(harborcluster.Spec.ImageSource.TagSuffix),
+		)
+	} else {
+		options = append(options,
+			image.WithConfigstore(m.ConfigStore),
+			image.WithConfigImageKey(MinIOClientConfigImageKey),
+		)
+	}
+
+	image, err := image.GetImage(ctx, MinIOClientComponentName, options...)
+	if err != nil {
+		return "", err
+	}
+
+	return image, nil
+}
+
+func (m *MinIOController) getMinIOClientImagePullPolicy(_ context.Context, harborcluster *goharborv1.HarborCluster) corev1.PullPolicy {
+	spec := harborcluster.Spec.InClusterStorage.MinIOSpec.MinIOClientSpec
+	if spec != nil && spec.ImagePullPolicy != nil {
+		return *spec.ImagePullPolicy
+	}
+
+	if harborcluster.Spec.ImageSource != nil && harborcluster.Spec.ImageSource.ImagePullPolicy != nil {
+		return *harborcluster.Spec.ImageSource.ImagePullPolicy
+	}
+
+	return config.DefaultImagePullPolicy
+}
+
+func (m *MinIOController) getMinIOClientImagePullSecrets(_ context.Context, harborcluster *goharborv1.HarborCluster) []corev1.LocalObjectReference {
+	spec := harborcluster.Spec.InClusterStorage.MinIOSpec.MinIOClientSpec
+	if spec != nil && len(spec.ImagePullSecrets) > 0 {
+		return spec.ImagePullSecrets
+	}
+
+	if harborcluster.Spec.ImageSource != nil && len(harborcluster.Spec.ImageSource.ImagePullSecrets) > 0 {
+		return harborcluster.Spec.ImageSource.ImagePullSecrets
+	}
+
+	return nil
+}