Distribute CRDs using helm chart (#1244)

* Distribute CRDs using helm chart

CRDs for now were being deployed using controller, we
should make sure CRDs are being distributed by helm
chart preferrably but if we are consuming kanister not
as sub chart in that case they can set an env var
in the kanister pod to make sure CRDs are being created.

* Remove crd source files

* Fix minor helm template issue

* Make sure RBAC for CRD creation is only done if createCRDs is true

* Address review comments

- rename helm field `createCRDs` to `updateCRDs`
- refactor order of embedded files

* Address review comments

* Refactor, revert part of last review commentss

* Change default value of `updateCRDs` to `true`

When we didn't have this `updateCRDs` flag in values.yaml
controller was responsible to create/update the CRDs. To
make sure that we follow the same default path we are changing
the default value of `updateCRDs` to `true`, so that kanister
controller will be responsible to manager crds. If thats
not required/possible, this flag can be set to false.

* Address review comment, default to true for create crd

Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>

helm/kanister-operator/crds/actionset.yaml

@@ -0,0 +1 @@
+../../../pkg/customresource/actionset.yaml

helm/kanister-operator/crds/blueprint.yaml

@@ -0,0 +1 @@
+../../../pkg/customresource/blueprint.yaml

helm/kanister-operator/crds/profile.yaml

@@ -0,0 +1 @@
+../../../pkg/customresource/profile.yaml

helm/kanister-operator/templates/deployment.yaml

@@ -26,6 +26,9 @@ spec:
         volumeMounts:
           - name: webhook-certs
             mountPath: /var/run/webhook/serving-cert
+        env:
+        - name: CREATEORUPDATE_CRDS
+          value: {{ .Values.controller.updateCRDs | quote }}
 {{- if .Values.resources }}
         resources:
 {{ toYaml .Values.resources | indent 12 }}

helm/kanister-operator/templates/rbac.yaml

@@ -18,12 +18,14 @@ rules:
   - events
   verbs:
   - create
+{{- if .Values.controller.updateCRDs }}
 - apiGroups:
   - apiextensions.k8s.io
   resources:
   - customresourcedefinitions
   verbs:
   - "*"
+{{- end }}
 - apiGroups:
   - snapshot.storage.k8s.io
   resources:

helm/kanister-operator/values.yaml

@@ -13,6 +13,10 @@ serviceAccount:
 controller:
   service:
     port: 443
+  # updateCRDs specifies if kanister controller should create/update the CRDs
+  # false : CRDs would be created by helm
+  # true : CRDs would be created by kanister controller
+  updateCRDs: true
 resources:
 # We usually recommend not to specify default resources and to leave this as a conscious
 # choice for the user. This also increases chances charts run on environments with little

pkg/customresource/actionset-crd.go → pkg/customresource/actionset.yaml

@@ -1,6 +1,3 @@
-package customresource
-
-const actionsetCRD = `
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
@@ -252,4 +249,3 @@ status:
     plural: ""
   conditions: []
   storedVersions: []
-`

pkg/customresource/blueprint-crd.go → pkg/customresource/blueprint.yaml

@@ -1,6 +1,3 @@
-package customresource
-
-const blueprintCRD = `
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
@@ -107,4 +104,3 @@ status:
     plural: ""
   conditions: []
   storedVersions: []
-`

pkg/customresource/customresource.go

@@ -21,17 +21,16 @@ import (
 	"time"
 
 	"github.com/Masterminds/semver"
+	"github.com/pkg/errors"
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
-	"k8s.io/apimachinery/pkg/api/errors"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/serializer"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/kubernetes"
 
-	"github.com/kanisterio/kanister/pkg/consts"
-
 	// importing go check to bypass the testing flags
 	_ "gopkg.in/check.v1"
 )
@@ -114,15 +113,21 @@ func decodeSpecIntoObject(spec []byte, intoObj runtime.Object) error {
 }
 
 func createCRD(context Context, resource CustomResource) error {
-	crd, err := getCRDFromSpec(specFromResName(resource.Name))
+	c, err := rawCRDFromFile(fmt.Sprintf("%s.yaml", resource.Name))
+	if err != nil {
+		return errors.Wrap(err, "Getting raw CRD from CRD manifests")
+	}
+
+	crd, err := getCRDFromSpec(c)
 	if err != nil {
-		return err
+		return errors.Wrap(err, "Getting CRD object from CRD bytes")
 	}
+
 	ctx := contextpkg.Background()
 	_, err = context.APIExtensionClientset.ApiextensionsV1().CustomResourceDefinitions().Create(ctx, crd, metav1.CreateOptions{})
 	if err != nil {
-		if !errors.IsAlreadyExists(err) {
-			return fmt.Errorf("failed to create %s CRD. %+v", resource.Name, err)
+		if !apierrors.IsAlreadyExists(err) {
+			return errors.Errorf("Failed to create %s CRD. %+v", resource.Name, err)
 		}
 
 		// if CRD already exists, get the resource version and create the CRD with that resource version
@@ -140,16 +145,9 @@ func createCRD(context Context, resource CustomResource) error {
 	return nil
 }
 
-func specFromResName(name string) []byte {
-	switch name {
-	case consts.ActionSetResourceName:
-		return []byte(actionsetCRD)
-	case consts.BlueprintResourceName:
-		return []byte(blueprintCRD)
-	case consts.ProfileResourceName:
-		return []byte(profileCRD)
-	}
-	return nil
+func rawCRDFromFile(path string) ([]byte, error) {
+	// yamls is the variable that has embeded custom resource manifest. More at `embed.go`
+	return yamls.ReadFile(path)
 }
 
 func waitForCRDInit(context Context, resource CustomResource) error {

pkg/customresource/embed.go

@@ -0,0 +1,16 @@
+package customresource
+
+import "embed"
+
+// embed.go embeds the CRD yamls (actionset, profile, blueprint) with the
+// controller binary so that we can read these manifests in runtime.
+
+// We need these manfiests at two places, at `pkg/customresource/` and at
+// `helm/kanister-operator/crds`. To make sure we are not duplicating the
+// things we have original files at `pkg/customresource` and have soft links
+// at `helm/kanister-operator/crds`.
+
+//go:embed actionset.yaml
+//go:embed blueprint.yaml
+//go:embed profile.yaml
+var yamls embed.FS

pkg/customresource/profile-crd.go → pkg/customresource/profile.yaml

@@ -1,6 +1,3 @@
-package customresource
-
-const profileCRD = `
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
@@ -174,4 +171,3 @@ status:
     plural: ""
   conditions: []
   storedVersions: []
-`

pkg/kancontroller/kancontroller.go

@@ -27,18 +27,24 @@ import (
 	"fmt"
 	"os"
 	"os/signal"
+	"strconv"
 	"syscall"
 
 	"k8s.io/client-go/rest"
 
 	"github.com/kanisterio/kanister/pkg/controller"
+	"github.com/kanisterio/kanister/pkg/field"
 	_ "github.com/kanisterio/kanister/pkg/function"
 	"github.com/kanisterio/kanister/pkg/handler"
 	"github.com/kanisterio/kanister/pkg/kube"
 	"github.com/kanisterio/kanister/pkg/log"
 	"github.com/kanisterio/kanister/pkg/resource"
 )
 
+const (
+	createOrUpdateCRDEnvVar = "CREATEORUPDATE_CRDS"
+)
+
 func Execute() {
 	ctx := context.Background()
 	// Initialize the clients.
@@ -73,10 +79,12 @@ func Execute() {
 		}()
 	}
 
-	// Make sure the CRD's exist.
-	if err := resource.CreateCustomResources(ctx, config); err != nil {
-		log.WithError(err).Print("Failed to create CustomResources.")
-		return
+	// CRDs should only be created/updated if the env var CREATEORUPDATE_CRDS is set to true
+	if createOrUpdateCRDs() {
+		if err := resource.CreateCustomResources(ctx, config); err != nil {
+			log.WithError(err).Print("Failed to create CustomResources.")
+			return
+		}
 	}
 
 	ns, err := kube.GetControllerNamespace()
@@ -112,3 +120,18 @@ func isCACertMounted() bool {
 
 	return true
 }
+
+func createOrUpdateCRDs() bool {
+	createOrUpdateCRD := os.Getenv(createOrUpdateCRDEnvVar)
+	if createOrUpdateCRD == "" {
+		return true
+	}
+
+	c, err := strconv.ParseBool(createOrUpdateCRD)
+	if err != nil {
+		log.Print("environment variable", field.M{"CREATEORUPDATE_CRDS": createOrUpdateCRD})
+		return true
+	}
+
+	return c
+}