Support for CLI and JSON-styled startup configs (#37)

* typo fix

* remove unused kustomization file

* added cimments about startup config predefined name

* renamed main reconciliation function

* initial skeleton for startupcfg handler and status

* rename found->pod

* added readiness probe and skeleton for scrpaligo provisioning

* refactor startup config handling and controller status updates

* added docs on how to stream controller logs

* address linter

* address more linter checks

* remove unused received and used var declaration on empty slice

* do startup config check at a caller side

* added e2e test for startup config

* address lint

* added matrix test for e2e

* increase timeout

* rename func

* rename

* defer driver closing in parent func

* added checkpoint provisioning

* added commit save to persist the config

* checkout before setting go

* crank up ready timer even more

* added ready check for bare CR

* split e2e test functions since gh actions borked

* added readme and bump version

.github/workflows/e2e.yml

@@ -21,12 +21,23 @@ jobs:
   e2e:
     name: End-to-end test
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        test:
+          [
+            "TestSrlinuxReconciler_BareSrlinuxCR",
+            "TestSrlinuxReconciler_WithJSONStartupConfig",
+            "TestSrlinuxReconciler_WithCLIStartupConfig",
+          ]
 
     steps:
       - name: Set env vars
         run: |
           echo "KNE_REF=${{ inputs.kne_ref }}" >> $GITHUB_ENV
 
+      - uses: actions/checkout@v3
+
       - name: Set up Go
         uses: actions/setup-go@v4
         with:
@@ -39,12 +50,10 @@ jobs:
           version: ${{ inputs.kind_version }}
           skipClusterCreation: true
 
-      - uses: actions/checkout@v3
-
       - name: Prepare e2e environment
         run: make prepare-e2e-env
 
       - name: Run e2e tests
         # this test ensures that srl-controller (built from referenced source) can be succesfully installed on a KNE cluster
         # for a using specified versions of KNE/KinD
-        run: make test-e2e
+        run: E2E_TEST_NAME=${{ matrix.test }} make test-e2e

.mk/e2e.mk

@@ -36,14 +36,21 @@ install-srl-controller: ## Install srl-controller from current working dir
 	@echo "wait for controller manager to be ready"
 	kubectl -n srlinux-controller wait --for=condition=Available deployment.apps/srlinux-controller-controller-manager
 
+.PHONY: uninstall-srl-controller
+uninstall-srl-controller: ## Uninstall srl-controller from current working dir
+	kubectl delete -k config/default
+
 .PHONY: kind-load-image
 kind-load-image:  ## Load SR Linux container image to kind cluster
 	docker pull ${SRL_IMAGE}
-	kind load docker-image ${SRL_IMAGE} --name srl-test
+	kind load docker-image ${SRL_IMAGE} --name ${KIND_CLUSTER_NAME}
+
+.PHONY: start-kne-cluster
+start-kne-cluster: install-kne kne-test-deployment-cfg-file deploy-kne kind-load-image ## Deploy KNE kind cluster but do not install any controllers
 
 .PHONY: prepare-e2e-env
 prepare-e2e-env: install-kne kne-test-deployment-cfg-file deploy-kne temp-docker-build install-srl-controller kind-load-image ## Install srl-controller from current working dir
 
 .PHONY: test-e2e
-test-e2e: ## Test e2e using kind
-	go test -v github.com/srl-labs/srl-controller/tests/e2e
+test-e2e: ## Test e2e using kind and a provided test name
+	go test -timeout 5m -v github.com/srl-labs/srl-controller/tests/e2e -run ${E2E_TEST_NAME}

README.md

@@ -62,6 +62,12 @@ Once the kne+kind cluster is created and the `srl-controller` is installed onto
 kne create examples/srlinux/2node-srl-with-config.pbtxt
 ```
 
+Note, that controller logs can be viewed live with:
+
+```bash
+kubectl logs --follow -n srlinux-controller $(kubectl get pods -A | grep srlinux-controller | awk '{print $2}')
+```
+
 This will deploy the SR Linux nodes and will create k8s services as per the topology configuration. The services will be exposed via MetalLB and can be queried as:
 
 ```text
@@ -106,30 +112,27 @@ When a request to create a `Srlinux` resource named `r1` in namespace `ns` comes
 
 1. Checks if the pods exist within a namespace `ns` with a name `r1`
 2. If the pod hasn't been found, then the controller first ensures that the necessary config maps exist in namespace `ns` and creates them otherwise.
-3. When config maps are sorted out, the controller schedules a pod with the name `r1` and requeue the request
-4. In a requeue run, the pod is now found and the controller updates the status of `Srlinux` resource with the image name that was used in the pod spec.
+3. When config maps are sorted out, the controller schedules a pod with the name `r1` and requeues the request.
+4. If a startup-config was provided, the controller loads this config using SSH into the pod, creates a named checkpoint "initial" and requeues the request.
+5. In a requeue run, the pod is now found and the controller updates the status of `Srlinux` resource.
 
 ### Deletion
 
 When a deletion happens on `Srlinux` resource, the reconcile loop does nothing.
 
-### API access
-
-This repo contains a clientset for API access to the `Srlinux` custom resource. Check [kne repo](https://github.com/openconfig/kne/blob/fc195a73035bcbf344791979ca3e067be47a249c/topo/node/srl/srl.go#L46) to see how this can be done.
-
 ## Building `srl-controller` container image
 
 To build `srl-controller` container image, execute:
 
 ```bash
 # don't forget to set the correct tag
-# for example make docker-build IMG=ghcr.io/srl-labs/srl-controller:0.4.3
+# for example make docker-build IMG=ghcr.io/srl-labs/srl-controller:v0.6.0
 make docker-build IMG=ghcr.io/srl-labs/srl-controller:${tag}
 ```
 
 > build process will try to remove license headers for some manifests, discard those changes.
 
-Next update the controller version in [manager/kustomization.yaml](config/manager/kustomization.yaml) kustomization file to match the newly built version.
+Next, update the controller version in [manager/kustomization.yaml](config/manager/kustomization.yaml) kustomization file to match the newly built version.
 
 Finally, upload the container image to the registry:
 
@@ -141,6 +144,12 @@ docker tag ghcr.io/srl-labs/srl-controller:${tag} ghcr.io/srl-labs/srl-controlle
 docker push ghcr.io/srl-labs/srl-controller:latest
 ```
 
+Note, update the SR Linux manifest in the [KNE repo](https://github.com/openconfig/kne/) to use the new version of the controller. To generate the manifest, run:
+
+```bash
+kustomize build config/default
+```
+
 ## Developers guide
 
 Developers should deploy the controller onto a cluster from the source code. Ensure that the `srl-controller` is [uninstalled](#uninstall) from the cluster before proceeding.

api/v1/srlinux_types.go

@@ -56,16 +56,31 @@ type SrlinuxSpec struct {
 
 // SrlinuxStatus defines the observed state of Srlinux.
 type SrlinuxStatus struct {
+	// Status is the status of the srlinux custom resource.
+	// Can be one of: "created", "running", "error".
+	Status string `json:"status,omitempty"`
 	// Image used to run srlinux pod
 	Image string `json:"image,omitempty"`
+	// StartupConfig contains the status of the startup-config.
+	StartupConfig StartupConfigStatus `json:"startup-config,omitempty"`
+	// Ready is true if the srlinux NOS is ready to receive config.
+	// This is when management server is running and initial commit is processed.
+	Ready bool `json:"ready,omitempty"`
+}
+
+type StartupConfigStatus struct {
+	// Phase is the phase startup-config is in. Can be one of: "pending", "loaded", "not-provided", "failed".
+	Phase string `json:"phase,omitempty"`
 }
 
 //+kubebuilder:object:root=true
 //+kubebuilder:subresource:status
 
 // Srlinux is the Schema for the srlinuxes API.
-// +kubebuilder:printcolumn:name="Image",type="string",JSONPath=".status.image"
 // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp"
+// +kubebuilder:printcolumn:name="Image",type="string",JSONPath=".status.image"
+// +kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.status"
+// +kubebuilder:printcolumn:name="Ready",type="boolean",JSONPath=".status.ready"
 type Srlinux struct {
 	metav1.TypeMeta   `json:",inline"`
 	metav1.ObjectMeta `json:"metadata,omitempty"`

config/crd/bases/kne.srlinux.dev_srlinuxes.yaml

@@ -16,12 +16,18 @@ spec:
   scope: Namespaced
   versions:
   - additionalPrinterColumns:
-    - jsonPath: .status.image
-      name: Image
-      type: string
     - jsonPath: .metadata.creationTimestamp
       name: Age
       type: date
+    - jsonPath: .status.image
+      name: Image
+      type: string
+    - jsonPath: .status.status
+      name: Status
+      type: string
+    - jsonPath: .status.ready
+      name: Ready
+      type: boolean
     name: v1
     schema:
       openAPIV3Schema:
@@ -122,6 +128,23 @@ spec:
               image:
                 description: Image used to run srlinux pod
                 type: string
+              ready:
+                description: Ready is true if the srlinux NOS is ready to receive
+                  config. This is when management server is running and initial commit
+                  is processed.
+                type: boolean
+              startup-config:
+                description: StartupConfig contains the status of the startup-config.
+                properties:
+                  phase:
+                    description: 'Phase is the phase startup-config is in. Can be
+                      one of: "pending", "loaded", "not-provided", "failed".'
+                    type: string
+                type: object
+              status:
+                description: 'Status is the status of the srlinux custom resource.
+                  Can be one of: "created", "running", "error".'
+                type: string
             type: object
         type: object
     served: true

config/manager/kustomization.yaml

@@ -10,4 +10,4 @@ kind: Kustomization
 images:
   - name: controller
     newName: ghcr.io/srl-labs/srl-controller
-    newTag: v0.5.0
+    newTag: v0.6.0

controllers/pod.go

@@ -8,7 +8,6 @@ import (
 	"context"
 	"fmt"
 
-	"github.com/go-logr/logr"
 	knenode "github.com/openconfig/kne/topo/node"
 	srlinuxv1 "github.com/srl-labs/srl-controller/api/v1"
 	corev1 "k8s.io/api/core/v1"
@@ -24,6 +23,10 @@ const (
 	licenseFileName               = "license.key"
 	licenseMntPath                = "/opt/srlinux/etc/license.key"
 	licenseMntSubPath             = "license.key"
+	readinessFile                 = "/etc/opt/srlinux/devices/app_ephemeral.mgmt_server.ready_for_config"
+	readinessInitialDelay         = 10
+	readinessPeriodSeconds        = 5
+	readinessFailureThreshold     = 10
 )
 
 // podForSrlinux returns a srlinux Pod object.
@@ -52,7 +55,9 @@ func (r *SrlinuxReconciler) podForSrlinux(
 	}
 
 	// handle startup config volume mounts if the startup config was defined
-	handleStartupConfig(s, pod, log)
+	if s.Spec.Config.ConfigDataPresent {
+		createStartupConfigVolumesAndMounts(s, pod, log)
+	}
 
 	//nolint:godox
 	// TODO: handle the error
@@ -98,6 +103,19 @@ func createContainers(s *srlinuxv1.Srlinux) []corev1.Container {
 			RunAsUser:  pointer.Int64(0),
 		},
 		VolumeMounts: createVolumeMounts(s),
+		ReadinessProbe: &corev1.Probe{
+			ProbeHandler: corev1.ProbeHandler{
+				Exec: &corev1.ExecAction{
+					Command: []string{
+						"cat",
+						readinessFile,
+					},
+				},
+			},
+			InitialDelaySeconds: readinessInitialDelay,
+			PeriodSeconds:       readinessPeriodSeconds,
+			FailureThreshold:    readinessFailureThreshold,
+		},
 	}}
 }
 
@@ -171,50 +189,6 @@ func createVolumes(s *srlinuxv1.Srlinux) []corev1.Volume {
 	return vols
 }
 
-// handleStartupConfig creates volume mounts and volumes for srlinux pod
-// if the (startup) config file was provided in the spec.
-// Volume mounts happens in the /tmp/startup-config directory and not in the /etc/opt/srlinux
-// because we need to support renaming operations on config.json, and bind mount paths are not allowing this.
-// Hence the temp location, from which the config file is then copied to /etc/opt/srlinux by the kne-entrypoint.sh.
-func handleStartupConfig(s *srlinuxv1.Srlinux, pod *corev1.Pod, log logr.Logger) {
-	// initialize config path and config file variables
-	cfgPath := defaultConfigPath
-	if p := s.Spec.GetConfig().ConfigPath; p != "" {
-		cfgPath = p
-	}
-
-	// only create startup config mounts if the config data was set in kne
-	if s.Spec.Config.ConfigDataPresent {
-		log.Info(
-			"Adding volume for startup config to pod spec",
-			"volume.name",
-			"startup-config-volume",
-			"mount.path",
-			cfgPath,
-		)
-
-		pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{
-			Name: "startup-config-volume",
-			VolumeSource: corev1.VolumeSource{
-				ConfigMap: &corev1.ConfigMapVolumeSource{
-					LocalObjectReference: corev1.LocalObjectReference{
-						Name: fmt.Sprintf("%s-config", s.Name),
-					},
-				},
-			},
-		})
-
-		pod.Spec.Containers[0].VolumeMounts = append(
-			pod.Spec.Containers[0].VolumeMounts,
-			corev1.VolumeMount{
-				Name:      "startup-config-volume",
-				MountPath: cfgPath,
-				ReadOnly:  false,
-			},
-		)
-	}
-}
-
 func createVolumeMounts(s *srlinuxv1.Srlinux) []corev1.VolumeMount {
 	vms := []corev1.VolumeMount{
 		{