feat(run): upload attempt logs (#266)

api/v1alpha1/terraformrun_types.go

@@ -42,12 +42,18 @@ type TerraformRunLayer struct {
 
 // TerraformRunStatus defines the observed state of TerraformRun
 type TerraformRunStatus struct {
-	Conditions   []metav1.Condition `json:"conditions,omitempty"`
-	State        string             `json:"state,omitempty"`
-	Retries      int                `json:"retries"`
-	LastRun      string             `json:"lastRun,omitempty"`
-	RunnerPod    string             `json:"runnerPod,omitempty"`
-	PlanArtifact string             `json:"planArtifact,omitempty"`
+	Conditions []metav1.Condition `json:"conditions,omitempty"`
+	State      string             `json:"state,omitempty"`
+	Retries    int                `json:"retries"`
+	LastRun    string             `json:"lastRun,omitempty"`
+	Attempts   []Attempt          `json:"attempts,omitempty"`
+	RunnerPod  string             `json:"runnerPod,omitempty"`
+}
+
+type Attempt struct {
+	PodName      string `json:"podName,omitempty"`
+	Number       int    `json:"number,omitempty"`
+	LogsUploaded bool   `json:"logsUploaded,omitempty"`
 }
 
 // +kubebuilder:object:root=true

internal/controllers/manager.go

@@ -19,7 +19,9 @@ package controllers
 import (
 	"os"
 
+	logClient "k8s.io/client-go/kubernetes"
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
+	"k8s.io/client-go/rest"
 
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
@@ -88,6 +90,14 @@ func (c *Controllers) Exec() {
 		log.Fatalf("unable to start manager: %s", err)
 	}
 	datastoreClient := datastore.NewDefaultClient()
+	config, err := rest.InClusterConfig()
+	if err != nil {
+		panic(err.Error())
+	}
+	clientset, err := logClient.NewForConfig(config)
+	if err != nil {
+		panic(err.Error())
+	}
 
 	for _, ctrlType := range c.config.Controller.Types {
 		switch ctrlType {
@@ -113,10 +123,12 @@ func (c *Controllers) Exec() {
 			log.Infof("repository controller started successfully")
 		case "run":
 			if err = (&terraformrun.Reconciler{
-				Client:   mgr.GetClient(),
-				Scheme:   mgr.GetScheme(),
-				Recorder: mgr.GetEventRecorderFor("Burrito"),
-				Config:   c.config,
+				Client:       mgr.GetClient(),
+				Scheme:       mgr.GetScheme(),
+				Recorder:     mgr.GetEventRecorderFor("Burrito"),
+				Config:       c.config,
+				Datastore:    datastoreClient,
+				K8SLogClient: clientset,
 			}).SetupWithManager(mgr); err != nil {
 				log.Fatalf("unable to create run controller: %s", err)
 			}

internal/controllers/terraformrun/controller.go

@@ -19,9 +19,13 @@ package terraformrun
 import (
 	"context"
 	"math"
+	"strconv"
 	"time"
 
 	"github.com/google/go-cmp/cmp"
+	datastore "github.com/padok-team/burrito/internal/datastore/client"
+	logClient "k8s.io/client-go/kubernetes"
+
 	log "github.com/sirupsen/logrus"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
@@ -50,9 +54,11 @@ func (c RealClock) Now() time.Time {
 // RunReconcilier reconciles a TerraformRun object
 type Reconciler struct {
 	client.Client
-	Scheme   *runtime.Scheme
-	Config   *config.Config
-	Recorder record.EventRecorder
+	K8SLogClient *logClient.Clientset
+	Scheme       *runtime.Scheme
+	Config       *config.Config
+	Recorder     record.EventRecorder
+	Datastore    datastore.Client
 	Clock
 }
 
@@ -98,12 +104,26 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 	}
 	state, conditions := r.GetState(ctx, run, layer, repo)
 	result, runInfo := state.getHandler()(ctx, r, run, layer, repo)
+	if runInfo.NewPod {
+		attempt := configv1alpha1.Attempt{
+			PodName:      runInfo.RunnerPod,
+			LogsUploaded: false,
+			Number:       runInfo.Retries,
+		}
+		run.Status.Attempts = append(run.Status.Attempts, attempt)
+	}
 	run.Status = configv1alpha1.TerraformRunStatus{
 		Conditions: conditions,
 		State:      getStateString(state),
 		Retries:    runInfo.Retries,
 		LastRun:    runInfo.LastRun,
 		RunnerPod:  runInfo.RunnerPod,
+		Attempts:   run.Status.Attempts,
+	}
+	err = r.uploadLogs(run)
+	if err != nil {
+		r.Recorder.Event(run, corev1.EventTypeWarning, "Reconciliation", "Failed to upload logs")
+		log.Errorf("failed to upload logs for run %s: %s", run.Name, err)
 	}
 	err = r.Client.Status().Update(ctx, run)
 	if err != nil {
@@ -114,6 +134,43 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 	return result, nil
 }
 
+func (r *Reconciler) uploadLogs(run *configv1alpha1.TerraformRun) error {
+	for i, attempt := range run.Status.Attempts {
+		if attempt.LogsUploaded {
+			continue
+		}
+		pod := &corev1.Pod{}
+		err := r.Client.Get(context.Background(), types.NamespacedName{
+			Namespace: run.Namespace,
+			Name:      attempt.PodName,
+		}, pod)
+		if errors.IsNotFound(err) {
+			log.Infof("pod %s not found, ignoring...", attempt.PodName)
+			continue
+		}
+		if err != nil {
+			log.Errorf("failed to get pod %s: %s", attempt.PodName, err)
+			continue
+		}
+		if pod.Status.Phase != corev1.PodSucceeded && pod.Status.Phase != corev1.PodFailed {
+			log.Infof("pod %s is not in a terminal state, ignoring...", attempt.PodName)
+			continue
+		}
+		// Upload logs
+		logs, err := r.K8SLogClient.CoreV1().Pods(run.Namespace).GetLogs(pod.Name, &corev1.PodLogOptions{}).Do(context.Background()).Raw()
+		if err != nil {
+			log.Errorf("failed to get logs for pod %s: %s", pod.Name, err)
+			continue
+		}
+		err = r.Datastore.PutLogs(run.Namespace, run.Spec.Layer.Name, run.Name, strconv.Itoa(attempt.Number), logs)
+		if err != nil {
+			return err
+		}
+		run.Status.Attempts[i].LogsUploaded = true
+	}
+	return nil
+}
+
 // SetupWithManager sets up the controller with the Manager.
 func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error {
 	r.Clock = RealClock{}

internal/controllers/terraformrun/controller_test.go

@@ -14,9 +14,11 @@ import (
 
 	configv1alpha1 "github.com/padok-team/burrito/api/v1alpha1"
 	controller "github.com/padok-team/burrito/internal/controllers/terraformrun"
+	datastore "github.com/padok-team/burrito/internal/datastore/client"
 	utils "github.com/padok-team/burrito/internal/testing"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
+	logClient "k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/record"
@@ -62,16 +64,19 @@ var _ = BeforeSuite(func() {
 
 	err = configv1alpha1.AddToScheme(scheme.Scheme)
 	Expect(err).NotTo(HaveOccurred())
-
+	logClient, err := logClient.NewForConfig(cfg)
+	Expect(err).NotTo(HaveOccurred())
 	//+kubebuilder:scaffold:scheme
 
 	k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme})
 	utils.LoadResources(k8sClient, "testdata")
 	reconciler = &controller.Reconciler{
-		Client: k8sClient,
-		Scheme: scheme.Scheme,
-		Config: config.TestConfig(),
-		Clock:  &MockClock{},
+		Client:       k8sClient,
+		Scheme:       scheme.Scheme,
+		Config:       config.TestConfig(),
+		Clock:        &MockClock{},
+		Datastore:    datastore.NewMockClient(),
+		K8SLogClient: logClient,
 		Recorder: record.NewBroadcasterForTests(1*time.Second).NewRecorder(scheme.Scheme, corev1.EventSource{
 			Component: "burrito",
 		}),

internal/controllers/terraformrun/states.go

@@ -18,6 +18,7 @@ type RunInfo struct {
 	Retries   int
 	LastRun   string
 	RunnerPod string
+	NewPod    bool
 }
 
 func getRunInfo(run *configv1alpha1.TerraformRun) RunInfo {
@@ -89,6 +90,7 @@ func (s *Initial) getHandler() Handler {
 			Retries:   0,
 			LastRun:   r.Clock.Now().Format(time.UnixDate),
 			RunnerPod: pod.Name,
+			NewPod:    true,
 		}
 		r.Recorder.Event(run, corev1.EventTypeNormal, "Run", fmt.Sprintf("Successfully created pod %s for initial run", pod.Name))
 		// Minimal time (1s) to transit from Initial state to Running state
@@ -142,6 +144,7 @@ func (s *Retrying) getHandler() Handler {
 			Retries:   runInfo.Retries + 1,
 			LastRun:   r.Clock.Now().Format(time.UnixDate),
 			RunnerPod: pod.Name,
+			NewPod:    true,
 		}
 		r.Recorder.Event(run, corev1.EventTypeNormal, "Run", fmt.Sprintf("Successfully created pod %s for retry run", pod.Name))
 		// Minimal time (1s) to transit from Retrying state to Running state

internal/server/api/logs.go

@@ -8,23 +8,19 @@ import (
 	"github.com/labstack/echo/v4"
 )
 
-type Attempt struct {
-	AttemptNumber string `param:"attempt"`
-	Namespace     string `param:"namespace"`
-	Layer         string `param:"layer"`
-	Run           string `param:"run"`
-}
-
 type GetLogsResponse struct {
 	Results []string `json:"results"`
 }
 
 func getLogsArgs(c echo.Context) (string, string, string, string, error) {
-	attempt := Attempt{}
-	if err := c.Bind(attempt); err != nil {
+	namespace := c.Param("namespace")
+	layer := c.Param("layer")
+	run := c.Param("run")
+	attempt := c.Param("attempt")
+	if namespace == "" || layer == "" || run == "" || attempt == "" {
 		return "", "", "", "", fmt.Errorf("missing query parameters")
 	}
-	return attempt.Namespace, attempt.Layer, attempt.Run, attempt.AttemptNumber, nil
+	return namespace, layer, run, attempt, nil
 }
 
 // logs/${namespace}/${layer}/${runId}/${attemptId}

internal/server/api/runs.go

@@ -11,18 +11,14 @@ type GetAttemptsResponse struct {
 	Count int `json:"count"`
 }
 
-type Run struct {
-	Namespace string `param:"namespace"`
-	Layer     string `param:"layer"`
-	Run       string `param:"run"`
-}
-
 func getRunAttemptArgs(c echo.Context) (string, string, string, error) {
-	run := Run{}
-	if err := c.Bind(run); err != nil {
+	namespace := c.Param("namespace")
+	layer := c.Param("layer")
+	run := c.Param("run")
+	if namespace == "" || layer == "" || run == "" {
 		return "", "", "", fmt.Errorf("missing query parameters")
 	}
-	return run.Namespace, run.Layer, run.Run, nil
+	return namespace, layer, run, nil
 }
 
 func (a *API) GetAttemptsHandler(c echo.Context) error {

manifests/crds/config.terraform.padok.cloud_terraformruns.yaml

@@ -77,6 +77,17 @@ spec:
           status:
             description: TerraformRunStatus defines the observed state of TerraformRun
             properties:
+              attempts:
+                items:
+                  properties:
+                    logsUploaded:
+                      type: boolean
+                    number:
+                      type: integer
+                    podName:
+                      type: string
+                  type: object
+                type: array
               conditions:
                 items:
                   description: "Condition contains details for one aspect of the current
@@ -148,8 +159,6 @@ spec:
                 type: array
               lastRun:
                 type: string
-              planArtifact:
-                type: string
               retries:
                 type: integer
               runnerPod:

manifests/install.yaml

@@ -4235,6 +4235,17 @@ spec:
           status:
             description: TerraformRunStatus defines the observed state of TerraformRun
             properties:
+              attempts:
+                items:
+                  properties:
+                    logsUploaded:
+                      type: boolean
+                    number:
+                      type: integer
+                    podName:
+                      type: string
+                  type: object
+                type: array
               conditions:
                 items:
                   description: "Condition contains details for one aspect of the current state of this API Resource.\n---\nThis struct is intended for direct use as an array at the field path .status.conditions.  For example,\n\n\n\ttype FooStatus struct{\n\t    // Represents the observations of a foo's current state.\n\t    // Known .status.conditions.type are: \"Available\", \"Progressing\", and \"Degraded\"\n\t    // +patchMergeKey=type\n\t    // +patchStrategy=merge\n\t    // +listType=map\n\t    // +listMapKey=type\n\t    Conditions []metav1.Condition `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`\n\n\n\t    // other fields\n\t}"
@@ -4297,8 +4308,6 @@ spec:
                 type: array
               lastRun:
                 type: string
-              planArtifact:
-                type: string
               retries:
                 type: integer
               runnerPod: