diff --git a/api/v1alpha1/terraformrun_types.go b/api/v1alpha1/terraformrun_types.go index 37bcac0d..07976c31 100644 --- a/api/v1alpha1/terraformrun_types.go +++ b/api/v1alpha1/terraformrun_types.go @@ -42,12 +42,18 @@ type TerraformRunLayer struct { // TerraformRunStatus defines the observed state of TerraformRun type TerraformRunStatus struct { - Conditions []metav1.Condition `json:"conditions,omitempty"` - State string `json:"state,omitempty"` - Retries int `json:"retries"` - LastRun string `json:"lastRun,omitempty"` - RunnerPod string `json:"runnerPod,omitempty"` - PlanArtifact string `json:"planArtifact,omitempty"` + Conditions []metav1.Condition `json:"conditions,omitempty"` + State string `json:"state,omitempty"` + Retries int `json:"retries"` + LastRun string `json:"lastRun,omitempty"` + Attempts []Attempt `json:"attempts,omitempty"` + RunnerPod string `json:"runnerPod,omitempty"` +} + +type Attempt struct { + PodName string `json:"podName,omitempty"` + Number int `json:"number,omitempty"` + LogsUploaded bool `json:"logsUploaded,omitempty"` } // +kubebuilder:object:root=true diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index c92b9a21..418e033f 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -1,5 +1,4 @@ //go:build !ignore_autogenerated -// +build !ignore_autogenerated /* Copyright 2022. @@ -42,6 +41,21 @@ func (in *Artifact) DeepCopy() *Artifact { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Attempt) DeepCopyInto(out *Attempt) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Attempt. +func (in *Attempt) DeepCopy() *Attempt { + if in == nil { + return nil + } + out := new(Attempt) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MetadataOverride) DeepCopyInto(out *MetadataOverride) { *out = *in @@ -676,6 +690,11 @@ func (in *TerraformRunStatus) DeepCopyInto(out *TerraformRunStatus) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.Attempts != nil { + in, out := &in.Attempts, &out.Attempts + *out = make([]Attempt, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TerraformRunStatus. diff --git a/internal/controllers/manager.go b/internal/controllers/manager.go index 66bcc551..924be78b 100644 --- a/internal/controllers/manager.go +++ b/internal/controllers/manager.go @@ -19,7 +19,9 @@ package controllers import ( "os" + logClient "k8s.io/client-go/kubernetes" _ "k8s.io/client-go/plugin/pkg/client/auth" + "k8s.io/client-go/rest" "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" @@ -88,6 +90,14 @@ func (c *Controllers) Exec() { log.Fatalf("unable to start manager: %s", err) } datastoreClient := datastore.NewDefaultClient() + config, err := rest.InClusterConfig() + if err != nil { + panic(err.Error()) + } + clientset, err := logClient.NewForConfig(config) + if err != nil { + panic(err.Error()) + } for _, ctrlType := range c.config.Controller.Types { switch ctrlType { @@ -113,10 +123,12 @@ func (c *Controllers) Exec() { log.Infof("repository controller started successfully") case "run": if err = (&terraformrun.Reconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - Recorder: mgr.GetEventRecorderFor("Burrito"), - Config: c.config, + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: mgr.GetEventRecorderFor("Burrito"), + Config: c.config, + Datastore: datastoreClient, + K8SLogClient: clientset, }).SetupWithManager(mgr); err != nil { log.Fatalf("unable to create run controller: %s", err) } diff --git a/internal/controllers/terraformrun/controller.go b/internal/controllers/terraformrun/controller.go index ba3845a3..d4577968 100644 --- a/internal/controllers/terraformrun/controller.go +++ b/internal/controllers/terraformrun/controller.go @@ -19,9 +19,13 @@ package terraformrun import ( "context" "math" + "strconv" "time" "github.com/google/go-cmp/cmp" + datastore "github.com/padok-team/burrito/internal/datastore/client" + logClient "k8s.io/client-go/kubernetes" + log "github.com/sirupsen/logrus" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -50,9 +54,11 @@ func (c RealClock) Now() time.Time { // RunReconcilier reconciles a TerraformRun object type Reconciler struct { client.Client - Scheme *runtime.Scheme - Config *config.Config - Recorder record.EventRecorder + K8SLogClient *logClient.Clientset + Scheme *runtime.Scheme + Config *config.Config + Recorder record.EventRecorder + Datastore datastore.Client Clock } @@ -98,12 +104,26 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu } state, conditions := r.GetState(ctx, run, layer, repo) result, runInfo := state.getHandler()(ctx, r, run, layer, repo) + if runInfo.NewPod { + attempt := configv1alpha1.Attempt{ + PodName: runInfo.RunnerPod, + LogsUploaded: false, + Number: runInfo.Retries, + } + run.Status.Attempts = append(run.Status.Attempts, attempt) + } run.Status = configv1alpha1.TerraformRunStatus{ Conditions: conditions, State: getStateString(state), Retries: runInfo.Retries, LastRun: runInfo.LastRun, RunnerPod: runInfo.RunnerPod, + Attempts: run.Status.Attempts, + } + err = r.uploadLogs(run) + if err != nil { + r.Recorder.Event(run, corev1.EventTypeWarning, "Reconciliation", "Failed to upload logs") + log.Errorf("failed to upload logs for run %s: %s", run.Name, err) } err = r.Client.Status().Update(ctx, run) if err != nil { @@ -114,6 +134,43 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu return result, nil } +func (r *Reconciler) uploadLogs(run *configv1alpha1.TerraformRun) error { + for i, attempt := range run.Status.Attempts { + if attempt.LogsUploaded { + continue + } + pod := &corev1.Pod{} + err := r.Client.Get(context.Background(), types.NamespacedName{ + Namespace: run.Namespace, + Name: attempt.PodName, + }, pod) + if errors.IsNotFound(err) { + log.Infof("pod %s not found, ignoring...", attempt.PodName) + continue + } + if err != nil { + log.Errorf("failed to get pod %s: %s", attempt.PodName, err) + continue + } + if pod.Status.Phase != corev1.PodSucceeded && pod.Status.Phase != corev1.PodFailed { + log.Infof("pod %s is not in a terminal state, ignoring...", attempt.PodName) + continue + } + // Upload logs + logs, err := r.K8SLogClient.CoreV1().Pods(run.Namespace).GetLogs(pod.Name, &corev1.PodLogOptions{}).Do(context.Background()).Raw() + if err != nil { + log.Errorf("failed to get logs for pod %s: %s", pod.Name, err) + continue + } + err = r.Datastore.PutLogs(run.Namespace, run.Spec.Layer.Name, run.Name, strconv.Itoa(attempt.Number), logs) + if err != nil { + return err + } + run.Status.Attempts[i].LogsUploaded = true + } + return nil +} + // SetupWithManager sets up the controller with the Manager. func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { r.Clock = RealClock{} diff --git a/internal/controllers/terraformrun/controller_test.go b/internal/controllers/terraformrun/controller_test.go index 9af3ddeb..872f6079 100644 --- a/internal/controllers/terraformrun/controller_test.go +++ b/internal/controllers/terraformrun/controller_test.go @@ -14,9 +14,11 @@ import ( configv1alpha1 "github.com/padok-team/burrito/api/v1alpha1" controller "github.com/padok-team/burrito/internal/controllers/terraformrun" + datastore "github.com/padok-team/burrito/internal/datastore/client" utils "github.com/padok-team/burrito/internal/testing" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" + logClient "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" "k8s.io/client-go/tools/record" @@ -62,16 +64,19 @@ var _ = BeforeSuite(func() { err = configv1alpha1.AddToScheme(scheme.Scheme) Expect(err).NotTo(HaveOccurred()) - + logClient, err := logClient.NewForConfig(cfg) + Expect(err).NotTo(HaveOccurred()) //+kubebuilder:scaffold:scheme k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) utils.LoadResources(k8sClient, "testdata") reconciler = &controller.Reconciler{ - Client: k8sClient, - Scheme: scheme.Scheme, - Config: config.TestConfig(), - Clock: &MockClock{}, + Client: k8sClient, + Scheme: scheme.Scheme, + Config: config.TestConfig(), + Clock: &MockClock{}, + Datastore: datastore.NewMockClient(), + K8SLogClient: logClient, Recorder: record.NewBroadcasterForTests(1*time.Second).NewRecorder(scheme.Scheme, corev1.EventSource{ Component: "burrito", }), diff --git a/internal/controllers/terraformrun/states.go b/internal/controllers/terraformrun/states.go index 2462b3cb..eb20da37 100644 --- a/internal/controllers/terraformrun/states.go +++ b/internal/controllers/terraformrun/states.go @@ -18,6 +18,7 @@ type RunInfo struct { Retries int LastRun string RunnerPod string + NewPod bool } func getRunInfo(run *configv1alpha1.TerraformRun) RunInfo { @@ -89,6 +90,7 @@ func (s *Initial) getHandler() Handler { Retries: 0, LastRun: r.Clock.Now().Format(time.UnixDate), RunnerPod: pod.Name, + NewPod: true, } r.Recorder.Event(run, corev1.EventTypeNormal, "Run", fmt.Sprintf("Successfully created pod %s for initial run", pod.Name)) // Minimal time (1s) to transit from Initial state to Running state @@ -142,6 +144,7 @@ func (s *Retrying) getHandler() Handler { Retries: runInfo.Retries + 1, LastRun: r.Clock.Now().Format(time.UnixDate), RunnerPod: pod.Name, + NewPod: true, } r.Recorder.Event(run, corev1.EventTypeNormal, "Run", fmt.Sprintf("Successfully created pod %s for retry run", pod.Name)) // Minimal time (1s) to transit from Retrying state to Running state diff --git a/internal/server/api/logs.go b/internal/server/api/logs.go index 679b8765..81799d86 100644 --- a/internal/server/api/logs.go +++ b/internal/server/api/logs.go @@ -8,23 +8,19 @@ import ( "github.com/labstack/echo/v4" ) -type Attempt struct { - AttemptNumber string `param:"attempt"` - Namespace string `param:"namespace"` - Layer string `param:"layer"` - Run string `param:"run"` -} - type GetLogsResponse struct { Results []string `json:"results"` } func getLogsArgs(c echo.Context) (string, string, string, string, error) { - attempt := Attempt{} - if err := c.Bind(attempt); err != nil { + namespace := c.Param("namespace") + layer := c.Param("layer") + run := c.Param("run") + attempt := c.Param("attempt") + if namespace == "" || layer == "" || run == "" || attempt == "" { return "", "", "", "", fmt.Errorf("missing query parameters") } - return attempt.Namespace, attempt.Layer, attempt.Run, attempt.AttemptNumber, nil + return namespace, layer, run, attempt, nil } // logs/${namespace}/${layer}/${runId}/${attemptId} diff --git a/internal/server/api/runs.go b/internal/server/api/runs.go index 2fc82b6d..5579fb6f 100644 --- a/internal/server/api/runs.go +++ b/internal/server/api/runs.go @@ -11,18 +11,14 @@ type GetAttemptsResponse struct { Count int `json:"count"` } -type Run struct { - Namespace string `param:"namespace"` - Layer string `param:"layer"` - Run string `param:"run"` -} - func getRunAttemptArgs(c echo.Context) (string, string, string, error) { - run := Run{} - if err := c.Bind(run); err != nil { + namespace := c.Param("namespace") + layer := c.Param("layer") + run := c.Param("run") + if namespace == "" || layer == "" || run == "" { return "", "", "", fmt.Errorf("missing query parameters") } - return run.Namespace, run.Layer, run.Run, nil + return namespace, layer, run, nil } func (a *API) GetAttemptsHandler(c echo.Context) error { diff --git a/manifests/crds/config.terraform.padok.cloud_terraformruns.yaml b/manifests/crds/config.terraform.padok.cloud_terraformruns.yaml index b0b0a862..f7ce36c8 100644 --- a/manifests/crds/config.terraform.padok.cloud_terraformruns.yaml +++ b/manifests/crds/config.terraform.padok.cloud_terraformruns.yaml @@ -77,6 +77,17 @@ spec: status: description: TerraformRunStatus defines the observed state of TerraformRun properties: + attempts: + items: + properties: + logsUploaded: + type: boolean + number: + type: integer + podName: + type: string + type: object + type: array conditions: items: description: "Condition contains details for one aspect of the current @@ -148,8 +159,6 @@ spec: type: array lastRun: type: string - planArtifact: - type: string retries: type: integer runnerPod: diff --git a/manifests/install.yaml b/manifests/install.yaml index cddba10e..c3f25c05 100644 --- a/manifests/install.yaml +++ b/manifests/install.yaml @@ -4235,6 +4235,17 @@ spec: status: description: TerraformRunStatus defines the observed state of TerraformRun properties: + attempts: + items: + properties: + logsUploaded: + type: boolean + number: + type: integer + podName: + type: string + type: object + type: array conditions: items: description: "Condition contains details for one aspect of the current state of this API Resource.\n---\nThis struct is intended for direct use as an array at the field path .status.conditions. For example,\n\n\n\ttype FooStatus struct{\n\t // Represents the observations of a foo's current state.\n\t // Known .status.conditions.type are: \"Available\", \"Progressing\", and \"Degraded\"\n\t // +patchMergeKey=type\n\t // +patchStrategy=merge\n\t // +listType=map\n\t // +listMapKey=type\n\t Conditions []metav1.Condition `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`\n\n\n\t // other fields\n\t}" @@ -4297,8 +4308,6 @@ spec: type: array lastRun: type: string - planArtifact: - type: string retries: type: integer runnerPod: