feat: make drift check and apply with custom runner code (#2)

* chore(wip): wip

* chore(evaluate): change function signature

* feat: add compute hash func

* chore(signature): again changing function signature

* feat: add evaluate for isTerraformRunning and isPlanArtifactUpToDDate

* feat(conditions): terraformFailure TerraformApplyUpToDate

* feat: use const prefixes for cahce keys

* chore(tests): add some units tests for conditions

* build: generate deepcopy and manifests

* feat: fix timestamp conversion

* test: implement testing for all evaluate methods of terraform conditions

* chore: rename structs

* chore: uniformize every sub evaluate mehtod

* test: start implementing terraform layer conditions tests

* test: implement terraform layer conditions testing

* chore: remove unused struct

* feat: start working on job

* feat: fix init default pod spec

* feat: fix init default pod spec

* chore(wip): runner

* feat: finish cache logic for plan pod creation

* feat: add command for apply pod

* feat: implement redis cache

* improvement(runner): add runner code, init cobra, move to cache package

* feat: remove applied bin cahce key

* feat(cobra): implement cobra launch

* chore(runner): modify pod creation to use burrito binary

* chore(docker): and crd generation

* feat: use config to setup cahce in layer controller

* feat: run exec on cotnrollers start

* feat: use custom image for runeer and generate random name

* fix: do not inpu error 2 times to logger

* fix(runner): pod generation

* fix(runner): pod generation remove command

* fix: change prefix in pod env vars

* feat: catch set lock error

* feat: use commen generate key function for cahce keys

* fix(all): add some logs on controller and runner, fix some cache deletion issues

* feat(redis): implement delete func

* improvement(runner): no sé

* feat: make runner put last plan date in cache

* fix: use unix timestamp as value for last plan date key in cache

Co-authored-by: Alan <alanl@padok.fr>

Dockerfile

@@ -15,19 +15,24 @@ RUN go mod download
 COPY main.go main.go
 COPY api/ api/
 COPY controllers/ controllers/
+COPY burrito/ burrito/
+COPY cmd/ cmd/
+COPY runner/ runner/
+COPY cache/ cache 
 
 # Build
 # the GOARCH has not a default value to allow the binary be built according to the host where the command
 # was called. For example, if we call make docker-build in a local env which has the Apple Silicon M1 SO
 # the docker BUILDPLATFORM arg will be linux/arm64 when for Apple x86 it will be linux/amd64. Therefore,
 # by leaving it empty we can ensure that the container and binary shipped on it will have the same platform.
-RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o manager main.go
+RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o bin/burrito main.go
+
+FROM golang:alpine
+
+RUN apk add --update git bash openssh
 
-# Use distroless as minimal base image to package the manager binary
-# Refer to https://github.com/GoogleContainerTools/distroless for more details
-FROM gcr.io/distroless/static:nonroot
 WORKDIR /
-COPY --from=builder /workspace/manager .
+COPY --from=builder /workspace/bin/burrito .
 USER 65532:65532
 
-ENTRYPOINT ["/manager"]
+ENTRYPOINT ["/burrito"]

Makefile

@@ -109,7 +109,7 @@ test: manifests generate fmt vet envtest ## Run tests.
 
 .PHONY: build
 build: generate fmt vet ## Build manager binary.
-	go build -o bin/manager main.go
+	go build -o bin/burrito main.go
 
 .PHONY: run
 run: manifests generate fmt vet ## Run a controller from your host.

api/v1alpha1/terraformlayer_types.go

@@ -28,14 +28,30 @@ type TerraformLayerSpec struct {
 	// INSERT ADDITIONAL SPEC FIELDS - desired state of cluster
 	// Important: Run "make" to regenerate code after modifying this file
 
-	// Foo is an example field of TerraformLayer. Edit terraformlayer_types.go to remove/update
-	Foo string `json:"foo,omitempty"`
+	Path                string                            `json:"path,omitempty"`
+	Branch              string                            `json:"branch,omitempty"`
+	TerraformVersion    string                            `json:"terraformVersion,omitempty"`
+	Repository          TerraformLayerRepository          `json:"repository,omitempty"`
+	RemediationStrategy TerraformLayerRemediationStrategy `json:"remediationStrategy,omitempty"`
+	PlanOnPullRequest   bool                              `json:"planOnPullRequest,omitempty"`
+	// RunnerPodTemplate   corev1.PodSpec                    `json:"template,omitempty"`
+}
+
+type TerraformLayerRemediationStrategy struct {
+	PlanOnDrift  bool `json:"planOnDrift,omitempty"`
+	ApplyOnDrift bool `json:"applyOnDrift,omitempty"`
+	ApplyOnPush  bool `json:"applyOnPush,omitempty"`
+}
+
+type TerraformLayerRepository struct {
+	Kind      string `json:"kind,omitempty"`
+	Name      string `json:"name,omitempty"`
+	Namespace string `json:"namespace,omitempty"`
 }
 
 // TerraformLayerStatus defines the observed state of TerraformLayer
 type TerraformLayerStatus struct {
-	// INSERT ADDITIONAL STATUS FIELD - define observed state of cluster
-	// Important: Run "make" to regenerate code after modifying this file
+	Conditions []metav1.Condition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type" protobuf:"bytes,1,rep,name=conditions"`
 }
 
 //+kubebuilder:object:root=true

api/v1alpha1/terraformrepository_types.go

@@ -17,6 +17,7 @@ limitations under the License.
 package v1alpha1
 
 import (
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
@@ -28,14 +29,17 @@ type TerraformRepositorySpec struct {
 	// INSERT ADDITIONAL SPEC FIELDS - desired state of cluster
 	// Important: Run "make" to regenerate code after modifying this file
 
-	// Foo is an example field of TerraformRepository. Edit terraformrepository_types.go to remove/update
-	Foo string `json:"foo,omitempty"`
+	Repository TerraformRepositoryRepository `json:"repository,omitempty"`
+}
+
+type TerraformRepositoryRepository struct {
+	Url       string                 `json:"url,omitempty"`
+	SecretRef corev1.SecretReference `json:"secretRef,omitempty"`
 }
 
 // TerraformRepositoryStatus defines the observed state of TerraformRepository
 type TerraformRepositoryStatus struct {
-	// INSERT ADDITIONAL STATUS FIELD - define observed state of cluster
-	// Important: Run "make" to regenerate code after modifying this file
+	Conditions []metav1.Condition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type" protobuf:"bytes,1,rep,name=conditions"`
 }
 
 //+kubebuilder:object:root=true

burrito/burrito.go

@@ -0,0 +1,40 @@
+package burrito
+
+import (
+	"io"
+	"os"
+
+	"github.com/padok-team/burrito/burrito/config"
+	"github.com/padok-team/burrito/controllers"
+	"github.com/padok-team/burrito/runner"
+)
+
+type App struct {
+	Config *config.Config
+
+	Runner      Runner
+	Controllers Controllers
+
+	Out io.Writer
+	Err io.Writer
+}
+
+type Runner interface {
+	Exec()
+}
+
+type Controllers interface {
+	Exec()
+}
+
+func New() (*App, error) {
+	c := &config.Config{}
+	app := &App{
+		Config:      c,
+		Runner:      runner.New(c),
+		Controllers: controllers.New(c),
+		Out:         os.Stdout,
+		Err:         os.Stderr,
+	}
+	return app, nil
+}

burrito/config/config.go

@@ -0,0 +1,112 @@
+package config
+
+import (
+	"fmt"
+	"os"
+	"reflect"
+	"strings"
+
+	"github.com/spf13/pflag"
+	"github.com/spf13/viper"
+)
+
+type Config struct {
+	Runner     RunnerConfig     `yaml:"runner"`
+	Controller ControllerConfig `yaml:"controller"`
+	Redis      Redis            `yaml:"redis"`
+}
+
+type ControllerConfig struct {
+	WatchedNamespaces []string `yaml:"namespaces"`
+}
+
+type RepositoryConfig struct {
+	URL      string `yaml:"url"`
+	SSH      string `yaml:"ssh"`
+	Username string `yaml:"username"`
+	Password string `yaml:"password"`
+}
+
+type RunnerConfig struct {
+	Path       string           `yaml:"path"`
+	Branch     string           `yaml:"branch"`
+	Version    string           `yaml:"version"`
+	Action     string           `yaml:"action"`
+	Repository RepositoryConfig `yaml:"repository"`
+	Layer      LayerConfig      `yaml:"layer"`
+}
+
+type LayerConfig struct {
+	Lock     string `yaml:"lock"`
+	PlanSum  string `yaml:"planSum"`
+	PlanBin  string `yaml:"planBin"`
+	ApplySum string `yaml:"applySum"`
+	PlanDate string `yaml:"planDate"`
+}
+
+type Redis struct {
+	URL      string `yaml:"url"`
+	Password string `yaml:"password"`
+	Database int    `yaml:"database"`
+}
+
+func (c *Config) Load(flags *pflag.FlagSet) error {
+	v := viper.New()
+
+	// burrito looks for configuration files called config.yaml, config.json,
+	// config.toml, config.hcl, etc.
+	v.SetConfigName("config")
+
+	// burrito looks for configuration files in the common configuration
+	// directories.
+	v.AddConfigPath("/etc/burrito/")
+	v.AddConfigPath("$HOME/.burrito/")
+
+	// Viper logs the configuration file it uses, if any.
+	if err := v.ReadInConfig(); err == nil {
+		fmt.Fprintf(os.Stderr, "Using config file: %s\n", v.ConfigFileUsed())
+	}
+
+	// burrito can be configured with environment variables that start with
+	// burrito_.
+	v.SetEnvPrefix("burrito")
+	v.AutomaticEnv()
+
+	// Options with dashes in flag names have underscores when set inside a
+	// configuration file or with environment variables.
+	flags.SetNormalizeFunc(func(fs *pflag.FlagSet, name string) pflag.NormalizedName {
+		name = strings.ReplaceAll(name, "-", "_")
+		return pflag.NormalizedName(name)
+	})
+	v.BindPFlags(flags)
+
+	// Nested configuration options set with environment variables use an
+	// underscore as a separator.
+	v.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
+	bindEnvironmentVariables(v, *c)
+
+	return v.Unmarshal(c)
+}
+
+// bindEnvironmentVariables inspects iface's structure and recursively binds its
+// fields to environment variables. This is a workaround to a limitation of
+// Viper, found here:
+// https://github.com/spf13/viper/issues/188#issuecomment-399884438
+func bindEnvironmentVariables(v *viper.Viper, iface interface{}, parts ...string) {
+	ifv := reflect.ValueOf(iface)
+	ift := reflect.TypeOf(iface)
+	for i := 0; i < ift.NumField(); i++ {
+		val := ifv.Field(i)
+		typ := ift.Field(i)
+		tv, ok := typ.Tag.Lookup("yaml")
+		if !ok {
+			continue
+		}
+		switch val.Kind() {
+		case reflect.Struct:
+			bindEnvironmentVariables(v, val.Interface(), append(parts, tv)...)
+		default:
+			v.BindEnv(strings.Join(append(parts, tv), "."))
+		}
+	}
+}

burrito/controllers.go

@@ -0,0 +1,6 @@
+package burrito
+
+func (app *App) StartController() error {
+	app.Controllers.Exec()
+	return nil
+}

burrito/runner.go

@@ -0,0 +1,5 @@
+package burrito
+
+func (app *App) StartRunner() {
+	app.Runner.Exec()
+}