feat(appset): Implement Plugin Generator (#13017)

* add internal http package Signed-off-by: Maxence Laude <maxence@laude.pro> * add services plugin Signed-off-by: Maxence Laude <maxence@laude.pro> * add generator plugin Signed-off-by: Maxence Laude <maxence@laude.pro> * adapted matrix && merge generator Signed-off-by: Maxence Laude <maxence@laude.pro> * adapted plugin to webhook Signed-off-by: Maxence Laude <maxence@laude.pro> * update applicationset controller and types for plugin Signed-off-by: Maxence Laude <maxence@laude.pro> * add proposal for applicationset plugin generator Signed-off-by: Maxence Laude <maxence@laude.pro> * execute codegen Signed-off-by: Maxence Laude <maxence@laude.pro> * First draft of documentation Signed-off-by: Maxence Laude <maxence@laude.pro> * Fix wrong expected error on client_test Signed-off-by: Maxence Laude <maxence@laude.pro> * docs(plugin-generator): minor improvements Signed-off-by: Sébastien Crocquesel <88554524+scrocquesel@users.noreply.github.com> * Improvement * changes Signed-off-by: Michael Crenshaw <350466+crenshaw-dev@users.noreply.github.com> * fix docs Signed-off-by: Michael Crenshaw <350466+crenshaw-dev@users.noreply.github.com> * wrap output Signed-off-by: Michael Crenshaw <350466+crenshaw-dev@users.noreply.github.com> * fix test Signed-off-by: Michael Crenshaw <350466+crenshaw-dev@users.noreply.github.com> * fix tests Signed-off-by: Michael Crenshaw <350466+crenshaw-dev@users.noreply.github.com> * nested parameters Signed-off-by: Michael Crenshaw <350466+crenshaw-dev@users.noreply.github.com> * simplify Signed-off-by: Michael Crenshaw <350466+crenshaw-dev@users.noreply.github.com> * docs Signed-off-by: Michael Crenshaw <350466+crenshaw-dev@users.noreply.github.com> --------- Signed-off-by: Michael Crenshaw <350466+crenshaw-dev@users.noreply.github.com> * Add plugin to GetRequeueAfter function (merge && matrix) Signed-off-by: Maxence Laude <maxence@laude.pro> * Improvement : renaming * more changes Signed-off-by: Michael Crenshaw <350466+crenshaw-dev@users.noreply.github.com> * clearer docs Signed-off-by: Michael Crenshaw <350466+crenshaw-dev@users.noreply.github.com> * abstract Signed-off-by: Michael Crenshaw <350466+crenshaw-dev@users.noreply.github.com> * naming Signed-off-by: Michael Crenshaw <350466+crenshaw-dev@users.noreply.github.com> * revert accidental change Signed-off-by: Michael Crenshaw <350466+crenshaw-dev@users.noreply.github.com> * ugh Signed-off-by: Michael Crenshaw <350466+crenshaw-dev@users.noreply.github.com> * fix accidental renames Signed-off-by: Michael Crenshaw <350466+crenshaw-dev@users.noreply.github.com> --------- Signed-off-by: Michael Crenshaw <350466+crenshaw-dev@users.noreply.github.com> * Fix typo renaming Signed-off-by: Maxence Laude <maxence@laude.pro> * Improve docs Signed-off-by: Maxence Laude <maxence@laude.pro> * Webhook implementation Signed-off-by: Maxence Laude <maxence@laude.pro> * Typo docs Signed-off-by: Maxence Laude <maxence@laude.pro> * fix plugin generator nil panic Signed-off-by: Michael Crenshaw <350466+crenshaw-dev@users.noreply.github.com> * Add company to USERS.md Signed-off-by: Maxence Laude <maxence@laude.pro> * input.parameters * fix plugin generator nil panic Signed-off-by: Michael Crenshaw <350466+crenshaw-dev@users.noreply.github.com> * input.parameters Signed-off-by: Michael Crenshaw <350466+crenshaw-dev@users.noreply.github.com> --------- Signed-off-by: Michael Crenshaw <350466+crenshaw-dev@users.noreply.github.com> * Change param structure * change param structure Signed-off-by: Michael Crenshaw <350466+crenshaw-dev@users.noreply.github.com> * nest parameters Signed-off-by: Michael Crenshaw <350466+crenshaw-dev@users.noreply.github.com> --------- Signed-off-by: Michael Crenshaw <350466+crenshaw-dev@users.noreply.github.com> * Fix conflicts Signed-off-by: Maxence Laude <maxence@laude.pro> * Fix docs Signed-off-by: Maxence Laude <maxence@laude.pro> * Fix docs Signed-off-by: Maxence Laude <maxence@laude.pro> --------- Signed-off-by: Maxence Laude <maxence@laude.pro> Signed-off-by: Sébastien Crocquesel <88554524+scrocquesel@users.noreply.github.com> Signed-off-by: Michael Crenshaw <350466+crenshaw-dev@users.noreply.github.com> Co-authored-by: Sébastien Crocquesel <88554524+scrocquesel@users.noreply.github.com> Co-authored-by: Michael Crenshaw <350466+crenshaw-dev@users.noreply.github.com>
argoproj · Jun 2, 2023 · ec2340a · ec2340a
1 parent 8e8970e
commit ec2340a
Show file tree

Hide file tree

Showing 34 changed files with 14,980 additions and 5,344 deletions.
diff --git a/USERS.md b/USERS.md
@@ -175,6 +175,7 @@ Currently, the following organizations are **officially** using Argo CD:
 1. [Objective](https://www.objective.com.br/)
 1. [OCCMundial](https://occ.com.mx)
 1. [Octadesk](https://octadesk.com)
+1. [Olfeo](https://www.olfeo.com/)
 1. [omegaUp](https://omegaUp.com)
 1. [Omni](https://omni.se/)
 1. [openEuler](https://openeuler.org)

diff --git a/applicationset/generators/matrix.go b/applicationset/generators/matrix.go
@@ -97,6 +97,7 @@ func (m *MatrixGenerator) getParams(appSetBaseGenerator argoprojiov1alpha1.Appli
 			SCMProvider:             appSetBaseGenerator.SCMProvider,
 			ClusterDecisionResource: appSetBaseGenerator.ClusterDecisionResource,
 			PullRequest:             appSetBaseGenerator.PullRequest,
+			Plugin:                  appSetBaseGenerator.Plugin,
 			Matrix:                  matrixGen,
 			Merge:                   mergeGen,
 			Selector:                appSetBaseGenerator.Selector,
@@ -135,6 +136,7 @@ func (m *MatrixGenerator) GetRequeueAfter(appSetGenerator *argoprojiov1alpha1.Ap
 			Clusters:    r.Clusters,
 			Git:         r.Git,
 			PullRequest: r.PullRequest,
+			Plugin:      r.Plugin,
 			Matrix:      matrixGen,
 			Merge:       mergeGen,
 		}

diff --git a/applicationset/generators/matrix_test.go b/applicationset/generators/matrix_test.go
@@ -5,7 +5,6 @@ import (
 	"testing"
 	"time"
 
-	"github.com/argoproj/argo-cd/v2/applicationset/services/mocks"
 	"github.com/stretchr/testify/require"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -14,6 +13,8 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 
+	"github.com/argoproj/argo-cd/v2/applicationset/services/mocks"
+
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
@@ -848,7 +849,7 @@ func TestMatrixGenerateListElementsYaml(t *testing.T) {
 	}
 
 	listGenerator := &argoprojiov1alpha1.ListGenerator{
-		Elements: []apiextensionsv1.JSON{},
+		Elements:     []apiextensionsv1.JSON{},
 		ElementsYaml: "{{ .foo.bar | toJson }}",
 	}
 
@@ -870,60 +871,59 @@ func TestMatrixGenerateListElementsYaml(t *testing.T) {
 			},
 			expected: []map[string]interface{}{
 				{
-					"chart":         "a",
-					"version":         "1",
+					"chart":   "a",
+					"version": "1",
 					"foo": map[string]interface{}{
 						"bar": []interface{}{
 							map[string]interface{}{
-								"chart": "a",
+								"chart":   "a",
 								"version": "1",
 							},
 							map[string]interface{}{
-								"chart": "b",
+								"chart":   "b",
 								"version": "2",
 							},
 						},
 					},
 					"path": map[string]interface{}{
-						"basename": "dir",
+						"basename":           "dir",
 						"basenameNormalized": "dir",
-						"filename": "file_name.yaml",
+						"filename":           "file_name.yaml",
 						"filenameNormalized": "file-name.yaml",
-						"path": "path/dir",
-						"segments": []string {
+						"path":               "path/dir",
+						"segments": []string{
 							"path",
 							"dir",
 						},
 					},
 				},
 				{
-					"chart":         "b",
-					"version":         "2",
+					"chart":   "b",
+					"version": "2",
 					"foo": map[string]interface{}{
 						"bar": []interface{}{
 							map[string]interface{}{
-								"chart": "a",
+								"chart":   "a",
 								"version": "1",
 							},
 							map[string]interface{}{
-								"chart": "b",
+								"chart":   "b",
 								"version": "2",
 							},
 						},
 					},
 					"path": map[string]interface{}{
-						"basename": "dir",
+						"basename":           "dir",
 						"basenameNormalized": "dir",
-						"filename": "file_name.yaml",
+						"filename":           "file_name.yaml",
 						"filenameNormalized": "file-name.yaml",
-						"path": "path/dir",
-						"segments": []string {
+						"path":               "path/dir",
+						"segments": []string{
 							"path",
 							"dir",
 						},
 					},
 				},
-
 			},
 		},
 	}
@@ -952,27 +952,26 @@ func TestMatrixGenerateListElementsYaml(t *testing.T) {
 					"foo": map[string]interface{}{
 						"bar": []interface{}{
 							map[string]interface{}{
-								"chart": "a",
+								"chart":   "a",
 								"version": "1",
 							},
 							map[string]interface{}{
-								"chart": "b",
+								"chart":   "b",
 								"version": "2",
 							},
 						},
 					},
 					"path": map[string]interface{}{
-						"basename": "dir",
+						"basename":           "dir",
 						"basenameNormalized": "dir",
-						"filename": "file_name.yaml",
+						"filename":           "file_name.yaml",
 						"filenameNormalized": "file-name.yaml",
-						"path": "path/dir",
-						"segments": []string {
+						"path":               "path/dir",
+						"segments": []string{
 							"path",
 							"dir",
 						},
 					},
-
 				}}, nil)
 				genMock.On("GetTemplate", &gitGeneratorSpec).
 					Return(&argoprojiov1alpha1.ApplicationSetTemplate{})

diff --git a/applicationset/generators/merge.go b/applicationset/generators/merge.go
@@ -154,6 +154,7 @@ func (m *MergeGenerator) getParams(appSetBaseGenerator argoprojiov1alpha1.Applic
 			SCMProvider:             appSetBaseGenerator.SCMProvider,
 			ClusterDecisionResource: appSetBaseGenerator.ClusterDecisionResource,
 			PullRequest:             appSetBaseGenerator.PullRequest,
+			Plugin:                  appSetBaseGenerator.Plugin,
 			Matrix:                  matrixGen,
 			Merge:                   mergeGen,
 			Selector:                appSetBaseGenerator.Selector,
@@ -190,6 +191,7 @@ func (m *MergeGenerator) GetRequeueAfter(appSetGenerator *argoprojiov1alpha1.App
 			Clusters:    r.Clusters,
 			Git:         r.Git,
 			PullRequest: r.PullRequest,
+			Plugin:      r.Plugin,
 			Matrix:      matrixGen,
 			Merge:       mergeGen,
 		}

diff --git a/applicationset/generators/plugin.go b/applicationset/generators/plugin.go
@@ -0,0 +1,211 @@
+package generators
+
+import (
+	"context"
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/jeremywohl/flatten"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/client-go/kubernetes"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	argoprojiov1alpha1 "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1"
+	"github.com/argoproj/argo-cd/v2/util/settings"
+
+	"github.com/argoproj/argo-cd/v2/applicationset/services/plugin"
+)
+
+const (
+	DefaultPluginRequeueAfterSeconds = 30 * time.Minute
+)
+
+var _ Generator = (*PluginGenerator)(nil)
+
+type PluginGenerator struct {
+	client    client.Client
+	ctx       context.Context
+	clientset kubernetes.Interface
+	namespace string
+}
+
+func NewPluginGenerator(client client.Client, ctx context.Context, clientset kubernetes.Interface, namespace string) Generator {
+	g := &PluginGenerator{
+		client:    client,
+		ctx:       ctx,
+		clientset: clientset,
+		namespace: namespace,
+	}
+	return g
+}
+
+func (g *PluginGenerator) GetRequeueAfter(appSetGenerator *argoprojiov1alpha1.ApplicationSetGenerator) time.Duration {
+	// Return a requeue default of 30 minutes, if no default is specified.
+
+	if appSetGenerator.Plugin.RequeueAfterSeconds != nil {
+		return time.Duration(*appSetGenerator.Plugin.RequeueAfterSeconds) * time.Second
+	}
+
+	return DefaultPluginRequeueAfterSeconds
+}
+
+func (g *PluginGenerator) GetTemplate(appSetGenerator *argoprojiov1alpha1.ApplicationSetGenerator) *argoprojiov1alpha1.ApplicationSetTemplate {
+	return &appSetGenerator.Plugin.Template
+}
+
+func (g *PluginGenerator) GenerateParams(appSetGenerator *argoprojiov1alpha1.ApplicationSetGenerator, applicationSetInfo *argoprojiov1alpha1.ApplicationSet) ([]map[string]interface{}, error) {
+
+	if appSetGenerator == nil {
+		return nil, EmptyAppSetGeneratorError
+	}
+
+	if appSetGenerator.Plugin == nil {
+		return nil, EmptyAppSetGeneratorError
+	}
+
+	ctx := context.Background()
+
+	providerConfig := appSetGenerator.Plugin
+
+	pluginClient, err := g.getPluginFromGenerator(ctx, applicationSetInfo.Name, providerConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	list, err := pluginClient.List(ctx, providerConfig.Input.Parameters)
+	if err != nil {
+		return nil, fmt.Errorf("error listing params: %w", err)
+	}
+
+	res, err := g.generateParams(appSetGenerator, applicationSetInfo, list.Output.Parameters, appSetGenerator.Plugin.Input.Parameters, applicationSetInfo.Spec.GoTemplate)
+	if err != nil {
+		return nil, err
+	}
+
+	return res, nil
+}
+
+func (g *PluginGenerator) getPluginFromGenerator(ctx context.Context, appSetName string, generatorConfig *argoprojiov1alpha1.PluginGenerator) (*plugin.Service, error) {
+	cm, err := g.getConfigMap(ctx, generatorConfig.ConfigMapRef.Name)
+	if err != nil {
+		return nil, fmt.Errorf("error fetching ConfigMap: %w", err)
+	}
+	token, err := g.getToken(ctx, cm["token"])
+	if err != nil {
+		return nil, fmt.Errorf("error fetching Secret token: %v", err)
+	}
+
+	var requestTimeout int
+	requestTimeoutStr, ok := cm["requestTimeout"]
+	if ok {
+		requestTimeout, err = strconv.Atoi(requestTimeoutStr)
+		if err != nil {
+			return nil, fmt.Errorf("error set requestTimeout : %w", err)
+		}
+	}
+
+	pluginClient, err := plugin.NewPluginService(ctx, appSetName, cm["baseUrl"], token, requestTimeout)
+	if err != nil {
+		return nil, err
+	}
+	return pluginClient, nil
+}
+
+func (g *PluginGenerator) generateParams(appSetGenerator *argoprojiov1alpha1.ApplicationSetGenerator, appSet *argoprojiov1alpha1.ApplicationSet, objectsFound []map[string]interface{}, pluginParams argoprojiov1alpha1.PluginParameters, useGoTemplate bool) ([]map[string]interface{}, error) {
+	res := []map[string]interface{}{}
+
+	for _, objectFound := range objectsFound {
+
+		params := map[string]interface{}{}
+
+		if useGoTemplate {
+			for k, v := range objectFound {
+				params[k] = v
+			}
+		} else {
+			flat, err := flatten.Flatten(objectFound, "", flatten.DotStyle)
+			if err != nil {
+				return nil, err
+			}
+			for k, v := range flat {
+				params[k] = fmt.Sprintf("%v", v)
+			}
+		}
+
+		params["generator"] = map[string]interface{}{
+			"input": map[string]argoprojiov1alpha1.PluginParameters{
+				"parameters": pluginParams,
+			},
+		}
+
+		err := appendTemplatedValues(appSetGenerator.Plugin.Values, params, appSet.Spec.GoTemplate, appSet.Spec.GoTemplateOptions)
+		if err != nil {
+			return nil, err
+		}
+
+		res = append(res, params)
+	}
+
+	return res, nil
+}
+
+func (g *PluginGenerator) getToken(ctx context.Context, tokenRef string) (string, error) {
+
+	if tokenRef == "" || !strings.HasPrefix(tokenRef, "$") {
+		return "", fmt.Errorf("token is empty, or does not reference a secret key starting with '$': %v", tokenRef)
+	}
+
+	secretName, tokenKey := plugin.ParseSecretKey(tokenRef)
+
+	secret := &corev1.Secret{}
+	err := g.client.Get(
+		ctx,
+		client.ObjectKey{
+			Name:      secretName,
+			Namespace: g.namespace,
+		},
+		secret)
+
+	if err != nil {
+		return "", fmt.Errorf("error fetching secret %s/%s: %v", g.namespace, secretName, err)
+	}
+
+	secretValues := make(map[string]string, len(secret.Data))
+
+	for k, v := range secret.Data {
+		secretValues[k] = string(v)
+	}
+
+	token := settings.ReplaceStringSecret(tokenKey, secretValues)
+
+	return token, err
+}
+
+func (g *PluginGenerator) getConfigMap(ctx context.Context, configMapRef string) (map[string]string, error) {
+	cm := &corev1.ConfigMap{}
+	err := g.client.Get(
+		ctx,
+		client.ObjectKey{
+			Name:      configMapRef,
+			Namespace: g.namespace,
+		},
+		cm)
+
+	if err != nil {
+		return nil, err
+	}
+
+	baseUrl, ok := cm.Data["baseUrl"]
+	if !ok || baseUrl == "" {
+		return nil, fmt.Errorf("baseUrl not found in ConfigMap")
+	}
+
+	token, ok := cm.Data["token"]
+	if !ok || token == "" {
+		return nil, fmt.Errorf("token not found in ConfigMap")
+	}
+
+	return cm.Data, nil
+}