feat: support json patch in the module generate method (#1127)

go.mod

@@ -20,6 +20,7 @@ require (
 	github.com/davecgh/go-spew v1.1.1
 	github.com/djherbis/times v1.5.0
 	github.com/evanphx/json-patch v4.12.0+incompatible
+	github.com/evanphx/json-patch/v5 v5.9.0
 	github.com/fluxcd/pkg/sourceignore v0.5.0
 	github.com/fluxcd/pkg/tar v0.4.0
 	github.com/go-chi/chi/v5 v5.0.12

go.sum

@@ -463,8 +463,8 @@ github.com/envoyproxy/protoc-gen-validate v1.0.4 h1:gVPz/FMfvh57HdSJQyvBtF00j8JU
 github.com/envoyproxy/protoc-gen-validate v1.0.4/go.mod h1:qys6tmnRsYrQqIhm2bvKZH4Blx/1gTIZ2UKVY1M+Yew=
 github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84=
 github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
-github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww=
-github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4=
+github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg=
+github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w=
 github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=

pkg/apis/api.kusion.io/v1/types.go

@@ -357,16 +357,16 @@ type Accessory map[string]interface{}
 //		   # extend accessories module base
 //	       accessories: {
 //	           # Built-in module, key represents the module source
-//	           "kusionstack/mysql@v0.1" : d.MySQL {
+//	           "mysql" : d.MySQL {
 //	               type: "cloud"
 //	               version: "8.0"
 //	           }
 //	           # Built-in module, key represents the module source
-//	           "kusionstack/prometheus@v0.1" : m.Prometheus {
+//	           "prometheus" : m.Prometheus {
 //	               path: "/metrics"
 //	           }
 //	           # Customized module, key represents the module source
-//	           "foo/customize": customizedModule {
+//	           "customize": customizedModule {
 //	               ...
 //	           }
 //	       }
@@ -391,25 +391,57 @@ type AppConfiguration struct {
 	// Workload defines how to run your application code.
 	Workload *Workload `json:"workload" yaml:"workload"`
 	// Accessories defines a collection of accessories that will be attached to the workload.
-	// The key in this map represents the module source. e.g. kusionstack/mysql@v0.1
+	// The key in this map represents the module name
 	Accessories map[string]Accessory `json:"accessories,omitempty" yaml:"accessories,omitempty"`
 	// Labels and Annotations can be used to attach arbitrary metadata as key-value pairs to resources.
 	Labels      map[string]string `json:"labels,omitempty" yaml:"labels,omitempty"`
 	Annotations map[string]string `json:"annotations,omitempty" yaml:"annotations,omitempty"`
 }
 
-// Patcher contains fields should be patched into the workload corresponding fields
+// Patcher primarily contains patches for fields associated with Workloads, and additionally offers the capability to patch other resources.
 type Patcher struct {
 	// Environments represent the environment variables patched to all containers in the workload.
-	Environments []v1.EnvVar `json:"environments" yaml:"environments"`
+	Environments []v1.EnvVar `json:"environments,omitempty" yaml:"environments,omitempty"`
 	// Labels represent the labels patched to the workload.
-	Labels map[string]string `json:"labels" yaml:"labels"`
+	Labels map[string]string `json:"labels,omitempty" yaml:"labels,omitempty"`
 	// PodLabels represent the labels patched to the pods.
-	PodLabels map[string]string `json:"podLabels" yaml:"podLabels"`
+	PodLabels map[string]string `json:"podLabels,omitempty" yaml:"podLabels,omitempty"`
 	// Annotations represent the annotations patched to the workload.
-	Annotations map[string]string `json:"annotations" yaml:"annotations"`
+	Annotations map[string]string `json:"annotations,omitempty" yaml:"annotations,omitempty"`
 	// PodAnnotations represent the annotations patched to the pods.
-	PodAnnotations map[string]string `json:"podAnnotations" yaml:"podAnnotations"`
+	PodAnnotations map[string]string `json:"podAnnotations,omitempty" yaml:"podAnnotations,omitempty"`
+	// JSONPatchers represents patchers that can be patched to an arbitrary resource.
+	// The key of this map represents the ResourceId of the resource to be patched.
+	JSONPatchers map[string]JSONPatcher `json:"jsonPatcher,omitempty" yaml:"jsonPatcher,omitempty"`
+}
+
+type PatchType string
+
+const (
+	MergePatch PatchType = "MergePatch"
+	JSONPatch  PatchType = "JSONPatch"
+)
+
+// JSONPatcher represents the patcher that can be patched to an arbitrary resource.
+// The patch algorithm follows the RFC6902 JSON patch and RFC7396 JSON merge patches.
+type JSONPatcher struct {
+	// PatchType
+	Type PatchType `json:"type" yaml:"type"`
+	// Payload is the patch content.
+	//
+	// JSONPatch Example:
+	// original := []byte(`{"name": "John", "age": 24, "height": 3.21}`)
+	// payload := []byte(`[
+	//		{"op": "replace", "path": "/name", "value": "Jane"},
+	//		{"op": "remove", "path": "/height"}
+	//	]`)
+	// result: {"age":24,"name":"Jane"}
+	//
+	// MergePatch Example:
+	// original := []byte(`{"name": "Tina", "age": 28, "height": 3.75}`)
+	// payload := []byte(`{"height":null,"name":"Jane"}`)
+	// result: {"age":28,"name":"Jane"}
+	Payload []byte `json:"payload" yaml:"payload"`
 }
 
 const ConfigBackends = "backends"

pkg/modules/generators/app_configurations_generator.go

@@ -2,10 +2,12 @@ package generators
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"strings"
 
+	jsonpatch "github.com/evanphx/json-patch/v5"
 	yamlv2 "gopkg.in/yaml.v2"
 	"gopkg.in/yaml.v3"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@@ -17,6 +19,7 @@ import (
 	"kusionstack.io/kusion/pkg/modules"
 	"kusionstack.io/kusion/pkg/modules/generators/workload"
 	"kusionstack.io/kusion/pkg/modules/proto"
+	jsonutil "kusionstack.io/kusion/pkg/util/json"
 	"kusionstack.io/kusion/pkg/workspace"
 )
 
@@ -128,21 +131,24 @@ func (g *appConfigurationGenerator) Generate(spec *v1.Spec) error {
 	wl := spec.Resources[1]
 
 	// call modules to generate customized resources
-	resources, patchers, err := g.callModules(projectModuleConfigs)
+	resources, patcher, err := g.callModules(projectModuleConfigs)
 	if err != nil {
 		return err
 	}
 
-	// patch workload with resource patchers
-	for _, p := range patchers {
-		if err = PatchWorkload(&wl, &p); err != nil {
+	// append the generated resources to the spec
+	spec.Resources = append(spec.Resources, resources...)
+
+	// patch workload with resource patcher
+	if patcher != nil {
+		if err = PatchWorkload(&wl, patcher); err != nil {
+			return err
+		}
+		if err = JSONPatch(spec.Resources, patcher); err != nil {
 			return err
 		}
 	}
 
-	// append the generated resources to the spec
-	spec.Resources = append(spec.Resources, resources...)
-
 	// The OrderedResourcesGenerator should be executed after all resources are generated.
 	if err = modules.CallGenerators(spec, NewOrderedResourcesGeneratorFunc()); err != nil {
 		return err
@@ -151,6 +157,51 @@ func (g *appConfigurationGenerator) Generate(spec *v1.Spec) error {
 	return nil
 }
 
+func JSONPatch(resources v1.Resources, patcher *v1.Patcher) error {
+	if resources == nil || patcher == nil {
+		return nil
+	}
+
+	resIndex := resources.Index()
+
+	if patcher.JSONPatchers != nil {
+		for id, jsonPatcher := range patcher.JSONPatchers {
+			res, ok := resIndex[id]
+			if !ok {
+				return fmt.Errorf("target patch resource %s not found", id)
+			}
+
+			target := jsonutil.Marshal2String(res.Attributes)
+			switch jsonPatcher.Type {
+			case v1.MergePatch:
+				modified, err := jsonpatch.MergePatch([]byte(target), jsonPatcher.Payload)
+				if err != nil {
+					return fmt.Errorf("merge patch to:%s failed", id)
+				}
+				if err = json.Unmarshal(modified, &res.Attributes); err != nil {
+					return err
+				}
+			case v1.JSONPatch:
+				patch, err := jsonpatch.DecodePatch(jsonPatcher.Payload)
+				if err != nil {
+					return fmt.Errorf("decode json patch:%s failed", jsonPatcher.Payload)
+				}
+
+				modified, err := patch.Apply([]byte(target))
+				if err != nil {
+					return fmt.Errorf("apply json patch to:%s failed", id)
+				}
+				if err = json.Unmarshal(modified, &res.Attributes); err != nil {
+					return err
+				}
+			default:
+				return fmt.Errorf("unsupported patch type:%s", jsonPatcher.Type)
+			}
+		}
+	}
+	return nil
+}
+
 func PatchWorkload(workload *v1.Resource, patcher *v1.Patcher) error {
 	if patcher == nil {
 		return nil
@@ -270,7 +321,7 @@ type moduleConfig struct {
 	ctx            v1.GenericConfig
 }
 
-func (g *appConfigurationGenerator) callModules(projectModuleConfigs map[string]v1.GenericConfig) (resources []v1.Resource, patchers []v1.Patcher, err error) {
+func (g *appConfigurationGenerator) callModules(projectModuleConfigs map[string]v1.GenericConfig) (resources []v1.Resource, patcher *v1.Patcher, err error) {
 	pluginMap := make(map[string]*modules.Plugin)
 	defer func() {
 		for _, plugin := range pluginMap {
@@ -337,17 +388,13 @@ func (g *appConfigurationGenerator) callModules(projectModuleConfigs map[string]
 		}
 
 		// parse patcher
-		for _, patcher := range response.Patchers {
-			temp := &v1.Patcher{}
-			err = yaml.Unmarshal(patcher, temp)
-			if err != nil {
-				return nil, nil, err
-			}
-			patchers = append(patchers, *temp)
+		err = yaml.Unmarshal(response.Patcher, patcher)
+		if err != nil {
+			return nil, nil, err
 		}
 	}
 
-	return resources, patchers, nil
+	return resources, patcher, nil
 }
 
 func (g *appConfigurationGenerator) buildModuleConfigIndex(platformModuleConfigs map[string]v1.GenericConfig) (map[string]moduleConfig, error) {

pkg/modules/generators/app_configurations_generator_test.go

@@ -417,3 +417,64 @@ func TestAppConfigurationGenerator_CallModules(t *testing.T) {
 		assert.Error(t, err)
 	})
 }
+
+func TestJsonPatch(t *testing.T) {
+	t.Run("ResourcesNil", func(t *testing.T) {
+		err := JSONPatch(nil, &v1.Patcher{})
+		assert.NoError(t, err)
+	})
+
+	t.Run("PatcherNil", func(t *testing.T) {
+		err := JSONPatch([]v1.Resource{{ID: "test"}}, nil)
+		assert.NoError(t, err)
+	})
+
+	t.Run("JsonPatchersNil", func(t *testing.T) {
+		err := JSONPatch([]v1.Resource{{ID: "test"}}, &v1.Patcher{})
+		assert.NoError(t, err)
+	})
+
+	t.Run("ResourceNotFound", func(t *testing.T) {
+		err := JSONPatch([]v1.Resource{{ID: "test"}}, &v1.Patcher{
+			JSONPatchers: map[string]v1.JSONPatcher{
+				"notfound": {Type: v1.MergePatch, Payload: []byte(`{"key": "value"}`)},
+			},
+		})
+		assert.Error(t, err)
+	})
+
+	t.Run("MergePatch", func(t *testing.T) {
+		resources := []v1.Resource{
+			{ID: "test", Attributes: map[string]interface{}{"key": "old"}},
+		}
+		err := JSONPatch(resources, &v1.Patcher{
+			JSONPatchers: map[string]v1.JSONPatcher{
+				"test": {Type: v1.MergePatch, Payload: []byte(`{"key": "new"}`)},
+			},
+		})
+		assert.NoError(t, err)
+		assert.Equal(t, "new", resources[0].Attributes["key"])
+	})
+
+	t.Run("JSONPatch", func(t *testing.T) {
+		resources := []v1.Resource{
+			{ID: "test", Attributes: map[string]interface{}{"key": "old"}},
+		}
+		err := JSONPatch(resources, &v1.Patcher{
+			JSONPatchers: map[string]v1.JSONPatcher{
+				"test": {Type: v1.JSONPatch, Payload: []byte(`[{"op": "replace", "path": "/key", "value": "new"}]`)},
+			},
+		})
+		assert.NoError(t, err)
+		assert.Equal(t, "new", resources[0].Attributes["key"])
+	})
+
+	t.Run("UnsupportedPatchType", func(t *testing.T) {
+		err := JSONPatch([]v1.Resource{{ID: "test"}}, &v1.Patcher{
+			JSONPatchers: map[string]v1.JSONPatcher{
+				"test": {Type: "unsupported", Payload: []byte(`{"key": "value"}`)},
+			},
+		})
+		assert.Error(t, err)
+	})
+}

pkg/modules/proto/module.proto

@@ -15,7 +15,7 @@ message GeneratorRequest {
   bytes dev_config = 5;
   // PlatformModuleConfig is the platform engineer's inputs of this module
   bytes platform_config = 6;
- // context contains workspace-level configurations, such as topologies, server endpoints, metadata, etc.
+  // context contains workspace-level configurations, such as topologies, server endpoints, metadata, etc.
   bytes context = 7;
 }
 
@@ -24,7 +24,7 @@ message GeneratorResponse {
   // Resources is a v1.Resource array, which represents the generated resources by this module.
   repeated bytes resources = 1;
   // Patcher contains fields should be patched into the workload corresponding fields
-  repeated bytes patchers = 2;
+  bytes patcher = 2;
 }
 
 service Module {

pkg/util/json/json.go

@@ -6,7 +6,6 @@ import (
 	"kusionstack.io/kusion/pkg/util"
 )
 
-// https://github.com/ksonnet/ksonnet/blob/master/pkg/kubecfg/diff.go
 func removeFields(config, live interface{}) interface{} {
 	switch c := config.(type) {
 	case map[string]interface{}: