diff --git a/go.mod b/go.mod index e58816af..87c91ddf 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( github.com/davecgh/go-spew v1.1.1 github.com/djherbis/times v1.5.0 github.com/evanphx/json-patch v4.12.0+incompatible + github.com/evanphx/json-patch/v5 v5.9.0 github.com/fluxcd/pkg/sourceignore v0.5.0 github.com/fluxcd/pkg/tar v0.4.0 github.com/go-chi/chi/v5 v5.0.12 diff --git a/go.sum b/go.sum index f2d64662..a945457d 100644 --- a/go.sum +++ b/go.sum @@ -463,8 +463,8 @@ github.com/envoyproxy/protoc-gen-validate v1.0.4 h1:gVPz/FMfvh57HdSJQyvBtF00j8JU github.com/envoyproxy/protoc-gen-validate v1.0.4/go.mod h1:qys6tmnRsYrQqIhm2bvKZH4Blx/1gTIZ2UKVY1M+Yew= github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= -github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= +github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg= +github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= diff --git a/pkg/apis/api.kusion.io/v1/types.go b/pkg/apis/api.kusion.io/v1/types.go index 2dded034..59c5f915 100644 --- a/pkg/apis/api.kusion.io/v1/types.go +++ b/pkg/apis/api.kusion.io/v1/types.go @@ -357,16 +357,16 @@ type Accessory map[string]interface{} // # extend accessories module base // accessories: { // # Built-in module, key represents the module source -// "kusionstack/mysql@v0.1" : d.MySQL { +// "mysql" : d.MySQL { // type: "cloud" // version: "8.0" // } // # Built-in module, key represents the module source -// "kusionstack/prometheus@v0.1" : m.Prometheus { +// "prometheus" : m.Prometheus { // path: "/metrics" // } // # Customized module, key represents the module source -// "foo/customize": customizedModule { +// "customize": customizedModule { // ... // } // } @@ -391,25 +391,57 @@ type AppConfiguration struct { // Workload defines how to run your application code. Workload *Workload `json:"workload" yaml:"workload"` // Accessories defines a collection of accessories that will be attached to the workload. - // The key in this map represents the module source. e.g. kusionstack/mysql@v0.1 + // The key in this map represents the module name Accessories map[string]Accessory `json:"accessories,omitempty" yaml:"accessories,omitempty"` // Labels and Annotations can be used to attach arbitrary metadata as key-value pairs to resources. Labels map[string]string `json:"labels,omitempty" yaml:"labels,omitempty"` Annotations map[string]string `json:"annotations,omitempty" yaml:"annotations,omitempty"` } -// Patcher contains fields should be patched into the workload corresponding fields +// Patcher primarily contains patches for fields associated with Workloads, and additionally offers the capability to patch other resources. type Patcher struct { // Environments represent the environment variables patched to all containers in the workload. - Environments []v1.EnvVar `json:"environments" yaml:"environments"` + Environments []v1.EnvVar `json:"environments,omitempty" yaml:"environments,omitempty"` // Labels represent the labels patched to the workload. - Labels map[string]string `json:"labels" yaml:"labels"` + Labels map[string]string `json:"labels,omitempty" yaml:"labels,omitempty"` // PodLabels represent the labels patched to the pods. - PodLabels map[string]string `json:"podLabels" yaml:"podLabels"` + PodLabels map[string]string `json:"podLabels,omitempty" yaml:"podLabels,omitempty"` // Annotations represent the annotations patched to the workload. - Annotations map[string]string `json:"annotations" yaml:"annotations"` + Annotations map[string]string `json:"annotations,omitempty" yaml:"annotations,omitempty"` // PodAnnotations represent the annotations patched to the pods. - PodAnnotations map[string]string `json:"podAnnotations" yaml:"podAnnotations"` + PodAnnotations map[string]string `json:"podAnnotations,omitempty" yaml:"podAnnotations,omitempty"` + // JSONPatchers represents patchers that can be patched to an arbitrary resource. + // The key of this map represents the ResourceId of the resource to be patched. + JSONPatchers map[string]JSONPatcher `json:"jsonPatcher,omitempty" yaml:"jsonPatcher,omitempty"` +} + +type PatchType string + +const ( + MergePatch PatchType = "MergePatch" + JSONPatch PatchType = "JSONPatch" +) + +// JSONPatcher represents the patcher that can be patched to an arbitrary resource. +// The patch algorithm follows the RFC6902 JSON patch and RFC7396 JSON merge patches. +type JSONPatcher struct { + // PatchType + Type PatchType `json:"type" yaml:"type"` + // Payload is the patch content. + // + // JSONPatch Example: + // original := []byte(`{"name": "John", "age": 24, "height": 3.21}`) + // payload := []byte(`[ + // {"op": "replace", "path": "/name", "value": "Jane"}, + // {"op": "remove", "path": "/height"} + // ]`) + // result: {"age":24,"name":"Jane"} + // + // MergePatch Example: + // original := []byte(`{"name": "Tina", "age": 28, "height": 3.75}`) + // payload := []byte(`{"height":null,"name":"Jane"}`) + // result: {"age":28,"name":"Jane"} + Payload []byte `json:"payload" yaml:"payload"` } const ConfigBackends = "backends" diff --git a/pkg/modules/generators/app_configurations_generator.go b/pkg/modules/generators/app_configurations_generator.go index e021a4df..213e8cc3 100644 --- a/pkg/modules/generators/app_configurations_generator.go +++ b/pkg/modules/generators/app_configurations_generator.go @@ -2,10 +2,12 @@ package generators import ( "context" + "encoding/json" "errors" "fmt" "strings" + jsonpatch "github.com/evanphx/json-patch/v5" yamlv2 "gopkg.in/yaml.v2" "gopkg.in/yaml.v3" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -17,6 +19,7 @@ import ( "kusionstack.io/kusion/pkg/modules" "kusionstack.io/kusion/pkg/modules/generators/workload" "kusionstack.io/kusion/pkg/modules/proto" + jsonutil "kusionstack.io/kusion/pkg/util/json" "kusionstack.io/kusion/pkg/workspace" ) @@ -128,21 +131,24 @@ func (g *appConfigurationGenerator) Generate(spec *v1.Spec) error { wl := spec.Resources[1] // call modules to generate customized resources - resources, patchers, err := g.callModules(projectModuleConfigs) + resources, patcher, err := g.callModules(projectModuleConfigs) if err != nil { return err } - // patch workload with resource patchers - for _, p := range patchers { - if err = PatchWorkload(&wl, &p); err != nil { + // append the generated resources to the spec + spec.Resources = append(spec.Resources, resources...) + + // patch workload with resource patcher + if patcher != nil { + if err = PatchWorkload(&wl, patcher); err != nil { + return err + } + if err = JSONPatch(spec.Resources, patcher); err != nil { return err } } - // append the generated resources to the spec - spec.Resources = append(spec.Resources, resources...) - // The OrderedResourcesGenerator should be executed after all resources are generated. if err = modules.CallGenerators(spec, NewOrderedResourcesGeneratorFunc()); err != nil { return err @@ -151,6 +157,51 @@ func (g *appConfigurationGenerator) Generate(spec *v1.Spec) error { return nil } +func JSONPatch(resources v1.Resources, patcher *v1.Patcher) error { + if resources == nil || patcher == nil { + return nil + } + + resIndex := resources.Index() + + if patcher.JSONPatchers != nil { + for id, jsonPatcher := range patcher.JSONPatchers { + res, ok := resIndex[id] + if !ok { + return fmt.Errorf("target patch resource %s not found", id) + } + + target := jsonutil.Marshal2String(res.Attributes) + switch jsonPatcher.Type { + case v1.MergePatch: + modified, err := jsonpatch.MergePatch([]byte(target), jsonPatcher.Payload) + if err != nil { + return fmt.Errorf("merge patch to:%s failed", id) + } + if err = json.Unmarshal(modified, &res.Attributes); err != nil { + return err + } + case v1.JSONPatch: + patch, err := jsonpatch.DecodePatch(jsonPatcher.Payload) + if err != nil { + return fmt.Errorf("decode json patch:%s failed", jsonPatcher.Payload) + } + + modified, err := patch.Apply([]byte(target)) + if err != nil { + return fmt.Errorf("apply json patch to:%s failed", id) + } + if err = json.Unmarshal(modified, &res.Attributes); err != nil { + return err + } + default: + return fmt.Errorf("unsupported patch type:%s", jsonPatcher.Type) + } + } + } + return nil +} + func PatchWorkload(workload *v1.Resource, patcher *v1.Patcher) error { if patcher == nil { return nil @@ -270,7 +321,7 @@ type moduleConfig struct { ctx v1.GenericConfig } -func (g *appConfigurationGenerator) callModules(projectModuleConfigs map[string]v1.GenericConfig) (resources []v1.Resource, patchers []v1.Patcher, err error) { +func (g *appConfigurationGenerator) callModules(projectModuleConfigs map[string]v1.GenericConfig) (resources []v1.Resource, patcher *v1.Patcher, err error) { pluginMap := make(map[string]*modules.Plugin) defer func() { for _, plugin := range pluginMap { @@ -337,17 +388,13 @@ func (g *appConfigurationGenerator) callModules(projectModuleConfigs map[string] } // parse patcher - for _, patcher := range response.Patchers { - temp := &v1.Patcher{} - err = yaml.Unmarshal(patcher, temp) - if err != nil { - return nil, nil, err - } - patchers = append(patchers, *temp) + err = yaml.Unmarshal(response.Patcher, patcher) + if err != nil { + return nil, nil, err } } - return resources, patchers, nil + return resources, patcher, nil } func (g *appConfigurationGenerator) buildModuleConfigIndex(platformModuleConfigs map[string]v1.GenericConfig) (map[string]moduleConfig, error) { diff --git a/pkg/modules/generators/app_configurations_generator_test.go b/pkg/modules/generators/app_configurations_generator_test.go index b59bdc61..62e417e9 100644 --- a/pkg/modules/generators/app_configurations_generator_test.go +++ b/pkg/modules/generators/app_configurations_generator_test.go @@ -417,3 +417,64 @@ func TestAppConfigurationGenerator_CallModules(t *testing.T) { assert.Error(t, err) }) } + +func TestJsonPatch(t *testing.T) { + t.Run("ResourcesNil", func(t *testing.T) { + err := JSONPatch(nil, &v1.Patcher{}) + assert.NoError(t, err) + }) + + t.Run("PatcherNil", func(t *testing.T) { + err := JSONPatch([]v1.Resource{{ID: "test"}}, nil) + assert.NoError(t, err) + }) + + t.Run("JsonPatchersNil", func(t *testing.T) { + err := JSONPatch([]v1.Resource{{ID: "test"}}, &v1.Patcher{}) + assert.NoError(t, err) + }) + + t.Run("ResourceNotFound", func(t *testing.T) { + err := JSONPatch([]v1.Resource{{ID: "test"}}, &v1.Patcher{ + JSONPatchers: map[string]v1.JSONPatcher{ + "notfound": {Type: v1.MergePatch, Payload: []byte(`{"key": "value"}`)}, + }, + }) + assert.Error(t, err) + }) + + t.Run("MergePatch", func(t *testing.T) { + resources := []v1.Resource{ + {ID: "test", Attributes: map[string]interface{}{"key": "old"}}, + } + err := JSONPatch(resources, &v1.Patcher{ + JSONPatchers: map[string]v1.JSONPatcher{ + "test": {Type: v1.MergePatch, Payload: []byte(`{"key": "new"}`)}, + }, + }) + assert.NoError(t, err) + assert.Equal(t, "new", resources[0].Attributes["key"]) + }) + + t.Run("JSONPatch", func(t *testing.T) { + resources := []v1.Resource{ + {ID: "test", Attributes: map[string]interface{}{"key": "old"}}, + } + err := JSONPatch(resources, &v1.Patcher{ + JSONPatchers: map[string]v1.JSONPatcher{ + "test": {Type: v1.JSONPatch, Payload: []byte(`[{"op": "replace", "path": "/key", "value": "new"}]`)}, + }, + }) + assert.NoError(t, err) + assert.Equal(t, "new", resources[0].Attributes["key"]) + }) + + t.Run("UnsupportedPatchType", func(t *testing.T) { + err := JSONPatch([]v1.Resource{{ID: "test"}}, &v1.Patcher{ + JSONPatchers: map[string]v1.JSONPatcher{ + "test": {Type: "unsupported", Payload: []byte(`{"key": "value"}`)}, + }, + }) + assert.Error(t, err) + }) +} diff --git a/pkg/modules/proto/module.pb.go b/pkg/modules/proto/module.pb.go index 1bf541cc..f45c5ea1 100644 --- a/pkg/modules/proto/module.pb.go +++ b/pkg/modules/proto/module.pb.go @@ -132,7 +132,7 @@ type GeneratorResponse struct { // Resources is a v1.Resource array, which represents the generated resources by this module. Resources [][]byte `protobuf:"bytes,1,rep,name=resources,proto3" json:"resources,omitempty"` // Patcher contains fields should be patched into the workload corresponding fields - Patchers [][]byte `protobuf:"bytes,2,rep,name=patchers,proto3" json:"patchers,omitempty"` + Patcher []byte `protobuf:"bytes,2,opt,name=patcher,proto3" json:"patcher,omitempty"` } func (x *GeneratorResponse) Reset() { @@ -174,9 +174,9 @@ func (x *GeneratorResponse) GetResources() [][]byte { return nil } -func (x *GeneratorResponse) GetPatchers() [][]byte { +func (x *GeneratorResponse) GetPatcher() []byte { if x != nil { - return x.Patchers + return x.Patcher } return nil } @@ -198,17 +198,17 @@ var file_module_proto_rawDesc = []byte{ 0x66, 0x69, 0x67, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0e, 0x70, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x74, - 0x65, 0x78, 0x74, 0x22, 0x4d, 0x0a, 0x11, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x6f, 0x72, + 0x65, 0x78, 0x74, 0x22, 0x4b, 0x0a, 0x11, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x09, 0x72, 0x65, 0x73, - 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x61, 0x74, 0x63, 0x68, 0x65, - 0x72, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x08, 0x70, 0x61, 0x74, 0x63, 0x68, 0x65, - 0x72, 0x73, 0x32, 0x3b, 0x0a, 0x06, 0x4d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x12, 0x31, 0x0a, 0x08, - 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x12, 0x11, 0x2e, 0x47, 0x65, 0x6e, 0x65, 0x72, - 0x61, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x12, 0x2e, 0x47, 0x65, - 0x6e, 0x65, 0x72, 0x61, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, - 0x0a, 0x5a, 0x08, 0x2e, 0x2e, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x33, + 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x61, 0x74, 0x63, 0x68, 0x65, + 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x70, 0x61, 0x74, 0x63, 0x68, 0x65, 0x72, + 0x32, 0x3b, 0x0a, 0x06, 0x4d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x12, 0x31, 0x0a, 0x08, 0x47, 0x65, + 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x12, 0x11, 0x2e, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, + 0x6f, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x12, 0x2e, 0x47, 0x65, 0x6e, 0x65, + 0x72, 0x61, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x0a, 0x5a, + 0x08, 0x2e, 0x2e, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x33, } var ( diff --git a/pkg/modules/proto/module.proto b/pkg/modules/proto/module.proto index 2d1f8aaa..40877291 100644 --- a/pkg/modules/proto/module.proto +++ b/pkg/modules/proto/module.proto @@ -15,7 +15,7 @@ message GeneratorRequest { bytes dev_config = 5; // PlatformModuleConfig is the platform engineer's inputs of this module bytes platform_config = 6; - // context contains workspace-level configurations, such as topologies, server endpoints, metadata, etc. + // context contains workspace-level configurations, such as topologies, server endpoints, metadata, etc. bytes context = 7; } @@ -24,7 +24,7 @@ message GeneratorResponse { // Resources is a v1.Resource array, which represents the generated resources by this module. repeated bytes resources = 1; // Patcher contains fields should be patched into the workload corresponding fields - repeated bytes patchers = 2; + bytes patcher = 2; } service Module { diff --git a/pkg/modules/proto/scrip.sh b/pkg/modules/proto/scrip.sh old mode 100644 new mode 100755 diff --git a/pkg/util/json/json.go b/pkg/util/json/json.go index fc467739..01bf3907 100644 --- a/pkg/util/json/json.go +++ b/pkg/util/json/json.go @@ -6,7 +6,6 @@ import ( "kusionstack.io/kusion/pkg/util" ) -// https://github.com/ksonnet/ksonnet/blob/master/pkg/kubecfg/diff.go func removeFields(config, live interface{}) interface{} { switch c := config.(type) { case map[string]interface{}: