feat: 3 ways merge base on json merge patch (#55)

* feat: runtime apply use 3 ways merge base on json merge patch

* refactor: add live state to ChangeStep to do 3-way diff if need

go.sum

@@ -1352,7 +1352,6 @@ gopkg.in/yaml.v3 v3.0.0-20191120175047-4206685974f2/go.mod h1:K4uyk7z7BCEPqu6E+C
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20220512140231-539c8e751b99 h1:dbuHpmKjkDzSOMKAWl10QNlgaZUd3V1q99xc81tt2Kc=
 gopkg.in/yaml.v3 v3.0.0-20220512140231-539c8e751b99/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0 h1:hjy8E9ON/egN1tAYqKb61G10WtihqetD4sz2H+8nIeA=
 gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

pkg/engine/operation/change.go

@@ -19,15 +19,17 @@ import (
 )
 
 type ChangeStep struct {
-	ID     string      // the resource id
-	Action ActionType  // the operation performed by this step.
-	Old    interface{} // the state of the resource before performing this step.
-	New    interface{} // the state of the resource after performing this step.
+	ID       string      // the resource id
+	Action   ActionType  // the operation performed by this step.
+	Original interface{} // local stored resource
+	Modified interface{} // planed resource
+	Current  interface{} // live resource
 }
 
+// TODO: 3-way diff
 func (cs *ChangeStep) Diff() (string, error) {
 	// Generate diff report
-	diffReport, err := diffToReport(cs.Old, cs.New)
+	diffReport, err := diffToReport(cs.Original, cs.Modified)
 	if err != nil {
 		log.Errorf("failed to compute diff with ChangeStep ID: %s", cs.ID)
 		return "", err
@@ -59,12 +61,13 @@ func (cs *ChangeStep) Diff() (string, error) {
 	return buf.String(), nil
 }
 
-func NewChangeStep(id string, op ActionType, oldData, newData interface{}) *ChangeStep {
+func NewChangeStep(id string, op ActionType, original, modified, current interface{}) *ChangeStep {
 	return &ChangeStep{
-		ID:     id,
-		Action: op,
-		Old:    oldData,
-		New:    newData,
+		ID:       id,
+		Action:   op,
+		Original: original,
+		Modified: modified,
+		Current:  current,
 	}
 }
 

pkg/engine/operation/change_test.go

@@ -11,10 +11,10 @@ import (
 )
 
 var (
-	TestChangeStepOpCreate   = NewChangeStep("id", Create, nil, nil)
-	TestChangeStepOpDelete   = NewChangeStep("id", Delete, nil, nil)
-	TestChangeStepOpUpdate   = NewChangeStep("id", Update, nil, nil)
-	TestChangeStepOpUnChange = NewChangeStep("id", UnChange, nil, nil)
+	TestChangeStepOpCreate   = NewChangeStep("id", Create, nil, nil, nil)
+	TestChangeStepOpDelete   = NewChangeStep("id", Delete, nil, nil, nil)
+	TestChangeStepOpUpdate   = NewChangeStep("id", Update, nil, nil, nil)
+	TestChangeStepOpUnChange = NewChangeStep("id", UnChange, nil, nil, nil)
 	TestStepKeys             = []string{"test-key-1", "test-key-2", "test-key-3", "test-key-4"}
 	TestChangeSteps          = map[string]*ChangeStep{
 		"test-key-1": TestChangeStepOpCreate,
@@ -128,10 +128,10 @@ func TestChangeStep_Diff(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			cs := &ChangeStep{
-				ID:     tt.fields.ID,
-				Action: tt.fields.Op,
-				Old:    tt.fields.Old,
-				New:    tt.fields.New,
+				ID:       tt.fields.ID,
+				Action:   tt.fields.Op,
+				Original: tt.fields.Old,
+				Modified: tt.fields.New,
 			}
 			got, err := cs.Diff()
 			if (err != nil) != tt.wantErr {
@@ -145,59 +145,6 @@ func TestChangeStep_Diff(t *testing.T) {
 	}
 }
 
-func TestNewChangeStep(t *testing.T) {
-	type args struct {
-		id  string
-		op  ActionType
-		old interface{}
-		new interface{}
-	}
-	tests := []struct {
-		name string
-		args args
-		want *ChangeStep
-	}{
-		{
-			name: "t1",
-			args: args{
-				id:  "id",
-				op:  Create,
-				old: nil,
-				new: nil,
-			},
-			want: &ChangeStep{
-				ID:     "id",
-				Action: Create,
-				Old:    nil,
-				New:    nil,
-			},
-		},
-		{
-			name: "t2",
-			args: args{
-				id:  "id[0]",
-				op:  Create,
-				old: nil,
-				new: nil,
-			},
-			want: &ChangeStep{
-				ID:     "id[0]",
-				Action: Create,
-				Old:    nil,
-				New:    nil,
-			},
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			if got := NewChangeStep(tt.args.id, tt.args.op, tt.args.old, tt.args.new); !reflect.DeepEqual(got,
-				tt.want) {
-				t.Errorf("NewChangeStep() = %v, want %v", got, tt.want)
-			}
-		})
-	}
-}
-
 func TestChanges_Get(t *testing.T) {
 	type fields struct {
 		order   *ChangeOrder

pkg/engine/operation/preview.go

@@ -80,6 +80,7 @@ func (o *Operation) Preview(request *PreviewRequest, operation Type) (rsp *Previ
 			PriorStateResourceIndex: priorStateResourceIndex,
 			StateResourceIndex:      priorStateResourceIndex,
 			Order:                   o.Order,
+			Runtime:                 o.Runtime, // preview need get the latest manifest from runtime
 			resultState:             resultState,
 			lock:                    &sync.Mutex{},
 		},

pkg/engine/operation/preview_test.go

@@ -1,6 +1,7 @@
 package operation
 
 import (
+	"context"
 	"reflect"
 	"sync"
 	"testing"
@@ -30,6 +31,26 @@ var (
 	}
 )
 
+var _ runtime.Runtime = (*fakePreviewRuntime)(nil)
+
+type fakePreviewRuntime struct{}
+
+func (f *fakePreviewRuntime) Apply(ctx context.Context, priorState, planState *models.Resource) (*models.Resource, status.Status) {
+	return planState, nil
+}
+
+func (f *fakePreviewRuntime) Read(ctx context.Context, resourceState *models.Resource) (*models.Resource, status.Status) {
+	return resourceState, nil
+}
+
+func (f *fakePreviewRuntime) Delete(ctx context.Context, resourceState *models.Resource) status.Status {
+	return nil
+}
+
+func (f *fakePreviewRuntime) Watch(ctx context.Context, resourceState *models.Resource) (*models.Resource, status.Status) {
+	return resourceState, nil
+}
+
 func TestOperation_Preview(t *testing.T) {
 	type fields struct {
 		OperationType           Type
@@ -57,14 +78,14 @@ func TestOperation_Preview(t *testing.T) {
 		{
 			name: "success-when-apply",
 			fields: fields{
-				Runtime:      &runtime.KubernetesRuntime{},
+				Runtime:      &fakePreviewRuntime{},
 				StateStorage: &states.FileSystemState{Path: states.KusionState},
 				Order:        &ChangeOrder{StepKeys: []string{}, ChangeSteps: map[string]*ChangeStep{}},
 			},
 			args: args{
 				request: &PreviewRequest{
 					Request: Request{
-						Tenant:   "fake-tennat",
+						Tenant:   "fake-tenant",
 						Stack:    "fake-stack",
 						Project:  "fake-project",
 						Operator: "fake-operator",
@@ -82,10 +103,11 @@ func TestOperation_Preview(t *testing.T) {
 					StepKeys: []string{"fake-id"},
 					ChangeSteps: map[string]*ChangeStep{
 						"fake-id": {
-							ID:     "fake-id",
-							Action: Create,
-							Old:    (*models.Resource)(nil),
-							New:    &FakeResourceState,
+							ID:       "fake-id",
+							Action:   Create,
+							Original: (*models.Resource)(nil),
+							Modified: &FakeResourceState,
+							Current:  (*models.Resource)(nil),
 						},
 					},
 				},
@@ -95,14 +117,14 @@ func TestOperation_Preview(t *testing.T) {
 		{
 			name: "success-when-destroy",
 			fields: fields{
-				Runtime:      &runtime.KubernetesRuntime{},
+				Runtime:      &fakePreviewRuntime{},
 				StateStorage: &states.FileSystemState{Path: states.KusionState},
 				Order:        &ChangeOrder{},
 			},
 			args: args{
 				request: &PreviewRequest{
 					Request: Request{
-						Tenant:   "fake-tennat",
+						Tenant:   "fake-tenant",
 						Stack:    "fake-stack",
 						Project:  "fake-project",
 						Operator: "fake-operator",
@@ -120,10 +142,11 @@ func TestOperation_Preview(t *testing.T) {
 					StepKeys: []string{"fake-id"},
 					ChangeSteps: map[string]*ChangeStep{
 						"fake-id": {
-							ID:     "fake-id",
-							Action: Delete,
-							Old:    &FakeResourceState,
-							New:    (*models.Resource)(nil),
+							ID:       "fake-id",
+							Action:   Delete,
+							Original: &FakeResourceState,
+							Modified: (*models.Resource)(nil),
+							Current:  &FakeResourceState,
 						},
 					},
 				},
@@ -133,7 +156,7 @@ func TestOperation_Preview(t *testing.T) {
 		{
 			name: "fail-because-empty-models",
 			fields: fields{
-				Runtime:      &runtime.KubernetesRuntime{},
+				Runtime:      &fakePreviewRuntime{},
 				StateStorage: &states.FileSystemState{Path: states.KusionState},
 				Order:        &ChangeOrder{},
 			},
@@ -151,7 +174,7 @@ func TestOperation_Preview(t *testing.T) {
 		{
 			name: "fail-because-nonexistent-id",
 			fields: fields{
-				Runtime:      &runtime.KubernetesRuntime{},
+				Runtime:      &fakePreviewRuntime{},
 				StateStorage: &states.FileSystemState{Path: states.KusionState},
 				Order:        &ChangeOrder{},
 			},

pkg/engine/operation/resource_node.go

@@ -39,23 +39,29 @@ func (rn *ResourceNode) Execute(operation *Operation) status.Status {
 		planedState = nil
 	}
 
-	// 2. get State
+	// 2. get prior state which is stored in kusion_state.json
 	key := rn.state.ResourceKey()
 	priorState := operation.PriorStateResourceIndex[key]
 
-	// 3. compute ActionType of current resource node
-	if priorState == nil {
+	// get the latest resource from runtime
+	liveState, s := operation.Runtime.Read(context.Background(), priorState)
+	if status.IsErr(s) {
+		return s
+	}
+
+	// 3. compute ActionType of current resource node between planState and liveState
+	if liveState == nil {
 		rn.Action = Create
 	} else if planedState == nil {
 		rn.Action = Delete
-	} else if reflect.DeepEqual(priorState, planedState) {
+	} else if reflect.DeepEqual(liveState, planedState) {
 		rn.Action = UnChange
 	} else {
 		rn.Action = Update
 	}
 
 	if operation.OperationType == Preview {
-		fillResponseChangeSteps(operation, rn, priorState, planedState)
+		fillResponseChangeSteps(operation, rn, priorState, planedState, liveState)
 		return nil
 	}
 	// 4. apply
@@ -73,8 +79,8 @@ func (rn *ResourceNode) Execute(operation *Operation) status.Status {
 	return nil
 }
 
-func (rn *ResourceNode) applyResource(operation *Operation, priorState *models.Resource, planedState *models.Resource) status.Status {
-	log.Infof("PriorAttributes and PlanAttributes are not equal. operation:%v, prior:%v, plan:%v", rn.Action,
+func (rn *ResourceNode) applyResource(operation *Operation, priorState, planedState *models.Resource) status.Status {
+	log.Infof("operation:%v, prior:%v, plan:%v, live:%v", rn.Action,
 		jsonUtil.Marshal2String(priorState), jsonUtil.Marshal2String(planedState))
 
 	var res *models.Resource
@@ -122,7 +128,8 @@ func NewResourceNode(key string, state *models.Resource, action ActionType) *Res
 	return &ResourceNode{BaseNode: BaseNode{ID: key}, Action: action, state: state}
 }
 
-func fillResponseChangeSteps(operation *Operation, rn *ResourceNode, prior, plan interface{}) {
+// save change steps in DAG walking order so that we can preview a full applying list
+func fillResponseChangeSteps(operation *Operation, rn *ResourceNode, prior, plan, live interface{}) {
 	defer operation.lock.Unlock()
 	operation.lock.Lock()
 
@@ -137,7 +144,7 @@ func fillResponseChangeSteps(operation *Operation, rn *ResourceNode, prior, plan
 		order.ChangeSteps = make(map[string]*ChangeStep)
 	}
 	order.StepKeys = append(order.StepKeys, rn.ID)
-	order.ChangeSteps[rn.ID] = NewChangeStep(rn.ID, rn.Action, prior, plan)
+	order.ChangeSteps[rn.ID] = NewChangeStep(rn.ID, rn.Action, prior, plan, live)
 }
 
 var ImplicitReplaceFun = func(resourceIndex map[string]*models.Resource, refPath string) (reflect.Value, status.Status) {

pkg/engine/operation/resource_node_test.go

@@ -171,6 +171,10 @@ func TestResourceNode_Execute(t *testing.T) {
 				func(k *runtime.KubernetesRuntime, ctx context.Context, priorState *models.Resource) status.Status {
 					return nil
 				})
+			monkey.PatchInstanceMethod(reflect.TypeOf(tt.args.operation.Runtime), "Read",
+				func(k *runtime.KubernetesRuntime, ctx context.Context, resourceState *models.Resource) (*models.Resource, status.Status) {
+					return resourceState, nil
+				})
 			monkey.PatchInstanceMethod(reflect.TypeOf(tt.args.operation.StateStorage), "Apply",
 				func(f *states.FileSystemState, state *states.State) error {
 					return nil