feat: use kubeConfig in extensions to init kubernetes runtime (#672)

pkg/apis/intent/resource.go

@@ -13,6 +13,9 @@ const (
 	// ResourceExtensionGVK is the key for resource extension, which is used to
 	// store the GVK of the resource.
 	ResourceExtensionGVK = "GVK"
+	// ResourceExtensionKubeConfig is the key for resource extension, which is used
+	// to indicate the path of kubeConfig for Kubernetes type resource.
+	ResourceExtensionKubeConfig = "kubeConfig"
 )
 
 type Resources []Resource

pkg/engine/operation/apply.go

@@ -88,7 +88,7 @@ func (ao *ApplyOperation) Apply(request *ApplyRequest) (rsp *ApplyResponse, st s
 
 	resources := request.Intent.Resources
 	resources = append(resources, priorState.Resources...)
-	runtimesMap, s := runtimeinit.Runtimes(resources, o.Stack)
+	runtimesMap, s := runtimeinit.Runtimes(resources)
 	if status.IsErr(s) {
 		return nil, s
 	}

pkg/engine/operation/apply_test.go

@@ -176,7 +176,6 @@ func TestOperation_Apply(t *testing.T) {
 			}).Build()
 			mockey.Mock(runtimeinit.Runtimes).To(func(
 				resources intent.Resources,
-				stack *stack.Stack,
 			) (map[intent.Type]runtime.Runtime, status.Status) {
 				return map[intent.Type]runtime.Runtime{runtime.Kubernetes: &kubernetes.KubernetesRuntime{}}, nil
 			}).Build()

pkg/engine/operation/destory.go

@@ -54,7 +54,7 @@ func (do *DestroyOperation) Destroy(request *DestroyRequest) (st status.Status)
 
 	// only destroy resources we have recorded
 	resources := priorState.Resources
-	runtimesMap, s := runtimeinit.Runtimes(resources, o.Stack)
+	runtimesMap, s := runtimeinit.Runtimes(resources)
 	if status.IsErr(s) {
 		return s
 	}

pkg/engine/operation/preview.go

@@ -64,7 +64,7 @@ func (po *PreviewOperation) Preview(request *PreviewRequest) (rsp *PreviewRespon
 	// Kusion is a multi-runtime system. We initialize runtimes dynamically by resource types
 	resources := request.Intent.Resources
 	resources = append(resources, priorState.Resources...)
-	runtimesMap, s := runtimeinit.Runtimes(resources, o.Stack)
+	runtimesMap, s := runtimeinit.Runtimes(resources)
 	if status.IsErr(s) {
 		return nil, s
 	}

pkg/engine/operation/preview_test.go

@@ -267,7 +267,6 @@ func TestOperation_Preview(t *testing.T) {
 
 			mockey.Mock(runtimeinit.Runtimes).To(func(
 				resources intent.Resources,
-				stack *stack.Stack,
 			) (map[intent.Type]runtime.Runtime, status.Status) {
 				return map[intent.Type]runtime.Runtime{runtime.Kubernetes: &fakePreviewRuntime{}}, nil
 			}).Build()

pkg/engine/operation/watch.go

@@ -36,7 +36,7 @@ func (wo *WatchOperation) Watch(req *WatchRequest) error {
 
 	// init runtimes
 	resources := req.Intent.Resources
-	runtimes, s := runtimeinit.Runtimes(resources, req.Stack)
+	runtimes, s := runtimeinit.Runtimes(resources)
 	if status.IsErr(s) {
 		return errors.New(s.Message())
 	}

pkg/engine/operation/watch_test.go

@@ -10,7 +10,6 @@ import (
 	k8sWatch "k8s.io/apimachinery/pkg/watch"
 
 	"kusionstack.io/kusion/pkg/apis/intent"
-	"kusionstack.io/kusion/pkg/apis/stack"
 	"kusionstack.io/kusion/pkg/apis/status"
 	opsmodels "kusionstack.io/kusion/pkg/engine/operation/models"
 	"kusionstack.io/kusion/pkg/engine/runtime"
@@ -34,7 +33,6 @@ func TestWatchOperation_Watch(t *testing.T) {
 		}
 		mockey.Mock(runtimeinit.Runtimes).To(func(
 			resources intent.Resources,
-			stack *stack.Stack,
 		) (map[intent.Type]runtime.Runtime, status.Status) {
 			return map[intent.Type]runtime.Runtime{runtime.Kubernetes: fooRuntime}, nil
 		}).Build()

pkg/engine/runtime/init/init.go

@@ -5,10 +5,10 @@ import (
 	"reflect"
 
 	"kusionstack.io/kusion/pkg/apis/intent"
-	"kusionstack.io/kusion/pkg/apis/stack"
 	"kusionstack.io/kusion/pkg/apis/status"
 	"kusionstack.io/kusion/pkg/engine/runtime"
 	"kusionstack.io/kusion/pkg/engine/runtime/kubernetes"
+	"kusionstack.io/kusion/pkg/engine/runtime/kubernetes/kubeops"
 	"kusionstack.io/kusion/pkg/engine/runtime/terraform"
 )
 
@@ -18,31 +18,50 @@ var SupportRuntimes = map[intent.Type]InitFn{
 }
 
 // InitFn runtime init func
-type InitFn func(stack *stack.Stack) (runtime.Runtime, error)
+type InitFn func(resource *intent.Resource) (runtime.Runtime, error)
 
-func Runtimes(resources intent.Resources, stack *stack.Stack) (map[intent.Type]runtime.Runtime, status.Status) {
+func Runtimes(resources intent.Resources) (map[intent.Type]runtime.Runtime, status.Status) {
 	runtimesMap := map[intent.Type]runtime.Runtime{}
 	if resources == nil {
 		return runtimesMap, nil
 	}
+	if errStatus := validResources(resources); errStatus != nil {
+		return nil, errStatus
+	}
 
 	for _, resource := range resources {
 		rt := resource.Type
-		if rt == "" {
-			return nil, status.NewErrorStatusWithCode(status.IllegalManifest, fmt.Errorf("no resource type in resource: %v", resource.ID))
-		}
-
-		if SupportRuntimes[rt] == nil {
-			return nil, status.NewErrorStatusWithCode(status.IllegalManifest, fmt.Errorf("unknow resource type: %s. Currently supported resource types are: %v",
-				rt, reflect.ValueOf(SupportRuntimes).MapKeys()))
-		} else if runtimesMap[rt] == nil {
-			r, err := SupportRuntimes[rt](stack)
+		if runtimesMap[rt] == nil {
+			r, err := SupportRuntimes[rt](&resource)
 			if err != nil {
 				return nil, status.NewErrorStatus(fmt.Errorf("init %s runtime failed. %w", rt, err))
 			}
 			runtimesMap[rt] = r
 		}
 	}
-
 	return runtimesMap, nil
 }
+
+func validResources(resources intent.Resources) status.Status {
+	var kubeConfig string
+	for _, resource := range resources {
+		rt := resource.Type
+		if rt == "" {
+			return status.NewErrorStatusWithCode(status.IllegalManifest, fmt.Errorf("no resource type in resource: %v", resource.ID))
+		}
+		if SupportRuntimes[rt] == nil {
+			return status.NewErrorStatusWithCode(status.IllegalManifest, fmt.Errorf("unknown resource type: %s. Currently supported resource types are: %v",
+				rt, reflect.ValueOf(SupportRuntimes).MapKeys()))
+		}
+		if rt == intent.Kubernetes {
+			config := kubeops.GetKubeConfig(&resource)
+			if kubeConfig != "" && kubeConfig != config {
+				return status.NewErrorStatusWithCode(status.IllegalManifest, fmt.Errorf("different kubeConfig in different resources"))
+			}
+			if kubeConfig == "" {
+				kubeConfig = config
+			}
+		}
+	}
+	return nil
+}

pkg/engine/runtime/init/init_test.go

@@ -0,0 +1,99 @@
+package init
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"kusionstack.io/kusion/pkg/apis/intent"
+)
+
+func TestValidResources(t *testing.T) {
+	testcases := []struct {
+		name      string
+		success   bool
+		resources intent.Resources
+	}{
+		{
+			name:    "valid resources",
+			success: true,
+			resources: []intent.Resource{
+				{
+					ID:   "mock-id",
+					Type: "Kubernetes",
+					Attributes: map[string]any{
+						"mock-key": "mock-value",
+					},
+					Extensions: map[string]any{
+						"kubeConfig": "/etc/kubeConfig.yaml",
+					},
+				},
+			},
+		},
+		{
+			name:    "invalid resources empty type",
+			success: false,
+			resources: []intent.Resource{
+				{
+					ID:   "mock-id",
+					Type: "",
+					Attributes: map[string]any{
+						"mock-key": "mock-value",
+					},
+					Extensions: map[string]any{
+						"kubeConfig": "/etc/kubeConfig.yaml",
+					},
+				},
+			},
+		},
+		{
+			name:    "invalid resources unsupported type",
+			success: false,
+			resources: []intent.Resource{
+				{
+					ID:   "mock-id",
+					Type: "Unsupported",
+					Attributes: map[string]any{
+						"mock-key": "mock-value",
+					},
+					Extensions: map[string]any{
+						"kubeConfig": "/etc/kubeConfig.yaml",
+					},
+				},
+			},
+		},
+		{
+			name:    "invalid resources multiple kubeConfig",
+			success: false,
+			resources: []intent.Resource{
+				{
+					ID:   "mock-id",
+					Type: "Kubernetes",
+					Attributes: map[string]any{
+						"mock-key": "mock-value",
+					},
+					Extensions: map[string]any{
+						"kubeConfig": "/etc/kubeConfig.yaml",
+					},
+				},
+				{
+					ID:   "mock-id",
+					Type: "Kubernetes",
+					Attributes: map[string]any{
+						"mock-key": "mock-value",
+					},
+					Extensions: map[string]any{
+						"kubeConfig": "/etc/kubeConfig_2.yaml",
+					},
+				},
+			},
+		},
+	}
+
+	for _, tc := range testcases {
+		t.Run(tc.name, func(t *testing.T) {
+			err := validResources(tc.resources)
+			assert.Equal(t, tc.success, err == nil)
+		})
+	}
+}

pkg/util/kube/config/config.go → pkg/engine/runtime/kubernetes/kubeops/config.go

@@ -1,12 +1,12 @@
-package config
+package kubeops
 
 import (
 	"os"
 	"path/filepath"
 
 	"k8s.io/client-go/util/homedir"
 
-	"kusionstack.io/kusion/pkg/apis/stack"
+	"kusionstack.io/kusion/pkg/apis/intent"
 )
 
 const (
@@ -20,16 +20,21 @@ var (
 	RecommendedKubeConfigFile = filepath.Join(RecommendedConfigDir, RecommendedKubeConfigFileName)
 )
 
+// GetKubeConfig gets kubeConfig in the following order:
 // 1. If $KUBECONFIG environment variable is set, then it is used.
-// 2. If not, and the `kubeConfig` in stack configuration is set, then it is used.
+// 2. If not, and the `kubeConfig` in resource extensions is set, then it is used.
 // 3. Otherwise, ${HOME}/.kube/config is used.
-func GetKubeConfig(stack *stack.Stack) string {
+func GetKubeConfig(resource *intent.Resource) string {
 	if kubeConfigFile := os.Getenv(RecommendedConfigPathEnvVar); kubeConfigFile != "" {
 		return kubeConfigFile
-	} else if stack.KubeConfig != "" {
-		kubeConfigFile, _ := filepath.Abs(stack.KubeConfig)
-		if kubeConfigFile != "" {
-			return kubeConfigFile
+	}
+	if resource != nil {
+		kubeConfig, ok := resource.Extensions[intent.ResourceExtensionKubeConfig].(string)
+		if ok && kubeConfig != "" {
+			kubeConfigFile, _ := filepath.Abs(kubeConfig)
+			if kubeConfigFile != "" {
+				return kubeConfigFile
+			}
 		}
 	}
 	return RecommendedKubeConfigFile

pkg/util/kube/config/config_test.go → pkg/engine/runtime/kubernetes/kubeops/config_test.go

@@ -1,4 +1,4 @@
-package config
+package kubeops
 
 import (
 	"os"
@@ -7,7 +7,7 @@ import (
 	"github.com/bytedance/mockey"
 	"github.com/stretchr/testify/assert"
 
-	"kusionstack.io/kusion/pkg/apis/stack"
+	"kusionstack.io/kusion/pkg/apis/intent"
 )
 
 func mockGetenv(result string) {
@@ -17,24 +17,23 @@ func mockGetenv(result string) {
 }
 
 func TestGetKubeConfig(t *testing.T) {
-	stack := &stack.Stack{
-		Configuration: stack.Configuration{
-			KubeConfig: "",
+	resource := &intent.Resource{
+		Extensions: map[string]any{
+			"kubeConfig": "/home/test/kubeconfig",
 		},
 	}
 
 	// Mock
 	mockey.PatchConvey("test null env config", t, func() {
 		mockGetenv("")
-		assert.Equal(t, RecommendedKubeConfigFile, GetKubeConfig(stack))
+		assert.Equal(t, RecommendedKubeConfigFile, GetKubeConfig(nil))
 	})
 	mockey.PatchConvey("test env config", t, func() {
 		mockGetenv("test")
-		assert.Equal(t, "test", GetKubeConfig(stack))
+		assert.Equal(t, "test", GetKubeConfig(resource))
 	})
-	mockey.PatchConvey("test stack config", t, func() {
+	mockey.PatchConvey("test resource config", t, func() {
 		mockGetenv("")
-		stack.KubeConfig = "/home/test/kubeconfig"
-		assert.Equal(t, "/home/test/kubeconfig", GetKubeConfig(stack))
+		assert.Equal(t, "/home/test/kubeconfig", GetKubeConfig(resource))
 	})
 }

pkg/engine/runtime/kubernetes/kubernetes_runtime.go

@@ -26,14 +26,13 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
 
 	"kusionstack.io/kusion/pkg/apis/intent"
-	"kusionstack.io/kusion/pkg/apis/stack"
 	"kusionstack.io/kusion/pkg/apis/status"
 	"kusionstack.io/kusion/pkg/engine"
 	"kusionstack.io/kusion/pkg/engine/printers/convertor"
 	"kusionstack.io/kusion/pkg/engine/runtime"
+	"kusionstack.io/kusion/pkg/engine/runtime/kubernetes/kubeops"
 	"kusionstack.io/kusion/pkg/log"
 	jsonutil "kusionstack.io/kusion/pkg/util/json"
-	"kusionstack.io/kusion/pkg/util/kube/config"
 )
 
 var _ runtime.Runtime = (*KubernetesRuntime)(nil)
@@ -44,8 +43,8 @@ type KubernetesRuntime struct {
 }
 
 // NewKubernetesRuntime create a new KubernetesRuntime
-func NewKubernetesRuntime(stack *stack.Stack) (runtime.Runtime, error) {
-	client, mapper, err := getKubernetesClient(stack)
+func NewKubernetesRuntime(resource *intent.Resource) (runtime.Runtime, error) {
+	client, mapper, err := getKubernetesClient(resource)
 	if err != nil {
 		return nil, err
 	}
@@ -376,9 +375,9 @@ func (k *KubernetesRuntime) Watch(ctx context.Context, request *runtime.WatchReq
 }
 
 // getKubernetesClient get kubernetes client
-func getKubernetesClient(stack *stack.Stack) (dynamic.Interface, meta.RESTMapper, error) {
+func getKubernetesClient(resource *intent.Resource) (dynamic.Interface, meta.RESTMapper, error) {
 	// build config
-	cfg, err := clientcmd.BuildConfigFromFlags("", config.GetKubeConfig(stack))
+	cfg, err := clientcmd.BuildConfigFromFlags("", kubeops.GetKubeConfig(resource))
 	if err != nil {
 		return nil, nil, err
 	}

pkg/engine/runtime/terraform/terraform_runtime.go

@@ -9,7 +9,6 @@ import (
 	"github.com/spf13/afero"
 
 	"kusionstack.io/kusion/pkg/apis/intent"
-	"kusionstack.io/kusion/pkg/apis/stack"
 	"kusionstack.io/kusion/pkg/apis/status"
 	"kusionstack.io/kusion/pkg/engine/runtime"
 	"kusionstack.io/kusion/pkg/engine/runtime/terraform/tfops"
@@ -23,7 +22,7 @@ type TerraformRuntime struct {
 	mu *sync.Mutex
 }
 
-func NewTerraformRuntime(_ *stack.Stack) (runtime.Runtime, error) {
+func NewTerraformRuntime(_ *intent.Resource) (runtime.Runtime, error) {
 	fs := afero.Afero{Fs: afero.NewOsFs()}
 	ws := tfops.NewWorkSpace(fs)
 	TFRuntime := &TerraformRuntime{