feat: invoke modules by go-plugin

pkg/apis/core/v1/appconfiguration.go

@@ -36,7 +36,7 @@ type Accessory map[string]interface{}
 //		       }
 //		   }
 //
-//		   # extend accessories module base
+//		# A collection of accessories that will be attached to the workload
 //	    accessories: {
 //	       # Built-in module, key represents the module source
 //	       "kusionstack/mysql@v0.1" : d.MySQL {
@@ -53,7 +53,7 @@ type Accessory map[string]interface{}
 //	       }
 //	   }
 //
-//		   # extend pipeline module base
+//		   # pipeline modules
 //		   pipeline: {
 //		       # Step is a module
 //		       "step" : Step {
@@ -74,7 +74,7 @@ type AppConfiguration struct {
 	Workload *workload.Workload `json:"workload" yaml:"workload"`
 	// Accessories defines a collection of accessories that will be attached to the workload.
 	// The key in this map represents the module source. e.g. kusionstack/mysql@v0.1
-	Accessories map[string]*Accessory `json:"accessories,omitempty" yaml:"accessories,omitempty"`
+	Accessories map[string]Accessory `json:"accessories,omitempty" yaml:"accessories,omitempty"`
 	// Labels and Annotations can be used to attach arbitrary metadata as key-value pairs to resources.
 	Labels      map[string]string `json:"labels,omitempty" yaml:"labels,omitempty"`
 	Annotations map[string]string `json:"annotations,omitempty" yaml:"annotations,omitempty"`

pkg/cmd/build/builders/appconfig_builder.go

@@ -22,7 +22,7 @@ func (acg *AppsConfigBuilder) Build(
 
 	var gfs []modules.NewGeneratorFunc
 	err := modules.ForeachOrdered(acg.Apps, func(appName string, app v1.AppConfiguration) error {
-		gfs = append(gfs, generators.NewAppConfigurationGeneratorFunc(project, stack, appName, &app, acg.Workspace))
+		gfs = append(gfs, generators.NewAppConfigurationGeneratorFunc(project.Name, stack.Name, appName, &app, acg.Workspace))
 		return nil
 	})
 	if err != nil {

pkg/cmd/build/util.go

@@ -115,6 +115,7 @@ func buildAppConfigs(o *builders.Options, stack *v1.Stack) (map[string]v1.AppCon
 	// environment variables, thus we unmarshal appConfigs with yaml.v2 rather than yaml.v3.
 	err = yaml.Unmarshal([]byte(out), appConfigs)
 	if err != nil {
+		// todo wrap error to human readable messages
 		return nil, err
 	}
 

pkg/modules/generators/app_configurations_generator.go

@@ -1,34 +1,41 @@
 package generators
 
 import (
+	"encoding/json"
 	"errors"
 	"fmt"
 
-	apiv1 "kusionstack.io/kusion/pkg/apis/core/v1"
+	"kusionstack.io/kusion/pkg/apis/core/v1"
 	"kusionstack.io/kusion/pkg/modules"
 	"kusionstack.io/kusion/pkg/modules/generators/workload"
+	"kusionstack.io/kusion/pkg/modules/proto"
+	jsonutil "kusionstack.io/kusion/pkg/util/json"
 	"kusionstack.io/kusion/pkg/workspace"
 )
 
 type appConfigurationGenerator struct {
-	project *apiv1.Project
-	stack   *apiv1.Stack
+	project string
+	stack   string
 	appName string
-	app     *apiv1.AppConfiguration
-	ws      *apiv1.Workspace
+	app     *v1.AppConfiguration
+	ws      *v1.Workspace
 }
 
 func NewAppConfigurationGenerator(
-	project *apiv1.Project,
-	stack *apiv1.Stack,
+	project string,
+	stack string,
 	appName string,
-	app *apiv1.AppConfiguration,
-	ws *apiv1.Workspace,
+	app *v1.AppConfiguration,
+	ws *v1.Workspace,
 ) (modules.Generator, error) {
-	if len(project.Name) == 0 {
+	if len(project) == 0 {
 		return nil, fmt.Errorf("project name must not be empty")
 	}
 
+	if len(stack) == 0 {
+		return nil, fmt.Errorf("stack name must not be empty")
+	}
+
 	if len(appName) == 0 {
 		return nil, fmt.Errorf("app name must not be empty")
 	}
@@ -42,7 +49,7 @@ func NewAppConfigurationGenerator(
 	}
 
 	if err := workspace.ValidateWorkspace(ws); err != nil {
-		return nil, fmt.Errorf("invalid config of workspace %s, %w", stack.Name, err)
+		return nil, fmt.Errorf("invalid config of workspace %s, %w", stack, err)
 	}
 
 	return &appConfigurationGenerator{
@@ -55,74 +62,136 @@ func NewAppConfigurationGenerator(
 }
 
 func NewAppConfigurationGeneratorFunc(
-	project *apiv1.Project,
-	stack *apiv1.Stack,
+	project string,
+	stack string,
 	appName string,
-	app *apiv1.AppConfiguration,
-	ws *apiv1.Workspace,
+	app *v1.AppConfiguration,
+	ws *v1.Workspace,
 ) modules.NewGeneratorFunc {
 	return func() (modules.Generator, error) {
 		return NewAppConfigurationGenerator(project, stack, appName, app, ws)
 	}
 }
 
-func (g *appConfigurationGenerator) Generate(i *apiv1.Intent) error {
-	if i.Resources == nil {
-		i.Resources = make(apiv1.Resources, 0)
+func (g *appConfigurationGenerator) Generate(spec *v1.Intent) error {
+	if spec.Resources == nil {
+		spec.Resources = make(v1.Resources, 0)
 	}
 	g.app.Name = g.appName
 
 	// retrieve the module configs of the specified project
-	platformConfigs, err := workspace.GetProjectModuleConfigs(g.ws.Modules, g.project.Name)
+	projectModuleConfigs, err := workspace.GetProjectModuleConfigs(g.ws.Modules, g.project)
 	if err != nil {
 		return err
 	}
 
 	// todo: is namespace a module? how to retrieve it? Currently, it is configured in the workspace file.
-	namespace := g.getNamespaceName(platformConfigs)
+	namespace := g.getNamespaceName(projectModuleConfigs)
 
 	// Generate built-in resources
 	gfs := []modules.NewGeneratorFunc{
 		NewNamespaceGeneratorFunc(namespace),
 		workload.NewWorkloadGeneratorFunc(&workload.Generator{
-			Project:         g.project.Name,
-			Stack:           g.stack.Name,
+			Project:         g.project,
+			Stack:           g.stack,
 			App:             g.appName,
 			Namespace:       namespace,
 			Workload:        g.app.Workload,
-			PlatformConfigs: platformConfigs,
+			PlatformConfigs: projectModuleConfigs,
 		}),
 	}
-	if err = modules.CallGenerators(i, gfs...); err != nil {
+	if err = modules.CallGenerators(spec, gfs...); err != nil {
 		return err
 	}
 
-	// Generate customized module resources
-	for range platformConfigs {
-		// todo: invoke kusion module generators for each module
+	// Call modules to generate customized resources
+	resources, err := g.callModules(projectModuleConfigs)
+	if err != nil {
+		return err
 	}
+	spec.Resources = append(spec.Resources, resources...)
 
 	// The OrderedResourcesGenerator should be executed after all resources are generated.
-	if err := modules.CallGenerators(i, NewOrderedResourcesGeneratorFunc()); err != nil {
+	if err = modules.CallGenerators(spec, NewOrderedResourcesGeneratorFunc()); err != nil {
 		return err
 	}
 
 	// Add kubeConfig from workspace if exist
-	modules.AddKubeConfigIf(i, g.ws)
-
+	modules.AddKubeConfigIf(spec, g.ws)
 	return nil
 }
 
+func (g *appConfigurationGenerator) callModules(projectModuleConfigs map[string]v1.GenericConfig) ([]v1.Resource, error) {
+	var resources []v1.Resource
+
+	pluginMap := make(map[string]*modules.Plugin)
+	defer func() {
+		for _, plugin := range pluginMap {
+			plugin.KillPluginClient()
+		}
+	}()
+
+	// Generate customized module resources
+	for t, config := range projectModuleConfigs {
+		// init the plugin
+		if pluginMap[t] == nil {
+			plugin, err := modules.NewPlugin(t)
+			if err != nil {
+				return nil, err
+			}
+			if plugin == nil {
+				return nil, fmt.Errorf("init plugin for module %s failed", t)
+			}
+			pluginMap[t] = plugin
+		}
+		plugin := pluginMap[t]
+
+		// prepare the request
+		devConfig := jsonutil.Marshal2String(g.app.Accessories[t])
+		platformConfig := jsonutil.Marshal2String(config)
+		wsConfig := jsonutil.Marshal2String(g.ws)
+		protoRequest := &proto.GeneratorRequest{
+			Project:              g.project,
+			Stack:                g.stack,
+			App:                  g.appName,
+			DevModuleConfig:      []byte(devConfig),
+			PlatformModuleConfig: []byte(platformConfig),
+			RuntimeConfig:        []byte(wsConfig),
+		}
+
+		// invoke the plugin
+		response, err := plugin.Module.Generate(protoRequest)
+		if err != nil {
+			return nil, err
+		}
+		if response == nil {
+			return nil, fmt.Errorf("empty response from module %s", t)
+		}
+		for _, res := range response.Resources {
+			temp := &v1.Resource{}
+			err = json.Unmarshal(res, temp)
+			if err != nil {
+				return nil, err
+			}
+			// todo: validate resources format in response
+			// todo parse patches in the resources
+			resources = append(resources, *temp)
+		}
+	}
+
+	return resources, nil
+}
+
 // getNamespaceName obtains the final namespace name using the following precedence
 // (from lower to higher):
 // - Project name
 // - Namespace module config (specified in corresponding workspace file)
-func (g *appConfigurationGenerator) getNamespaceName(moduleConfigs map[string]apiv1.GenericConfig) string {
+func (g *appConfigurationGenerator) getNamespaceName(moduleConfigs map[string]v1.GenericConfig) string {
 	if moduleConfigs == nil {
-		return g.project.Name
+		return g.project
 	}
 
-	namespaceName := g.project.Name
+	namespaceName := g.project
 	namespaceModuleConfigs, exist := moduleConfigs["namespace"]
 	if exist {
 		if name, ok := namespaceModuleConfigs["name"]; ok {