feat: separate module logs into different files (#949)

pkg/log/log.go

@@ -87,7 +87,7 @@ func GetLevelFromStr(level string) Level {
 	}
 }
 
-func GetLogDir() LogDir {
+func GetLogDir() Dir {
 	return log.GetLogDir()
 }
 

pkg/log/types.go

@@ -24,6 +24,6 @@ type Logger interface {
 	Fatalf(format string, args ...interface{})
 	Fatal(args ...interface{})
 	SetLevel(level Level)
-	GetLogDir() LogDir
+	GetLogDir() Dir
 	With(args ...interface{}) Logger
 }

pkg/log/zap.go

@@ -13,11 +13,11 @@ import (
 )
 
 var (
-	logFolder    = "logs"
-	logDirConfig LogDir
+	Folder    = "logs"
+	dirConfig Dir
 )
 
-type LogDir struct {
+type Dir struct {
 	DefaultLogDir string
 	ErrorLogDir   string
 	DebugLogDir   string
@@ -56,14 +56,14 @@ func newZapLogger() (Logger, error) {
 	if v := os.Getenv("LOG_DIR"); v != "" {
 		kusionDataDir = v
 	}
-	logDirConfig = LogDir{
-		DefaultLogDir: filepath.Join(kusionDataDir, logFolder, "kusion.log"),
-		ErrorLogDir:   filepath.Join(kusionDataDir, logFolder, "kusion_error.log"),
-		DebugLogDir:   filepath.Join(kusionDataDir, logFolder, "kusion_debug.log"),
+	dirConfig = Dir{
+		DefaultLogDir: filepath.Join(kusionDataDir, Folder, "kusion.log"),
+		ErrorLogDir:   filepath.Join(kusionDataDir, Folder, "kusion_error.log"),
+		DebugLogDir:   filepath.Join(kusionDataDir, Folder, "kusion_debug.log"),
 	}
-	debugCore, debugAtom := newZapCore(logDirConfig.DebugLogDir, zapcore.DebugLevel)
-	defaultCore, defaultAtom := newZapCore(logDirConfig.DefaultLogDir, zapcore.InfoLevel)
-	errorCore, errorAtom := newZapCore(logDirConfig.ErrorLogDir, zapcore.ErrorLevel)
+	debugCore, debugAtom := newZapCore(dirConfig.DebugLogDir, zapcore.DebugLevel)
+	defaultCore, defaultAtom := newZapCore(dirConfig.DefaultLogDir, zapcore.InfoLevel)
+	errorCore, errorAtom := newZapCore(dirConfig.ErrorLogDir, zapcore.ErrorLevel)
 	combinedCore := zapcore.NewTee(defaultCore, errorCore, debugCore)
 	// AddCallerSkip skips 2 number of callers since the file that gets
 	// logged will always be the wrapped file.
@@ -131,8 +131,8 @@ func (l *zapLogger) SetLevel(level Level) {
 	}
 }
 
-func (l *zapLogger) GetLogDir() LogDir {
-	return logDirConfig
+func (l *zapLogger) GetLogDir() Dir {
+	return dirConfig
 }
 
 func (l *zapLogger) With(args ...interface{}) Logger {

pkg/modules/plugin.go

@@ -9,8 +9,10 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/hashicorp/go-hclog"
 	"github.com/hashicorp/go-plugin"
 
+	"kusionstack.io/kusion/pkg/log"
 	"kusionstack.io/kusion/pkg/util/kfile"
 )
 
@@ -71,11 +73,15 @@ func (p *Plugin) initModule() error {
 	if err != nil {
 		return err
 	}
-	client := newPluginClient(pluginPath)
+	pluginName := prefix[0] + "-" + prefix[1]
+	client, err := newPluginClient(pluginPath, pluginName)
+	if err != nil {
+		return err
+	}
 	p.client = client
 	rpcClient, err := client.Client()
 	if err != nil {
-		return err
+		return fmt.Errorf("init kusion module plugin: %s failed. %w", key, err)
 	}
 
 	// dispense the plugin to get the real module
@@ -112,18 +118,43 @@ func buildPluginPath(namespace, resourceType, version string) (string, error) {
 	return p, nil
 }
 
-func newPluginClient(path string) *plugin.Client {
-	// We're a host! Start by launching the plugin process.
-	// need to defer kill
+func newPluginClient(modulePluginPath, moduleName string) (*plugin.Client, error) {
+	// create the plugin log file
+	var logFilePath string
+	dir, err := kfile.KusionDataFolder()
+	if err != nil {
+		return nil, err
+	}
+	logDir := path.Join(dir, log.Folder, Dir)
+	if _, err := os.Stat(logDir); os.IsNotExist(err) {
+		if err := os.MkdirAll(logDir, os.ModePerm); err != nil {
+			return nil, fmt.Errorf("failed to create module log dir: %w", err)
+		}
+	}
+	logFilePath = path.Join(logDir, moduleName+".log")
+	logFile, err := os.Create(logFilePath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create module log file: %w", err)
+	}
+
+	// write log to a separate file
+	logger := hclog.New(&hclog.LoggerOptions{
+		Name:   moduleName,
+		Output: logFile,
+		Level:  hclog.Debug,
+	})
+
+	// We're a host! Start by launching the plugin process.Need to defer kill
 	client := plugin.NewClient(&plugin.ClientConfig{
 		HandshakeConfig: HandshakeConfig,
 		Plugins:         PluginMap,
-		Cmd:             exec.Command(path),
+		Cmd:             exec.Command(modulePluginPath),
 		AllowedProtocols: []plugin.Protocol{
 			plugin.ProtocolGRPC,
 		},
+		Logger: logger,
 	})
-	return client
+	return client, nil
 }
 
 func (p *Plugin) KillPluginClient() error {