feature/2268: added support for merging fields from multiple secrets files

core/cmd/app.go

@@ -11,7 +11,6 @@ import (
 	"github.com/urfave/cli"
 
 	"github.com/smartcontractkit/chainlink/v2/core/build"
-	"github.com/smartcontractkit/chainlink/v2/core/config/env"
 	"github.com/smartcontractkit/chainlink/v2/core/logger"
 	"github.com/smartcontractkit/chainlink/v2/core/services/chainlink"
 	"github.com/smartcontractkit/chainlink/v2/core/static"
@@ -61,16 +60,16 @@ func NewApp(s *Shell) *cli.App {
 			// Note: we cannot use the EnvVar field since it will combine with the flags.
 			Hidden: true,
 		},
-		cli.StringFlag{
+		cli.StringSliceFlag{
 			Name:   "secrets, s",
-			Usage:  "TOML configuration file for secrets. Must be set if and only if config is set.",
+			Usage:  "TOML configuration file for secrets. Must be set if and only if config is set. Multiple files can be used (-s secretsA.toml -s secretsB.toml), and they are applied in order. No overrides are allowed.",
 			Hidden: true,
 		},
 	}
 	app.Before = func(c *cli.Context) error {
 		s.configFiles = c.StringSlice("config")
 		s.configFilesIsSet = c.IsSet("config")
-		s.secretsFile = c.String("secrets")
+		s.secretsFiles = c.StringSlice("secrets")
 		s.secretsFileIsSet = c.IsSet("secrets")
 
 		// Default to using a stdout logger only.
@@ -119,7 +118,7 @@ func NewApp(s *Shell) *cli.App {
 
 		// Allow for initServerConfig to be called if the flag is provided.
 		if c.Bool("applyInitServerConfig") {
-			cfg, err = initServerConfig(&opts, s.configFiles, s.secretsFile)
+			cfg, err = initServerConfig(&opts, s.configFiles, s.secretsFiles)
 			if err != nil {
 				return err
 			}
@@ -200,9 +199,9 @@ func NewApp(s *Shell) *cli.App {
 					Name:  "config, c",
 					Usage: "TOML configuration file(s) via flag, or raw TOML via env var. If used, legacy env vars must not be set. Multiple files can be used (-c configA.toml -c configB.toml), and they are applied in order with duplicated fields overriding any earlier values. If the 'CL_CONFIG' env var is specified, it is always processed last with the effect of being the final override. [$CL_CONFIG]",
 				},
-				cli.StringFlag{
+				cli.StringSliceFlag{
 					Name:  "secrets, s",
-					Usage: "TOML configuration file for secrets. Must be set if and only if config is set.",
+					Usage: "TOML configuration file for secrets. Must be set if and only if config is set. Multiple files can be used (-s secretsA.toml -s secretsB.toml), and fields from the files will be merged. No overrides are allowed.",
 				},
 			},
 			Before: func(c *cli.Context) error {
@@ -219,12 +218,12 @@ func NewApp(s *Shell) *cli.App {
 					if s.configFilesIsSet || s.secretsFileIsSet {
 						return errNoDuplicateFlags
 					} else {
-						s.secretsFile = c.String("secrets")
+						s.secretsFiles = c.StringSlice("secrets")
 					}
 				}
 
 				// flags here, or ENV VAR only
-				cfg, err := initServerConfig(&opts, s.configFiles, s.secretsFile)
+				cfg, err := initServerConfig(&opts, s.configFiles, s.secretsFiles)
 				if err != nil {
 					return err
 				}
@@ -312,33 +311,10 @@ func format(s string) string {
 	return string(whitespace.ReplaceAll([]byte(s), []byte(" ")))
 }
 
-func initServerConfig(opts *chainlink.GeneralConfigOpts, configFiles []string, secretsFile string) (chainlink.GeneralConfig, error) {
-	configs := []string{}
-	for _, fileName := range configFiles {
-		b, err := os.ReadFile(fileName)
-		if err != nil {
-			return nil, errors.Wrapf(err, "failed to read config file: %s", fileName)
-		}
-		configs = append(configs, string(b))
-	}
-
-	if configTOML := env.Config.Get(); configTOML != "" {
-		configs = append(configs, configTOML)
+func initServerConfig(opts *chainlink.GeneralConfigOpts, configFiles []string, secretsFiles []string) (chainlink.GeneralConfig, error) {
+	err := opts.Setup(configFiles, secretsFiles)
+	if err != nil {
+		return nil, err
 	}
-
-	opts.ConfigStrings = configs
-
-	secrets := ""
-	if secretsFile != "" {
-		b, err := os.ReadFile(secretsFile)
-		if err != nil {
-			return nil, errors.Wrapf(err, "failed to read secrets file: %s", secretsFile)
-		}
-
-		secrets = string(b)
-	}
-
-	opts.SecretsString = secrets
-
 	return opts.New()
 }

core/cmd/app_test.go

@@ -2,16 +2,14 @@ package cmd
 
 import (
 	"fmt"
-	"os"
-	"path/filepath"
 	"testing"
 
-	gotoml "github.com/pelletier/go-toml/v2"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
 	"github.com/smartcontractkit/chainlink/v2/core/config/env"
 	"github.com/smartcontractkit/chainlink/v2/core/config/toml"
+	testtomlutils "github.com/smartcontractkit/chainlink/v2/core/internal/testutils/configtest/v2/toml"
 	"github.com/smartcontractkit/chainlink/v2/core/services/chainlink"
 	"github.com/smartcontractkit/chainlink/v2/core/store/models"
 )
@@ -51,17 +49,6 @@ var (
 	}
 )
 
-func makeTestFile(t *testing.T, contents any, fileName string) string {
-	d := t.TempDir()
-	p := filepath.Join(d, fileName)
-
-	b, err := gotoml.Marshal(contents)
-	require.NoError(t, err)
-
-	require.NoError(t, os.WriteFile(p, b, 0777))
-	return p
-}
-
 func withDefaults(t *testing.T, c chainlink.Config, s chainlink.Secrets) chainlink.GeneralConfig {
 	cfg, err := chainlink.GeneralConfigOpts{Config: c, Secrets: s}.New()
 	require.NoError(t, err)
@@ -70,10 +57,10 @@ func withDefaults(t *testing.T, c chainlink.Config, s chainlink.Secrets) chainli
 
 func Test_initServerConfig(t *testing.T) {
 	type args struct {
-		opts        *chainlink.GeneralConfigOpts
-		fileNames   []string
-		secretsFile string
-		envVar      string
+		opts         *chainlink.GeneralConfigOpts
+		fileNames    []string
+		secretsFiles []string
+		envVar       string
 	}
 	tests := []struct {
 		name    string
@@ -101,7 +88,7 @@ func Test_initServerConfig(t *testing.T) {
 			name: "files only",
 			args: args{
 				opts:      new(chainlink.GeneralConfigOpts),
-				fileNames: []string{makeTestFile(t, testConfigFileContents, "test.toml")},
+				fileNames: []string{testtomlutils.WriteTOMLFile(t, testConfigFileContents, "test.toml")},
 			},
 			wantCfg: withDefaults(t, testConfigFileContents, chainlink.Secrets{}),
 		},
@@ -117,7 +104,7 @@ func Test_initServerConfig(t *testing.T) {
 			name: "env overlay of file",
 			args: args{
 				opts:      new(chainlink.GeneralConfigOpts),
-				fileNames: []string{makeTestFile(t, testConfigFileContents, "test.toml")},
+				fileNames: []string{testtomlutils.WriteTOMLFile(t, testConfigFileContents, "test.toml")},
 				envVar:    testEnvContents,
 			},
 			wantCfg: withDefaults(t, chainlink.Config{
@@ -136,32 +123,136 @@ func Test_initServerConfig(t *testing.T) {
 		{
 			name: "failed to read secrets",
 			args: args{
-				opts:        new(chainlink.GeneralConfigOpts),
-				fileNames:   []string{makeTestFile(t, testConfigFileContents, "test.toml")},
-				secretsFile: "/doesnt-exist",
+				opts:         new(chainlink.GeneralConfigOpts),
+				fileNames:    []string{testtomlutils.WriteTOMLFile(t, testConfigFileContents, "test.toml")},
+				secretsFiles: []string{"/doesnt-exist"},
 			},
 			wantErr: true,
 		},
 		{
 			name: "reading secrets",
 			args: args{
-				opts:        new(chainlink.GeneralConfigOpts),
-				fileNames:   []string{makeTestFile(t, testConfigFileContents, "test.toml")},
-				secretsFile: makeTestFile(t, testSecretsFileContents, "test_secrets.toml"),
+				opts:         new(chainlink.GeneralConfigOpts),
+				fileNames:    []string{testtomlutils.WriteTOMLFile(t, testConfigFileContents, "test.toml")},
+				secretsFiles: []string{testtomlutils.WriteTOMLFile(t, testSecretsFileContents, "test_secrets.toml")},
 			},
 			wantCfg: withDefaults(t, testConfigFileContents, testSecretsRedactedContents),
 		},
+		{
+			name: "reading multiple secrets",
+			args: args{
+				opts:      new(chainlink.GeneralConfigOpts),
+				fileNames: []string{testtomlutils.WriteTOMLFile(t, testConfigFileContents, "test.toml")},
+				secretsFiles: []string{
+					"../services/chainlink/testdata/mergingsecretsdata/secrets-database.toml",
+					"../services/chainlink/testdata/mergingsecretsdata/secrets-explorer.toml",
+					"../services/chainlink/testdata/mergingsecretsdata/secrets-password.toml",
+					"../services/chainlink/testdata/mergingsecretsdata/secrets-pyroscope.toml",
+					"../services/chainlink/testdata/mergingsecretsdata/secrets-prometheus.toml",
+					"../services/chainlink/testdata/mergingsecretsdata/secrets-mercury-split-one.toml",
+					"../services/chainlink/testdata/mergingsecretsdata/secrets-mercury-split-two.toml",
+					"../services/chainlink/testdata/mergingsecretsdata/secrets-threshold.toml",
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "reading multiple secrets with overrides: Database",
+			args: args{
+				opts:      new(chainlink.GeneralConfigOpts),
+				fileNames: []string{testtomlutils.WriteTOMLFile(t, testConfigFileContents, "test.toml")},
+				secretsFiles: []string{
+					"../testdata/mergingsecretsdata/secrets-database.toml",
+					"../testdata/mergingsecretsdata/secrets-database.toml",
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "reading multiple secrets with overrides: Explorer",
+			args: args{
+				opts:      new(chainlink.GeneralConfigOpts),
+				fileNames: []string{testtomlutils.WriteTOMLFile(t, testConfigFileContents, "test.toml")},
+				secretsFiles: []string{
+					"../testdata/mergingsecretsdata/secrets-explorer.toml",
+					"../testdata/mergingsecretsdata/secrets-explorer.toml",
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "reading multiple secrets with overrides: Password",
+			args: args{
+				opts:      new(chainlink.GeneralConfigOpts),
+				fileNames: []string{testtomlutils.WriteTOMLFile(t, testConfigFileContents, "test.toml")},
+				secretsFiles: []string{
+					"../testdata/mergingsecretsdata/secrets-password.toml",
+					"../testdata/mergingsecretsdata/secrets-password.toml",
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "reading multiple secrets with overrides: Pyroscope",
+			args: args{
+				opts:      new(chainlink.GeneralConfigOpts),
+				fileNames: []string{testtomlutils.WriteTOMLFile(t, testConfigFileContents, "test.toml")},
+				secretsFiles: []string{
+					"../testdata/mergingsecretsdata/secrets-pyroscope.toml",
+					"../testdata/mergingsecretsdata/secrets-pyroscope.toml",
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "reading multiple secrets with overrides: Prometheus",
+			args: args{
+				opts:      new(chainlink.GeneralConfigOpts),
+				fileNames: []string{testtomlutils.WriteTOMLFile(t, testConfigFileContents, "test.toml")},
+				secretsFiles: []string{
+					"../testdata/mergingsecretsdata/secrets-prometheus.toml",
+					"../testdata/mergingsecretsdata/secrets-prometheus.toml",
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "reading multiple secrets with overrides: Mercury",
+			args: args{
+				opts:      new(chainlink.GeneralConfigOpts),
+				fileNames: []string{testtomlutils.WriteTOMLFile(t, testConfigFileContents, "test.toml")},
+				secretsFiles: []string{
+					"../testdata/mergingsecretsdata/secrets-mercury-split-one.toml",
+					"../testdata/mergingsecretsdata/secrets-mercury-split-one.toml",
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "reading multiple secrets with overrides: Threshold",
+			args: args{
+				opts:      new(chainlink.GeneralConfigOpts),
+				fileNames: []string{testtomlutils.WriteTOMLFile(t, testConfigFileContents, "test.toml")},
+				secretsFiles: []string{
+					"../testdata/mergingsecretsdata/secrets-threshold.toml",
+					"../testdata/mergingsecretsdata/secrets-threshold.toml",
+				},
+			},
+			wantErr: true,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			if tt.args.envVar != "" {
 				t.Setenv(string(env.Config), tt.args.envVar)
 			}
-			cfg, err := initServerConfig(tt.args.opts, tt.args.fileNames, tt.args.secretsFile)
+			cfg, err := initServerConfig(tt.args.opts, tt.args.fileNames, tt.args.secretsFiles)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("loadOpts() error = %v, wantErr %v", err, tt.wantErr)
 			}
-			assert.Equal(t, cfg, tt.wantCfg)
+			if tt.wantCfg != nil {
+				assert.Equal(t, tt.wantCfg, cfg)
+			}
 		})
 	}
 }

core/cmd/shell.go

@@ -100,7 +100,7 @@ type Shell struct {
 
 	configFiles      []string
 	configFilesIsSet bool
-	secretsFile      string
+	secretsFiles     []string
 	secretsFileIsSet bool
 }