core/plugins: add per-plugin env vars (#11526)

* core/plugins: add per-plugin env vars

* Adding coverage (#11851)

* add TestIntegration_OCR2_plugins behind build tag; swap in freeport for fixed const

---------

Co-authored-by: Patrick <patrick.huie@smartcontract.com>

.github/workflows/ci-core.yml

@@ -71,6 +71,17 @@ jobs:
         run: go build -o chainlink.test .
       - name: Setup DB
         run: ./chainlink.test local db preparetest
+      - name: Install LOOP Plugins
+        run: |
+          pushd $(go list -m -f "{{.Dir}}" github.com/smartcontractkit/chainlink-feeds)
+          go install ./cmd/chainlink-feeds
+          popd
+          pushd $(go list -m -f "{{.Dir}}" github.com/smartcontractkit/chainlink-solana)
+          go install ./pkg/solana/cmd/chainlink-solana
+          popd
+          pushd $(go list -m -f "{{.Dir}}" github.com/smartcontractkit/chainlink-starknet/relayer)
+          go install ./pkg/chainlink/cmd/chainlink-starknet
+          popd
       - name: Increase Race Timeout
         if: github.event.schedule != ''
         run: |

core/cmd/shell_test.go

@@ -380,6 +380,17 @@ func TestSetupSolanaRelayer(t *testing.T) {
 		}
 	})
 
+	t2Config := configtest.NewGeneralConfig(t, func(c *chainlink.Config, s *chainlink.Secrets) {
+		c.Solana = solana.TOMLConfigs{
+			&solana.TOMLConfig{
+				ChainID: ptr[string]("solana-id-1"),
+				Enabled: ptr(true),
+				Chain:   solcfg.Chain{},
+				Nodes:   []*solcfg.Node{},
+			},
+		}
+	})
+
 	rf := chainlink.RelayerFactory{
 		Logger:       lggr,
 		LoopRegistry: reg,
@@ -435,6 +446,23 @@ func TestSetupSolanaRelayer(t *testing.T) {
 		_, err := rf.NewSolana(ks, duplicateConfig.SolanaConfigs())
 		require.Error(t, err)
 	})
+
+	t.Run("plugin env parsing fails", func(t *testing.T) {
+		t.Setenv("CL_SOLANA_CMD", "phony_solana_cmd")
+		t.Setenv("CL_SOLANA_ENV", "fake_path")
+
+		_, err := rf.NewSolana(ks, t2Config.SolanaConfigs())
+		require.Error(t, err)
+		require.Contains(t, err.Error(), "failed to parse Solana env file")
+	})
+
+	t.Run("plugin already registered", func(t *testing.T) {
+		t.Setenv("CL_SOLANA_CMD", "phony_solana_cmd")
+
+		_, err := rf.NewSolana(ks, tConfig.SolanaConfigs())
+		require.Error(t, err)
+		require.Contains(t, err.Error(), "failed to create Solana LOOP command")
+	})
 }
 
 func TestSetupStarkNetRelayer(t *testing.T) {
@@ -465,6 +493,17 @@ func TestSetupStarkNetRelayer(t *testing.T) {
 			},
 		}
 	})
+
+	t2Config := configtest.NewGeneralConfig(t, func(c *chainlink.Config, s *chainlink.Secrets) {
+		c.Starknet = stkcfg.TOMLConfigs{
+			&stkcfg.TOMLConfig{
+				ChainID: ptr[string]("starknet-id-3"),
+				Enabled: ptr(true),
+				Chain:   stkcfg.Chain{},
+				Nodes:   []*config.Node{},
+			},
+		}
+	})
 	rf := chainlink.RelayerFactory{
 		Logger:       lggr,
 		LoopRegistry: reg,
@@ -520,6 +559,23 @@ func TestSetupStarkNetRelayer(t *testing.T) {
 		_, err := rf.NewStarkNet(ks, duplicateConfig.StarknetConfigs())
 		require.Error(t, err)
 	})
+
+	t.Run("plugin env parsing fails", func(t *testing.T) {
+		t.Setenv("CL_STARKNET_CMD", "phony_starknet_cmd")
+		t.Setenv("CL_STARKNET_ENV", "fake_path")
+
+		_, err := rf.NewStarkNet(ks, t2Config.StarknetConfigs())
+		require.Error(t, err)
+		require.Contains(t, err.Error(), "failed to parse Starknet env file")
+	})
+
+	t.Run("plugin already registered", func(t *testing.T) {
+		t.Setenv("CL_STARKNET_CMD", "phony_starknet_cmd")
+
+		_, err := rf.NewStarkNet(ks, tConfig.StarknetConfigs())
+		require.Error(t, err)
+		require.Contains(t, err.Error(), "failed to create StarkNet LOOP command")
+	})
 }
 
 // flagSetApplyFromAction applies the flags from action to the flagSet.

core/config/env/env.go

@@ -1,19 +1,32 @@
 package env
 
 import (
+	"fmt"
 	"os"
 	"strings"
 
 	"github.com/smartcontractkit/chainlink/v2/core/store/models"
 )
 
 var (
-	Config = Var("CL_CONFIG")
+	Config                       = Var("CL_CONFIG")
+	DatabaseAllowSimplePasswords = Var("CL_DATABASE_ALLOW_SIMPLE_PASSWORDS")
+	DatabaseURL                  = Secret("CL_DATABASE_URL")
+	DatabaseBackupURL            = Secret("CL_DATABASE_BACKUP_URL")
+	PasswordKeystore             = Secret("CL_PASSWORD_KEYSTORE")
+	PasswordVRF                  = Secret("CL_PASSWORD_VRF")
+	PyroscopeAuthToken           = Secret("CL_PYROSCOPE_AUTH_TOKEN")
+	PrometheusAuthToken          = Secret("CL_PROMETHEUS_AUTH_TOKEN")
+	ThresholdKeyShare            = Secret("CL_THRESHOLD_KEY_SHARE")
+	// Migrations env vars
+	EVMChainIDNotNullMigration0195 = "CL_EVM_CHAINID_NOT_NULL_MIGRATION_0195"
+)
 
-	// LOOPP commands and vars
-	MedianPluginCmd   = Var("CL_MEDIAN_CMD")
-	SolanaPluginCmd   = Var("CL_SOLANA_CMD")
-	StarknetPluginCmd = Var("CL_STARKNET_CMD")
+// LOOPP commands and vars
+var (
+	MedianPlugin   = NewPlugin("median")
+	SolanaPlugin   = NewPlugin("solana")
+	StarknetPlugin = NewPlugin("starknet")
 	// PrometheusDiscoveryHostName is the externally accessible hostname
 	// published by the node in the `/discovery` endpoint. Generally, it is expected to match
 	// the public hostname of node.
@@ -22,24 +35,13 @@ var (
 	// In house we observed that the resolved value of os.Hostname was not accessible to
 	// outside of the given pod
 	PrometheusDiscoveryHostName = Var("CL_PROMETHEUS_DISCOVERY_HOSTNAME")
-	// EnvLooopHostName is the hostname used for HTTP communication between the
+	// LOOPPHostName is the hostname used for HTTP communication between the
 	// node and LOOPps. In most cases this does not need to be set explicitly.
 	LOOPPHostName = Var("CL_LOOPP_HOSTNAME")
 	// Work around for Solana LOOPPs configured with zero values.
 	MinOCR2MaxDurationQuery = Var("CL_MIN_OCR2_MAX_DURATION_QUERY")
 	// PipelineOvertime is an undocumented escape hatch for overriding the default padding in pipeline executions.
 	PipelineOvertime = Var("CL_PIPELINE_OVERTIME")
-
-	DatabaseAllowSimplePasswords = Var("CL_DATABASE_ALLOW_SIMPLE_PASSWORDS")
-	DatabaseURL                  = Secret("CL_DATABASE_URL")
-	DatabaseBackupURL            = Secret("CL_DATABASE_BACKUP_URL")
-	PasswordKeystore             = Secret("CL_PASSWORD_KEYSTORE")
-	PasswordVRF                  = Secret("CL_PASSWORD_VRF")
-	PyroscopeAuthToken           = Secret("CL_PYROSCOPE_AUTH_TOKEN")
-	PrometheusAuthToken          = Secret("CL_PROMETHEUS_AUTH_TOKEN")
-	ThresholdKeyShare            = Secret("CL_THRESHOLD_KEY_SHARE")
-	// Migrations env vars
-	EVMChainIDNotNullMigration0195 = "CL_EVM_CHAINID_NOT_NULL_MIGRATION_0195"
 )
 
 type Var string
@@ -54,3 +56,16 @@ func (e Var) IsTrue() bool { return strings.ToLower(e.Get()) == "true" }
 type Secret string
 
 func (e Secret) Get() models.Secret { return models.Secret(os.Getenv(string(e))) }
+
+type Plugin struct {
+	Cmd Var
+	Env Var
+}
+
+func NewPlugin(kind string) Plugin {
+	kind = strings.ToUpper(kind)
+	return Plugin{
+		Cmd: Var(fmt.Sprintf("CL_%s_CMD", kind)),
+		Env: Var(fmt.Sprintf("CL_%s_ENV", kind)),
+	}
+}

core/config/env/env_test.go

@@ -0,0 +1,24 @@
+package env
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestNewPlugin(t *testing.T) {
+	for _, tt := range []struct {
+		name string
+		kind string
+		exp  Plugin
+	}{
+		{"lower", "foo", Plugin{Cmd: "CL_FOO_CMD", Env: "CL_FOO_ENV"}},
+		{"upper", "BAR", Plugin{Cmd: "CL_BAR_CMD", Env: "CL_BAR_ENV"}},
+		{"mixed", "Baz", Plugin{Cmd: "CL_BAZ_CMD", Env: "CL_BAZ_ENV"}},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			got := NewPlugin(tt.kind)
+			require.Equal(t, tt.exp, got)
+		})
+	}
+}

core/internal/features/ocr2/features_ocr2_plugin_test.go

@@ -0,0 +1,14 @@
+//go:build integration
+
+package ocr2_test
+
+import (
+	"testing"
+
+	"github.com/smartcontractkit/chainlink/v2/core/config/env"
+)
+
+func TestIntegration_OCR2_plugins(t *testing.T) {
+	t.Setenv(string(env.MedianPlugin.Cmd), "chainlink-feeds")
+	testIntegration_OCR2(t)
+}

core/internal/features/ocr2/features_ocr2_test.go

@@ -189,7 +189,10 @@ func setupNodeOCR2(
 
 func TestIntegration_OCR2(t *testing.T) {
 	t.Parallel()
+	testIntegration_OCR2(t)
+}
 
+func testIntegration_OCR2(t *testing.T) {
 	for _, test := range []struct {
 		name                string
 		chainReaderAndCodec bool
@@ -199,8 +202,6 @@ func TestIntegration_OCR2(t *testing.T) {
 	} {
 		test := test
 		t.Run(test.name, func(t *testing.T) {
-			t.Parallel()
-
 			owner, b, ocrContractAddress, ocrContract := setupOCR2Contracts(t)
 
 			lggr := logger.TestLogger(t)

core/scripts/go.mod

@@ -166,6 +166,8 @@ require (
 	github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect
 	github.com/gtank/merlin v0.1.1 // indirect
 	github.com/gtank/ristretto255 v0.1.2 // indirect
+	github.com/hashicorp/consul/sdk v0.14.1 // indirect
+	github.com/hashicorp/go-envparse v0.1.0 // indirect
 	github.com/hashicorp/go-hclog v1.5.0 // indirect
 	github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
 	github.com/hashicorp/go-plugin v1.5.2 // indirect

core/scripts/go.sum

@@ -694,6 +694,8 @@ github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtng
 github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
 github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
 github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
+github.com/hashicorp/go-envparse v0.1.0 h1:bE++6bhIsNCPLvgDZkYqo3nA+/PFI51pkrHdmPSDFPY=
+github.com/hashicorp/go-envparse v0.1.0/go.mod h1:OHheN1GoygLlAkTlXLXvAdnXdZxy8JUweQ1rAXx1xnc=
 github.com/hashicorp/go-getter v1.7.1 h1:SWiSWN/42qdpR0MdhaOc/bLR48PLuP1ZQtYLRlM69uY=
 github.com/hashicorp/go-getter v1.7.1/go.mod h1:W7TalhMmbPmsSMdNjD0ZskARur/9GJ17cfHTRtXV744=
 github.com/hashicorp/go-hclog v1.5.0 h1:bI2ocEMgcVlz55Oj1xZNBsVi900c7II+fWDyV9o+13c=
@@ -711,8 +713,8 @@ github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerX
 github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
 github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
-github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE=
-github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8=
+github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek=
 github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=

core/services/chainlink/relayer_factory.go

@@ -116,7 +116,7 @@ func (r *RelayerFactory) NewSolana(ks keystore.Solana, chainCfgs solana.TOMLConf
 
 		lggr := solLggr.Named(relayID.ChainID)
 
-		if cmdName := env.SolanaPluginCmd.Get(); cmdName != "" {
+		if cmdName := env.SolanaPlugin.Cmd.Get(); cmdName != "" {
 
 			// setup the solana relayer to be a LOOP
 			cfgTOML, err := toml.Marshal(struct {
@@ -126,10 +126,14 @@ func (r *RelayerFactory) NewSolana(ks keystore.Solana, chainCfgs solana.TOMLConf
 			if err != nil {
 				return nil, fmt.Errorf("failed to marshal Solana configs: %w", err)
 			}
-
+			envVars, err := plugins.ParseEnvFile(env.SolanaPlugin.Env.Get())
+			if err != nil {
+				return nil, fmt.Errorf("failed to parse Solana env file: %w", err)
+			}
 			solCmdFn, err := plugins.NewCmdFactory(r.Register, plugins.CmdConfig{
 				ID:  relayID.Name(),
 				Cmd: cmdName,
+				Env: envVars,
 			})
 			if err != nil {
 				return nil, fmt.Errorf("failed to create Solana LOOP command: %w", err)
@@ -187,7 +191,7 @@ func (r *RelayerFactory) NewStarkNet(ks keystore.StarkNet, chainCfgs config.TOML
 
 		lggr := starkLggr.Named(relayID.ChainID)
 
-		if cmdName := env.StarknetPluginCmd.Get(); cmdName != "" {
+		if cmdName := env.StarknetPlugin.Cmd.Get(); cmdName != "" {
 			// setup the starknet relayer to be a LOOP
 			cfgTOML, err := toml.Marshal(struct {
 				Starknet config.TOMLConfig
@@ -196,9 +200,14 @@ func (r *RelayerFactory) NewStarkNet(ks keystore.StarkNet, chainCfgs config.TOML
 				return nil, fmt.Errorf("failed to marshal StarkNet configs: %w", err)
 			}
 
+			envVars, err := plugins.ParseEnvFile(env.StarknetPlugin.Env.Get())
+			if err != nil {
+				return nil, fmt.Errorf("failed to parse Starknet env file: %w", err)
+			}
 			starknetCmdFn, err := plugins.NewCmdFactory(r.Register, plugins.CmdConfig{
 				ID:  relayID.Name(),
 				Cmd: cmdName,
+				Env: envVars,
 			})
 			if err != nil {
 				return nil, fmt.Errorf("failed to create StarkNet LOOP command: %w", err)

core/services/ocr2/delegate.go

@@ -25,6 +25,7 @@ import (
 	ocr2keepers20runner "github.com/smartcontractkit/chainlink-automation/pkg/v2/runner"
 	ocr2keepers21config "github.com/smartcontractkit/chainlink-automation/pkg/v3/config"
 	ocr2keepers21 "github.com/smartcontractkit/chainlink-automation/pkg/v3/plugin"
+	"github.com/smartcontractkit/chainlink/v2/core/config/env"
 
 	"github.com/smartcontractkit/chainlink-vrf/altbn_128"
 	dkgpkg "github.com/smartcontractkit/chainlink-vrf/dkg"
@@ -504,12 +505,6 @@ type connProvider interface {
 	ClientConn() grpc.ClientConnInterface
 }
 
-func defaultPathFromPluginName(pluginName string) string {
-	// By default we install the command on the system path, in the
-	// form: `chainlink-<plugin name>`
-	return fmt.Sprintf("chainlink-%s", pluginName)
-}
-
 func (d *Delegate) newServicesGenericPlugin(
 	ctx context.Context,
 	lggr logger.SugaredLogger,
@@ -530,9 +525,11 @@ func (d *Delegate) newServicesGenericPlugin(
 		return nil, err
 	}
 
+	plugEnv := env.NewPlugin(p.PluginName)
+
 	command := p.Command
 	if command == "" {
-		command = defaultPathFromPluginName(p.PluginName)
+		command = plugEnv.Cmd.Get()
 	}
 
 	// Add the default pipeline to the pluginConfig
@@ -587,8 +584,22 @@ func (d *Delegate) newServicesGenericPlugin(
 		OffchainConfigDigester:       provider.OffchainConfigDigester(),
 	}
 
+	envVars, err := plugins.ParseEnvFile(plugEnv.Env.Get())
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse median env file: %w", err)
+	}
+	if len(p.EnvVars) > 0 {
+		for k, v := range p.EnvVars {
+			envVars = append(envVars, k+"="+v)
+		}
+	}
+
 	pluginLggr := lggr.Named(p.PluginName).Named(spec.ContractID).Named(spec.GetID())
-	cmdFn, grpcOpts, err := d.cfg.RegisterLOOP(fmt.Sprintf("%s-%s-%s", p.PluginName, spec.ContractID, spec.GetID()), command)
+	cmdFn, grpcOpts, err := d.cfg.RegisterLOOP(plugins.CmdConfig{
+		ID:  fmt.Sprintf("%s-%s-%s", p.PluginName, spec.ContractID, spec.GetID()),
+		Cmd: command,
+		Env: envVars,
+	})
 	if err != nil {
 		return nil, fmt.Errorf("failed to register loop: %w", err)
 	}