non-dev geth container building (#10213)

integration-tests/actions/actions.go

@@ -85,7 +85,7 @@ func FundChainlinkNodesAddress(
 
 // FundChainlinkNodesAddress will fund all of the provided Chainlink nodes addresses with a set amount of native currency
 func FundChainlinkNodesAddresses(
-	nodes []*client.ChainlinkK8sClient,
+	nodes []*client.ChainlinkClient,
 	client blockchain.EVMClient,
 	amount *big.Float,
 ) error {

integration-tests/client/chainlink_models.go

@@ -598,7 +598,7 @@ schedule          = "{{.Schedule}}"
 observationSource = """
 {{.ObservationSource}}
 """`
-	return marshallTemplate(c, "CRON Job", cronJobTemplateString)
+	return MarshallTemplate(c, "CRON Job", cronJobTemplateString)
 }
 
 // PipelineSpec common API call pipeline
@@ -618,7 +618,7 @@ func (d *PipelineSpec) String() (string, error) {
 		fetch [type=bridge name="{{.BridgeTypeAttributes.Name}}" requestData="{{.BridgeTypeAttributes.RequestData}}"];
 		parse [type=jsonparse path="{{.DataPath}}"];
 		fetch -> parse;`
-	return marshallTemplate(d, "API call pipeline template", sourceString)
+	return MarshallTemplate(d, "API call pipeline template", sourceString)
 }
 
 // VRFV2TxPipelineSpec VRFv2 request with tx callback
@@ -655,7 +655,7 @@ simulate [type=ethcall
           contract="{{ .Address }}"
           data="$(vrf.output)"]
 decode_log->vrf->estimate_gas->simulate`
-	return marshallTemplate(d, "VRFV2 pipeline template", sourceString)
+	return MarshallTemplate(d, "VRFV2 pipeline template", sourceString)
 }
 
 // VRFTxPipelineSpec VRF request with tx callback
@@ -687,7 +687,7 @@ submit_tx  [type=ethtx to="{{.Address}}"
             data="$(encode_tx)"
             txMeta="{\\"requestTxHash\\": $(jobRun.logTxHash),\\"requestID\\": $(decode_log.requestID),\\"jobID\\": $(jobSpec.databaseID)}"]
 decode_log->vrf->encode_tx->submit_tx`
-	return marshallTemplate(d, "VRF pipeline template", sourceString)
+	return MarshallTemplate(d, "VRF pipeline template", sourceString)
 }
 
 // DirectRequestTxPipelineSpec oracle request with tx callback
@@ -719,7 +719,7 @@ func (d *DirectRequestTxPipelineSpec) String() (string, error) {
 			parse  [type=jsonparse path="{{.DataPath}}"]
             submit [type=ethtx to="$(decode_log.requester)" data="$(encode_tx)" failOnRevert=true]
 			decode_log -> fetch -> parse -> encode_tx -> submit`
-	return marshallTemplate(d, "Direct request pipeline template", sourceString)
+	return MarshallTemplate(d, "Direct request pipeline template", sourceString)
 }
 
 // DirectRequestJobSpec represents a direct request spec
@@ -746,7 +746,7 @@ minIncomingConfirmations = {{.MinIncomingConfirmations}}
 observationSource = """
 {{.ObservationSource}}
 """`
-	return marshallTemplate(d, "Direct Request Job", directRequestTemplateString)
+	return MarshallTemplate(d, "Direct Request Job", directRequestTemplateString)
 }
 
 // FluxMonitorJobSpec represents a flux monitor spec
@@ -788,7 +788,7 @@ maxTaskDuration = {{if not .Precision}} "180s" {{else}} {{.Precision}} {{end}}
 observationSource = """
 {{.ObservationSource}}
 """`
-	return marshallTemplate(f, "Flux Monitor Job", fluxMonitorTemplateString)
+	return MarshallTemplate(f, "Flux Monitor Job", fluxMonitorTemplateString)
 }
 
 // KeeperJobSpec represents a V2 keeper spec
@@ -812,7 +812,7 @@ contractAddress          = "{{.ContractAddress}}"
 fromAddress              = "{{.FromAddress}}"
 minIncomingConfirmations = {{.MinIncomingConfirmations}}
 `
-	return marshallTemplate(k, "Keeper Job", keeperTemplateString)
+	return MarshallTemplate(k, "Keeper Job", keeperTemplateString)
 }
 
 // OCRBootstrapJobSpec represents the spec for bootstrapping an OCR job, given to one node that then must be linked
@@ -843,7 +843,7 @@ contractAddress                        = "{{.ContractAddress}}"
 p2pBootstrapPeers                      = []
 isBootstrapPeer                        = {{.IsBootstrapPeer}}
 p2pPeerID                              = "{{.P2PPeerID}}"`
-	return marshallTemplate(o, "OCR Bootstrap Job", ocrTemplateString)
+	return MarshallTemplate(o, "OCR Bootstrap Job", ocrTemplateString)
 }
 
 // OCRTaskJobSpec represents an OCR job that is given to other nodes, meant to communicate with the bootstrap node,
@@ -954,7 +954,7 @@ observationSource                      = """
 {{.ObservationSource}}
 """`
 
-	return marshallTemplate(specWrap, "OCR Job", ocrTemplateString)
+	return MarshallTemplate(specWrap, "OCR Job", ocrTemplateString)
 }
 
 // OCR2TaskJobSpec represents an OCR2 job that is given to other nodes, meant to communicate with the bootstrap node,
@@ -1062,7 +1062,7 @@ observationSource                      = """
 [relayConfig]{{range $key, $value := .RelayConfig}}
 {{$key}} = {{$value}}{{end}}
 `
-	return marshallTemplate(specWrap, "OCR2 Job", ocr2TemplateString)
+	return MarshallTemplate(specWrap, "OCR2 Job", ocr2TemplateString)
 }
 
 // VRFV2JobSpec represents a VRFV2 job
@@ -1102,7 +1102,7 @@ observationSource = """
 {{.ObservationSource}}
 """
 `
-	return marshallTemplate(v, "VRFV2 Job", vrfTemplateString)
+	return MarshallTemplate(v, "VRFV2 Job", vrfTemplateString)
 }
 
 // VRFJobSpec represents a VRF job
@@ -1132,7 +1132,7 @@ observationSource = """
 {{.ObservationSource}}
 """
 `
-	return marshallTemplate(v, "VRF Job", vrfTemplateString)
+	return MarshallTemplate(v, "VRF Job", vrfTemplateString)
 }
 
 // BlockhashStoreJobSpec represents a blockhashstore job
@@ -1164,7 +1164,7 @@ pollPeriod               = "{{.PollPeriod}}"
 runTimeout               = "{{.RunTimeout}}"
 evmChainID               = "{{.EVMChainID}}"
 `
-	return marshallTemplate(b, "BlockhashStore Job", vrfTemplateString)
+	return MarshallTemplate(b, "BlockhashStore Job", vrfTemplateString)
 }
 
 // WebhookJobSpec reprsents a webhook job
@@ -1189,7 +1189,7 @@ externalInitiators = [
 observationSource = """
 {{.ObservationSource}}
 """`
-	return marshallTemplate(w, "Webhook Job", webHookTemplateString)
+	return MarshallTemplate(w, "Webhook Job", webHookTemplateString)
 }
 
 // ObservationSourceSpecHTTP creates a http GET task spec for json data
@@ -1209,7 +1209,7 @@ func ObservationSourceSpecBridge(bta *BridgeTypeAttributes) string {
 }
 
 // marshallTemplate Helper to marshall templates
-func marshallTemplate(jobSpec interface{}, name, templateString string) (string, error) {
+func MarshallTemplate(jobSpec interface{}, name, templateString string) (string, error) {
 	var buf bytes.Buffer
 	tmpl, err := template.New(name).Parse(templateString)
 	if err != nil {

integration-tests/docker/test_env/cl_node.go

@@ -5,6 +5,7 @@ import (
 	"crypto/ed25519"
 	"encoding/hex"
 	"fmt"
+	"math/big"
 	"net/url"
 	"os"
 	"strings"
@@ -15,22 +16,21 @@ import (
 	"github.com/pelletier/go-toml/v2"
 	"github.com/pkg/errors"
 
-	"math/big"
-
 	"github.com/ethereum/go-ethereum"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/rs/zerolog/log"
 	"github.com/smartcontractkit/chainlink-testing-framework/blockchain"
 	"github.com/smartcontractkit/chainlink-testing-framework/logwatch"
+	ocrtypes "github.com/smartcontractkit/libocr/offchainreporting2/types"
+	"github.com/smartcontractkit/libocr/offchainreporting2plus/confighelper"
+	tc "github.com/testcontainers/testcontainers-go"
+	tcwait "github.com/testcontainers/testcontainers-go/wait"
+
 	"github.com/smartcontractkit/chainlink/integration-tests/client"
 	"github.com/smartcontractkit/chainlink/integration-tests/utils"
 	"github.com/smartcontractkit/chainlink/integration-tests/utils/templates"
 	"github.com/smartcontractkit/chainlink/v2/core/services/chainlink"
 	"github.com/smartcontractkit/chainlink/v2/core/services/keystore/chaintype"
-	ocrtypes "github.com/smartcontractkit/libocr/offchainreporting2/types"
-	"github.com/smartcontractkit/libocr/offchainreporting2plus/confighelper"
-	tc "github.com/testcontainers/testcontainers-go"
-	tcwait "github.com/testcontainers/testcontainers-go/wait"
 )
 
 var (
@@ -257,6 +257,7 @@ func (n *ClNode) StartContainer() error {
 		return errors.Wrap(err, ErrConnectNodeClient)
 	}
 
+	clClient.Config.InternalIP = n.ContainerName
 	n.Container = container
 	n.API = clClient
 

integration-tests/docker/test_env/mockserver.go

@@ -92,7 +92,7 @@ func (ms *MockServer) StartContainer() error {
 func (ms *MockServer) getContainerRequest() tc.ContainerRequest {
 	return tc.ContainerRequest{
 		Name:         ms.ContainerName,
-		Image:        "mockserver/mockserver:5.11.2",
+		Image:        "mockserver/mockserver:5.15.0",
 		ExposedPorts: []string{"1080/tcp"},
 		Env: map[string]string{
 			"SERVER_PORT": "1080",

integration-tests/docker/test_env/test_env.go

@@ -1,22 +1,25 @@
 package test_env
 
 import (
-	"github.com/smartcontractkit/chainlink-testing-framework/blockchain"
-	"github.com/smartcontractkit/chainlink/integration-tests/contracts"
+	"math/big"
 	"sync"
 
-	"math/big"
+	"github.com/smartcontractkit/chainlink-testing-framework/blockchain"
+	"github.com/smartcontractkit/chainlink-testing-framework/docker/test_env"
+
+	"github.com/smartcontractkit/chainlink/integration-tests/contracts"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog/log"
 	"github.com/smartcontractkit/chainlink-testing-framework/logwatch"
+	tc "github.com/testcontainers/testcontainers-go"
+	"go.uber.org/multierr"
+
 	"github.com/smartcontractkit/chainlink/integration-tests/client"
 	"github.com/smartcontractkit/chainlink/integration-tests/docker"
 	"github.com/smartcontractkit/chainlink/integration-tests/utils"
 	"github.com/smartcontractkit/chainlink/v2/core/services/chainlink"
-	tc "github.com/testcontainers/testcontainers-go"
-	"go.uber.org/multierr"
 )
 
 var (
@@ -31,7 +34,8 @@ type CLClusterTestEnv struct {
 
 	/* components */
 	CLNodes          []*ClNode
-	Geth             *Geth
+	Geth             *Geth                       // for tests using --dev networks
+	PrivateGethChain []test_env.PrivateGethChain // for tests using non-dev networks
 	MockServer       *MockServer
 	EVMClient        blockchain.EVMClient
 	ContractDeployer contracts.ContractDeployer
@@ -72,6 +76,30 @@ func (te *CLClusterTestEnv) ParallelTransactions(enabled bool) {
 	te.EVMClient.ParallelTransactions(enabled)
 }
 
+func (m *CLClusterTestEnv) WithPrivateGethChain(evmNetworks []blockchain.EVMNetwork) *CLClusterTestEnv {
+	var chains []test_env.PrivateGethChain
+	for _, evmNetwork := range evmNetworks {
+		n := evmNetwork
+		chains = append(chains, test_env.NewPrivateGethChain(&n, []string{m.Network.Name}))
+	}
+	m.PrivateGethChain = chains
+	return m
+}
+
+func (m *CLClusterTestEnv) StartPrivateGethChain() error {
+	for _, chain := range m.PrivateGethChain {
+		err := chain.PrimaryNode.Start()
+		if err != nil {
+			return err
+		}
+		err = chain.PrimaryNode.ConnectToClient()
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func (te *CLClusterTestEnv) StartGeth() (blockchain.EVMNetwork, InternalDockerUrls, error) {
 	return te.Geth.StartContainer()
 }

integration-tests/docker/test_env/test_env_builder.go

@@ -1,15 +1,18 @@
 package test_env
 
 import (
+	"math/big"
+	"os"
+
+	"github.com/pkg/errors"
 	"github.com/smartcontractkit/chainlink-testing-framework/blockchain"
+	"github.com/smartcontractkit/chainlink-testing-framework/logwatch"
+
 	"github.com/smartcontractkit/chainlink/integration-tests/contracts"
 	"github.com/smartcontractkit/chainlink/integration-tests/networks"
-	"os"
-
-	"math/big"
 
 	"github.com/rs/zerolog/log"
-	"github.com/smartcontractkit/chainlink-testing-framework/logwatch"
+
 	"github.com/smartcontractkit/chainlink/integration-tests/types/config/node"
 	"github.com/smartcontractkit/chainlink/v2/core/services/chainlink"
 )
@@ -20,6 +23,7 @@ type CLTestEnvBuilder struct {
 	hasMockServer        bool
 	hasForwarders        bool
 	clNodeConfig         *chainlink.Config
+	nonDevGethNetworks   []blockchain.EVMNetwork
 	clNodesCount         int
 	externalAdapterCount int
 	customNodeCsaKeys    []string
@@ -65,6 +69,11 @@ func (b *CLTestEnvBuilder) WithGeth() *CLTestEnvBuilder {
 	return b
 }
 
+func (m *CLTestEnvBuilder) WithPrivateGethChains(evmNetworks []blockchain.EVMNetwork) *CLTestEnvBuilder {
+	m.nonDevGethNetworks = evmNetworks
+	return m
+}
+
 func (b *CLTestEnvBuilder) WithCLNodeConfig(cfg *chainlink.Config) *CLTestEnvBuilder {
 	b.clNodeConfig = cfg
 	return b
@@ -130,13 +139,36 @@ func (b *CLTestEnvBuilder) buildNewEnv(cfg *TestEnvConfig) (*CLClusterTestEnv, e
 			return nil, err
 		}
 	}
+	if b.nonDevGethNetworks != nil {
+		te.WithPrivateGethChain(b.nonDevGethNetworks)
+		err := te.StartPrivateGethChain()
+		if err != nil {
+			return te, err
+		}
+		var nonDevGethNetworks []blockchain.EVMNetwork
+		for i, n := range te.PrivateGethChain {
+			nonDevGethNetworks = append(nonDevGethNetworks, *n.NetworkConfig)
+			nonDevGethNetworks[i].URLs = []string{n.PrimaryNode.InternalWsUrl}
+			nonDevGethNetworks[i].HTTPURLs = []string{n.PrimaryNode.InternalHttpUrl}
+		}
+		if nonDevGethNetworks == nil {
+			return nil, errors.New("cannot create nodes with custom config without nonDevGethNetworks")
+		}
+
+		err = te.StartClNodes(b.clNodeConfig, b.clNodesCount)
+		if err != nil {
+			return nil, err
+		}
+		return te, nil
+	}
 	networkConfig := networks.SelectedNetwork
 	var internalDockerUrls InternalDockerUrls
 	if b.hasGeth && networkConfig.Simulated {
 		networkConfig, internalDockerUrls, err = te.StartGeth()
 		if err != nil {
 			return nil, err
 		}
+
 	}
 
 	bc, err := blockchain.NewEVMClientFromNetwork(networkConfig)

integration-tests/go.mod

@@ -20,7 +20,7 @@ require (
 	github.com/rs/zerolog v1.29.1
 	github.com/slack-go/slack v0.12.2
 	github.com/smartcontractkit/chainlink-env v0.36.0
-	github.com/smartcontractkit/chainlink-testing-framework v1.15.2
+	github.com/smartcontractkit/chainlink-testing-framework v1.15.3
 	github.com/smartcontractkit/chainlink/v2 v2.0.0-00010101000000-000000000000
 	github.com/smartcontractkit/libocr v0.0.0-20230802221916-2271752fa829
 	github.com/smartcontractkit/ocr2keepers v0.7.9

integration-tests/go.sum

@@ -2247,8 +2247,8 @@ github.com/smartcontractkit/chainlink-solana v1.0.3-0.20230802143301-165000751a8
 github.com/smartcontractkit/chainlink-solana v1.0.3-0.20230802143301-165000751a85/go.mod h1:H3/j2l84FsxYevCLNERdVasI7FVr+t2mkpv+BCJLSVw=
 github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20230802150127-d2c95679d61a h1:b3rjvZLpTV45TmCV+ALX+EDDslf91pnDUugP54Lu9FA=
 github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20230802150127-d2c95679d61a/go.mod h1:LL+FLf10gOUHrF3aUsRGEZlT/w8DaW5T/eEo/54W68c=
-github.com/smartcontractkit/chainlink-testing-framework v1.15.2 h1:WhZvHHz104RKC9u5u+zHeP4YwyyFVG3bW0DVRSe4FnE=
-github.com/smartcontractkit/chainlink-testing-framework v1.15.2/go.mod h1:WxEBntXbe20FgTZllV5fo4fKLd0TM8NfYdWkX5kt6Mg=
+github.com/smartcontractkit/chainlink-testing-framework v1.15.3 h1:phuO4wpbT3GR2ZFIzgwQ3DjfLaZwtHOvJkuEJOC8vxk=
+github.com/smartcontractkit/chainlink-testing-framework v1.15.3/go.mod h1:m9p6TzNHs4SXfsS3MJHjTk2vF8jsqHb2ipSSYkjT6zs=
 github.com/smartcontractkit/go-plugin v0.0.0-20230605132010-0f4d515d1472 h1:x3kNwgFlDmbE/n0gTSRMt9GBDfsfGrs4X9b9arPZtFI=
 github.com/smartcontractkit/go-plugin v0.0.0-20230605132010-0f4d515d1472/go.mod h1:6/1TEzT0eQznvI/gV2CM29DLSkAK/e58mUWKVsPaph0=
 github.com/smartcontractkit/grpc-proxy v0.0.0-20230731113816-f1be6620749f h1:hgJif132UCdjo8u43i7iPN1/MFnu49hv7lFGFftCHKU=