Merge branch 'develop' into VRF-892

.github/workflows/automation-nightly-tests.yml

@@ -54,18 +54,30 @@ jobs:
     env:
       CHAINLINK_COMMIT_SHA: ${{ github.sha }}
       CHAINLINK_ENV_USER: ${{ github.actor }}
-      TEST_LOG_LEVEL: debug
+      TEST_LOG_LEVEL: info
       SELECTED_NETWORKS: "SIMULATED"
     strategy:
       fail-fast: false
       matrix:
         tests:
-          - name: Upgrade
+          - name: Upgrade 2.0
             suite: smoke
-            nodes: 6
+            nodes: 1
             os: ubuntu20.04-8cores-32GB
             network: SIMULATED
-            command: -run ^TestAutomationNodeUpgrade$ ./smoke
+            command: -run ^TestAutomationNodeUpgrade/registry_2_0 ./smoke
+          - name: Upgrade 2.1
+            suite: smoke
+            nodes: 5
+            os: ubuntu20.04-8cores-32GB
+            network: SIMULATED
+            command: -run ^TestAutomationNodeUpgrade/registry_2_1 ./smoke
+          - name: Upgrade 2.2
+            suite: smoke
+            nodes: 5
+            os: ubuntu20.04-8cores-32GB
+            network: SIMULATED
+            command: -run ^TestAutomationNodeUpgrade/registry_2_2 ./smoke
     runs-on: ${{ matrix.tests.os }}
     name: Automation ${{ matrix.tests.name }} Test
     steps:
@@ -184,7 +196,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        name: [ Upgrade ]
+        name: [ Upgrade 2.0, Upgrade 2.1, Upgrade 2.2 ]
     steps:
       - name: Get Results
         id: test-results

.github/workflows/automation-ondemand-tests.yml

@@ -121,32 +121,46 @@ jobs:
     env:
       CHAINLINK_COMMIT_SHA: ${{ github.sha }}
       CHAINLINK_ENV_USER: ${{ github.actor }}
-      TEST_LOG_LEVEL: debug
+      TEST_LOG_LEVEL: info
     strategy:
       fail-fast: false
       matrix:
         tests:
           - name: chaos
             suite: chaos
-            nodes: 5
+            nodes: 15
             os: ubuntu-latest
             pyroscope_env: ci-automation-on-demand-chaos
             network: SIMULATED
             command: -run ^TestAutomationChaos$ ./chaos
           - name: reorg
             suite: reorg
-            nodes: 1
+            nodes: 5
             os: ubuntu-latest
             pyroscope_env: ci-automation-on-demand-reorg
             network: SIMULATED_NONDEV
             command: -run ^TestAutomationReorg$ ./reorg
-          - name: upgrade
+          - name: upgrade 2.0
+            suite: smoke
+            nodes: 1
+            os: ubuntu20.04-8cores-32GB
+            pyroscope_env: ci-automation-on-demand-upgrade
+            network: SIMULATED
+            command: -run ^TestAutomationNodeUpgrade/registry_2_0 ./smoke
+          - name: upgrade 2.1
             suite: smoke
-            nodes: 6
+            nodes: 1
+            os: ubuntu20.04-8cores-32GB
+            pyroscope_env: ci-automation-on-demand-upgrade
+            network: SIMULATED
+            command: -run ^TestAutomationNodeUpgrade/registry_2_1 ./smoke
+          - name: upgrade 2.2
+            suite: smoke
+            nodes: 1
             os: ubuntu20.04-8cores-32GB
             pyroscope_env: ci-automation-on-demand-upgrade
             network: SIMULATED
-            command: -run ^TestAutomationNodeUpgrade$ ./smoke
+            command: -run ^TestAutomationNodeUpgrade/registry_2_2 ./smoke
     runs-on: ${{ matrix.tests.os }}
     name: Automation On Demand ${{ matrix.tests.name }} Test
     steps:

core/capabilities/targets/write_target.go

@@ -22,14 +22,15 @@ import (
 	evmtypes "github.com/smartcontractkit/chainlink/v2/core/chains/evm/types"
 	"github.com/smartcontractkit/chainlink/v2/core/chains/legacyevm"
 	"github.com/smartcontractkit/chainlink/v2/core/gethwrappers/keystone/generated/forwarder"
+	"github.com/smartcontractkit/chainlink/v2/core/logger"
 	"github.com/smartcontractkit/chainlink/v2/core/services/pipeline"
 )
 
 var forwardABI = evmtypes.MustGetABI(forwarder.KeystoneForwarderMetaData.ABI)
 
-func InitializeWrite(registry commontypes.CapabilitiesRegistry, legacyEVMChains legacyevm.LegacyChainContainer) error {
+func InitializeWrite(registry commontypes.CapabilitiesRegistry, legacyEVMChains legacyevm.LegacyChainContainer, lggr logger.Logger) error {
 	for _, chain := range legacyEVMChains.Slice() {
-		capability := NewEvmWrite(chain)
+		capability := NewEvmWrite(chain, lggr)
 		if err := registry.Add(context.TODO(), capability); err != nil {
 			return err
 		}
@@ -41,12 +42,15 @@ var (
 	_ capabilities.ActionCapability = &EvmWrite{}
 )
 
+const defaultGasLimit = 200000
+
 type EvmWrite struct {
 	chain legacyevm.Chain
 	capabilities.CapabilityInfo
+	lggr logger.Logger
 }
 
-func NewEvmWrite(chain legacyevm.Chain) *EvmWrite {
+func NewEvmWrite(chain legacyevm.Chain, lggr logger.Logger) *EvmWrite {
 	// generate ID based on chain selector
 	name := fmt.Sprintf("write_%v", chain.ID())
 	chainName, err := chainselectors.NameFromChainId(chain.ID().Uint64())
@@ -64,6 +68,7 @@ func NewEvmWrite(chain legacyevm.Chain) *EvmWrite {
 	return &EvmWrite{
 		chain,
 		info,
+		lggr.Named("EvmWrite"),
 	}
 }
 
@@ -153,6 +158,7 @@ func encodePayload(args []any, rawSelector string) ([]byte, error) {
 }
 
 func (cap *EvmWrite) Execute(ctx context.Context, callback chan<- capabilities.CapabilityResponse, request capabilities.CapabilityRequest) error {
+	cap.lggr.Debugw("Execute", "request", request)
 	// TODO: idempotency
 
 	// TODO: extract into ChainWriter?
@@ -184,8 +190,6 @@ func (cap *EvmWrite) Execute(ctx context.Context, callback chan<- capabilities.C
 
 	// TODO: validate encoded report is prefixed with workflowID and executionID that match the request meta
 
-	// unlimited gas in the MVP demo
-	gasLimit := 0
 	// No signature validation in the MVP demo
 	signatures := [][]byte{}
 
@@ -208,7 +212,7 @@ func (cap *EvmWrite) Execute(ctx context.Context, callback chan<- capabilities.C
 		FromAddress:    config.FromAddress().Address(),
 		ToAddress:      config.ForwarderAddress().Address(),
 		EncodedPayload: calldata,
-		FeeLimit:       uint32(gasLimit),
+		FeeLimit:       uint32(defaultGasLimit),
 		Meta:           txMeta,
 		Strategy:       strategy,
 		Checker:        checker,

core/capabilities/targets/write_target_test.go

@@ -15,6 +15,7 @@ import (
 	"github.com/smartcontractkit/chainlink/v2/core/internal/testutils"
 	"github.com/smartcontractkit/chainlink/v2/core/internal/testutils/configtest"
 	"github.com/smartcontractkit/chainlink/v2/core/internal/testutils/evmtest"
+	"github.com/smartcontractkit/chainlink/v2/core/logger"
 	"github.com/smartcontractkit/chainlink/v2/core/services/chainlink"
 	"github.com/smartcontractkit/chainlink/v2/core/services/keystore/keys/ethkey"
 
@@ -45,7 +46,7 @@ func TestEvmWrite(t *testing.T) {
 	evmcfg := evmtest.NewChainScopedConfig(t, cfg)
 	chain.On("Config").Return(evmcfg)
 
-	capability := targets.NewEvmWrite(chain)
+	capability := targets.NewEvmWrite(chain, logger.TestLogger(t))
 	ctx := testutils.Context(t)
 
 	config, err := values.NewMap(map[string]any{

core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/active_list.go

@@ -9,6 +9,7 @@ import (
 	ocr2keepers "github.com/smartcontractkit/chainlink-common/pkg/types/automation"
 
 	"github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/core"
+	"github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/prommetrics"
 )
 
 // ActiveUpkeepList is a list to manage active upkeep IDs
@@ -49,9 +50,10 @@ func (al *activeList) Reset(ids ...*big.Int) {
 	for _, id := range ids {
 		al.items[id.String()] = true
 	}
+	prommetrics.AutomationActiveUpkeeps.Set(float64(len(al.items)))
 }
 
-// Add adds new entries to the list
+// Add adds new entries to the list. Returns the number of items added
 func (al *activeList) Add(ids ...*big.Int) int {
 	al.lock.Lock()
 	defer al.lock.Unlock()
@@ -63,10 +65,11 @@ func (al *activeList) Add(ids ...*big.Int) int {
 			al.items[key] = true
 		}
 	}
+	prommetrics.AutomationActiveUpkeeps.Set(float64(len(al.items)))
 	return count
 }
 
-// Remove removes entries from the list
+// Remove removes entries from the list. Returns the number of items removed
 func (al *activeList) Remove(ids ...*big.Int) int {
 	al.lock.Lock()
 	defer al.lock.Unlock()
@@ -79,6 +82,7 @@ func (al *activeList) Remove(ids ...*big.Int) int {
 			delete(al.items, key)
 		}
 	}
+	prommetrics.AutomationActiveUpkeeps.Set(float64(len(al.items)))
 	return count
 }
 

core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/logprovider/buffer.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/smartcontractkit/chainlink/v2/core/chains/evm/logpoller"
 	"github.com/smartcontractkit/chainlink/v2/core/logger"
+	"github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/prommetrics"
 )
 
 var (
@@ -230,6 +231,7 @@ func (b *logEventBuffer) enqueue(id *big.Int, logs ...logpoller.Log) int {
 	}
 	if added > 0 {
 		lggr.Debugw("Added logs to buffer", "addedLogs", added, "dropped", dropped, "latestBlock", latestBlock)
+		prommetrics.AutomationLogsInLogBuffer.Add(float64(added - dropped))
 	}
 
 	return added - dropped
@@ -331,6 +333,7 @@ func (b *logEventBuffer) dequeueRange(start, end int64, upkeepLimit, totalLimit
 
 	if len(results) > 0 {
 		b.lggr.Debugw("Dequeued logs", "results", len(results), "start", start, "end", end)
+		prommetrics.AutomationLogsInLogBuffer.Sub(float64(len(results)))
 	}
 
 	return results

core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/logprovider/provider.go

@@ -24,6 +24,7 @@ import (
 	"github.com/smartcontractkit/chainlink/v2/core/gethwrappers/generated/automation_utils_2_1"
 	"github.com/smartcontractkit/chainlink/v2/core/logger"
 	"github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/core"
+	"github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/prommetrics"
 	"github.com/smartcontractkit/chainlink/v2/core/services/pg"
 	"github.com/smartcontractkit/chainlink/v2/core/utils"
 )
@@ -162,6 +163,7 @@ func (p *logEventProvider) GetLatestPayloads(ctx context.Context) ([]ocr2keepers
 	if err != nil {
 		return nil, fmt.Errorf("%w: %s", ErrHeadNotAvailable, err)
 	}
+	prommetrics.AutomationLogProviderLatestBlock.Set(float64(latest.BlockNumber))
 	start := latest.BlockNumber - p.opts.LookbackBlocks
 	if start <= 0 {
 		start = 1

core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/logprovider/recoverer.go

@@ -27,6 +27,7 @@ import (
 	"github.com/smartcontractkit/chainlink/v2/core/chains/evm/logpoller"
 	"github.com/smartcontractkit/chainlink/v2/core/logger"
 	"github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/core"
+	"github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/prommetrics"
 	"github.com/smartcontractkit/chainlink/v2/core/services/pg"
 	"github.com/smartcontractkit/chainlink/v2/core/utils"
 )
@@ -305,7 +306,7 @@ func (r *logRecoverer) GetRecoveryProposals(ctx context.Context) ([]ocr2keepers.
 	var results, pending []ocr2keepers.UpkeepPayload
 	for _, payload := range r.pending {
 		if allLogsCounter >= MaxProposals {
-			// we have enough proposals, pushed the rest are pushed back to pending
+			// we have enough proposals, the rest are pushed back to pending
 			pending = append(pending, payload)
 			continue
 		}
@@ -321,6 +322,7 @@ func (r *logRecoverer) GetRecoveryProposals(ctx context.Context) ([]ocr2keepers.
 	}
 
 	r.pending = pending
+	prommetrics.AutomationRecovererPendingPayloads.Set(float64(len(r.pending)))
 
 	r.lggr.Debugf("found %d recoverable payloads", len(results))
 
@@ -417,6 +419,7 @@ func (r *logRecoverer) recoverFilter(ctx context.Context, f upkeepFilter, startB
 	added, alreadyPending, ok := r.populatePending(f, filteredLogs)
 	if added > 0 {
 		r.lggr.Debugw("found missed logs", "added", added, "alreadyPending", alreadyPending, "upkeepID", f.upkeepID)
+		prommetrics.AutomationRecovererMissedLogs.Add(float64(added))
 	}
 	if !ok {
 		r.lggr.Debugw("failed to add all logs to pending", "upkeepID", f.upkeepID)
@@ -673,6 +676,7 @@ func (r *logRecoverer) addPending(payload ocr2keepers.UpkeepPayload) error {
 	}
 	if !exist {
 		r.pending = append(pending, payload)
+		prommetrics.AutomationRecovererPendingPayloads.Inc()
 	}
 	return nil
 }
@@ -684,6 +688,8 @@ func (r *logRecoverer) removePending(workID string) {
 	for _, p := range r.pending {
 		if p.WorkID != workID {
 			updated = append(updated, p)
+		} else {
+			prommetrics.AutomationRecovererPendingPayloads.Dec()
 		}
 	}
 	r.pending = updated

core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/prommetrics/metrics.go

@@ -0,0 +1,38 @@
+package prommetrics
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+// AutomationNamespace is the namespace for all Automation related metrics
+const AutomationLogTriggerNamespace = "automation_log_trigger"
+
+// Automation metrics
+var (
+	AutomationLogsInLogBuffer = promauto.NewGauge(prometheus.GaugeOpts{
+		Namespace: AutomationLogTriggerNamespace,
+		Name:      "num_logs_in_log_buffer",
+		Help:      "The total number of logs currently being stored in the log buffer",
+	})
+	AutomationRecovererMissedLogs = promauto.NewCounter(prometheus.CounterOpts{
+		Namespace: AutomationLogTriggerNamespace,
+		Name:      "num_recoverer_missed_logs",
+		Help:      "How many valid log triggers were identified as being missed by the recoverer",
+	})
+	AutomationRecovererPendingPayloads = promauto.NewGauge(prometheus.GaugeOpts{
+		Namespace: AutomationLogTriggerNamespace,
+		Name:      "num_recoverer_pending_payloads",
+		Help:      "How many log trigger payloads are currently pending in the recoverer",
+	})
+	AutomationActiveUpkeeps = promauto.NewGauge(prometheus.GaugeOpts{
+		Namespace: AutomationLogTriggerNamespace,
+		Name:      "num_active_upkeeps",
+		Help:      "How many log trigger upkeeps are currently active",
+	})
+	AutomationLogProviderLatestBlock = promauto.NewGauge(prometheus.GaugeOpts{
+		Namespace: AutomationLogTriggerNamespace,
+		Name:      "log_provider_latest_block",
+		Help:      "The latest block number the log provider has seen",
+	})
+)