Skip to content

Commit

Permalink
Offchain continue execution (#10195)
Browse files Browse the repository at this point in the history 
* simplify block subscriber to only use broadcaster

* add automation custom telem to job

* not expose constants

* addressed comments

* update channel size

* address comments

* verify check block and log block (#10154)

* verify check block and log block

* fix lint issue

* add some tests

* add more tests

* update comments

* refactor

* fix lint issue

* make check pipeline robust

* more refactoring

* fix lint issues

* fix tests

* make simulating perform upkeeps robust

* make feed lookup robust

* make mercury request robust

* fix tests

* use select

* add check upkeeps tests

* add simulate perform tests
  • Loading branch information
@FelixFan1992
FelixFan1992 authored Aug 15, 2023
1 parent 1f7a1fd commit 9550be0
Show file tree
Hide file tree
Showing 6 changed files with 559 additions and 176 deletions.
40 changes: 22 additions & 18 deletions core/services/ocr2/plugins/ocr2keeper/evm21/abi.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,12 +36,16 @@ const (
UpkeepFailureReasonTxHashNoLongerExists UpkeepFailureReason = 33

// pipeline execution error
NoPipelineError PipelineExecutionState = 0
CheckBlockTooOld PipelineExecutionState = 1
CheckBlockInvalid PipelineExecutionState = 2
RpcFlakyFailure PipelineExecutionState = 3
MercuryFlakyFailure PipelineExecutionState = 4
PackUnpackFailed PipelineExecutionState = 5
NoPipelineError PipelineExecutionState = 0
CheckBlockTooOld PipelineExecutionState = 1
CheckBlockInvalid PipelineExecutionState = 2
RpcFlakyFailure PipelineExecutionState = 3
MercuryFlakyFailure PipelineExecutionState = 4
PackUnpackDecodeFailed PipelineExecutionState = 5
MercuryUnmarshalError PipelineExecutionState = 6
InvalidMercuryRequest PipelineExecutionState = 7
FailedToReadMercuryResponse PipelineExecutionState = 8
InvalidRevertDataInput PipelineExecutionState = 9
)

var utilsABI = types.MustGetABI(automation_utils_2_1.AutomationUtilsABI)
Expand All @@ -62,19 +66,19 @@ func NewEvmRegistryPackerV2_1(abi abi.ABI, utilsAbi abi.ABI) *evmRegistryPackerV
}

func (rp *evmRegistryPackerV2_1) UnpackCheckResult(p ocr2keepers.UpkeepPayload, raw string) (ocr2keepers.CheckResult, error) {
var result ocr2keepers.CheckResult

b, err := hexutil.Decode(raw)
if err != nil {
return result, err
// decode failed, not retryable
return getIneligibleCheckResultWithoutPerformData(p, UpkeepFailureReasonNone, PackUnpackDecodeFailed, false), fmt.Errorf("upkeepId %s failed to decode checkUpkeep result %s: %s", p.UpkeepID.String(), raw, err)
}

out, err := rp.abi.Methods["checkUpkeep"].Outputs.UnpackValues(b)
if err != nil {
return result, fmt.Errorf("%w: unpack checkUpkeep return: %s", err, raw)
// unpack failed, not retryable
return getIneligibleCheckResultWithoutPerformData(p, UpkeepFailureReasonNone, PackUnpackDecodeFailed, false), fmt.Errorf("upkeepId %s failed to unpack checkUpkeep result %s: %s", p.UpkeepID.String(), raw, err)
}

result = ocr2keepers.CheckResult{
result := ocr2keepers.CheckResult{
Eligible: *abi.ConvertType(out[0], new(bool)).(*bool),
Retryable: false,
GasAllocated: uint64((*abi.ConvertType(out[4], new(*big.Int)).(**big.Int)).Int64()),
Expand All @@ -96,31 +100,31 @@ func (rp *evmRegistryPackerV2_1) UnpackCheckResult(p ocr2keepers.UpkeepPayload,
return result, nil
}

func (rp *evmRegistryPackerV2_1) UnpackCheckCallbackResult(callbackResp []byte) (bool, []byte, uint8, *big.Int, error) {
func (rp *evmRegistryPackerV2_1) UnpackCheckCallbackResult(callbackResp []byte) (PipelineExecutionState, bool, []byte, uint8, *big.Int, error) {
out, err := rp.abi.Methods["checkCallback"].Outputs.UnpackValues(callbackResp)
if err != nil {
return false, nil, 0, nil, fmt.Errorf("%w: unpack checkUpkeep return: %s", err, hexutil.Encode(callbackResp))
return PackUnpackDecodeFailed, false, nil, 0, nil, fmt.Errorf("%w: unpack checkUpkeep return: %s", err, hexutil.Encode(callbackResp))
}

upkeepNeeded := *abi.ConvertType(out[0], new(bool)).(*bool)
rawPerformData := *abi.ConvertType(out[1], new([]byte)).(*[]byte)
failureReason := *abi.ConvertType(out[2], new(uint8)).(*uint8)
gasUsed := *abi.ConvertType(out[3], new(*big.Int)).(**big.Int)
return upkeepNeeded, rawPerformData, failureReason, gasUsed, nil
return NoPipelineError, upkeepNeeded, rawPerformData, failureReason, gasUsed, nil
}

func (rp *evmRegistryPackerV2_1) UnpackPerformResult(raw string) (bool, error) {
func (rp *evmRegistryPackerV2_1) UnpackPerformResult(raw string) (PipelineExecutionState, bool, error) {
b, err := hexutil.Decode(raw)
if err != nil {
return false, err
return PackUnpackDecodeFailed, false, err
}

out, err := rp.abi.Methods["simulatePerformUpkeep"].Outputs.UnpackValues(b)
if err != nil {
return false, fmt.Errorf("%w: unpack simulatePerformUpkeep return: %s", err, raw)
return PackUnpackDecodeFailed, false, err
}

return *abi.ConvertType(out[0], new(bool)).(*bool), nil
return NoPipelineError, *abi.ConvertType(out[0], new(bool)).(*bool), nil
}

func (rp *evmRegistryPackerV2_1) UnpackUpkeepInfo(id *big.Int, raw string) (UpkeepInfo, error) {
Expand Down
57 changes: 50 additions & 7 deletions core/services/ocr2/plugins/ocr2keeper/evm21/abi_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package evm

import (
"fmt"
"math/big"
"strings"
"testing"
Expand All @@ -20,52 +21,88 @@ func TestUnpackCheckResults(t *testing.T) {
uid, _ := new(big.Int).SetString("1843548457736589226156809205796175506139185429616502850435279853710366065936", 10)
upkeepId := ocr2keepers.UpkeepIdentifier{}
upkeepId.FromBigInt(uid)
payload1, _ := core.NewUpkeepPayload(uid, ocr2keepers.NewTrigger(19447615, common.HexToHash("0x0")), []byte{})
p1, _ := core.NewUpkeepPayload(uid, ocr2keepers.NewTrigger(19447615, common.HexToHash("0x0")), []byte{})

payload2, _ := core.NewUpkeepPayload(uid, ocr2keepers.NewTrigger(19448272, common.HexToHash("0x0")), []byte{})
p2, _ := core.NewUpkeepPayload(uid, ocr2keepers.NewTrigger(19448272, common.HexToHash("0x0")), []byte{})

tests := []struct {
Name string
Payload ocr2keepers.UpkeepPayload
RawData string
ExpectedResult ocr2keepers.CheckResult
ExpectedError error
}{
{
Name: "upkeep not needed",
Payload: payload1,
Payload: p1,
RawData: "0x000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000c00000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000421c000000000000000000000000000000000000000000000000000000003b9aca00000000000000000000000000000000000000000000000000000c8caf37f3b3890000000000000000000000000000000000000000000000000000000000000000",
ExpectedResult: ocr2keepers.CheckResult{
UpkeepID: upkeepId,
Eligible: false,
IneligibilityReason: uint8(UpkeepFailureReasonUpkeepNotNeeded),
Trigger: ocr2keepers.NewLogTrigger(ocr2keepers.BlockNumber(19447615), [32]byte{}, nil),
WorkID: "e54c524132d9c8d87b7e43b76f6d769face19ffd2ff93fc24f123dd745d3ce1e",
PerformData: nil,
FastGasWei: big.NewInt(1000000000),
LinkNative: big.NewInt(3532383906411401),
},
},
{
Name: "target check reverted",
Payload: payload2,
Payload: p2,
RawData: "0x000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000c000000000000000000000000000000000000000000000000000000000000000030000000000000000000000000000000000000000000000000000000000007531000000000000000000000000000000000000000000000000000000003b9aca00000000000000000000000000000000000000000000000000000c8caf37f3b3890000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000008914039bf676e20aad43a5642485e666575ed0d927a4b5679745e947e7d125ee2687c10000000000000000000000000000000000000000000000000000000000000060000000000000000000000000000000000000000000000000000000000000024462e8a50d00000000000000000000000000000000000000000000000000000000000000a000000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000001c0000000000000000000000000000000000000000000000000000000000128c1d000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000009666565644944537472000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000040000000000000000000000000000000000000000000000000000000000000008000000000000000000000000000000000000000000000000000000000000000184554482d5553442d415242495452554d2d544553544e4554000000000000000000000000000000000000000000000000000000000000000000000000000000184254432d5553442d415242495452554d2d544553544e45540000000000000000000000000000000000000000000000000000000000000000000000000000000b626c6f636b4e756d6265720000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000014000000000000000000000000000000000000006400000000000000000000000000000000000000000000000000000000000000000000000000000000",
ExpectedResult: ocr2keepers.CheckResult{
UpkeepID: upkeepId,
Eligible: false,
IneligibilityReason: uint8(UpkeepFailureReasonTargetCheckReverted),
Trigger: ocr2keepers.NewLogTrigger(ocr2keepers.BlockNumber(19448272), [32]byte{}, nil),
WorkID: "e54c524132d9c8d87b7e43b76f6d769face19ffd2ff93fc24f123dd745d3ce1e",
PerformData: []byte{98, 232, 165, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 40, 193, 208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 102, 101, 101, 100, 73, 68, 83, 116, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 69, 84, 72, 45, 85, 83, 68, 45, 65, 82, 66, 73, 84, 82, 85, 77, 45, 84, 69, 83, 84, 78, 69, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 66, 84, 67, 45, 85, 83, 68, 45, 65, 82, 66, 73, 84, 82, 85, 77, 45, 84, 69, 83, 84, 78, 69, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 98, 108, 111, 99, 107, 78, 117, 109, 98, 101, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
FastGasWei: big.NewInt(1000000000),
LinkNative: big.NewInt(3532383906411401),
},
},
{
Name: "decode failed",
Payload: p2,
RawData: "invalid_raw_data",
ExpectedResult: ocr2keepers.CheckResult{
UpkeepID: upkeepId,
PipelineExecutionState: uint8(PackUnpackDecodeFailed),
Trigger: p2.Trigger,
WorkID: p2.WorkID,
},
ExpectedError: fmt.Errorf("upkeepId %s failed to decode checkUpkeep result invalid_raw_data: hex string without 0x prefix", p2.UpkeepID.String()),
},
{
Name: "unpack failed",
Payload: p2,
RawData: "0x123123",
ExpectedResult: ocr2keepers.CheckResult{
UpkeepID: upkeepId,
PipelineExecutionState: uint8(PackUnpackDecodeFailed),
Trigger: p2.Trigger,
WorkID: p2.WorkID,
},
ExpectedError: fmt.Errorf("upkeepId %s failed to unpack checkUpkeep result 0x123123: abi: cannot marshal in to go type: length insufficient 3 require 32", p2.UpkeepID.String()),
},
}
for _, test := range tests {
t.Run(test.Name, func(t *testing.T) {
packer, err := newPacker()
assert.NoError(t, err)
rs, err := packer.UnpackCheckResult(test.Payload, test.RawData)
assert.Nil(t, err)
if test.ExpectedError != nil {
assert.Equal(t, test.ExpectedError.Error(), err.Error())
} else {
assert.Nil(t, err)
}
assert.Equal(t, test.ExpectedResult.UpkeepID, rs.UpkeepID)
assert.Equal(t, test.ExpectedResult.Eligible, rs.Eligible)
assert.Equal(t, test.ExpectedResult.Trigger, rs.Trigger)
assert.Equal(t, test.ExpectedResult.WorkID, rs.WorkID)
assert.Equal(t, test.ExpectedResult.IneligibilityReason, rs.IneligibilityReason)
assert.Equal(t, test.ExpectedResult.PipelineExecutionState, rs.PipelineExecutionState)
})
}
}
Expand All @@ -74,19 +111,22 @@ func TestPacker_UnpackPerformResult(t *testing.T) {
tests := []struct {
Name string
RawData string
State PipelineExecutionState
}{
{
Name: "unpack success",
RawData: "0x0000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000a52d",
State: NoPipelineError,
},
}
for _, test := range tests {
t.Run(test.Name, func(t *testing.T) {
packer, err := newPacker()
assert.NoError(t, err)
rs, err := packer.UnpackPerformResult(test.RawData)
state, rs, err := packer.UnpackPerformResult(test.RawData)
assert.Nil(t, err)
assert.True(t, rs)
assert.Equal(t, test.State, state)
})
}
}
Expand All @@ -100,6 +140,7 @@ func TestPacker_UnpackCheckCallbackResult(t *testing.T) {
FailureReason uint8
GasUsed *big.Int
ErrorString string
State PipelineExecutionState
}{
{
Name: "unpack upkeep needed",
Expand All @@ -123,14 +164,15 @@ func TestPacker_UnpackCheckCallbackResult(t *testing.T) {
UpkeepNeeded: false,
PerformData: nil,
ErrorString: "abi: improperly encoded boolean value: unpack checkUpkeep return: ",
State: PackUnpackDecodeFailed,
},
}
for _, test := range tests {
t.Run(test.Name, func(t *testing.T) {
packer, err := newPacker()
assert.NoError(t, err)

needed, pd, failureReason, gasUsed, err := packer.UnpackCheckCallbackResult(test.CallbackResp)
state, needed, pd, failureReason, gasUsed, err := packer.UnpackCheckCallbackResult(test.CallbackResp)

if test.ErrorString != "" {
assert.EqualError(t, err, test.ErrorString+hexutil.Encode(test.CallbackResp))
Expand All @@ -141,6 +183,7 @@ func TestPacker_UnpackCheckCallbackResult(t *testing.T) {
assert.Equal(t, test.PerformData, pd)
assert.Equal(t, test.FailureReason, failureReason)
assert.Equal(t, test.GasUsed, gasUsed)
assert.Equal(t, test.State, state)
})
}
}
Expand Down
Loading

0 comments on commit 9550be0

Please sign in to comment.