Merge branch 'develop' into fix/log-poller-created-at

.github/workflows/ci-core.yml

@@ -264,8 +264,15 @@ jobs:
         env:
           GRAFANA_CLOUD_BASIC_AUTH: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }}
           GRAFANA_CLOUD_HOST: ${{ secrets.GRAFANA_CLOUD_HOST }}
+          GITHUB_EVENT_PATH: ${{ github.event_path }}
         run: |
-          ./runner -grafana_auth=$GRAFANA_CLOUD_BASIC_AUTH -grafana_host=$GRAFANA_CLOUD_HOST -command go_core_tests `ls -R ./artifacts/go_core_tests*/output.txt`
+          ./runner \
+            -grafana_auth=$GRAFANA_CLOUD_BASIC_AUTH \
+            -grafana_host=$GRAFANA_CLOUD_HOST \
+            -gh_sha=$GITHUB_SHA \
+            -gh_event_path=$GITHUB_EVENT_PATH \
+            -command=./tools/bin/go_core_tests \
+            `ls -R ./artifacts/go_core_tests*/output.txt`
       - name: Store logs artifacts
         if: always()
         uses: actions/upload-artifact@3cea5372237819ed00197afe530f5a7ea3e805c8 # v3.1.0

core/cmd/shell.go

@@ -35,7 +35,6 @@ import (
 	"github.com/smartcontractkit/chainlink-relay/pkg/loop"
 	pkgsolana "github.com/smartcontractkit/chainlink-solana/pkg/solana"
 	pkgstarknet "github.com/smartcontractkit/chainlink-starknet/relayer/pkg/chainlink"
-
 	"github.com/smartcontractkit/chainlink/v2/core/build"
 	"github.com/smartcontractkit/chainlink/v2/core/chains/cosmos"
 	"github.com/smartcontractkit/chainlink/v2/core/chains/evm"
@@ -45,6 +44,7 @@ import (
 	"github.com/smartcontractkit/chainlink/v2/core/config/env"
 	"github.com/smartcontractkit/chainlink/v2/core/logger"
 	"github.com/smartcontractkit/chainlink/v2/core/logger/audit"
+	"github.com/smartcontractkit/chainlink/v2/core/services"
 	"github.com/smartcontractkit/chainlink/v2/core/services/chainlink"
 	"github.com/smartcontractkit/chainlink/v2/core/services/keystore"
 	"github.com/smartcontractkit/chainlink/v2/core/services/periodicbackup"
@@ -135,7 +135,7 @@ type ChainlinkAppFactory struct{}
 func (n ChainlinkAppFactory) NewApplication(ctx context.Context, cfg chainlink.GeneralConfig, appLggr logger.Logger, db *sqlx.DB) (app chainlink.Application, err error) {
 	initGlobals(cfg.Prometheus())
 
-	err = handleNodeVersioning(db, appLggr, cfg.RootDir(), cfg.Database())
+	err = handleNodeVersioning(db, appLggr, cfg.RootDir(), cfg.Database(), cfg.WebServer().HTTPPort())
 	if err != nil {
 		return nil, err
 	}
@@ -335,7 +335,7 @@ func (r relayerFactory) NewStarkNet(ks keystore.StarkNet) (loop.Relayer, error)
 }
 
 // handleNodeVersioning is a setup-time helper to encapsulate version changes and db migration
-func handleNodeVersioning(db *sqlx.DB, appLggr logger.Logger, rootDir string, cfg config.Database) error {
+func handleNodeVersioning(db *sqlx.DB, appLggr logger.Logger, rootDir string, cfg config.Database, healthReportPort uint16) error {
 	var err error
 	// Set up the versioning Configs
 	verORM := versioning.NewORM(db, appLggr, cfg.DefaultQueryTimeout())
@@ -352,7 +352,7 @@ func handleNodeVersioning(db *sqlx.DB, appLggr logger.Logger, rootDir string, cf
 		// Need to do this BEFORE migration
 		backupCfg := cfg.Backup()
 		if backupCfg.Mode() != config.DatabaseBackupModeNone && backupCfg.OnVersionUpgrade() {
-			if err = takeBackupIfVersionUpgrade(cfg.URL(), rootDir, cfg.Backup(), appLggr, appv, dbv); err != nil {
+			if err = takeBackupIfVersionUpgrade(cfg.URL(), rootDir, cfg.Backup(), appLggr, appv, dbv, healthReportPort); err != nil {
 				if errors.Is(err, sql.ErrNoRows) {
 					appLggr.Debugf("Failed to find any node version in the DB: %w", err)
 				} else if strings.Contains(err.Error(), "relation \"node_versions\" does not exist") {
@@ -381,7 +381,7 @@ func handleNodeVersioning(db *sqlx.DB, appLggr logger.Logger, rootDir string, cf
 	return nil
 }
 
-func takeBackupIfVersionUpgrade(dbUrl url.URL, rootDir string, cfg periodicbackup.BackupConfig, lggr logger.Logger, appv, dbv *semver.Version) (err error) {
+func takeBackupIfVersionUpgrade(dbUrl url.URL, rootDir string, cfg periodicbackup.BackupConfig, lggr logger.Logger, appv, dbv *semver.Version, healthReportPort uint16) (err error) {
 	if appv == nil {
 		lggr.Debug("Application version is missing, skipping automatic DB backup.")
 		return nil
@@ -400,7 +400,14 @@ func takeBackupIfVersionUpgrade(dbUrl url.URL, rootDir string, cfg periodicbacku
 	if err != nil {
 		return errors.Wrap(err, "takeBackupIfVersionUpgrade failed")
 	}
-	return databaseBackup.RunBackup(appv.String())
+
+	//Because backups can take a long time we must start a "fake" health report to prevent
+	//node shutdown because of healthcheck fail/timeout
+	ibhr := services.NewInBackupHealthReport(healthReportPort, lggr)
+	ibhr.Start()
+	defer ibhr.Stop()
+	err = databaseBackup.RunBackup(appv.String())
+	return err
 }
 
 // Runner implements the Run method.

core/scripts/common/helpers.go

@@ -486,6 +486,8 @@ func CalculateLatestBlockHeader(env Environment, blockNumberInput int) (err erro
 		}
 	}
 
+	// GetRLPHeaders method increments the blockNum sent by 1 and then fetches
+	// block headers for the child block.
 	blockNumber = blockNumber - 1
 
 	blockNumberBigInts := []*big.Int{big.NewInt(int64(blockNumber))}

core/scripts/vrfv2plus/testnet/proofs.go

@@ -9,6 +9,7 @@ import (
 	"os"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/shopspring/decimal"
 	helpers "github.com/smartcontractkit/chainlink/core/scripts/common"
 	"github.com/smartcontractkit/chainlink/v2/core/services/keystore/keys/vrfkey"
@@ -102,7 +103,7 @@ func generateProofForV2Plus(e helpers.Environment) {
 
 	parsedSubId, ok := new(big.Int).SetString(*subId, 10)
 	if !ok {
-		helpers.PanicErr(fmt.Errorf("unable to parse subID: %s %w", subId, err))
+		helpers.PanicErr(fmt.Errorf("unable to parse subID: %s %w", *subId, err))
 	}
 	extraArgs, err := extraargs.ExtraArgsV1(*nativePayment)
 	helpers.PanicErr(err)
@@ -154,6 +155,6 @@ func generateProofForV2Plus(e helpers.Environment) {
 		rc.CallbackGasLimit,
 		rc.NumWords,
 		rc.Sender,
-		rc.NativePayment,
+		hexutil.Encode(rc.ExtraArgs),
 	)
 }

core/scripts/vrfv2plus/testnet/util.go

@@ -212,7 +212,7 @@ func wrapperConfigure(
 	wrapper, err := vrfv2plus_wrapper.NewVRFV2PlusWrapper(wrapperAddress, e.Ec)
 	helpers.PanicErr(err)
 
-	tx, err := wrapper.SetConfig0(
+	tx, err := wrapper.SetConfig(
 		e.Owner,
 		uint32(wrapperGasOverhead),
 		uint32(coordinatorGasOverhead),

core/services/health.go

@@ -1,13 +1,17 @@
 package services
 
 import (
+	"context"
+	"fmt"
+	"net/http"
 	"sync"
 	"time"
 
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 
+	"github.com/smartcontractkit/chainlink/v2/core/logger"
 	"github.com/smartcontractkit/chainlink/v2/core/static"
 	"github.com/smartcontractkit/chainlink/v2/core/utils"
 )
@@ -230,3 +234,42 @@ func (c *checker) IsHealthy() (healthy bool, errors map[string]error) {
 
 	return
 }
+
+type InBackupHealthReport struct {
+	server http.Server
+	lggr   logger.Logger
+}
+
+func (i *InBackupHealthReport) Stop() {
+	shutdownCtx, shutdownRelease := context.WithTimeout(context.Background(), time.Second)
+	defer shutdownRelease()
+	if err := i.server.Shutdown(shutdownCtx); err != nil {
+		i.lggr.Errorf("InBackupHealthReport shutdown error: %v", err)
+	}
+	i.lggr.Info("InBackupHealthReport shutdown complete")
+}
+
+func (i *InBackupHealthReport) Start() {
+	go func() {
+		http.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
+			w.WriteHeader(http.StatusServiceUnavailable)
+			_, err := w.Write([]byte("Database backup in progress..."))
+			if err != nil {
+				i.lggr.Errorf("Cannot write response to /health")
+			}
+		})
+		i.lggr.Info("Starting InBackupHealthReport")
+		if err := i.server.ListenAndServe(); !errors.Is(err, http.ErrServerClosed) {
+			i.lggr.Errorf("InBackupHealthReport server error: %v", err)
+		}
+	}()
+}
+
+// NewInBackupHealthReport creates a new InBackupHealthReport that will serve the /health endpoint, useful for
+// preventing shutdowns due to health-checks when running long backup tasks
+func NewInBackupHealthReport(port uint16, lggr logger.Logger) *InBackupHealthReport {
+	return &InBackupHealthReport{
+		server: http.Server{Addr: fmt.Sprintf(":%d", port), ReadHeaderTimeout: time.Second * 5},
+		lggr:   lggr,
+	}
+}

core/services/health_test.go

@@ -2,12 +2,17 @@ package services_test
 
 import (
 	"fmt"
+	"io"
+	"net/http"
 	"testing"
+	"time"
 
 	"github.com/pkg/errors"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"go.uber.org/zap/zapcore"
 
+	"github.com/smartcontractkit/chainlink/v2/core/logger"
 	"github.com/smartcontractkit/chainlink/v2/core/services"
 )
 
@@ -65,3 +70,24 @@ func TestCheck(t *testing.T) {
 		assert.Equal(t, test.expected, results, "case %d", i)
 	}
 }
+
+func TestNewInBackupHealthReport(t *testing.T) {
+	lggr, observed := logger.TestLoggerObserved(t, zapcore.InfoLevel)
+	ibhr := services.NewInBackupHealthReport(1234, lggr)
+
+	ibhr.Start()
+	require.Eventually(t, func() bool { return observed.Len() >= 1 }, time.Second*5, time.Millisecond*100)
+	require.Equal(t, "Starting InBackupHealthReport", observed.TakeAll()[0].Message)
+
+	res, err := http.Get("http://localhost:1234/health")
+	require.NoError(t, err)
+	require.Equal(t, http.StatusServiceUnavailable, res.StatusCode)
+
+	resBody, err := io.ReadAll(res.Body)
+	require.NoError(t, err)
+	require.Equal(t, "Database backup in progress...", string(resBody))
+
+	ibhr.Stop()
+	require.Eventually(t, func() bool { return observed.Len() >= 1 }, time.Second*5, time.Millisecond*100)
+	require.Equal(t, "InBackupHealthReport shutdown complete", observed.TakeAll()[0].Message)
+}