From 08638ff2b963b1afe731de9910f73654914a45ff Mon Sep 17 00:00:00 2001 From: Sergey Kudasov Date: Thu, 8 Aug 2024 20:13:25 +0200 Subject: [PATCH] CRIB CI integration (#13924) * decouple CRIB vars * move CRIB vars to CTF, connect with GAP * bump deps * finalize deps * update go.mod * Spin up a separate GAP for crib and k8s * Change up the ports since 8080 is expected for CRIB connections * Use released version of setup-gap action * increase timeout * less logs * increase timeout * increase timeout even more * try creds for one hour * run without chaos * again * try to spin up CRIB * use GATI * update GATI secrets * use different port * fix working dir * update ref * try fixing working dir * another try * another try * another try * another try * nix develop * Fix nix develop * turn debug logs on * use local-dev-simulated-core-ocr1 profile * add teardown step * uppdate crib actions refs * add ref comments * reduce logging * add a confluence link * pin CI versions * temporary enable a nightly run --------- Co-authored-by: chainchad <96362174+chainchad@users.noreply.github.com> Co-authored-by: Radek Scheibinger --- .github/workflows/crib-integration-test.yml | 183 +++++++++++------- integration-tests/README.md | 6 + .../actions/vrf/common/actions.go | 2 +- integration-tests/client/chainlink.go | 16 +- integration-tests/client/chainlink_k8s.go | 7 +- integration-tests/client/chainlink_models.go | 11 +- integration-tests/crib/README.md | 17 +- integration-tests/crib/connect.go | 87 +++------ integration-tests/crib/ocr_test.go | 39 ++-- integration-tests/load/go.mod | 2 +- integration-tests/load/go.sum | 4 +- integration-tests/smoke/README.md | 6 - integration-tests/testsetups/ocr.go | 1 - 13 files changed, 199 insertions(+), 182 deletions(-) diff --git a/.github/workflows/crib-integration-test.yml b/.github/workflows/crib-integration-test.yml index 75b2215d2fc..248004636bc 100644 --- a/.github/workflows/crib-integration-test.yml +++ b/.github/workflows/crib-integration-test.yml @@ -1,74 +1,111 @@ -# this is disabled because of GAP limitations, should be re-enabled when github-actions-controller will be installed +name: CRIB Integration Tests +on: + schedule: + - cron: "0 1 * * *" + workflow_call: +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true +jobs: + test: + runs-on: ubuntu-latest + environment: integration + permissions: + id-token: write + contents: read + actions: read + steps: + - name: Checkout repository + uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 -#name: CRIB Integration Tests -#on: -# push: -# workflow_call: -#concurrency: -# group: ${{ github.workflow }}-${{ github.ref }} -# cancel-in-progress: true -#jobs: -# test: -# runs-on: ubuntu-latest -# environment: integration -# permissions: -# id-token: write -# contents: read -# actions: read -# steps: -# - name: Checkout repository -# uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 -# -# - name: Setup Nix + GATI environment -# uses: smartcontractkit/.github/actions/setup-nix-gati@514fe346780e2eddf7ea8b9f48120c2fba120d94 -# with: -# aws-role-arn: ${{ secrets.AWS_OIDC_CHAINLINK_AUTO_PR_TOKEN_ISSUER_ROLE_ARN }} -# aws-lambda-url: ${{ secrets.AWS_CORE_TOKEN_ISSUER_LAMBDA_URL }} # see https://github.com/smartcontractkit/ infra/blob/a79bcfb48315c4411023c182e98eb80ff9e9cda6/accounts/production/us-west-2/lambda/ github-app-token-issuer-production/teams/releng/config.json#L9 -# aws-region: ${{ secrets.AWS_REGION }} -# aws-role-duration-seconds: ${{ secrets.AWS_ROLE_DURATION_SECONDS }} -# enable-magic-cache: true -# -# - name: Nix Develop Action -# uses: nicknovitski/nix-develop@v1 -# with: -# arguments: "--accept-flake-config" -# - name: setup-gap -# uses: smartcontractkit/.github/actions/setup-gap@d316f66b2990ea4daa479daa3de6fc92b00f863e # setup-gap@0.3.2 -# with: -# aws-role-arn: ${{ secrets.AWS_OIDC_CRIB_ROLE_ARN_STAGE }} -# api-gateway-host: ${{ secrets.AWS_API_GW_HOST_K8S_STAGE }} -# aws-region: ${{ secrets.AWS_REGION }} -# ecr-private-registry: ${{ secrets.AWS_ACCOUNT_ID_PROD }} -# k8s-cluster-name: ${{ secrets.AWS_K8S_CLUSTER_NAME_STAGE }} -# use-private-ecr-registry: true -# use-k8s: true -# metrics-job-name: "k8s" -# gc-basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} -# gc-host: ${{ secrets.GRAFANA_INTERNAL_HOST }} -# gc-org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} -# - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 -# name: Checkout CRIB repository -# with: -# repository: 'smartcontractkit/crib' -# ref: 'main' -# - name: Generate Short UUID -# id: uuid -# run: echo "CRIB_NAMESPACE=$(uuidgen | cut -c1-5)" >> $GITHUB_ENV -# - name: Create a new CRIB environment -# run: |- -# devspace use namespace $CRIB_NAMESPACE -# devspace deploy --profile local-dev-simulated-core-ocr1 -# - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 -# - name: Setup go -# uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 -# with: -# go-version-file: "go.mod" -# - name: Run CRIB integration test -# working-directory: integration-tests/crib -# env: -# K8S_STAGING_INGRESS_SUFFIX: ${{ secrets.K8S_STAGING_INGRESS_SUFFIX }} -# CRIB_NAMESPACE: ${{ env.CRIB_NAMESPACE }} -# CRIB_NETWORK: geth -# CRIB_NODES: 5 -# run: |- -# go test -v -run TestCRIB \ No newline at end of file + - uses: cachix/install-nix-action@ba0dd844c9180cbf77aa72a116d6fbc515d0e87b # v27 + with: + nix_path: nixpkgs=channel:nixos-unstable + + - name: setup-gap crib + uses: smartcontractkit/.github/actions/setup-gap@00b58566e0ee2761e56d9db0ea72b783fdb89b8d # setup-gap@0.4.0 + with: + aws-role-duration-seconds: 3600 # 1 hour + aws-role-arn: ${{ secrets.AWS_OIDC_CRIB_ROLE_ARN_STAGE }} + api-gateway-host: ${{ secrets.AWS_API_GW_HOST_CRIB_STAGE }} + aws-region: ${{ secrets.AWS_REGION }} + ecr-private-registry: ${{ secrets.AWS_ACCOUNT_ID_PROD }} + k8s-cluster-name: ${{ secrets.AWS_K8S_CLUSTER_NAME_STAGE }} + gap-name: crib + use-private-ecr-registry: true + use-tls: true + proxy-port: 8080 + metrics-job-name: "test" + gc-basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} + gc-host: ${{ secrets.GRAFANA_INTERNAL_HOST }} + gc-org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} + + - name: setup-gap k8s + uses: smartcontractkit/.github/actions/setup-gap@00b58566e0ee2761e56d9db0ea72b783fdb89b8d # setup-gap@0.4.0 + with: + aws-role-duration-seconds: 3600 # 1 hour + aws-role-arn: ${{ secrets.AWS_OIDC_CRIB_ROLE_ARN_STAGE }} + api-gateway-host: ${{ secrets.AWS_API_GW_HOST_K8S_STAGE }} + aws-region: ${{ secrets.AWS_REGION }} + ecr-private-registry: ${{ secrets.AWS_ACCOUNT_ID_PROD }} + k8s-cluster-name: ${{ secrets.AWS_K8S_CLUSTER_NAME_STAGE }} + gap-name: k8s + use-private-ecr-registry: true + use-k8s: true + proxy-port: 8443 + metrics-job-name: "test" + gc-basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} + gc-host: ${{ secrets.GRAFANA_INTERNAL_HOST }} + gc-org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} + + - name: Setup GitHub token using GATI + id: token + uses: smartcontractkit/.github/actions/setup-github-token@c0b38e6c40d72d01b8d2f24f92623a2538b3dedb # main + with: + aws-role-arn: ${{ secrets.AWS_OIDC_GLOBAL_READ_ONLY_TOKEN_ISSUER_ROLE_ARN }} + aws-lambda-url: ${{ secrets.AWS_INFRA_RELENG_TOKEN_ISSUER_LAMBDA_URL }} + aws-region: ${{ secrets.AWS_REGION }} + aws-role-duration-seconds: "1800" + - name: Debug workspace dir + shell: bash + run: | + echo ${{ github.workspace }} + echo $GITHUB_WORKSPACE + + - name: Deploy and validate CRIB Environment for Core + uses: smartcontractkit/.github/actions/crib-deploy-environment@c0b38e6c40d72d01b8d2f24f92623a2538b3dedb # crib-deploy-environment@0.5.0 + id: deploy-crib + with: + github-token: ${{ steps.token.outputs.access-token }} + api-gateway-host: ${{ secrets.AWS_API_GW_HOST_K8S_STAGE }} + aws-region: ${{ secrets.AWS_REGION }} + aws-role-arn: ${{ secrets.AWS_OIDC_CRIB_ROLE_ARN_STAGE }} + ecr-private-registry-stage: ${{ secrets.AWS_ACCOUNT_ID_STAGE }} + ecr-private-registry: ${{ secrets.AWS_ACCOUNT_ID_PROD }} + ingress-base-domain: ${{ secrets.INGRESS_BASE_DOMAIN_STAGE }} + k8s-cluster-name: ${{ secrets.AWS_K8S_CLUSTER_NAME_STAGE }} + devspace-profiles: "local-dev-simulated-core-ocr1" + - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 + - name: Setup go + uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 + with: + go-version-file: "go.mod" + - name: Run CRIB integration test + working-directory: integration-tests/crib + env: + K8S_STAGING_INGRESS_SUFFIX: ${{ secrets.K8S_STAGING_INGRESS_SUFFIX }} + CRIB_NAMESPACE: ${{ steps.deploy-crib.outputs.devspace-namespace }} + CRIB_NETWORK: geth + CRIB_NODES: 5 + GAP_URL: ${{ secrets.GAP_URL }} +# SETH_LOG_LEVEL: debug +# RESTY_DEBUG: true +# TEST_PERSISTENCE: true + run: |- + go test -v -run TestCRIB + - name: Destroy CRIB Environment + id: destroy + if: always() && steps.deploy-crib.outputs.devspace-namespace != '' + uses: smartcontractkit/.github/actions/crib-purge-environment@c0b38e6c40d72d01b8d2f24f92623a2538b3dedb # crib-purge-environment@0.1.0 + with: + namespace: ${{ steps.deploy-crib.outputs.devspace-namespace }} \ No newline at end of file diff --git a/integration-tests/README.md b/integration-tests/README.md index 180021efeef..1510c8c91b7 100644 --- a/integration-tests/README.md +++ b/integration-tests/README.md @@ -129,3 +129,9 @@ Run soak/ocr_test.go with RPC network chaos by bringing down network to RPC node ```bash make test_soak_ocr_rpc_down_half_cl_nodes ``` + +### Debugging HTTP and RPC clients +```bash +export SETH_LOG_LEVEL=debug +export RESTY_DEBUG=true +``` diff --git a/integration-tests/actions/vrf/common/actions.go b/integration-tests/actions/vrf/common/actions.go index 1300ac8b726..5697a261766 100644 --- a/integration-tests/actions/vrf/common/actions.go +++ b/integration-tests/actions/vrf/common/actions.go @@ -429,7 +429,7 @@ type RPCRawClient struct { } func NewRPCRawClient(url string) *RPCRawClient { - isDebug := os.Getenv("DEBUG_RESTY") == "true" + isDebug := os.Getenv("RESTY_DEBUG") == "true" restyClient := resty.New().SetDebug(isDebug).SetBaseURL(url) return &RPCRawClient{ resty: restyClient, diff --git a/integration-tests/client/chainlink.go b/integration-tests/client/chainlink.go index 08a47101dc1..da17dcf0d75 100644 --- a/integration-tests/client/chainlink.go +++ b/integration-tests/client/chainlink.go @@ -2,6 +2,7 @@ package client import ( + "crypto/tls" "fmt" "math/big" "net/http" @@ -45,14 +46,10 @@ type ChainlinkClient struct { // NewChainlinkClient creates a new Chainlink model using a provided config func NewChainlinkClient(c *ChainlinkConfig, logger zerolog.Logger) (*ChainlinkClient, error) { - rc, err := initRestyClient(c.URL, c.Email, c.Password, c.HTTPTimeout) + rc, err := initRestyClient(c.URL, c.Email, c.Password, c.Headers, c.HTTPTimeout) if err != nil { return nil, err } - _, isSet := os.LookupEnv("CL_CLIENT_DEBUG") - if isSet { - rc.SetDebug(true) - } return &ChainlinkClient{ Config: c, APIClient: rc, @@ -61,8 +58,11 @@ func NewChainlinkClient(c *ChainlinkConfig, logger zerolog.Logger) (*ChainlinkCl }, nil } -func initRestyClient(url string, email string, password string, timeout *time.Duration) (*resty.Client, error) { - rc := resty.New().SetBaseURL(url) +func initRestyClient(url string, email string, password string, headers map[string]string, timeout *time.Duration) (*resty.Client, error) { + isDebug := os.Getenv("RESTY_DEBUG") == "true" + // G402 - TODO: certificates + //nolint + rc := resty.New().SetBaseURL(url).SetHeaders(headers).SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true}).SetDebug(isDebug) if timeout != nil { rc.SetTimeout(*timeout) } @@ -74,7 +74,7 @@ func initRestyClient(url string, email string, password string, timeout *time.Du for i := 0; i < retryCount; i++ { resp, err = rc.R().SetBody(session).Post("/sessions") if err != nil { - log.Debug().Err(err).Str("URL", url).Interface("Session Details", session).Msg("Error connecting to Chainlink node, retrying") + log.Warn().Err(err).Str("URL", url).Interface("Session Details", session).Msg("Error connecting to Chainlink node, retrying") time.Sleep(5 * time.Second) } else { break diff --git a/integration-tests/client/chainlink_k8s.go b/integration-tests/client/chainlink_k8s.go index 794e93f7276..077b8f7ca48 100644 --- a/integration-tests/client/chainlink_k8s.go +++ b/integration-tests/client/chainlink_k8s.go @@ -2,7 +2,6 @@ package client import ( - "os" "regexp" "github.com/rs/zerolog/log" @@ -23,14 +22,10 @@ type ChainlinkK8sClient struct { // NewChainlink creates a new Chainlink model using a provided config func NewChainlinkK8sClient(c *ChainlinkConfig, podName, chartName string) (*ChainlinkK8sClient, error) { - rc, err := initRestyClient(c.URL, c.Email, c.Password, c.HTTPTimeout) + rc, err := initRestyClient(c.URL, c.Email, c.Password, c.Headers, c.HTTPTimeout) if err != nil { return nil, err } - _, isSet := os.LookupEnv("CL_CLIENT_DEBUG") - if isSet { - rc.SetDebug(true) - } return &ChainlinkK8sClient{ ChainlinkClient: &ChainlinkClient{ APIClient: rc, diff --git a/integration-tests/client/chainlink_models.go b/integration-tests/client/chainlink_models.go index a0435d53368..86e9f75902d 100644 --- a/integration-tests/client/chainlink_models.go +++ b/integration-tests/client/chainlink_models.go @@ -20,11 +20,12 @@ type EIServiceConfig struct { // ChainlinkConfig represents the variables needed to connect to a Chainlink node type ChainlinkConfig struct { - URL string `toml:",omitempty"` - Email string `toml:",omitempty"` - Password string `toml:",omitempty"` - InternalIP string `toml:",omitempty"` - HTTPTimeout *time.Duration `toml:"-"` + URL string `toml:",omitempty"` + Email string `toml:",omitempty"` + Password string `toml:",omitempty"` + InternalIP string `toml:",omitempty"` + Headers map[string]string `toml:",omitempty"` + HTTPTimeout *time.Duration `toml:"-"` } // ResponseSlice is the generic model that can be used for all Chainlink API responses that are an slice diff --git a/integration-tests/crib/README.md b/integration-tests/crib/README.md index ecf393f780d..e895cca6763 100644 --- a/integration-tests/crib/README.md +++ b/integration-tests/crib/README.md @@ -1,4 +1,4 @@ -### CRIB Health Check Test +### Example e2e product test using CRIB ## Setup CRIB This is a simple smoke + chaos test for CRIB deployment. @@ -12,8 +12,15 @@ devspace deploy --debug --profile local-dev-simulated-core-ocr1 ## Run the tests ```shell -CRIB_NAMESPACE=crib-oh-my-crib -CRIB_NETWORK=geth # only "geth" is supported for now -CRIB_NODES=5 # min 5 nodes +export CRIB_NAMESPACE=crib-oh-my-crib +export CRIB_NETWORK=geth # only "geth" is supported for now +export CRIB_NODES=5 # min 5 nodes +#export SETH_LOG_LEVEL=debug # these two can be enabled to debug connection issues +#export RESTY_DEBUG=true +#export TEST_PERSISTENCE=true # to run the chaos test +export GAP_URL=https://localhost:8080/primary # only applicable in CI, unset the var to connect locally go test -v -run TestCRIB -``` \ No newline at end of file +``` + +## Configuring CI workflow +We are using GAP and GATI to access the infrastructure, please follow [configuration guide](https://smartcontract-it.atlassian.net/wiki/spaces/CRIB/pages/909967436/CRIB+CI+Integration) \ No newline at end of file diff --git a/integration-tests/crib/connect.go b/integration-tests/crib/connect.go index 91d7d8ee1a0..c180b2ff2ea 100644 --- a/integration-tests/crib/connect.go +++ b/integration-tests/crib/connect.go @@ -1,11 +1,11 @@ package crib import ( - "fmt" - "os" - "strconv" + "net/http" "time" + "github.com/smartcontractkit/chainlink-testing-framework/crib" + "github.com/pkg/errors" "github.com/smartcontractkit/seth" @@ -18,17 +18,7 @@ import ( "github.com/smartcontractkit/chainlink/integration-tests/client" ) -const ( - // these are constants for simulated CRIB that should never change - // CRIB: https://github.com/smartcontractkit/crib/tree/main/core - // Core Chart: https://github.com/smartcontractkit/infra-charts/tree/main/chainlink-cluster - mockserverCRIBTemplate = "https://%s-mockserver%s" - internalNodeDNSTemplate = "app-node%d" - ingressNetworkWSURLTemplate = "wss://%s-geth-1337-ws%s" - ingressNetworkHTTPURLTemplate = "https://%s-geth-1337-http%s" -) - -func setSethConfig(cfg tc.TestConfig, netWSURL string, netHTTPURL string) { +func setSethConfig(cfg tc.TestConfig, netWSURL string, netHTTPURL string, headers http.Header) { netName := "CRIB_SIMULATED" cfg.Network.SelectedNetworks = []string{netName} cfg.Network.RpcHttpUrls = map[string][]string{} @@ -36,6 +26,7 @@ func setSethConfig(cfg tc.TestConfig, netWSURL string, netHTTPURL string) { cfg.Network.RpcWsUrls = map[string][]string{} cfg.Network.RpcWsUrls[netName] = []string{netWSURL} cfg.Seth.EphemeralAddrs = ptr.Ptr(int64(0)) + cfg.Seth.RPCHeaders = headers } // ConnectRemote connects to a local environment, see https://github.com/smartcontractkit/crib/tree/main/core @@ -47,52 +38,33 @@ func ConnectRemote() ( []*client.ChainlinkK8sClient, error, ) { - ingressSuffix := os.Getenv("K8S_STAGING_INGRESS_SUFFIX") - if ingressSuffix == "" { - return nil, nil, nil, nil, errors.New("K8S_STAGING_INGRESS_SUFFIX must be set to connect to k8s ingresses") - } - cribNamespace := os.Getenv("CRIB_NAMESPACE") - if cribNamespace == "" { - return nil, nil, nil, nil, errors.New("CRIB_NAMESPACE must be set to connect") - } - cribNetwork := os.Getenv("CRIB_NETWORK") - if cribNetwork == "" { - return nil, nil, nil, nil, errors.New("CRIB_NETWORK must be set to connect, only 'geth' is supported for now") - } - cribNodes := os.Getenv("CRIB_NODES") - nodes, err := strconv.Atoi(cribNodes) + vars, err := crib.CoreDONSimulatedConnection() if err != nil { - return nil, nil, nil, nil, errors.New("CRIB_NODES must be a number, 5-19 nodes") + return nil, nil, nil, nil, err } + // TODO: move all the parts of ConnectRemote() to CTF when Seth config refactor is finalized config, err := tc.GetConfig([]string{"CRIB"}, tc.OCR) if err != nil { return nil, nil, nil, nil, err } - if nodes < 2 { - return nil, nil, nil, nil, fmt.Errorf("not enough chainlink nodes, need at least 2, TOML key: [CRIB.nodes]") - } - mockserverURL := fmt.Sprintf(mockserverCRIBTemplate, cribNamespace, ingressSuffix) var sethClient *seth.Client - switch cribNetwork { + switch vars.Network { case "geth": - netWSURL := fmt.Sprintf(ingressNetworkWSURLTemplate, cribNamespace, ingressSuffix) - netHTTPURL := fmt.Sprintf(ingressNetworkHTTPURLTemplate, cribNamespace, ingressSuffix) - setSethConfig(config, netWSURL, netHTTPURL) + setSethConfig(config, vars.NetworkWSURL, vars.NetworkHTTPURL, vars.BlockchainNodeHeaders) net := blockchain.EVMNetwork{ - Name: cribNetwork, - Simulated: true, - SupportsEIP1559: true, - ClientImplementation: blockchain.EthereumClientImplementation, - ChainID: 1337, - PrivateKeys: []string{ - "ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80", - }, - URLs: []string{netWSURL}, - HTTPURLs: []string{netHTTPURL}, + Name: vars.Network, + Simulated: true, + SupportsEIP1559: true, + ClientImplementation: blockchain.EthereumClientImplementation, + ChainID: vars.ChainID, + PrivateKeys: vars.PrivateKeys, + URLs: []string{vars.NetworkWSURL}, + HTTPURLs: []string{vars.NetworkHTTPURL}, ChainlinkTransactionLimit: 500000, Timeout: blockchain.StrDuration{Duration: 2 * time.Minute}, MinimumConfirmations: 1, GasEstimationBuffer: 10000, + Headers: vars.BlockchainNodeHeaders, } sethClient, err = seth_utils.GetChainClient(config, net) if err != nil { @@ -104,31 +76,34 @@ func ConnectRemote() ( // bootstrap node clClients := make([]*client.ChainlinkK8sClient, 0) c, err := client.NewChainlinkK8sClient(&client.ChainlinkConfig{ - URL: fmt.Sprintf("https://%s-node%d%s", cribNamespace, 1, ingressSuffix), Email: client.CLNodeTestEmail, - InternalIP: fmt.Sprintf(internalNodeDNSTemplate, 1), Password: client.CLNodeTestPassword, - }, fmt.Sprintf(internalNodeDNSTemplate, 1), cribNamespace) + URL: vars.NodeURLs[0], + InternalIP: vars.NodeInternalDNS[0], + Headers: vars.NodeHeaders[0], + }, vars.NodeInternalDNS[0], vars.Namespace) if err != nil { return nil, nil, nil, nil, err } clClients = append(clClients, c) // all the other nodes, indices of nodes in CRIB starts with 1 - for i := 2; i <= nodes; i++ { + for i := 1; i < vars.Nodes; i++ { cl, err := client.NewChainlinkK8sClient(&client.ChainlinkConfig{ - URL: fmt.Sprintf("https://%s-node%d%s", cribNamespace, i, ingressSuffix), Email: client.CLNodeTestEmail, - InternalIP: fmt.Sprintf(internalNodeDNSTemplate, i), Password: client.CLNodeTestPassword, - }, fmt.Sprintf(internalNodeDNSTemplate, i), cribNamespace) + URL: vars.NodeURLs[i], + InternalIP: vars.NodeInternalDNS[i], + Headers: vars.NodeHeaders[i], + }, vars.NodeInternalDNS[i], vars.Namespace) if err != nil { return nil, nil, nil, nil, err } clClients = append(clClients, cl) } mockServerClient := msClient.NewMockserverClient(&msClient.MockserverConfig{ - LocalURL: mockserverURL, - ClusterURL: mockserverURL, + LocalURL: vars.MockserverURL, + ClusterURL: "http://mockserver:1080", + Headers: vars.MockserverHeaders, }) //nolint:gosec // G602 - false positive https://github.com/securego/gosec/issues/1005 diff --git a/integration-tests/crib/ocr_test.go b/integration-tests/crib/ocr_test.go index b84af02a196..91a7a1d76b8 100644 --- a/integration-tests/crib/ocr_test.go +++ b/integration-tests/crib/ocr_test.go @@ -7,6 +7,7 @@ import ( "time" "github.com/smartcontractkit/havoc/k8schaos" + "github.com/stretchr/testify/require" "github.com/smartcontractkit/chainlink/integration-tests/actions" @@ -33,22 +34,24 @@ func TestCRIB(t *testing.T) { err = actions.WatchNewOCRRound(l, sethClient, 1, contracts.V1OffChainAgrregatorToOffChainAggregatorWithRounds(ocrInstances), 5*time.Minute) require.NoError(t, err, "Error watching for new OCR round") - ch, err := rebootCLNamespace( - 1*time.Second, - os.Getenv("CRIB_NAMESPACE"), - ) - ch.Create(context.Background()) - ch.AddListener(k8schaos.NewChaosLogger(l)) - t.Cleanup(func() { - err := ch.Delete(context.Background()) - require.NoError(t, err) - }) - require.Eventually(t, func() bool { - err = actions.WatchNewOCRRound(l, sethClient, 3, contracts.V1OffChainAgrregatorToOffChainAggregatorWithRounds(ocrInstances), 5*time.Minute) - if err != nil { - l.Info().Err(err).Msg("OCR round is not there yet") - return false - } - return true - }, 3*time.Minute, 5*time.Second) + if os.Getenv("TEST_PERSISTENCE") != "" { + ch, err := rebootCLNamespace( + 1*time.Second, + os.Getenv("CRIB_NAMESPACE"), + ) + ch.Create(context.Background()) + ch.AddListener(k8schaos.NewChaosLogger(l)) + t.Cleanup(func() { + err := ch.Delete(context.Background()) + require.NoError(t, err) + }) + require.Eventually(t, func() bool { + err = actions.WatchNewOCRRound(l, sethClient, 3, contracts.V1OffChainAgrregatorToOffChainAggregatorWithRounds(ocrInstances), 5*time.Minute) + if err != nil { + l.Info().Err(err).Msg("OCR round is not there yet") + return false + } + return true + }, 20*time.Minute, 5*time.Second) + } } diff --git a/integration-tests/load/go.mod b/integration-tests/load/go.mod index 2dec28df99f..bd47a97f48a 100644 --- a/integration-tests/load/go.mod +++ b/integration-tests/load/go.mod @@ -375,7 +375,7 @@ require ( github.com/smartcontractkit/chainlink-feeds v0.0.0-20240710170203-5b41615da827 // indirect github.com/smartcontractkit/chainlink-solana v1.1.1-0.20240806154405-8e5684f98564 // indirect github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240709043547-03612098f799 // indirect - github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240405215812-5a72bc9af239 // indirect + github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.2-0.20240805111647-acf86c1e347a // indirect github.com/smartcontractkit/havoc/k8schaos v0.0.0-20240409145249-e78d20847e37 // indirect github.com/smartcontractkit/tdh2/go/ocr2/decryptionplugin v0.0.0-20230906073235-9e478e5e19f1 // indirect github.com/smartcontractkit/wsrpc v0.7.3 // indirect diff --git a/integration-tests/load/go.sum b/integration-tests/load/go.sum index 484b5eb2482..7789b949449 100644 --- a/integration-tests/load/go.sum +++ b/integration-tests/load/go.sum @@ -1486,8 +1486,8 @@ github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.202407 github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240709043547-03612098f799/go.mod h1:UVFRacRkP7O7TQAzFmR52v5mUlxf+G1ovMlCQAB/cHU= github.com/smartcontractkit/chainlink-testing-framework v1.34.2 h1:YL3ft7KJB7SAopdmJeyeR4/kv0j4jOdagNihXq8OZ38= github.com/smartcontractkit/chainlink-testing-framework v1.34.2/go.mod h1:hRZEDh2+afO8MSZb9qYNscmWb+3mHZf01J5ACZuIdTQ= -github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240405215812-5a72bc9af239 h1:Kk5OVlx/5g9q3Z3lhxytZS4/f8ds1MiNM8yaHgK3Oe8= -github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240405215812-5a72bc9af239/go.mod h1:DC8sQMyTlI/44UCTL8QWFwb0bYNoXCfjwCv2hMivYZU= +github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.2-0.20240805111647-acf86c1e347a h1:8GtvGJaGyKzx/ar1yX74GxrzIYWTZVTyv4pYB/1ln8w= +github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.2-0.20240805111647-acf86c1e347a/go.mod h1:DC8sQMyTlI/44UCTL8QWFwb0bYNoXCfjwCv2hMivYZU= github.com/smartcontractkit/go-plugin v0.0.0-20240208201424-b3b91517de16 h1:TFe+FvzxClblt6qRfqEhUfa4kFQx5UobuoFGO2W4mMo= github.com/smartcontractkit/go-plugin v0.0.0-20240208201424-b3b91517de16/go.mod h1:lBS5MtSSBZk0SHc66KACcjjlU6WzEVP/8pwz68aMkCI= github.com/smartcontractkit/grpc-proxy v0.0.0-20230731113816-f1be6620749f h1:hgJif132UCdjo8u43i7iPN1/MFnu49hv7lFGFftCHKU= diff --git a/integration-tests/smoke/README.md b/integration-tests/smoke/README.md index 266720c7bc6..c4aa2b91a14 100644 --- a/integration-tests/smoke/README.md +++ b/integration-tests/smoke/README.md @@ -75,9 +75,3 @@ Then execute: go test -v -run ${TestName} ``` - - -### Debugging CL client API calls -```bash -export CL_CLIENT_DEBUG=true -``` \ No newline at end of file diff --git a/integration-tests/testsetups/ocr.go b/integration-tests/testsetups/ocr.go index b38c39eebe4..084ea5eca10 100644 --- a/integration-tests/testsetups/ocr.go +++ b/integration-tests/testsetups/ocr.go @@ -547,7 +547,6 @@ func (o *OCRSoakTest) LoadState() error { } o.mockServer = ctf_client.ConnectMockServerURL(testState.MockServerURL) - return err }