CRIB CI integration (#13924)

* decouple CRIB vars * move CRIB vars to CTF, connect with GAP * bump deps * finalize deps * update go.mod * Spin up a separate GAP for crib and k8s * Change up the ports since 8080 is expected for CRIB connections * Use released version of setup-gap action * increase timeout * less logs * increase timeout * increase timeout even more * try creds for one hour * run without chaos * again * try to spin up CRIB * use GATI * update GATI secrets * use different port * fix working dir * update ref * try fixing working dir * another try * another try * another try * another try * nix develop * Fix nix develop * turn debug logs on * use local-dev-simulated-core-ocr1 profile * add teardown step * uppdate crib actions refs * add ref comments * reduce logging * add a confluence link * pin CI versions * temporary enable a nightly run --------- Co-authored-by: chainchad <96362174+chainchad@users.noreply.github.com> Co-authored-by: Radek Scheibinger <radek.scheibinger@smartcontract.com>
smartcontractkit · Aug 8, 2024 · 08638ff · 08638ff
1 parent 84630b8
commit 08638ff
Show file tree

Hide file tree

Showing 13 changed files with 199 additions and 182 deletions.
diff --git a/.github/workflows/crib-integration-test.yml b/.github/workflows/crib-integration-test.yml
@@ -1,74 +1,111 @@
-# this is disabled because of GAP limitations, should be re-enabled when github-actions-controller will be installed
+name: CRIB Integration Tests
+on:
+  schedule:
+    - cron: "0 1 * * *"
+  workflow_call:
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    environment: integration
+    permissions:
+      id-token: write
+      contents: read
+      actions: read
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
 
-#name: CRIB Integration Tests
-#on:
-#  push:
-#  workflow_call:
-#concurrency:
-#  group: ${{ github.workflow }}-${{ github.ref }}
-#  cancel-in-progress: true
-#jobs:
-#  test:
-#    runs-on: ubuntu-latest
-#    environment: integration
-#    permissions:
-#      id-token: write
-#      contents: read
-#      actions: read
-#    steps:
-#      - name: Checkout repository
-#        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
-#
-#      - name: Setup Nix + GATI environment
-#        uses: smartcontractkit/.github/actions/setup-nix-gati@514fe346780e2eddf7ea8b9f48120c2fba120d94
-#        with:
-#          aws-role-arn: ${{ secrets.AWS_OIDC_CHAINLINK_AUTO_PR_TOKEN_ISSUER_ROLE_ARN }}
-#          aws-lambda-url: ${{ secrets.AWS_CORE_TOKEN_ISSUER_LAMBDA_URL }} # see https://github.com/smartcontractkit/  infra/blob/a79bcfb48315c4411023c182e98eb80ff9e9cda6/accounts/production/us-west-2/lambda/  github-app-token-issuer-production/teams/releng/config.json#L9
-#          aws-region: ${{ secrets.AWS_REGION }}
-#          aws-role-duration-seconds: ${{ secrets.AWS_ROLE_DURATION_SECONDS }}
-#          enable-magic-cache: true
-#
-#      - name: Nix Develop Action
-#        uses: nicknovitski/nix-develop@v1
-#        with:
-#          arguments: "--accept-flake-config"
-#      - name: setup-gap
-#        uses: smartcontractkit/.github/actions/setup-gap@d316f66b2990ea4daa479daa3de6fc92b00f863e # setup-gap@0.3.2
-#        with:
-#          aws-role-arn: ${{ secrets.AWS_OIDC_CRIB_ROLE_ARN_STAGE }}
-#          api-gateway-host: ${{ secrets.AWS_API_GW_HOST_K8S_STAGE }}
-#          aws-region: ${{ secrets.AWS_REGION }}
-#          ecr-private-registry: ${{ secrets.AWS_ACCOUNT_ID_PROD }}
-#          k8s-cluster-name: ${{ secrets.AWS_K8S_CLUSTER_NAME_STAGE }}
-#          use-private-ecr-registry: true
-#          use-k8s: true
-#          metrics-job-name: "k8s"
-#          gc-basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
-#          gc-host: ${{ secrets.GRAFANA_INTERNAL_HOST }}
-#          gc-org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
-#      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
-#        name: Checkout CRIB repository
-#        with:
-#          repository: 'smartcontractkit/crib'
-#          ref: 'main'
-#      - name: Generate Short UUID
-#        id: uuid
-#        run: echo "CRIB_NAMESPACE=$(uuidgen | cut -c1-5)" >> $GITHUB_ENV
-#      - name: Create a new CRIB environment
-#        run: |-
-#          devspace use namespace $CRIB_NAMESPACE
-#          devspace deploy --profile local-dev-simulated-core-ocr1
-#      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
-#      - name: Setup go
-#        uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
-#        with:
-#          go-version-file: "go.mod"
-#      - name: Run CRIB integration test
-#        working-directory: integration-tests/crib
-#        env:
-#          K8S_STAGING_INGRESS_SUFFIX: ${{ secrets.K8S_STAGING_INGRESS_SUFFIX }}
-#          CRIB_NAMESPACE: ${{ env.CRIB_NAMESPACE }}
-#          CRIB_NETWORK: geth
-#          CRIB_NODES: 5
-#        run: |-
-#          go test -v -run TestCRIB
+      - uses: cachix/install-nix-action@ba0dd844c9180cbf77aa72a116d6fbc515d0e87b # v27
+        with:
+          nix_path: nixpkgs=channel:nixos-unstable
+
+      - name: setup-gap crib
+        uses: smartcontractkit/.github/actions/setup-gap@00b58566e0ee2761e56d9db0ea72b783fdb89b8d # setup-gap@0.4.0
+        with:
+          aws-role-duration-seconds: 3600 # 1 hour
+          aws-role-arn: ${{ secrets.AWS_OIDC_CRIB_ROLE_ARN_STAGE }}
+          api-gateway-host: ${{ secrets.AWS_API_GW_HOST_CRIB_STAGE }}
+          aws-region: ${{ secrets.AWS_REGION }}
+          ecr-private-registry: ${{ secrets.AWS_ACCOUNT_ID_PROD }}
+          k8s-cluster-name: ${{ secrets.AWS_K8S_CLUSTER_NAME_STAGE }}
+          gap-name: crib
+          use-private-ecr-registry: true
+          use-tls: true
+          proxy-port: 8080
+          metrics-job-name: "test"
+          gc-basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
+          gc-host: ${{ secrets.GRAFANA_INTERNAL_HOST }}
+          gc-org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
+
+      - name: setup-gap k8s
+        uses: smartcontractkit/.github/actions/setup-gap@00b58566e0ee2761e56d9db0ea72b783fdb89b8d # setup-gap@0.4.0
+        with:
+          aws-role-duration-seconds: 3600 # 1 hour
+          aws-role-arn: ${{ secrets.AWS_OIDC_CRIB_ROLE_ARN_STAGE }}
+          api-gateway-host: ${{ secrets.AWS_API_GW_HOST_K8S_STAGE }}
+          aws-region: ${{ secrets.AWS_REGION }}
+          ecr-private-registry: ${{ secrets.AWS_ACCOUNT_ID_PROD }}
+          k8s-cluster-name: ${{ secrets.AWS_K8S_CLUSTER_NAME_STAGE }}
+          gap-name: k8s
+          use-private-ecr-registry: true
+          use-k8s: true
+          proxy-port: 8443
+          metrics-job-name: "test"
+          gc-basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
+          gc-host: ${{ secrets.GRAFANA_INTERNAL_HOST }}
+          gc-org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
+
+      - name: Setup GitHub token using GATI
+        id: token
+        uses: smartcontractkit/.github/actions/setup-github-token@c0b38e6c40d72d01b8d2f24f92623a2538b3dedb # main
+        with:
+          aws-role-arn: ${{ secrets.AWS_OIDC_GLOBAL_READ_ONLY_TOKEN_ISSUER_ROLE_ARN }}
+          aws-lambda-url: ${{ secrets.AWS_INFRA_RELENG_TOKEN_ISSUER_LAMBDA_URL }}
+          aws-region: ${{ secrets.AWS_REGION }}
+          aws-role-duration-seconds: "1800"
+      - name: Debug workspace dir
+        shell: bash
+        run: |
+          echo ${{ github.workspace }}
+          echo $GITHUB_WORKSPACE
+
+      - name: Deploy and validate CRIB Environment for Core
+        uses: smartcontractkit/.github/actions/crib-deploy-environment@c0b38e6c40d72d01b8d2f24f92623a2538b3dedb # crib-deploy-environment@0.5.0
+        id: deploy-crib
+        with:
+          github-token: ${{ steps.token.outputs.access-token }}
+          api-gateway-host: ${{ secrets.AWS_API_GW_HOST_K8S_STAGE }}
+          aws-region: ${{ secrets.AWS_REGION }}
+          aws-role-arn: ${{ secrets.AWS_OIDC_CRIB_ROLE_ARN_STAGE }}
+          ecr-private-registry-stage: ${{ secrets.AWS_ACCOUNT_ID_STAGE }}
+          ecr-private-registry: ${{ secrets.AWS_ACCOUNT_ID_PROD }}
+          ingress-base-domain: ${{ secrets.INGRESS_BASE_DOMAIN_STAGE }}
+          k8s-cluster-name: ${{ secrets.AWS_K8S_CLUSTER_NAME_STAGE }}
+          devspace-profiles: "local-dev-simulated-core-ocr1"
+      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+      - name: Setup go
+        uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
+        with:
+          go-version-file: "go.mod"
+      - name: Run CRIB integration test
+        working-directory: integration-tests/crib
+        env:
+          K8S_STAGING_INGRESS_SUFFIX: ${{ secrets.K8S_STAGING_INGRESS_SUFFIX }}
+          CRIB_NAMESPACE: ${{ steps.deploy-crib.outputs.devspace-namespace }}
+          CRIB_NETWORK: geth
+          CRIB_NODES: 5
+          GAP_URL: ${{ secrets.GAP_URL }}
+#          SETH_LOG_LEVEL: debug
+#          RESTY_DEBUG: true
+#          TEST_PERSISTENCE: true
+        run: |-
+          go test -v -run TestCRIB
+      - name: Destroy CRIB Environment
+        id: destroy
+        if: always() && steps.deploy-crib.outputs.devspace-namespace != ''
+        uses: smartcontractkit/.github/actions/crib-purge-environment@c0b38e6c40d72d01b8d2f24f92623a2538b3dedb # crib-purge-environment@0.1.0
+        with:
+          namespace: ${{ steps.deploy-crib.outputs.devspace-namespace }}
diff --git a/integration-tests/README.md b/integration-tests/README.md
@@ -129,3 +129,9 @@ Run soak/ocr_test.go with RPC network chaos by bringing down network to RPC node
 ```bash
 make test_soak_ocr_rpc_down_half_cl_nodes
 ```
+
+### Debugging HTTP and RPC clients
+```bash
+export SETH_LOG_LEVEL=debug
+export RESTY_DEBUG=true
+```
diff --git a/integration-tests/actions/vrf/common/actions.go b/integration-tests/actions/vrf/common/actions.go
@@ -429,7 +429,7 @@ type RPCRawClient struct {
 }
 
 func NewRPCRawClient(url string) *RPCRawClient {
-	isDebug := os.Getenv("DEBUG_RESTY") == "true"
+	isDebug := os.Getenv("RESTY_DEBUG") == "true"
 	restyClient := resty.New().SetDebug(isDebug).SetBaseURL(url)
 	return &RPCRawClient{
 		resty: restyClient,

diff --git a/integration-tests/client/chainlink.go b/integration-tests/client/chainlink.go
@@ -2,6 +2,7 @@
 package client
 
 import (
+	"crypto/tls"
 	"fmt"
 	"math/big"
 	"net/http"
@@ -45,14 +46,10 @@ type ChainlinkClient struct {
 
 // NewChainlinkClient creates a new Chainlink model using a provided config
 func NewChainlinkClient(c *ChainlinkConfig, logger zerolog.Logger) (*ChainlinkClient, error) {
-	rc, err := initRestyClient(c.URL, c.Email, c.Password, c.HTTPTimeout)
+	rc, err := initRestyClient(c.URL, c.Email, c.Password, c.Headers, c.HTTPTimeout)
 	if err != nil {
 		return nil, err
 	}
-	_, isSet := os.LookupEnv("CL_CLIENT_DEBUG")
-	if isSet {
-		rc.SetDebug(true)
-	}
 	return &ChainlinkClient{
 		Config:    c,
 		APIClient: rc,
@@ -61,8 +58,11 @@ func NewChainlinkClient(c *ChainlinkConfig, logger zerolog.Logger) (*ChainlinkCl
 	}, nil
 }
 
-func initRestyClient(url string, email string, password string, timeout *time.Duration) (*resty.Client, error) {
-	rc := resty.New().SetBaseURL(url)
+func initRestyClient(url string, email string, password string, headers map[string]string, timeout *time.Duration) (*resty.Client, error) {
+	isDebug := os.Getenv("RESTY_DEBUG") == "true"
+	// G402 - TODO: certificates
+	//nolint
+	rc := resty.New().SetBaseURL(url).SetHeaders(headers).SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true}).SetDebug(isDebug)
 	if timeout != nil {
 		rc.SetTimeout(*timeout)
 	}
@@ -74,7 +74,7 @@ func initRestyClient(url string, email string, password string, timeout *time.Du
 	for i := 0; i < retryCount; i++ {
 		resp, err = rc.R().SetBody(session).Post("/sessions")
 		if err != nil {
-			log.Debug().Err(err).Str("URL", url).Interface("Session Details", session).Msg("Error connecting to Chainlink node, retrying")
+			log.Warn().Err(err).Str("URL", url).Interface("Session Details", session).Msg("Error connecting to Chainlink node, retrying")
 			time.Sleep(5 * time.Second)
 		} else {
 			break

diff --git a/integration-tests/client/chainlink_k8s.go b/integration-tests/client/chainlink_k8s.go
@@ -2,7 +2,6 @@
 package client
 
 import (
-	"os"
 	"regexp"
 
 	"github.com/rs/zerolog/log"
@@ -23,14 +22,10 @@ type ChainlinkK8sClient struct {
 
 // NewChainlink creates a new Chainlink model using a provided config
 func NewChainlinkK8sClient(c *ChainlinkConfig, podName, chartName string) (*ChainlinkK8sClient, error) {
-	rc, err := initRestyClient(c.URL, c.Email, c.Password, c.HTTPTimeout)
+	rc, err := initRestyClient(c.URL, c.Email, c.Password, c.Headers, c.HTTPTimeout)
 	if err != nil {
 		return nil, err
 	}
-	_, isSet := os.LookupEnv("CL_CLIENT_DEBUG")
-	if isSet {
-		rc.SetDebug(true)
-	}
 	return &ChainlinkK8sClient{
 		ChainlinkClient: &ChainlinkClient{
 			APIClient: rc,

diff --git a/integration-tests/client/chainlink_models.go b/integration-tests/client/chainlink_models.go
@@ -20,11 +20,12 @@ type EIServiceConfig struct {
 
 // ChainlinkConfig represents the variables needed to connect to a Chainlink node
 type ChainlinkConfig struct {
-	URL         string         `toml:",omitempty"`
-	Email       string         `toml:",omitempty"`
-	Password    string         `toml:",omitempty"`
-	InternalIP  string         `toml:",omitempty"`
-	HTTPTimeout *time.Duration `toml:"-"`
+	URL         string            `toml:",omitempty"`
+	Email       string            `toml:",omitempty"`
+	Password    string            `toml:",omitempty"`
+	InternalIP  string            `toml:",omitempty"`
+	Headers     map[string]string `toml:",omitempty"`
+	HTTPTimeout *time.Duration    `toml:"-"`
 }
 
 // ResponseSlice is the generic model that can be used for all Chainlink API responses that are an slice

diff --git a/integration-tests/crib/README.md b/integration-tests/crib/README.md
@@ -1,4 +1,4 @@
-### CRIB Health Check Test
+### Example e2e product test using CRIB
 
 ## Setup CRIB
 This is a simple smoke + chaos test for CRIB deployment.
@@ -12,8 +12,15 @@ devspace deploy --debug --profile local-dev-simulated-core-ocr1
 
 ## Run the tests
 ```shell
-CRIB_NAMESPACE=crib-oh-my-crib
-CRIB_NETWORK=geth # only "geth" is supported for now
-CRIB_NODES=5 # min 5 nodes
+export CRIB_NAMESPACE=crib-oh-my-crib
+export CRIB_NETWORK=geth # only "geth" is supported for now
+export CRIB_NODES=5 # min 5 nodes
+#export SETH_LOG_LEVEL=debug # these two can be enabled to debug connection issues
+#export RESTY_DEBUG=true
+#export TEST_PERSISTENCE=true # to run the chaos test
+export GAP_URL=https://localhost:8080/primary # only applicable in CI, unset the var to connect locally
 go test -v -run TestCRIB
-```
+```
+
+## Configuring CI workflow
+We are using GAP and GATI to access the infrastructure, please follow [configuration guide](https://smartcontract-it.atlassian.net/wiki/spaces/CRIB/pages/909967436/CRIB+CI+Integration)