Skip to content

Commit

Permalink
[TT-590] Logstream with buffered streaming to Loki (#11477)
Browse files Browse the repository at this point in the history 
* testing out with local CTF

* print location for failed tests

* use logwatch with buffer

* try logwatch with loki and test targets in CI

* fix merge conflict

* fix lint issue

* adjust env vars in workflow

* add logwatch env vars as env and not with

* fix env var name

* fix go.mod

* use newer ctf

* fix go.sum

* use correct ctf version

* fix go.sum

* trigger tests

* small debug

* make the Gh summary print a bit nicer

* latest ctf, use var not secret for grafana url

* update ctf, change how we print test summary

* shutdown logwatch after flushing logs

* trigger tests

* use latest ctf where logwatch -> logstream

* update var name in GH workflow

* enable log stream for all smoke tests

* use better method for logstream shutdown and log flushing

* fix compile error

* latest ctf, remove comment

* remove replace from go.mod

* collect all logs to see what happens

* update ctf, better scoped dashboard link

* do not save logs if test doesn't fail

* print absolute log folder path

* cause test failure to make sure file logs are where they should be

* fix typo in var declaration in gh workflow

* run go mod tidy for integeration tests

* always execute test summary step

* remove failing vrfv2 test on purpose

* use taggeg CTF version

* connect mock adapter to logstream

* initialise killgrave only when necessary
  • Loading branch information
@Tofel
Tofel committed Dec 7, 2023
1 parent df54d26 commit 711987f
Show file tree
Hide file tree
Showing 22 changed files with 180 additions and 146 deletions.
28 changes: 28 additions & 0 deletions .github/workflows/integration-tests.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -435,6 +435,13 @@ jobs:
PYROSCOPE_SERVER: ${{ matrix.product.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725
PYROSCOPE_ENVIRONMENT: ${{ matrix.product.pyroscope_env }}
PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }}
LOKI_TENANT_ID: ${{ vars.LOKI_TENANT_ID }}
LOKI_URL: ${{ secrets.LOKI_URL }}
LOKI_BASIC_AUTH: ${{ secrets.LOKI_BASIC_AUTH }}
LOGSTREAM_LOG_TARGETS: ${{ vars.LOGSTREAM_LOG_TARGETS }}
GRAFANA_URL: ${{ vars.GRAFANA_URL }}
GRAFANA_DATASOURCE: ${{ vars.GRAFANA_DATASOURCE }}
RUN_ID: ${{ github.run_id }}
with:
test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -test.parallel=${{ matrix.product.nodes }} ${{ steps.build-go-test-command.outputs.run_command }} 2>&1 | tee /tmp/gotest.log | gotestfmt
test_download_vendor_packages_command: cd ./integration-tests && go mod download
Expand Down Expand Up @@ -487,6 +494,27 @@ jobs:
with:
name: trace-data
path: ./integration-tests/smoke/traces/trace-data.json
- name: Print failed test summary
if: always()
run: |
directory="./integration-tests/smoke/.test_summary"
files=("$directory"/*)
if [ -d "$directory" ]; then
echo "Test summary folder found"
if [ ${#files[@]} -gt 0 ]; then
first_file="${files[0]}"
echo "Name of the first test summary file: $(basename "$first_file")"
echo "### Failed Test Execution Logs Dashboard (over VPN):" >> $GITHUB_STEP_SUMMARY
cat "$first_file" | jq -r '.loki[] | "* [\(.test_name)](\(.value))"' >> $GITHUB_STEP_SUMMARY
if [ ${#files[@]} -gt 1 ]; then
echo "Found more than one test summary file. This is incorrect, there should be only one file"
fi
else
echo "Test summary directory is empty. This should not happen"
fi
else
echo "No test summary folder found. If no test failed or log collection wasn't explicitly requested this is correct. Exiting"
fi
### Used to check the required checks box when the matrix completes
eth-smoke-tests:
Expand Down
3 changes: 3 additions & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,3 +82,6 @@ go.work*

# This sometimes shows up for some reason
tools/flakeytests/coverage.txt

.test_summary/
.run.id
33 changes: 24 additions & 9 deletions integration-tests/docker/test_env/cl_node.go
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package test_env

import (
"context"
"fmt"
"math/big"
"net/url"
Expand All @@ -22,7 +23,7 @@ import (
"github.com/smartcontractkit/chainlink-testing-framework/docker"
"github.com/smartcontractkit/chainlink-testing-framework/docker/test_env"
"github.com/smartcontractkit/chainlink-testing-framework/logging"
"github.com/smartcontractkit/chainlink-testing-framework/logwatch"
"github.com/smartcontractkit/chainlink-testing-framework/logstream"
"github.com/smartcontractkit/chainlink-testing-framework/utils/testcontext"
"github.com/smartcontractkit/chainlink/v2/core/services/chainlink"
"github.com/smartcontractkit/chainlink/v2/core/services/keystore/chaintype"
Expand All @@ -47,7 +48,7 @@ type ClNode struct {
UserPassword string `json:"userPassword"`
t *testing.T
l zerolog.Logger
lw *logwatch.LogWatch
ls *logstream.LogStream
}

type ClNodeOption = func(c *ClNode)
Expand Down Expand Up @@ -76,9 +77,9 @@ func WithDbContainerName(name string) ClNodeOption {
}
}

func WithLogWatch(lw *logwatch.LogWatch) ClNodeOption {
func WithLogStream(ls *logstream.LogStream) ClNodeOption {
return func(c *ClNode) {
c.lw = lw
c.ls = ls
}
}

Expand Down Expand Up @@ -285,11 +286,7 @@ func (n *ClNode) StartContainer() error {
if err != nil {
return fmt.Errorf("%s err: %w", ErrStartCLNodeContainer, err)
}
if n.lw != nil {
if err := n.lw.ConnectContainer(testcontext.Get(n.t), container, "cl-node", true); err != nil {
return err
}
}

clEndpoint, err := test_env.GetEndpoint(testcontext.Get(n.t), container, "http")
if err != nil {
return err
Expand Down Expand Up @@ -410,5 +407,23 @@ func (n *ClNode) getContainerRequest(secrets string) (
FileMode: 0644,
},
},
LifecycleHooks: []tc.ContainerLifecycleHooks{
{PostStarts: []tc.ContainerHook{
func(ctx context.Context, c tc.Container) error {
if n.ls != nil {
return n.ls.ConnectContainer(ctx, c, "cl-node")
}
return nil
},
},
PostStops: []tc.ContainerHook{
func(ctx context.Context, c tc.Container) error {
if n.ls != nil {
return n.ls.DisconnectContainer(c)
}
return nil
},
}},
},
}, nil
}
131 changes: 30 additions & 101 deletions integration-tests/docker/test_env/test_env.go
View file
Original file line number Diff line number Diff line change
@@ -1,29 +1,25 @@
package test_env

import (
"context"
"encoding/json"
"fmt"
"io"
"math/big"
"os"
"path/filepath"
"runtime/debug"
"strings"
"testing"
"time"

"github.com/ethereum/go-ethereum/accounts/keystore"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
tc "github.com/testcontainers/testcontainers-go"
"golang.org/x/sync/errgroup"

"github.com/smartcontractkit/chainlink-testing-framework/blockchain"
"github.com/smartcontractkit/chainlink-testing-framework/docker"
"github.com/smartcontractkit/chainlink-testing-framework/docker/test_env"
"github.com/smartcontractkit/chainlink-testing-framework/logging"
"github.com/smartcontractkit/chainlink-testing-framework/logwatch"
"github.com/smartcontractkit/chainlink-testing-framework/utils/testcontext"
"github.com/smartcontractkit/chainlink-testing-framework/logstream"
"github.com/smartcontractkit/chainlink/v2/core/services/chainlink"

"github.com/smartcontractkit/chainlink/integration-tests/client"
Expand All @@ -35,9 +31,9 @@ var (
)

type CLClusterTestEnv struct {
Cfg *TestEnvConfig
Network *tc.DockerNetwork
LogWatch *logwatch.LogWatch
Cfg *TestEnvConfig
Network *tc.DockerNetwork
LogStream *logstream.LogStream

/* components */
ClCluster *ClCluster
Expand All @@ -58,27 +54,29 @@ func NewTestEnv() (*CLClusterTestEnv, error) {
if err != nil {
return nil, err
}
n := []string{network.Name}
return &CLClusterTestEnv{
MockAdapter: test_env.NewKillgrave(n, ""),
Network: network,
l: log.Logger,
Network: network,
l: log.Logger,
}, nil
}

// WithTestEnvConfig sets the test environment cfg.
// Sets up private ethereum chain and MockAdapter containers with the provided cfg.
func (te *CLClusterTestEnv) WithTestEnvConfig(cfg *TestEnvConfig) *CLClusterTestEnv {
te.Cfg = cfg
n := []string{te.Network.Name}
te.MockAdapter = test_env.NewKillgrave(n, te.Cfg.MockAdapter.ImpostersPath, test_env.WithContainerName(te.Cfg.MockAdapter.ContainerName))
if cfg.MockAdapter.ContainerName != "" {
n := []string{te.Network.Name}
te.MockAdapter = test_env.NewKillgrave(n, te.Cfg.MockAdapter.ImpostersPath, test_env.WithContainerName(te.Cfg.MockAdapter.ContainerName))
}
return te
}

func (te *CLClusterTestEnv) WithTestLogger(t *testing.T) *CLClusterTestEnv {
te.t = t
te.l = logging.GetTestLogger(t)
te.MockAdapter.WithTestLogger(t)
if te.MockAdapter != nil {
te.MockAdapter.WithTestLogger(t)
}
return te
}

Expand Down Expand Up @@ -174,6 +172,18 @@ func (te *CLClusterTestEnv) StartClCluster(nodeConfig *chainlink.Config, count i
}
}

if te.LogStream != nil {
for _, node := range te.ClCluster.Nodes {
node.ls = te.LogStream
}
if te.MockAdapter != nil {
err := te.LogStream.ConnectContainer(context.Background(), te.MockAdapter.Container, "mock-adapter")
if err != nil {
return err
}
}
}

// Start/attach node containers
return te.ClCluster.Start()
}
Expand Down Expand Up @@ -207,23 +217,6 @@ func (te *CLClusterTestEnv) Cleanup() error {

te.logWhetherAllContainersAreRunning()

// Getting the absolute path
wd, err := os.Getwd()
if err != nil {
return err
}

wd = filepath.Join(wd, "logs")
te.l.Info().Str("Working dir", wd).Msg("Would write test logs here")

// TODO: This is an imperfect and temporary solution, see TT-590 for a more sustainable solution
// Collect logs if the test fails, or if we just want them
if te.t.Failed() || os.Getenv("TEST_LOG_COLLECT") == "true" {
if err := te.collectTestLogs(); err != nil {
return err
}
}

if te.EVMClient == nil {
return fmt.Errorf("evm client is nil, unable to return funds from chainlink nodes during cleanup")
} else if te.EVMClient.NetworkSimulated() {
Expand All @@ -247,6 +240,10 @@ func (te *CLClusterTestEnv) Cleanup() error {

func (te *CLClusterTestEnv) logWhetherAllContainersAreRunning() {
for _, node := range te.ClCluster.Nodes {
if node.Container == nil {
continue
}

isCLRunning := node.Container.IsRunning()
isDBRunning := node.PostgresDb.Container.IsRunning()

Expand All @@ -260,74 +257,6 @@ func (te *CLClusterTestEnv) logWhetherAllContainersAreRunning() {
}
}

// collectTestLogs collects the logs from all the Chainlink nodes in the test environment and writes them to local files
func (te *CLClusterTestEnv) collectTestLogs() error {
te.l.Info().Msg("Collecting test logs")
sanitizedNetworkName := strings.ReplaceAll(te.EVMClient.GetNetworkName(), " ", "-")
folder := fmt.Sprintf("./logs/%s-%s-%s", te.t.Name(), sanitizedNetworkName, time.Now().Format("2006-01-02T15-04-05"))
if err := os.MkdirAll(folder, os.ModePerm); err != nil {
return err
}

eg := &errgroup.Group{}
for _, n := range te.ClCluster.Nodes {
node := n
eg.Go(func() error {
logFileName := filepath.Join(folder, fmt.Sprintf("node-%s.log", node.ContainerName))
logFile, err := os.OpenFile(logFileName, os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
return err
}
defer logFile.Close()
logReader, err := node.Container.Logs(testcontext.Get(te.t))
if err != nil {
return err
}
_, err = io.Copy(logFile, logReader)
if err != nil {
return err
}
te.l.Info().Str("Node", node.ContainerName).Str("File", logFileName).Msg("Wrote Logs")
return nil
})
}

if te.MockAdapter != nil {
eg.Go(func() error {
logFileName := filepath.Join(folder, "mock-adapter.log")
logFile, err := os.OpenFile(logFileName, os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
return err
}
defer logFile.Close()
logReader, err := te.MockAdapter.Container.Logs(testcontext.Get(te.t))
if err != nil {
return err
}
_, err = io.Copy(logFile, logReader)
if err != nil {
return err
}
te.l.Info().Str("Container", te.MockAdapter.ContainerName).Str("File", logFileName).Msg("Wrote Logs")
return nil
})
}

if err := eg.Wait(); err != nil {
return err
}

// Getting the absolute path
wd, err := os.Getwd()
if err != nil {
return err
}
absolutePath := filepath.Join(wd, folder)

te.l.Info().Str("Logs absolute Location", absolutePath).Msg("Wrote test logs")
return nil
}

func (te *CLClusterTestEnv) returnFunds() error {
te.l.Info().Msg("Attempting to return Chainlink node funds to default network wallets")
for _, chainlinkNode := range te.ClCluster.Nodes {
Expand Down
Loading

0 comments on commit 711987f

Please sign in to comment.