From a53b7f411c4a59f2d407961f8e7921ad75847dbc Mon Sep 17 00:00:00 2001 From: Dan Jaglowski Date: Wed, 31 May 2023 12:59:08 -0600 Subject: [PATCH 1/3] [chore][scraperinttest] Capture and logs errors from scrapers --- receiver/mysqlreceiver/integration_test.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/receiver/mysqlreceiver/integration_test.go b/receiver/mysqlreceiver/integration_test.go index 962f82c74064..9b49618906d6 100644 --- a/receiver/mysqlreceiver/integration_test.go +++ b/receiver/mysqlreceiver/integration_test.go @@ -1,9 +1,6 @@ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 -//go:build integration -// +build integration - package mysqlreceiver import ( From ce9b76424c2b5e8a789158efae5288e0475d29d5 Mon Sep 17 00:00:00 2001 From: Dan Jaglowski Date: Wed, 31 May 2023 16:08:21 -0600 Subject: [PATCH 2/3] wip --- .../coreinternal/scraperinttest/scraperint.go | 9 ++++++ receiver/mysqlreceiver/integration_test.go | 29 ++++++++++++++----- receiver/mysqlreceiver/scraper.go | 1 + .../testdata/integration/Dockerfile.mysql | 9 ------ .../testdata/integration/grant.sh | 16 ++++++++++ .../testdata/integration/init.sql | 13 +++++++++ 6 files changed, 60 insertions(+), 17 deletions(-) delete mode 100644 receiver/mysqlreceiver/testdata/integration/Dockerfile.mysql create mode 100644 receiver/mysqlreceiver/testdata/integration/grant.sh create mode 100644 receiver/mysqlreceiver/testdata/integration/init.sql diff --git a/internal/coreinternal/scraperinttest/scraperint.go b/internal/coreinternal/scraperinttest/scraperint.go index 55421515016c..fa37f69be6ca 100644 --- a/internal/coreinternal/scraperinttest/scraperint.go +++ b/internal/coreinternal/scraperinttest/scraperint.go @@ -122,6 +122,15 @@ func (it *IntegrationTest) Run(t *testing.T) { if len(allMetrics) == 0 { return false } + // if len(observedLogs.All()) > 0 { + // logs := strings.Builder{} + // for _, e := range observedLogs.All() { + // logs.WriteString(e.Message + "\n") + // } + // t.Errorf("full log:\n%s", logs.String()) + // t.FailNow() + // } + if it.writeExpected { require.NoError(t, golden.WriteMetrics(t, it.expectedFile, allMetrics[0])) return true diff --git a/receiver/mysqlreceiver/integration_test.go b/receiver/mysqlreceiver/integration_test.go index 9b49618906d6..6c7e781cdc1d 100644 --- a/receiver/mysqlreceiver/integration_test.go +++ b/receiver/mysqlreceiver/integration_test.go @@ -1,6 +1,9 @@ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 +//go:build integration +// +build integration + package mysqlreceiver import ( @@ -24,18 +27,28 @@ func TestIntegration(t *testing.T) { NewFactory(), scraperinttest.WithContainerRequest( testcontainers.ContainerRequest{ - FromDockerfile: testcontainers.FromDockerfile{ - Context: filepath.Join("testdata", "integration"), - Dockerfile: "Dockerfile.mysql", - }, + Image: "mysql:8.0.33", ExposedPorts: []string{mysqlPort}, WaitingFor: wait.ForListeningPort(mysqlPort). WithStartupTimeout(2 * time.Minute), - LifecycleHooks: []testcontainers.ContainerLifecycleHooks{{ - PostStarts: []testcontainers.ContainerHook{ - scraperinttest.RunScript([]string{"/setup.sh"}), + Env: map[string]string{ + "MYSQL_ROOT_PASSWORD": "otel", + "MYSQL_DATABASE": "otel", + "MYSQL_USER": "otel", + "MYSQL_PASSWORD": "otel", + }, + Files: []testcontainers.ContainerFile{ + { + HostFilePath: filepath.Join("testdata", "integration", "grant.sh"), + ContainerFilePath: "/docker-entrypoint-initdb.d/", + FileMode: 700, }, - }}, + { + HostFilePath: filepath.Join("testdata", "integration", "init.sql"), + ContainerFilePath: "/docker-entrypoint-initdb.d/", + FileMode: 700, + }, + }, }), scraperinttest.WithCustomConfig( func(t *testing.T, cfg component.Config, ci *scraperinttest.ContainerInfo) { diff --git a/receiver/mysqlreceiver/scraper.go b/receiver/mysqlreceiver/scraper.go index 0dadd833d47f..2ac4a53e8c5b 100644 --- a/receiver/mysqlreceiver/scraper.go +++ b/receiver/mysqlreceiver/scraper.go @@ -79,6 +79,7 @@ func (m *mySQLScraper) scrape(context.Context) (pmetric.Metrics, error) { innodbStats, innoErr := m.sqlclient.getInnodbStats() if innoErr != nil { m.logger.Error("Failed to fetch InnoDB stats", zap.Error(innoErr)) + return pmetric.Metrics{}, innoErr } errs := &scrapererror.ScrapeErrors{} diff --git a/receiver/mysqlreceiver/testdata/integration/Dockerfile.mysql b/receiver/mysqlreceiver/testdata/integration/Dockerfile.mysql deleted file mode 100644 index 6f60d14bc9cb..000000000000 --- a/receiver/mysqlreceiver/testdata/integration/Dockerfile.mysql +++ /dev/null @@ -1,9 +0,0 @@ -FROM mysql:8.0.33 - -COPY scripts/setup.sh /setup.sh -RUN chmod +x /setup.sh - -ENV MYSQL_DATABASE=otel -ENV MYSQL_USER=otel -ENV MYSQL_PASSWORD=otel -ENV MYSQL_ROOT_PASSWORD=otel diff --git a/receiver/mysqlreceiver/testdata/integration/grant.sh b/receiver/mysqlreceiver/testdata/integration/grant.sh new file mode 100644 index 000000000000..50fe62a0cb9b --- /dev/null +++ b/receiver/mysqlreceiver/testdata/integration/grant.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +set -e + +USER="otel" +ROOT_PASS="otel" + +# NOTE: -pPASSWORD is missing a space on purpose +mysql -u root -p"${ROOT_PASS}" -e "GRANT PROCESS ON *.* TO '${USER}'@'%'" > /dev/null +mysql -u root -p"${ROOT_PASS}" -e "GRANT SELECT ON information_schema.innodb_metrics TO '${USER}'@'%'" > /dev/null +mysql -u root -p"${ROOT_PASS}" -e "GRANT SELECT ON performance_schema.table_io_waits_summary_by_table TO '${USER}'@'%'" > /dev/null +mysql -u root -p"${ROOT_PASS}" -e "GRANT SELECT ON performance_schema.table_io_waits_summary_by_index_usage TO '${USER}'@'%'" > /dev/null +mysql -u root -p"${ROOT_PASS}" -e "FLUSH PRIVILEGES" > /dev/null diff --git a/receiver/mysqlreceiver/testdata/integration/init.sql b/receiver/mysqlreceiver/testdata/integration/init.sql new file mode 100644 index 000000000000..169928697f3e --- /dev/null +++ b/receiver/mysqlreceiver/testdata/integration/init.sql @@ -0,0 +1,13 @@ +USE otel; + +-- SET SESSION sql_mode = 'SUPER'; + +-- GRANT PROCESS ON *.* TO 'OTEL'@'%'; +-- GRANT SELECT ON INFORMATION_SCHEMA.INNODB_METRICS TO 'OTEL'@'%'; +-- -- -- GRANT SELECT ON PERFORMANCE_SCHEMA.TABLE_IO_WAITS_SUMMARY_BY_TABLE TO 'OTEL'@'%'; +-- -- -- GRANT SELECT ON PERFORMANCE_SCHEMA.TABLE_IO_WAITS_SUMMARY_BY_INDEX_USAGE TO 'OTEL'@'%'; +-- -- FLUSH PRIVILEGES: + +CREATE DATABASE a_schema; +CREATE TABLE a_schema.a_table (k int, v int); +CREATE INDEX an_index ON a_schema.a_table ((k + v)); From 4c548f58a4e7a178102aa541e16088b7e8546721 Mon Sep 17 00:00:00 2001 From: Dan Jaglowski Date: Thu, 1 Jun 2023 11:57:50 -0600 Subject: [PATCH 3/3] [chore][receiver/mysql] Migrate away from timing based setup in integration test --- .../coreinternal/scraperinttest/scraperint.go | 27 +++++++---- receiver/mysqlreceiver/README.md | 2 +- receiver/mysqlreceiver/integration_test.go | 9 +--- receiver/mysqlreceiver/scraper.go | 8 ++-- .../testdata/integration/grant.sh | 16 ------- .../testdata/integration/init.sh | 14 ++++++ .../testdata/integration/init.sql | 13 ------ .../testdata/integration/scripts/setup.sh | 46 ------------------- 8 files changed, 38 insertions(+), 97 deletions(-) delete mode 100644 receiver/mysqlreceiver/testdata/integration/grant.sh create mode 100644 receiver/mysqlreceiver/testdata/integration/init.sh delete mode 100644 receiver/mysqlreceiver/testdata/integration/init.sql delete mode 100644 receiver/mysqlreceiver/testdata/integration/scripts/setup.sh diff --git a/internal/coreinternal/scraperinttest/scraperint.go b/internal/coreinternal/scraperinttest/scraperint.go index fa37f69be6ca..18a95b13f697 100644 --- a/internal/coreinternal/scraperinttest/scraperint.go +++ b/internal/coreinternal/scraperinttest/scraperint.go @@ -58,7 +58,8 @@ type IntegrationTest struct { compareOptions []pmetrictest.CompareMetricsOption compareTimeout time.Duration - writeExpected bool + failOnErrorLogs bool + writeExpected bool } func (it *IntegrationTest) Run(t *testing.T) { @@ -122,14 +123,13 @@ func (it *IntegrationTest) Run(t *testing.T) { if len(allMetrics) == 0 { return false } - // if len(observedLogs.All()) > 0 { - // logs := strings.Builder{} - // for _, e := range observedLogs.All() { - // logs.WriteString(e.Message + "\n") - // } - // t.Errorf("full log:\n%s", logs.String()) - // t.FailNow() - // } + if it.failOnErrorLogs && len(observedLogs.All()) > 0 { + logs := strings.Builder{} + for _, e := range observedLogs.All() { + logs.WriteString(e.Message + "\n") + } + t.Errorf("full log:\n%s", logs.String()) + } if it.writeExpected { require.NoError(t, golden.WriteMetrics(t, it.expectedFile, allMetrics[0])) @@ -238,6 +238,15 @@ func WithExpectedFile(f string) TestOption { } } +// This option is useful for debugging scrapers but should not be used permanently +// because the logs do not correlate to a single scrape interval. In other words, +// when a retryable failure occurs, this setting will likely force a failure anyways. +func FailOnErrorLogs() TestOption { + return func(it *IntegrationTest) { + it.failOnErrorLogs = true + } +} + func WriteExpected() TestOption { return func(it *IntegrationTest) { it.writeExpected = true diff --git a/receiver/mysqlreceiver/README.md b/receiver/mysqlreceiver/README.md index f2a9c5b0d133..88f47d823ee9 100644 --- a/receiver/mysqlreceiver/README.md +++ b/receiver/mysqlreceiver/README.md @@ -18,7 +18,7 @@ This receiver queries MySQL's global status and InnoDB tables. This receiver supports MySQL version 8.0 -Collecting most metrics requires the ability to execute `SHOW GLOBAL STATUS`. The `buffer_pool_size` metric requires access to the `information_schema.innodb_metrics` table. Please refer to [setup.sh](./testdata/integration/scripts/setup.sh) for an example of how to configure these permissions. +Collecting most metrics requires the ability to execute `SHOW GLOBAL STATUS`. ## Configuration diff --git a/receiver/mysqlreceiver/integration_test.go b/receiver/mysqlreceiver/integration_test.go index 6c7e781cdc1d..dc58a2e31fd5 100644 --- a/receiver/mysqlreceiver/integration_test.go +++ b/receiver/mysqlreceiver/integration_test.go @@ -39,13 +39,8 @@ func TestIntegration(t *testing.T) { }, Files: []testcontainers.ContainerFile{ { - HostFilePath: filepath.Join("testdata", "integration", "grant.sh"), - ContainerFilePath: "/docker-entrypoint-initdb.d/", - FileMode: 700, - }, - { - HostFilePath: filepath.Join("testdata", "integration", "init.sql"), - ContainerFilePath: "/docker-entrypoint-initdb.d/", + HostFilePath: filepath.Join("testdata", "integration", "init.sh"), + ContainerFilePath: "/docker-entrypoint-initdb.d/init.sh", FileMode: 700, }, }, diff --git a/receiver/mysqlreceiver/scraper.go b/receiver/mysqlreceiver/scraper.go index 2ac4a53e8c5b..441e438552e8 100644 --- a/receiver/mysqlreceiver/scraper.go +++ b/receiver/mysqlreceiver/scraper.go @@ -79,7 +79,6 @@ func (m *mySQLScraper) scrape(context.Context) (pmetric.Metrics, error) { innodbStats, innoErr := m.sqlclient.getInnodbStats() if innoErr != nil { m.logger.Error("Failed to fetch InnoDB stats", zap.Error(innoErr)) - return pmetric.Metrics{}, innoErr } errs := &scrapererror.ScrapeErrors{} @@ -103,7 +102,7 @@ func (m *mySQLScraper) scrape(context.Context) (pmetric.Metrics, error) { m.scrapeGlobalStats(now, errs) // colect replicas status metrics. - m.scrapeReplicaStatusStats(now, errs) + m.scrapeReplicaStatusStats(now) m.mb.EmitForResource(metadata.WithMysqlInstanceEndpoint(m.config.Endpoint)) @@ -529,11 +528,10 @@ func (m *mySQLScraper) scrapeTableLockWaitEventStats(now pcommon.Timestamp, errs } } -func (m *mySQLScraper) scrapeReplicaStatusStats(now pcommon.Timestamp, errs *scrapererror.ScrapeErrors) { +func (m *mySQLScraper) scrapeReplicaStatusStats(now pcommon.Timestamp) { replicaStatusStats, err := m.sqlclient.getReplicaStatusStats() if err != nil { - m.logger.Error("Failed to fetch replica status stats", zap.Error(err)) - errs.AddPartial(8, err) + m.logger.Info("Failed to fetch replica status stats", zap.Error(err)) return } diff --git a/receiver/mysqlreceiver/testdata/integration/grant.sh b/receiver/mysqlreceiver/testdata/integration/grant.sh deleted file mode 100644 index 50fe62a0cb9b..000000000000 --- a/receiver/mysqlreceiver/testdata/integration/grant.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash - -# Copyright The OpenTelemetry Authors -# SPDX-License-Identifier: Apache-2.0 - -set -e - -USER="otel" -ROOT_PASS="otel" - -# NOTE: -pPASSWORD is missing a space on purpose -mysql -u root -p"${ROOT_PASS}" -e "GRANT PROCESS ON *.* TO '${USER}'@'%'" > /dev/null -mysql -u root -p"${ROOT_PASS}" -e "GRANT SELECT ON information_schema.innodb_metrics TO '${USER}'@'%'" > /dev/null -mysql -u root -p"${ROOT_PASS}" -e "GRANT SELECT ON performance_schema.table_io_waits_summary_by_table TO '${USER}'@'%'" > /dev/null -mysql -u root -p"${ROOT_PASS}" -e "GRANT SELECT ON performance_schema.table_io_waits_summary_by_index_usage TO '${USER}'@'%'" > /dev/null -mysql -u root -p"${ROOT_PASS}" -e "FLUSH PRIVILEGES" > /dev/null diff --git a/receiver/mysqlreceiver/testdata/integration/init.sh b/receiver/mysqlreceiver/testdata/integration/init.sh new file mode 100644 index 000000000000..b5151eb6223d --- /dev/null +++ b/receiver/mysqlreceiver/testdata/integration/init.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +set -e + +USER="otel" +ROOT_PASS="otel" + +# # NOTE: -pPASSWORD is missing a space on purpose +mysql -u root -p"${ROOT_PASS}" -e "GRANT PROCESS ON *.* TO ${USER}" > /dev/null +mysql -u root -p"${ROOT_PASS}" -e "GRANT SELECT ON performance_schema.* TO ${USER}" > /dev/null +mysql -u root -p"${ROOT_PASS}" -e "FLUSH PRIVILEGES" > /dev/null diff --git a/receiver/mysqlreceiver/testdata/integration/init.sql b/receiver/mysqlreceiver/testdata/integration/init.sql deleted file mode 100644 index 169928697f3e..000000000000 --- a/receiver/mysqlreceiver/testdata/integration/init.sql +++ /dev/null @@ -1,13 +0,0 @@ -USE otel; - --- SET SESSION sql_mode = 'SUPER'; - --- GRANT PROCESS ON *.* TO 'OTEL'@'%'; --- GRANT SELECT ON INFORMATION_SCHEMA.INNODB_METRICS TO 'OTEL'@'%'; --- -- -- GRANT SELECT ON PERFORMANCE_SCHEMA.TABLE_IO_WAITS_SUMMARY_BY_TABLE TO 'OTEL'@'%'; --- -- -- GRANT SELECT ON PERFORMANCE_SCHEMA.TABLE_IO_WAITS_SUMMARY_BY_INDEX_USAGE TO 'OTEL'@'%'; --- -- FLUSH PRIVILEGES: - -CREATE DATABASE a_schema; -CREATE TABLE a_schema.a_table (k int, v int); -CREATE INDEX an_index ON a_schema.a_table ((k + v)); diff --git a/receiver/mysqlreceiver/testdata/integration/scripts/setup.sh b/receiver/mysqlreceiver/testdata/integration/scripts/setup.sh deleted file mode 100644 index f67f0c995dd9..000000000000 --- a/receiver/mysqlreceiver/testdata/integration/scripts/setup.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env bash - -# Copyright The OpenTelemetry Authors -# SPDX-License-Identifier: Apache-2.0 - -set -e - -USER="otel" -ROOT_PASS="otel" -CODE=1 - - -setup_permissions() { - # NOTE: -pPASSWORD is missing a space on purpose - mysql -u root -p"${ROOT_PASS}" -e "GRANT PROCESS ON *.* TO ${USER}" > /dev/null - mysql -u root -p"${ROOT_PASS}" -e "GRANT SELECT ON INFORMATION_SCHEMA.INNODB_METRICS TO ${USER}" > /dev/null - mysql -u root -p"${ROOT_PASS}" -e "FLUSH PRIVILEGES" > /dev/null -} - -setup_data() { - mysql -u root -p"${ROOT_PASS}" -e "CREATE DATABASE a_schema" > /dev/null - mysql -u root -p"${ROOT_PASS}" -e "CREATE TABLE a_schema.a_table (k int, v int)" > /dev/null - mysql -u root -p"${ROOT_PASS}" -e "CREATE INDEX an_index ON a_schema.a_table ((k + v))" > /dev/null -} - -echo "Configuring ${USER} permissions. . ." -end=$((SECONDS+60)) -while [ $SECONDS -lt $end ]; do - result="$?" - if setup_permissions; then - echo "Permissions configured!" - while [ $SECONDS -lt $end ]; do - result="$?" - if setup_data; then - echo "Data created!" - exit 0 - fi - echo "Trying again in 5 seconds. . ." - sleep 5 - done - fi - echo "Trying again in 5 seconds. . ." - sleep 5 -done - -echo "Failed to configure permissions"