From 88bb41d45c2b10763a904586eae9632300deccfc Mon Sep 17 00:00:00 2001 From: Ricky Stewart Date: Wed, 5 May 2021 13:52:16 -0500 Subject: [PATCH 1/5] bazel: upgrade to bazel 4.0.0 in the builder image This will allow us to pick up the latest version of `rules_foreign_cc`, (for example, for #64334) which doesn't build with an earlier bazel (#64709). Also make sure the version assertion in `WORKSPACE` is up-to-date. Release note: None --- WORKSPACE | 2 +- build/bazelbuilder/Dockerfile | 4 ++-- build/teamcity-bazel-support.sh | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 796b31753e4a..2cd8607f81b1 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -122,7 +122,7 @@ yarn_install( # NB: @bazel_skylib comes from go_rules_dependencies(). load("@bazel_skylib//lib:versions.bzl", "versions") -versions.check(minimum_bazel_version = "3.5.0") +versions.check(minimum_bazel_version = "4.0.0") # Load gazelle dependencies. load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies") diff --git a/build/bazelbuilder/Dockerfile b/build/bazelbuilder/Dockerfile index 970dea367463..1c6af4fc8fa5 100644 --- a/build/bazelbuilder/Dockerfile +++ b/build/bazelbuilder/Dockerfile @@ -36,7 +36,7 @@ RUN echo "deb [arch=amd64] https://storage.googleapis.com/bazel-apt stable jdk1. && curl https://bazel.build/bazel-release.pub.gpg | apt-key add - \ && apt-get update \ && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - bazel-3.6.0 + bazel-4.0.0 # git - Upgrade to a more modern version RUN DEBIAN_FRONTEND=noninteractive apt-get install dh-autoreconf libcurl4-gnutls-dev libexpat1-dev gettext libz-dev libssl-dev -y && \ @@ -58,7 +58,7 @@ RUN apt-get purge -y \ RUN rm -rf /tmp/* /var/lib/apt/lists/* -RUN ln -s /usr/bin/bazel-3.6.0 /usr/bin/bazel +RUN ln -s /usr/bin/bazel-4.0.0 /usr/bin/bazel COPY entrypoint.sh /usr/bin ENTRYPOINT ["/usr/bin/entrypoint.sh"] diff --git a/build/teamcity-bazel-support.sh b/build/teamcity-bazel-support.sh index 59b650758b19..58febefd8bc3 100644 --- a/build/teamcity-bazel-support.sh +++ b/build/teamcity-bazel-support.sh @@ -1,4 +1,4 @@ -BAZEL_IMAGE=cockroachdb/bazel:20210408-113636 +BAZEL_IMAGE=cockroachdb/bazel:20210505-134517 # Call `run_bazel $NAME_OF_SCRIPT` to start an appropriately-configured Docker # container with the `cockroachdb/bazel` image running the given script. From 8a5865b606d52285b48a7e0573d7c2827e5ae0fb Mon Sep 17 00:00:00 2001 From: Ricky Stewart Date: Wed, 5 May 2021 16:49:51 -0500 Subject: [PATCH 2/5] bazel: use a wildcard instead of listing every stringer file for gazelle This excludes a single non-stringer file, `pkg/sql/sem/tree/parse_string.go`, so re-include it and mark it `# keep`. Release note: None --- BUILD.bazel | 41 +----------------------------------- pkg/sql/sem/tree/BUILD.bazel | 2 +- 2 files changed, 2 insertions(+), 41 deletions(-) diff --git a/BUILD.bazel b/BUILD.bazel index 13ff619872d2..80e8626b406e 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -86,46 +86,7 @@ load("@bazel_gazelle//:def.bzl", "gazelle") # gazelle:exclude vendor # gazelle:exclude pkg/security/securitytest/embedded.go # gazelle:exclude pkg/cmd/roachprod/vm/aws/embedded.go -# gazelle:exclude pkg/base/testclusterreplicationmode_string.go -# gazelle:exclude pkg/ccl/sqlproxyccl/errorcode_string.go -# gazelle:exclude pkg/cli/keytype_string.go -# gazelle:exclude pkg/clusterversion/key_string.go -# gazelle:exclude pkg/kv/kvclient/kvcoord/txnstate_string.go -# gazelle:exclude pkg/kv/kvserver/closedts/sidetransport/cantclosereason_string.go -# gazelle:exclude pkg/kv/kvserver/refreshraftreason_string.go -# gazelle:exclude pkg/roachpb/errordetailtype_string.go -# gazelle:exclude pkg/roachpb/method_string.go -# gazelle:exclude pkg/sql/advancecode_string.go -# gazelle:exclude pkg/sql/catalog/catalogkv/descriptorkind_string.go -# gazelle:exclude pkg/sql/catalog/descpb/formatversion_string.go -# gazelle:exclude pkg/sql/catalog/descpb/privilegedescversion_string.go -# gazelle:exclude pkg/sql/colfetcher/fetcherstate_string.go -# gazelle:exclude pkg/sql/execinfra/consumerstatus_string.go -# gazelle:exclude pkg/sql/execinfra/procstate_string.go -# gazelle:exclude pkg/sql/nodestatus_string.go -# gazelle:exclude pkg/sql/opt/optgen/lang/operator_string.go -# gazelle:exclude pkg/sql/opt/optgen/lang/token_string.go -# gazelle:exclude pkg/sql/opt/rule_name_string.go -# gazelle:exclude pkg/sql/pgwire/pgwirebase/clientmessagetype_string.go -# gazelle:exclude pkg/sql/pgwire/pgwirebase/formatcode_string.go -# gazelle:exclude pkg/sql/pgwire/pgwirebase/pgnumericsign_string.go -# gazelle:exclude pkg/sql/pgwire/pgwirebase/preparetype_string.go -# gazelle:exclude pkg/sql/pgwire/pgwirebase/servererrfieldtype_string.go -# gazelle:exclude pkg/sql/pgwire/pgwirebase/servermessagetype_string.go -# gazelle:exclude pkg/sql/privilege/kind_string.go -# gazelle:exclude pkg/sql/roleoption/option_string.go -# gazelle:exclude pkg/sql/schemachange/columnconversionkind_string.go -# gazelle:exclude pkg/sql/schemachanger/scop/type_string.go -# gazelle:exclude pkg/sql/sem/tree/createtypevariety_string.go -# gazelle:exclude pkg/sql/sem/tree/statementreturntype_string.go -# gazelle:exclude pkg/sql/sem/tree/statementtype_string.go -# gazelle:exclude pkg/sql/txnevent_string.go -# gazelle:exclude pkg/sql/txntype_string.go -# gazelle:exclude pkg/util/encoding/type_string.go -# gazelle:exclude pkg/util/timeutil/pgdate/field_string.go -# gazelle:exclude pkg/util/timeutil/pgdate/parsemode_string.go -# gazelle:exclude pkg/workload/schemachange/optype_string.go -# gazelle:exclude pkg/workload/schemachange/txstatus_string.go +# gazelle:exclude pkg/**/*_string.go # gazelle:exclude pkg/geo/wkt/wkt_generated.go # gazelle:exclude pkg/sql/schemachanger/scop/backfill_visitor_generated.go # gazelle:exclude pkg/sql/schemachanger/scop/mutation_visitor_generated.go diff --git a/pkg/sql/sem/tree/BUILD.bazel b/pkg/sql/sem/tree/BUILD.bazel index e78f8e8b10be..e10b789739fe 100644 --- a/pkg/sql/sem/tree/BUILD.bazel +++ b/pkg/sql/sem/tree/BUILD.bazel @@ -61,7 +61,7 @@ go_library( "operators.go", "overload.go", "parse_array.go", - "parse_string.go", + "parse_string.go", # keep "persistence.go", "pgwire_encode.go", "placeholders.go", From 3568d74348931ecbdfb68874677e4fa8e23f3fac Mon Sep 17 00:00:00 2001 From: Yahor Yuzefovich Date: Wed, 5 May 2021 22:33:05 -0700 Subject: [PATCH 3/5] sql: fix the session tracing in some cases When the session tracing is started, we might need to start two tracing spans: we always start a span for the connection, but also if we're inside of a txn, we start a separate span for the txn. Previously, the span of the previous txn wasn't properly cleaned up when the session tracing is started again outside of a txn, which resulted in old (irrelevant) entries being added to the newer trace. Now this is fixed. Additionally, this commit makes sure to finish the tracing spans (which wasn't done previously). Release note (bug fix): Previously, the session trace (i.e. `SHOW TRACE FOR SESSION`) could contain entries that corresponded to the previous trace (i.e. `SET TRACING=ON` didn't properly reset the trace). Now this is fixed. --- pkg/sql/exec_util.go | 18 +++-- .../testdata/autocommit_nonmetamorphic | 28 -------- .../testdata/show_trace_nonmetamorphic | 67 +++++++++++++++++++ 3 files changed, 75 insertions(+), 38 deletions(-) diff --git a/pkg/sql/exec_util.go b/pkg/sql/exec_util.go index 172a76405b4c..83f7d01f9a11 100644 --- a/pkg/sql/exec_util.go +++ b/pkg/sql/exec_util.go @@ -1612,10 +1612,11 @@ type SessionTracing struct { ex *connExecutor // firstTxnSpan is the span of the first txn that was active when session - // tracing was enabled. + // tracing was enabled. It is finished and unset in StopTracing. firstTxnSpan *tracing.Span - // connSpan is the connection's span. This is recording. + // connSpan is the connection's span. This is recording. It is finished and + // unset in StopTracing. connSpan *tracing.Span // lastRecording will collect the recording when stopping tracing. @@ -1731,7 +1732,6 @@ func (st *SessionTracing) StartTracing( } // StopTracing stops the trace that was started with StartTracing(). -// An error is returned if tracing was not active. func (st *SessionTracing) StopTracing() error { if !st.enabled { // We're not currently tracing. No-op. @@ -1742,18 +1742,16 @@ func (st *SessionTracing) StopTracing() error { st.showResults = false st.recordingType = tracing.RecordingOff + // Accumulate all recordings and finish the tracing spans. var spans []tracingpb.RecordedSpan - if st.firstTxnSpan != nil { spans = append(spans, st.firstTxnSpan.GetRecording()...) - st.firstTxnSpan.SetVerbose(false) + st.firstTxnSpan.Finish() + st.firstTxnSpan = nil } - st.connSpan.Finish() spans = append(spans, st.connSpan.GetRecording()...) - // NOTE: We're stopping recording on the connection's ctx only; the stopping - // is not inherited by children. If we are inside of a txn, that span will - // continue recording, even though nobody will collect its recording again. - st.connSpan.SetVerbose(false) + st.connSpan.Finish() + st.connSpan = nil st.ex.ctxHolder.unhijack() var err error diff --git a/pkg/sql/opt/exec/execbuilder/testdata/autocommit_nonmetamorphic b/pkg/sql/opt/exec/execbuilder/testdata/autocommit_nonmetamorphic index 4d735137293b..7247dac471b6 100644 --- a/pkg/sql/opt/exec/execbuilder/testdata/autocommit_nonmetamorphic +++ b/pkg/sql/opt/exec/execbuilder/testdata/autocommit_nonmetamorphic @@ -110,7 +110,6 @@ WHERE message LIKE '%r$rangeid: sending batch%' AND message NOT LIKE '%PushTxn%' AND message NOT LIKE '%QueryTxn%' ---- -dist sender send r37: sending batch 2 CPut to (n1,s1):1 dist sender send r37: sending batch 2 CPut, 1 EndTxn to (n1,s1):1 # TODO(radu): allow non-side-effecting projections. @@ -134,7 +133,6 @@ WHERE message LIKE '%r$rangeid: sending batch%' AND operation NOT LIKE '%async%' ---- dist sender send r37: sending batch 2 CPut to (n1,s1):1 -dist sender send r37: sending batch 2 CPut to (n1,s1):1 dist sender send r37: sending batch 1 EndTxn to (n1,s1):1 # Insert with RETURNING statement with side-effects should not auto-commit. @@ -159,7 +157,6 @@ WHERE message LIKE '%r$rangeid: sending batch%' AND operation NOT LIKE '%async%' ---- dist sender send r37: sending batch 2 CPut to (n1,s1):1 -dist sender send r37: sending batch 2 CPut to (n1,s1):1 dist sender send r37: sending batch 1 EndTxn to (n1,s1):1 # Another way to test the scenario above: generate an error and ensure that the @@ -195,7 +192,6 @@ WHERE message LIKE '%r$rangeid: sending batch%' AND message NOT LIKE '%PushTxn%' AND message NOT LIKE '%QueryTxn%' ---- -dist sender send r37: sending batch 2 CPut to (n1,s1):1 dist sender send r37: sending batch 1 Put, 1 EndTxn to (n1,s1):1 # Multi-row upsert should auto-commit. @@ -217,7 +213,6 @@ WHERE message LIKE '%r$rangeid: sending batch%' AND message NOT LIKE '%PushTxn%' AND message NOT LIKE '%QueryTxn%' ---- -dist sender send r37: sending batch 2 CPut to (n1,s1):1 dist sender send r37: sending batch 2 Put, 1 EndTxn to (n1,s1):1 # No auto-commit inside a transaction. @@ -266,7 +261,6 @@ WHERE message LIKE '%r$rangeid: sending batch%' AND message NOT LIKE '%PushTxn%' AND message NOT LIKE '%QueryTxn%' ---- -dist sender send r37: sending batch 2 Put to (n1,s1):1 dist sender send r37: sending batch 2 Put, 1 EndTxn to (n1,s1):1 # TODO(radu): allow non-side-effecting projections. @@ -290,7 +284,6 @@ WHERE message LIKE '%r$rangeid: sending batch%' AND operation NOT LIKE '%async%' ---- dist sender send r37: sending batch 2 Put to (n1,s1):1 -dist sender send r37: sending batch 2 Put to (n1,s1):1 dist sender send r37: sending batch 1 EndTxn to (n1,s1):1 # Upsert with RETURNING statement with side-effects should not auto-commit. @@ -315,7 +308,6 @@ WHERE message LIKE '%r$rangeid: sending batch%' AND operation NOT LIKE '%async%' ---- dist sender send r37: sending batch 2 Put to (n1,s1):1 -dist sender send r37: sending batch 2 Put to (n1,s1):1 dist sender send r37: sending batch 1 EndTxn to (n1,s1):1 # Another way to test the scenario above: generate an error and ensure that the @@ -351,7 +343,6 @@ WHERE message LIKE '%r$rangeid: sending batch%' AND message NOT LIKE '%PushTxn%' AND message NOT LIKE '%QueryTxn%' ---- -dist sender send r37: sending batch 2 Put to (n1,s1):1 dist sender send r37: sending batch 1 Scan to (n1,s1):1 dist sender send r37: sending batch 2 Put, 1 EndTxn to (n1,s1):1 @@ -403,8 +394,6 @@ WHERE message LIKE '%r$rangeid: sending batch%' AND message NOT LIKE '%QueryTxn%' ---- dist sender send r37: sending batch 1 Scan to (n1,s1):1 -dist sender send r37: sending batch 2 Put to (n1,s1):1 -dist sender send r37: sending batch 1 Scan to (n1,s1):1 dist sender send r37: sending batch 2 Put, 1 EndTxn to (n1,s1):1 # TODO(radu): allow non-side-effecting projections. @@ -429,8 +418,6 @@ WHERE message LIKE '%r$rangeid: sending batch%' ---- dist sender send r37: sending batch 1 Scan to (n1,s1):1 dist sender send r37: sending batch 2 Put to (n1,s1):1 -dist sender send r37: sending batch 1 Scan to (n1,s1):1 -dist sender send r37: sending batch 2 Put to (n1,s1):1 dist sender send r37: sending batch 1 EndTxn to (n1,s1):1 # Update with RETURNING statement with side-effects should not auto-commit. @@ -456,8 +443,6 @@ WHERE message LIKE '%r$rangeid: sending batch%' ---- dist sender send r37: sending batch 1 Scan to (n1,s1):1 dist sender send r37: sending batch 2 Put to (n1,s1):1 -dist sender send r37: sending batch 1 Scan to (n1,s1):1 -dist sender send r37: sending batch 2 Put to (n1,s1):1 dist sender send r37: sending batch 1 EndTxn to (n1,s1):1 # Another way to test the scenario above: generate an error and ensure that the @@ -493,8 +478,6 @@ WHERE message LIKE '%r$rangeid: sending batch%' AND message NOT LIKE '%PushTxn%' AND message NOT LIKE '%QueryTxn%' ---- -dist sender send r37: sending batch 1 Scan to (n1,s1):1 -dist sender send r37: sending batch 2 Put to (n1,s1):1 dist sender send r37: sending batch 1 DelRng, 1 EndTxn to (n1,s1):1 # Multi-row delete should auto-commit. @@ -516,8 +499,6 @@ WHERE message LIKE '%r$rangeid: sending batch%' AND message NOT LIKE '%PushTxn%' AND message NOT LIKE '%QueryTxn%' ---- -dist sender send r37: sending batch 1 Scan to (n1,s1):1 -dist sender send r37: sending batch 2 Put to (n1,s1):1 dist sender send r37: sending batch 1 DelRng, 1 EndTxn to (n1,s1):1 # No auto-commit inside a transaction. @@ -566,7 +547,6 @@ WHERE message LIKE '%r$rangeid: sending batch%' AND message NOT LIKE '%PushTxn%' AND message NOT LIKE '%QueryTxn%' ---- -dist sender send r37: sending batch 1 DelRng to (n1,s1):1 dist sender send r37: sending batch 1 Scan to (n1,s1):1 dist sender send r37: sending batch 2 Del, 1 EndTxn to (n1,s1):1 @@ -590,7 +570,6 @@ WHERE message LIKE '%r$rangeid: sending batch%' AND message NOT LIKE '%QueryTxn%' AND operation NOT LIKE '%async%' ---- -dist sender send r37: sending batch 1 DelRng to (n1,s1):1 dist sender send r37: sending batch 1 Scan to (n1,s1):1 dist sender send r37: sending batch 2 Del to (n1,s1):1 dist sender send r37: sending batch 1 EndTxn to (n1,s1):1 @@ -616,7 +595,6 @@ WHERE message LIKE '%r$rangeid: sending batch%' AND message NOT LIKE '%QueryTxn%' AND operation NOT LIKE '%async%' ---- -dist sender send r37: sending batch 1 DelRng to (n1,s1):1 dist sender send r37: sending batch 1 Scan to (n1,s1):1 dist sender send r37: sending batch 2 Del to (n1,s1):1 dist sender send r37: sending batch 1 EndTxn to (n1,s1):1 @@ -666,7 +644,6 @@ WHERE message LIKE '%r$rangeid: sending batch%' AND message NOT LIKE '%QueryTxn%' AND operation NOT LIKE '%async%' ---- -dist sender send r37: sending batch 1 DelRng to (n1,s1):1 dist sender send r37: sending batch 2 CPut to (n1,s1):1 dist sender send r37: sending batch 2 Get to (n1,s1):1 dist sender send r37: sending batch 1 EndTxn to (n1,s1):1 @@ -690,7 +667,6 @@ WHERE message LIKE '%r$rangeid: sending batch%' AND message NOT LIKE '%QueryTxn%' AND operation NOT LIKE '%async%' ---- -dist sender send r37: sending batch 1 DelRng to (n1,s1):1 dist sender send r37: sending batch 1 Scan to (n1,s1):1 dist sender send r37: sending batch 1 Put to (n1,s1):1 dist sender send r37: sending batch 1 Scan to (n1,s1):1 @@ -716,7 +692,6 @@ WHERE message LIKE '%r$rangeid: sending batch%' AND message NOT LIKE '%QueryTxn%' AND operation NOT LIKE '%async%' ---- -dist sender send r37: sending batch 1 DelRng to (n1,s1):1 dist sender send r37: sending batch 1 Get to (n1,s1):1 dist sender send r37: sending batch 1 Del to (n1,s1):1 dist sender send r37: sending batch 1 Scan to (n1,s1):1 @@ -745,7 +720,6 @@ WHERE message LIKE '%r$rangeid: sending batch%' AND operation NOT LIKE '%async%' ---- dist sender send r37: sending batch 1 DelRng to (n1,s1):1 -dist sender send r37: sending batch 1 DelRng to (n1,s1):1 dist sender send r37: sending batch 1 Scan to (n1,s1):1 dist sender send r37: sending batch 1 Del, 1 EndTxn to (n1,s1):1 @@ -775,7 +749,6 @@ WHERE message LIKE '%r$rangeid: sending batch%' AND message NOT LIKE '%QueryTxn%' AND operation NOT LIKE '%async%' ---- -dist sender send r37: sending batch 1 DelRng to (n1,s1):1 dist sender send r37: sending batch 2 CPut to (n1,s1):1 dist sender send r37: sending batch 2 CPut to (n1,s1):1 dist sender send r37: sending batch 1 EndTxn to (n1,s1):1 @@ -801,7 +774,6 @@ WHERE message LIKE '%r$rangeid: sending batch%' AND message NOT LIKE '%QueryTxn%' AND operation NOT LIKE '%async%' ---- -dist sender send r37: sending batch 1 DelRng to (n1,s1):1 dist sender send r37: sending batch 2 CPut to (n1,s1):1 dist sender send r37: sending batch 2 CPut to (n1,s1):1 dist sender send r37: sending batch 1 EndTxn to (n1,s1):1 diff --git a/pkg/sql/opt/exec/execbuilder/testdata/show_trace_nonmetamorphic b/pkg/sql/opt/exec/execbuilder/testdata/show_trace_nonmetamorphic index abc06ba472c2..e39b9b494d19 100644 --- a/pkg/sql/opt/exec/execbuilder/testdata/show_trace_nonmetamorphic +++ b/pkg/sql/opt/exec/execbuilder/testdata/show_trace_nonmetamorphic @@ -316,3 +316,70 @@ SET tracing = on; SELECT * FROM t; SET tracing = off # dist sender send r6: sending batch 1 Get to (n1,s1):1 # dist sender send querying next range at /Table/12/1 # dist sender send r8: sending batch 1 Scan to (n1,s1):1 + +# Regression tests for incorrect interaction between consecutive session traces +# (#59203, #60672). +statement ok +CREATE TABLE a (a INT PRIMARY KEY) + +# Get the range id. +let $rangeid +SELECT range_id FROM [ SHOW RANGES FROM TABLE a ] + +# Populate table descriptor cache. +query I +SELECT * FROM a +---- + +statement ok +BEGIN; +SET TRACING=ON; + INSERT INTO a VALUES (1); +ROLLBACK + +# The tracing is still enabled. Insert a couple of rows with auto-commit, and +# stop the tracing. +statement ok + INSERT INTO a VALUES (2), (3); +SET TRACING=OFF + +query TT +SELECT operation, message FROM [SHOW KV TRACE FOR SESSION] +WHERE message LIKE '%r$rangeid: sending batch%' + AND message NOT LIKE '%PushTxn%' + AND message NOT LIKE '%QueryTxn%' +---- +dist sender send r38: sending batch 1 CPut to (n1,s1):1 +dist sender send r38: sending batch 1 EndTxn to (n1,s1):1 +dist sender send r38: sending batch 2 CPut, 1 EndTxn to (n1,s1):1 + +# Make another session trace. +statement ok +BEGIN; +SET TRACING=ON; + INSERT INTO a VALUES (4), (5), (6); +SET TRACING=OFF; +COMMIT + +# Start the tracing again and insert a few rows with auto-commit. +statement ok +SET TRACING=ON; + INSERT INTO a VALUES (7), (8), (9), (10); + +# The tracing is still enabled. Insert a few rows, rollback the txn, and stop +# the tracing. +statement ok +BEGIN; + INSERT INTO a VALUES (11), (12), (13), (14), (15); +ROLLBACK; +SET TRACING=OFF; + +query TT +SELECT operation, message FROM [SHOW KV TRACE FOR SESSION] +WHERE message LIKE '%r$rangeid: sending batch%' + AND message NOT LIKE '%PushTxn%' + AND message NOT LIKE '%QueryTxn%' +---- +dist sender send r38: sending batch 4 CPut, 1 EndTxn to (n1,s1):1 +dist sender send r38: sending batch 5 CPut to (n1,s1):1 +dist sender send r38: sending batch 1 EndTxn to (n1,s1):1 From c8c1404516c2e74f9ecf515c8ab03f6de97d6316 Mon Sep 17 00:00:00 2001 From: Steven Danna Date: Thu, 6 May 2021 14:32:23 +0100 Subject: [PATCH 4/5] changefeedccl: add test for disabled outbound IO This adds a test to ensure that we do not allow sinkful changefeeds to be started when the --external-io-disabled flag has been set. Note that this only tests a single node setup. For the flag to be effective, it needs to be set on all nodes in a cluster. Release note: None --- pkg/base/test_server_args.go | 4 ++ pkg/ccl/changefeedccl/changefeed_test.go | 48 ++++++++++++++++++++++++ pkg/server/testserver.go | 1 + 3 files changed, 53 insertions(+) diff --git a/pkg/base/test_server_args.go b/pkg/base/test_server_args.go index eccb9d68c18f..dffd5adc3185 100644 --- a/pkg/base/test_server_args.go +++ b/pkg/base/test_server_args.go @@ -85,6 +85,10 @@ type TestServerArgs struct { // ExternalIODir is used to initialize field in cluster.Settings. ExternalIODir string + // ExternalIODirConfig is used to initialize the same-named + // field on the server.Config struct. + ExternalIODirConfig ExternalIODirConfig + // Fields copied to the server.Config. Insecure bool RetryOptions retry.Options // TODO(tbg): make testing knob. diff --git a/pkg/ccl/changefeedccl/changefeed_test.go b/pkg/ccl/changefeedccl/changefeed_test.go index c24b339f2238..a7a099049f2b 100644 --- a/pkg/ccl/changefeedccl/changefeed_test.go +++ b/pkg/ccl/changefeedccl/changefeed_test.go @@ -551,6 +551,54 @@ func TestChangefeedUserDefinedTypes(t *testing.T) { t.Run(`kafka`, kafkaTest(testFn)) } +func TestChangefeedExternalIODisabled(t *testing.T) { + defer leaktest.AfterTest(t)() + defer log.Scope(t).Close(t) + t.Run("sinkful changefeeds not allowed with disabled external io", func(t *testing.T) { + disallowedSinkProtos := []string{ + changefeedbase.SinkSchemeExperimentalSQL, + changefeedbase.SinkSchemeKafka, + changefeedbase.SinkSchemeNull, // Doesn't work because all sinkful changefeeds are disallowed + // Cloud sink schemes + "experimental-s3", + "experimental-gs", + "experimental-nodelocal", + "experimental-http", + "experimental-https", + "experimental-azure", + } + ctx := context.Background() + s, db, _ := serverutils.StartServer(t, base.TestServerArgs{ + ExternalIODirConfig: base.ExternalIODirConfig{ + DisableOutbound: true, + }, + }) + defer s.Stopper().Stop(ctx) + sqlDB := sqlutils.MakeSQLRunner(db) + sqlDB.Exec(t, "CREATE TABLE target_table (pk INT PRIMARY KEY)") + for _, proto := range disallowedSinkProtos { + sqlDB.ExpectErr(t, "Outbound IO is disabled by configuration, cannot create changefeed", + "CREATE CHANGEFEED FOR target_table INTO $1", + fmt.Sprintf("%s://does-not-matter", proto), + ) + } + }) + + withDisabledOutbound := func(args *base.TestServerArgs) { args.ExternalIODirConfig.DisableOutbound = true } + t.Run("sinkless changfeeds are allowed with disabled external io", + sinklessTestWithServerArgs(withDisabledOutbound, + func(t *testing.T, db *gosql.DB, f cdctest.TestFeedFactory) { + sqlDB := sqlutils.MakeSQLRunner(db) + sqlDB.Exec(t, "CREATE TABLE target_table (pk INT PRIMARY KEY)") + sqlDB.Exec(t, "INSERT INTO target_table VALUES (1)") + feed := feed(t, f, "CREATE CHANGEFEED FOR target_table") + defer closeFeed(t, feed) + assertPayloads(t, feed, []string{ + `target_table: [1]->{"after": {"pk": 1}}`, + }) + })) +} + // Test how Changefeeds react to schema changes that do not require a backfill // operation. func TestChangefeedSchemaChangeNoBackfill(t *testing.T) { diff --git a/pkg/server/testserver.go b/pkg/server/testserver.go index 439e341aa7ef..38945c680878 100644 --- a/pkg/server/testserver.go +++ b/pkg/server/testserver.go @@ -147,6 +147,7 @@ func makeTestConfigFromParams(params base.TestServerArgs) Config { cfg.JoinList = []string{params.JoinAddr} } cfg.ClusterName = params.ClusterName + cfg.ExternalIODirConfig = params.ExternalIODirConfig cfg.Insecure = params.Insecure cfg.AutoInitializeCluster = !params.NoAutoInitializeCluster cfg.SocketFile = params.SocketFile From 2053f173773b8f7502ba472044554cdb29554b9a Mon Sep 17 00:00:00 2001 From: richardjcai Date: Wed, 14 Apr 2021 14:48:21 -0400 Subject: [PATCH 5/5] roachtest: add roachtest against pg ruby driver. Release note: None --- pkg/cmd/roachtest/BUILD.bazel | 2 + pkg/cmd/roachtest/registry.go | 1 + pkg/cmd/roachtest/ruby_pg.go | 205 ++++++++++++++ pkg/cmd/roachtest/ruby_pg_blocklist.go | 194 +++++++++++++ pkg/cmd/roachtest/ruby_pg_helpers.rb | 377 +++++++++++++++++++++++++ pkg/testutils/lint/lint_test.go | 1 + 6 files changed, 780 insertions(+) create mode 100644 pkg/cmd/roachtest/ruby_pg.go create mode 100644 pkg/cmd/roachtest/ruby_pg_blocklist.go create mode 100644 pkg/cmd/roachtest/ruby_pg_helpers.rb diff --git a/pkg/cmd/roachtest/BUILD.bazel b/pkg/cmd/roachtest/BUILD.bazel index c8d14c7ff1f5..66bcad5d8628 100644 --- a/pkg/cmd/roachtest/BUILD.bazel +++ b/pkg/cmd/roachtest/BUILD.bazel @@ -93,6 +93,8 @@ go_library( "restart.go", "restore.go", "roachmart.go", + "ruby_pg.go", + "ruby_pg_blocklist.go", "schema_change_database_version_upgrade.go", "schemachange.go", "schemachange_random_load.go", diff --git a/pkg/cmd/roachtest/registry.go b/pkg/cmd/roachtest/registry.go index 35a09cc773db..671dfaafecfc 100644 --- a/pkg/cmd/roachtest/registry.go +++ b/pkg/cmd/roachtest/registry.go @@ -83,6 +83,7 @@ func registerTests(r *testRegistry) { registerRestoreNodeShutdown(r) registerRestore(r) registerRoachmart(r) + registerRubyPG(r) registerSchemaChangeBulkIngest(r) registerSchemaChangeDatabaseVersionUpgrade(r) registerSchemaChangeDuringKV(r) diff --git a/pkg/cmd/roachtest/ruby_pg.go b/pkg/cmd/roachtest/ruby_pg.go new file mode 100644 index 000000000000..a2881368d11a --- /dev/null +++ b/pkg/cmd/roachtest/ruby_pg.go @@ -0,0 +1,205 @@ +// Copyright 2021 The Cockroach Authors. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package main + +import ( + "bufio" + "bytes" + "context" + "fmt" + "regexp" + + "github.com/cockroachdb/cockroach/pkg/util/log" + "github.com/stretchr/testify/require" +) + +var rubyPGTestFailureRegex = regexp.MustCompile(`^rspec ./.*# .*`) +var rubyPGVersion = "v1.2.3" + +// This test runs Ruby PG's full test suite against a single cockroach node. +func registerRubyPG(r *testRegistry) { + runRubyPGTest := func( + ctx context.Context, + t *test, + c *cluster, + ) { + if c.isLocal() { + t.Fatal("cannot be run in local mode") + } + node := c.Node(1) + t.Status("setting up cockroach") + c.Put(ctx, cockroach, "./cockroach", c.All()) + if err := c.PutLibraries(ctx, "./lib"); err != nil { + t.Fatal(err) + } + c.Start(ctx, t, c.All()) + + version, err := fetchCockroachVersion(ctx, c, node[0], nil) + if err != nil { + t.Fatal(err) + } + + if err := alterZoneConfigAndClusterSettings(ctx, version, c, node[0], nil); err != nil { + t.Fatal(err) + } + + t.Status("cloning rails and installing prerequisites") + + c.l.Printf("Supported ruby-pg version is %s.", rubyPGVersion) + + if err := repeatRunE( + ctx, c, node, "update apt-get", `sudo apt-get -qq update`, + ); err != nil { + t.Fatal(err) + } + + if err := repeatRunE( + ctx, + c, + node, + "install dependencies", + `sudo apt-get -qq install ruby-full ruby-dev rubygems build-essential zlib1g-dev libpq-dev libsqlite3-dev`, + ); err != nil { + t.Fatal(err) + } + + if err := repeatRunE( + ctx, + c, + node, + "install ruby 2.7", + `mkdir -p ruby-install && \ + curl -fsSL https://github.com/postmodern/ruby-install/archive/v0.6.1.tar.gz | tar --strip-components=1 -C ruby-install -xz && \ + sudo make -C ruby-install install && \ + sudo ruby-install --system ruby 2.7.1 && \ + sudo gem update --system`, + ); err != nil { + t.Fatal(err) + } + + if err := repeatRunE( + ctx, c, node, "remove old ruby-pg", `sudo rm -rf /mnt/data1/ruby-pg`, + ); err != nil { + t.Fatal(err) + } + + if err := repeatGitCloneE( + ctx, + t.l, + c, + "https://github.com/ged/ruby-pg.git", + "/mnt/data1/ruby-pg", + rubyPGVersion, + node, + ); err != nil { + t.Fatal(err) + } + + t.Status("installing bundler") + if err := repeatRunE( + ctx, + c, + node, + "installing bundler", + `cd /mnt/data1/ruby-pg/ && sudo gem install bundler:2.1.4`, + ); err != nil { + t.Fatal(err) + } + + t.Status("installing gems") + if err := repeatRunE( + ctx, + c, + node, + "installing gems", + `cd /mnt/data1/ruby-pg/ && sudo bundle install`, + ); err != nil { + t.Fatal(err) + } + + if err := repeatRunE( + ctx, c, node, "remove old ruby-pg helpers.rb", `sudo rm /mnt/data1/ruby-pg/spec/helpers.rb`, + ); err != nil { + t.Fatal(err) + } + + // Write the cockroach config into the test suite to use. + err = c.PutE(ctx, c.l, "./pkg/cmd/roachtest/ruby_pg_helpers.rb", "/mnt/data1/ruby-pg/spec/helpers.rb", c.All()) + require.NoError(t, err) + + t.Status("running ruby-pg test suite") + // Note that this is expected to return an error, since the test suite + // will fail. And it is safe to swallow it here. + rawResults, _ := c.RunWithBuffer(ctx, t.l, node, + `cd /mnt/data1/ruby-pg/ && sudo rake`, + ) + + c.l.Printf("Test Results:\n%s", rawResults) + + // Find all the failed and errored tests. + results := newORMTestsResults() + blocklistName, expectedFailures, _, _ := rubyPGBlocklist.getLists(version) + if expectedFailures == nil { + t.Fatalf("No ruby-pg blocklist defined for cockroach version %s", version) + } + + scanner := bufio.NewScanner(bytes.NewReader(rawResults)) + for scanner.Scan() { + match := rubyPGTestFailureRegex.FindStringSubmatch(scanner.Text()) + if match == nil { + continue + } + if len(match) != 1 { + log.Fatalf(ctx, "expected one match for test name, found: %d", len(match)) + } + + // Take the first test name. + test := match[0] + + // This regex is used to get the name of the test. + // The test name follows the file name and a hashtag. + // ie. test.rb:99 # TEST NAME. + strs := regexp.MustCompile("^rspec .*.rb.*([0-9]|]) # ").Split(test, -1) + if len(strs) != 2 { + log.Fatalf(ctx, "expected test output line to be split into two strings") + } + test = strs[1] + + issue, expectedFailure := expectedFailures[test] + switch { + case expectedFailure: + results.results[test] = fmt.Sprintf("--- FAIL: %s - %s (expected)", + test, maybeAddGithubLink(issue), + ) + results.failExpectedCount++ + results.currentFailures = append(results.currentFailures, test) + case !expectedFailure: + results.results[test] = fmt.Sprintf("--- PASS: %s - %s (unexpected)", + test, maybeAddGithubLink(issue), + ) + } + results.runTests[test] = struct{}{} + } + + results.summarizeAll(t, "ruby-pg", blocklistName, expectedFailures, version, rubyPGVersion) + } + + r.Add(testSpec{ + MinVersion: "v20.1.0", + Name: "ruby-pg", + Owner: OwnerSQLExperience, + Cluster: makeClusterSpec(1), + Tags: []string{`default`, `orm`}, + Run: func(ctx context.Context, t *test, c *cluster) { + runRubyPGTest(ctx, t, c) + }, + }) +} diff --git a/pkg/cmd/roachtest/ruby_pg_blocklist.go b/pkg/cmd/roachtest/ruby_pg_blocklist.go new file mode 100644 index 000000000000..7b1d72895017 --- /dev/null +++ b/pkg/cmd/roachtest/ruby_pg_blocklist.go @@ -0,0 +1,194 @@ +// Copyright 2021 The Cockroach Authors. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package main + +var rubyPGBlocklist = blocklistsForVersion{ + {"v20.2", "rubyPGBlockList20_2", rubyPGBlockList20_2, "rubyPGIgnoreList20_2", rubyPGIgnoreList20_2}, + {"v21.1", "rubyPGBlockList21_1", rubyPGBlockList21_1, "rubyPGIgnoreList21_1", rubyPGIgnoreList21_1}, +} + +var rubyPGBlockList21_1 = rubyPGBlockList20_2 + +var rubyPGBlockList20_2 = blocklist{ + "PG::TypeMapByColumn will deny copy queries with different column count": "unknown", + "PG::TypeMapByColumn should gracefully handle not initialized state": "unknown", + "PG::TypeMapByColumn forwards get_copy_data conversions to another TypeMapByColumn as #default_type_map": "unknown", + "PG::TypeMapByOid should build a TypeMapByColumn when assigned and the number of rows is high enough": "unknown", + "PG::TypeMapByOid should allow mixed type conversions in binary format": "unknown", + "PG::TypeMapByOid should allow mixed type conversions in text format": "unknown", + "PG::TypeMapByOid should allow building new TypeMapByColumn for a given result": "unknown", + "running with sync_* methods PG::Connection calls the block supplied to wait_for_notify with the notify payload if it accepts three arguments": "unknown", + "running with sync_* methods PG::Connection calls a block for NOTIFY events if one is given": "unknown", + "running with sync_* methods PG::Connection sends nil as the payload if the notification wasn't given one": "unknown", + "running with sync_* methods PG::Connection described_class#block shouldn't block a second thread": "unknown", + "running with sync_* methods PG::Connection gracefully handle SQL statements while in #copy_data for output": "unknown", + "running with sync_* methods PG::Connection calls the block supplied to wait_for_notify with the notify payload if it accepts two arguments": "unknown", + "running with sync_* methods PG::Connection not read past the end of a large object": "unknown", + "running with sync_* methods PG::Connection automatically rolls back a transaction started with Connection#transaction if an exception is raised": "unknown", + "running with sync_* methods PG::Connection doesn't leave stale server connections after finish": "unknown", + "running with sync_* methods PG::Connection can handle server errors in #copy_data for input": "unknown", + "running with sync_* methods PG::Connection can process #copy_data output queries": "unknown", + "running with sync_* methods PG::Connection gracefully handle SQL statements while in #copy_data for input": "unknown", + "running with sync_* methods PG::Connection doesn't collapse sequential notifications": "unknown", + "running with sync_* methods PG::Connection can process #copy_data input queries": "unknown", + "running with sync_* methods PG::Connection can handle client errors in #copy_data for output": "unknown", + "running with sync_* methods PG::Connection correctly finishes COPY queries passed to #async_exec": "unknown", + "running with sync_* methods PG::Connection calls the block supplied to wait_for_notify with the notify payload if it doesn't accept arguments": "unknown", + "running with sync_* methods PG::Connection can handle client errors in #copy_data for input": "unknown", + "running with sync_* methods PG::Connection can handle server errors in #copy_data for output": "unknown", + "running with sync_* methods PG::Connection returns notifications which are already in the queue before wait_for_notify is called without waiting for the socket to become readable": "unknown", + "running with sync_* methods PG::Connection calls the block supplied to wait_for_notify with the notify payload if it accepts any number of arguments": "unknown", + "running with sync_* methods PG::Connection allows a query to be cancelled": "unknown", + "running with sync_* methods PG::Connection accepts nil as the timeout in #wait_for_notify ": "unknown", + "running with sync_* methods PG::Connection can handle incomplete #copy_data output queries": "unknown", + "running with sync_* methods PG::Connection can receive notices while waiting for NOTIFY without exceeding the timeout": "unknown", + "running with sync_* methods PG::Connection trace and untrace client-server communication": "unknown", + "running with sync_* methods PG::Connection returns the block result from Connection#transaction": "unknown", + "running with sync_* methods PG::Connection can wait for NOTIFY events": "unknown", + "running with sync_* methods PG::Connection set_single_row_mode should receive rows before entire query is finished": "unknown", + "running with sync_* methods PG::Connection set_single_row_mode should receive rows before entire query fails": "unknown", + "running with sync_* methods PG::Connection deprecated forms of methods should forward send_query to send_query_params": "unknown", + "running with sync_* methods PG::Connection deprecated forms of methods should forward exec to exec_params": "unknown", + "running with sync_* methods PG::Connection multinationalization support returns properly encoded text from notifies": "unknown", + "running with sync_* methods PG::Connection multinationalization support receives properly encoded text from wait_for_notify": "unknown", + "running with sync_* methods PG::Connection multinationalization support receives properly encoded messages in the notice callbacks": "unknown", + "running with sync_* methods PG::Connection multinationalization support encodes exception messages with the connection's encoding (#96)": "unknown", + "running with sync_* methods PG::Connection multinationalization support handles clearing result in or after set_notice_receiver": "unknown", + "running with sync_* methods PG::Connection multinationalization support respect and convert character encoding of input strings should convert strings to #send_describe_portal": "unknown", + "running with sync_* methods PG::Connection multinationalization support respect and convert character encoding of input strings should convert strings and parameters to #prepare and #exec_prepared": "unknown", + "running with sync_* methods PG::Connection multinationalization support respect and convert character encoding of input strings should convert query string and parameters to #send_query_params": "unknown", + "running with sync_* methods PG::Connection multinationalization support respect and convert character encoding of input strings should convert strings and parameters to #send_prepare and #send_query_prepared": "unknown", + "running with sync_* methods PG::Connection multinationalization support respect and convert character encoding of input strings should convert strings to #describe_portal": "unknown", + "running with sync_* methods PG::Connection multinationalization support respect and convert character encoding of input strings should convert query string and parameters to #exec_params": "unknown", + "running with sync_* methods PG::Connection multinationalization support respect and convert character encoding of input strings should convert error string to #put_copy_end": "unknown", + "running with sync_* methods PG::Connection multinationalization support rubyforge #22925: m17n support uses the client encoding for escaped literal": "unknown", + "running with sync_* methods PG::Connection multinationalization support rubyforge #22925: m17n support should return results in the same encoding as the client (EUC-JP)": "unknown", + "running with sync_* methods PG::Connection multinationalization support rubyforge #22925: m17n support the connection should use JOHAB dummy encoding when it's set to JOHAB": "unknown", + "running with sync_* methods PG::Connection multinationalization support rubyforge #22925: m17n support should return results in the same encoding as the client (iso-8859-1)": "unknown", + "running with sync_* methods PG::Connection multinationalization support rubyforge #22925: m17n support can use an encoding with high index for client encoding": "unknown", + "running with sync_* methods PG::Connection multinationalization support rubyforge #22925: m17n support uses the client encoding for escaped string": "unknown", + "running with sync_* methods PG::Connection multinationalization support rubyforge #22925: m17n support uses the previous string encoding for escaped string": "unknown", + "running with sync_* methods PG::Connection multinationalization support rubyforge #22925: m17n support uses the previous string encoding for quote_ident": "unknown", + "running with sync_* methods PG::Connection multinationalization support rubyforge #22925: m17n support uses the client encoding for quote_ident": "unknown", + "running with sync_* methods PG::Connection multinationalization support rubyforge #22925: m17n support the connection should return ASCII-8BIT when it's set to SQL_ASCII": "unknown", + "running with sync_* methods PG::Connection multinationalization support rubyforge #22925: m17n support uses the client encoding for escaped identifier": "unknown", + "running with sync_* methods PG::Connection multinationalization support rubyforge #22925: m17n support returns the results in the correct encoding even if the client_encoding has changed since the results were fetched": "unknown", + "running with sync_* methods PG::Connection multinationalization support rubyforge #22925: m17n support raises appropriate error if set_client_encoding is called with invalid arguments": "unknown", + "running with sync_* methods PG::Connection multinationalization support Ruby 1.9.x default_internal encoding allows users of the async interface to set the client_encoding to the default_internal": "unknown", + "running with sync_* methods PG::Connection multinationalization support Ruby 1.9.x default_internal encoding honors the Encoding.default_internal if it's set and the synchronous interface is used": "unknown", + "running with sync_* methods PG::Connection type casting shouldn't type map params unless requested": "unknown", + "running with sync_* methods PG::Connection type casting can type cast parameters to put_copy_data with explicit encoder": "unknown", + "running with sync_* methods PG::Connection type casting with default query type map can process #copy_data input queries with row encoder and respects character encoding": "unknown", + "running with sync_* methods PG::Connection type casting with default result type map should work with arbitrary number of params in conjunction with type casting": "unknown", + "running with sync_* methods PG::Connection type casting with default result type map can process #copy_data output with row decoder and respects character encoding": "unknown", + "running with sync_* methods PG::Connection type casting with default result type map should respect a type mapping for result": "unknown", + "running with sync_* methods PG::Connection type casting with default result type map can type cast #copy_data output with explicit decoder": "unknown", + "PG::Result raises a proper exception for a nonexistant schema": "unknown", + "PG::Result encapsulates errors in a PG::Error object": "unknown", + "PG::Result encapsulates database object names for integrity constraint violations": "unknown", + "Basic type mapping PG::BasicTypeMapForResults with usage of result oids for copy decoder selection can type cast #copy_data output with explicit decoder": "unknown", + "Basic type mapping PG::BasicTypeMapForResults connection wide type mapping should convert format 0 timestamps per TimestampLocal": "unknown", + "Basic type mapping PG::BasicTypeMapForResults connection wide type mapping should convert format 1 timestamps per TimestampUtc": "unknown", + "Basic type mapping PG::BasicTypeMapForResults connection wide type mapping should do float type conversions": "unknown", + "Basic type mapping PG::BasicTypeMapForResults connection wide type mapping should do cidr type conversions": "unknown", + "Basic type mapping PG::BasicTypeMapForResults connection wide type mapping should do text datetime without time zone type conversions": "unknown", + "Basic type mapping PG::BasicTypeMapForResults connection wide type mapping should convert format 0 timestamps per TimestampUtc": "unknown", + "Basic type mapping PG::BasicTypeMapForResults connection wide type mapping should convert format 1 timestamps per TimestampUtcToLocal": "unknown", + "Basic type mapping PG::BasicTypeMapForResults connection wide type mapping should convert format 0 timestamps per TimestampUtcToLocal": "unknown", + "Basic type mapping PG::BasicTypeMapForResults connection wide type mapping should do array type conversions": "unknown", + "Basic type mapping PG::BasicTypeMapForResults connection wide type mapping should convert format 1 timestamps with time zone": "unknown", + "Basic type mapping PG::BasicTypeMapForResults connection wide type mapping should convert format 0 timestamps with time zone": "unknown", + "Basic type mapping PG::BasicTypeMapForResults connection wide type mapping should convert format 1 timestamps per TimestampLocal": "unknown", + "Basic type mapping PG::BasicTypeMapBasedOnResult with usage of result oids for copy encoder selection can type cast #copy_data input with explicit encoder": "unknown", + "Basic type mapping PG::BasicTypeMapBasedOnResult with usage of result oids for bind params encoder selection can do JSON conversions": "unknown", + "Basic type mapping PG::BasicTypeMapForQueries should do IPAddr param encoding": "unknown", + "Basic type mapping PG::BasicTypeMapForQueries should do bigdecimal param encoding": "unknown", + "Basic type mapping PG::BasicTypeMapForQueries should do hash-as-json encoding": "unknown", + "Basic type mapping PG::BasicTypeMapForQueries should do array of string encoding on unknown classes": "unknown", + "Basic type mapping PG::BasicTypeMapForQueries should do default array-as-array param encoding with Time objects": "unknown", + "Basic type mapping PG::BasicTypeMapForQueries should do default array-as-array param encoding": "unknown", + "Basic type mapping PG::BasicTypeMapForQueries should do basic param encoding": "unknown", + "Basic type mapping PG::BasicTypeMapForQueries should do basic Time encoding": "unknown", + "Basic type mapping PG::BasicTypeMapForQueries should do basic param encoding of various float values": "unknown", + "Basic type mapping PG::BasicTypeMapForQueries should do array-as-json encoding": "unknown", + "Basic type mapping PG::BasicTypeMapForQueries Record encoding should do array-as-record encoding": "unknown", + "PG::TypeMapByClass should expire the cache after changes to the coders": "unknown", + "PG::Connection doesn't leave stale server connections after finish": "unknown", + "PG::Connection sends nil as the payload if the notification wasn't given one": "unknown", + "PG::Connection can handle client errors in #copy_data for input": "unknown", + "PG::Connection automatically rolls back a transaction started with Connection#transaction if an exception is raised": "unknown", + "PG::Connection returns the block result from Connection#transaction": "unknown", + "PG::Connection can handle server errors in #copy_data for output": "unknown", + "PG::Connection calls the block supplied to wait_for_notify with the notify payload if it doesn't accept arguments": "unknown", + "PG::Connection allows a query to be cancelled": "unknown", + "PG::Connection can process #copy_data output queries": "unknown", + "PG::Connection can handle client errors in #copy_data for output": "unknown", + "PG::Connection returns notifications which are already in the queue before wait_for_notify is called without waiting for the socket to become readable": "unknown", + "PG::Connection can process #copy_data input queries": "unknown", + "PG::Connection not read past the end of a large object": "unknown", + "PG::Connection doesn't collapse sequential notifications": "unknown", + "PG::Connection gracefully handle SQL statements while in #copy_data for output": "unknown", + "PG::Connection calls a block for NOTIFY events if one is given": "unknown", + "PG::Connection can wait for NOTIFY events": "unknown", + "PG::Connection can handle incomplete #copy_data output queries": "unknown", + "PG::Connection calls the block supplied to wait_for_notify with the notify payload if it accepts any number of arguments": "unknown", + "PG::Connection gracefully handle SQL statements while in #copy_data for input": "unknown", + "PG::Connection can handle server errors in #copy_data for input": "unknown", + "PG::Connection can receive notices while waiting for NOTIFY without exceeding the timeout": "unknown", + "PG::Connection correctly finishes COPY queries passed to #async_exec": "unknown", + "PG::Connection trace and untrace client-server communication": "unknown", + "PG::Connection accepts nil as the timeout in #wait_for_notify ": "unknown", + "PG::Connection calls the block supplied to wait_for_notify with the notify payload if it accepts three arguments": "unknown", + "PG::Connection described_class#block shouldn't block a second thread": "unknown", + "PG::Connection calls the block supplied to wait_for_notify with the notify payload if it accepts two arguments": "unknown", + "PG::Connection multinationalization support handles clearing result in or after set_notice_receiver": "unknown", + "PG::Connection multinationalization support receives properly encoded messages in the notice callbacks": "unknown", + "PG::Connection multinationalization support encodes exception messages with the connection's encoding (#96)": "unknown", + "PG::Connection multinationalization support returns properly encoded text from notifies": "unknown", + "PG::Connection multinationalization support receives properly encoded text from wait_for_notify": "unknown", + "PG::Connection multinationalization support Ruby 1.9.x default_internal encoding allows users of the async interface to set the client_encoding to the default_internal": "unknown", + "PG::Connection multinationalization support Ruby 1.9.x default_internal encoding honors the Encoding.default_internal if it's set and the synchronous interface is used": "unknown", + "PG::Connection multinationalization support rubyforge #22925: m17n support uses the previous string encoding for quote_ident": "unknown", + "PG::Connection multinationalization support rubyforge #22925: m17n support raises appropriate error if set_client_encoding is called with invalid arguments": "unknown", + "PG::Connection multinationalization support rubyforge #22925: m17n support can use an encoding with high index for client encoding": "unknown", + "PG::Connection multinationalization support rubyforge #22925: m17n support uses the client encoding for escaped string": "unknown", + "PG::Connection multinationalization support rubyforge #22925: m17n support uses the client encoding for escaped literal": "unknown", + "PG::Connection multinationalization support rubyforge #22925: m17n support returns the results in the correct encoding even if the client_encoding has changed since the results were fetched": "unknown", + "PG::Connection multinationalization support rubyforge #22925: m17n support uses the previous string encoding for escaped string": "unknown", + "PG::Connection multinationalization support rubyforge #22925: m17n support the connection should use JOHAB dummy encoding when it's set to JOHAB": "unknown", + "PG::Connection multinationalization support rubyforge #22925: m17n support uses the client encoding for escaped identifier": "unknown", + "PG::Connection multinationalization support rubyforge #22925: m17n support uses the client encoding for quote_ident": "unknown", + "PG::Connection multinationalization support rubyforge #22925: m17n support should return results in the same encoding as the client (iso-8859-1)": "unknown", + "PG::Connection multinationalization support rubyforge #22925: m17n support the connection should return ASCII-8BIT when it's set to SQL_ASCII": "unknown", + "PG::Connection multinationalization support rubyforge #22925: m17n support should return results in the same encoding as the client (EUC-JP)": "unknown", + "PG::Connection multinationalization support respect and convert character encoding of input strings should convert query string and parameters to #send_query_params": "unknown", + "PG::Connection multinationalization support respect and convert character encoding of input strings should convert strings and parameters to #send_prepare and #send_query_prepared": "unknown", + "PG::Connection multinationalization support respect and convert character encoding of input strings should convert query string and parameters to #exec_params": "unknown", + "PG::Connection multinationalization support respect and convert character encoding of input strings should convert strings to #describe_portal": "unknown", + "PG::Connection multinationalization support respect and convert character encoding of input strings should convert strings and parameters to #prepare and #exec_prepared": "unknown", + "PG::Connection multinationalization support respect and convert character encoding of input strings should convert strings to #send_describe_portal": "unknown", + "PG::Connection multinationalization support respect and convert character encoding of input strings should convert error string to #put_copy_end": "unknown", + "PG::Connection type casting can type cast parameters to put_copy_data with explicit encoder": "unknown", + "PG::Connection type casting shouldn't type map params unless requested": "unknown", + "PG::Connection type casting with default query type map can process #copy_data input queries with row encoder and respects character encoding": "unknown", + "PG::Connection type casting with default result type map can type cast #copy_data output with explicit decoder": "unknown", + "PG::Connection type casting with default result type map can process #copy_data output with row decoder and respects character encoding": "unknown", + "PG::Connection type casting with default result type map should respect a type mapping for result": "unknown", + "PG::Connection type casting with default result type map should work with arbitrary number of params in conjunction with type casting": "unknown", + "PG::Connection set_single_row_mode should receive rows before entire query is finished": "unknown", + "PG::Connection set_single_row_mode should receive rows before entire query fails": "unknown", + "PG::Connection deprecated forms of methods should forward send_query to send_query_params": "unknown", + "PG::Connection deprecated forms of methods should forward exec to exec_params": "unknown", +} + +var rubyPGIgnoreList21_1 = rubyPGIgnoreList20_2 + +var rubyPGIgnoreList20_2 = blocklist{} diff --git a/pkg/cmd/roachtest/ruby_pg_helpers.rb b/pkg/cmd/roachtest/ruby_pg_helpers.rb new file mode 100644 index 000000000000..f45c2a066720 --- /dev/null +++ b/pkg/cmd/roachtest/ruby_pg_helpers.rb @@ -0,0 +1,377 @@ +# This file was modified so setup_testing_db creates a CockroachDB instance +# with database "test". + +# -*- ruby -*- + +require 'pathname' +require 'rspec' +require 'shellwords' +require 'pg' + +DEFAULT_TEST_DIR_STR = File.join(Dir.pwd, "tmp_test_specs") +TEST_DIR_STR = ENV['RUBY_PG_TEST_DIR'] || DEFAULT_TEST_DIR_STR +TEST_DIRECTORY = Pathname.new(TEST_DIR_STR) + +module PG::TestingHelpers + + ### Automatically set up the database when it's used, and wrap a transaction around + ### examples that don't disable it. + def self::included( mod ) + super + + if mod.respond_to?( :around ) + + mod.before( :all ) { @conn = setup_testing_db(described_class ? described_class.name : mod.description) } + + mod.around( :each ) do |example| + begin + @conn.set_default_encoding + @conn.exec( 'BEGIN' ) unless example.metadata[:without_transaction] + desc = example.source_location.join(':') + @conn.exec %Q{SET application_name TO '%s'} % + [@conn.escape_string(desc.slice(-60))] + example.run + ensure + @conn.exec( 'ROLLBACK' ) unless example.metadata[:without_transaction] + end + end + + mod.after( :all ) { teardown_testing_db(@conn) } + end + + end + + + # + # Examples + # + + # Set some ANSI escape code constants (Shamelessly stolen from Perl's + # Term::ANSIColor by Russ Allbery and Zenin + ANSI_ATTRIBUTES = { + 'clear' => 0, + 'reset' => 0, + 'bold' => 1, + 'dark' => 2, + 'underline' => 4, + 'underscore' => 4, + 'blink' => 5, + 'reverse' => 7, + 'concealed' => 8, + + 'black' => 30, 'on_black' => 40, + 'red' => 31, 'on_red' => 41, + 'green' => 32, 'on_green' => 42, + 'yellow' => 33, 'on_yellow' => 43, + 'blue' => 34, 'on_blue' => 44, + 'magenta' => 35, 'on_magenta' => 45, + 'cyan' => 36, 'on_cyan' => 46, + 'white' => 37, 'on_white' => 47 + } + + + ############### + module_function + ############### + + ### Create a string that contains the ANSI codes specified and return it + def ansi_code( *attributes ) + attributes.flatten! + attributes.collect! {|at| at.to_s } + + return '' unless /(?:vt10[03]|xterm(?:-color)?|linux|screen)/i =~ ENV['TERM'] + attributes = ANSI_ATTRIBUTES.values_at( *attributes ).compact.join(';') + + # $stderr.puts " attr is: %p" % [attributes] + if attributes.empty? + return '' + else + return "\e[%sm" % attributes + end + end + + + ### Colorize the given +string+ with the specified +attributes+ and return it, handling + ### line-endings, color reset, etc. + def colorize( *args ) + string = '' + + if block_given? + string = yield + else + string = args.shift + end + + ending = string[/(\s)$/] || '' + string = string.rstrip + + return ansi_code( args.flatten ) + string + ansi_code( 'reset' ) + ending + end + + + ### Output a message with highlighting. + def message( *msg ) + $stderr.puts( colorize(:bold) { msg.flatten.join(' ') } ) + end + + + ### Output a logging message if $VERBOSE is true + def trace( *msg ) + return unless $VERBOSE + output = colorize( msg.flatten.join(' '), 'yellow' ) + $stderr.puts( output ) + end + + + ### Return the specified args as a string, quoting any that have a space. + def quotelist( *args ) + return args.flatten.collect {|part| part.to_s =~ /\s/ ? part.to_s.inspect : part.to_s } + end + + + ### Run the specified command +cmd+ with system(), failing if the execution + ### fails. + def run( *cmd ) + cmd.flatten! + + if cmd.length > 1 + trace( quotelist(*cmd) ) + else + trace( cmd ) + end + + system( *cmd ) + raise "Command failed: [%s]" % [cmd.join(' ')] unless $?.success? + end + + + ### Run the specified command +cmd+ after redirecting stdout and stderr to the specified + ### +logpath+, failing if the execution fails. + def log_and_run( logpath, *cmd ) + cmd.flatten! + + if cmd.length > 1 + trace( quotelist(*cmd) ) + else + trace( cmd ) + end + + # Eliminate the noise of creating/tearing down the database by + # redirecting STDERR/STDOUT to a logfile + logfh = File.open( logpath, File::WRONLY|File::CREAT|File::APPEND ) + system( *cmd, [STDOUT, STDERR] => logfh ) + + raise "Command failed: [%s]" % [cmd.join(' ')] unless $?.success? + end + + + ### Check the current directory for directories that look like they're + ### testing directories from previous tests, and tell any postgres instances + ### running in them to shut down. + def stop_existing_postmasters + # tmp_test_0.22329534700318 + pat = Pathname.getwd + 'tmp_test_*' + Pathname.glob( pat.to_s ).each do |testdir| + datadir = testdir + 'data' + pidfile = datadir + 'postmaster.pid' + if pidfile.exist? && pid = pidfile.read.chomp.to_i + trace "pidfile (%p) exists: %d" % [ pidfile, pid ] + begin + Process.kill( 0, pid ) + rescue Errno::ESRCH + trace "No postmaster running for %s" % [ datadir ] + # Process isn't alive, so don't try to stop it + else + trace "Stopping lingering database at PID %d" % [ pid ] + run 'pg_ctl', '-D', datadir.to_s, '-m', 'fast', 'stop' + end + else + trace "No pidfile (%p)" % [ pidfile ] + end + end + end + + + ### Set up a CockroachDB database instance for testing. + def setup_testing_db( description ) + require 'pg' + stop_existing_postmasters() + + trace "Setting up test database for #{description}" + @test_pgdata = TEST_DIRECTORY + 'data' + @test_pgdata.mkpath + + ENV['PGPORT'] ||= "26257" + @port = ENV['PGPORT'].to_i + ENV['PGHOST'] = 'localhost' + @conninfo = "host=localhost port=#{@port} dbname=test" + + @logfile = TEST_DIRECTORY + 'setup.log' + trace "Command output logged to #{@logfile}" + + begin + unless (@test_pgdata+"postgresql.conf").exist? + FileUtils.rm_rf( @test_pgdata, :verbose => $DEBUG ) + trace "GG" + trace "Running initdb" + end + + trace "Creating the test DB" + log_and_run @logfile, '/home/ubuntu/cockroach', 'sql', '--insecure', '-e', 'DROP DATABASE IF EXISTS test' + log_and_run @logfile, '/home/ubuntu/cockroach', 'sql', '--insecure', '-e', 'CREATE DATABASE test' + + rescue => err + $stderr.puts "%p during test setup: %s" % [ err.class, err.message ] + $stderr.puts "See #{@logfile} for details." + $stderr.puts err.backtrace if $DEBUG + fail + end + + conn = PG.connect( @conninfo ) + conn.set_notice_processor do |message| + $stderr.puts( description + ':' + message ) if $DEBUG + end + + return conn + end + + + def teardown_testing_db( conn ) + trace "Tearing down test database" + + if conn + check_for_lingering_connections( conn ) + conn.finish + end + + end + + + def check_for_lingering_connections( conn ) + conn.exec( "SELECT * FROM pg_stat_activity" ) do |res| + conns = res.find_all {|row| row['pid'].to_i != conn.backend_pid && ["client backend", nil].include?(row["backend_type"]) } + unless conns.empty? + puts "Lingering connections remain:" + conns.each do |row| + puts " [%s] {%s} %s -- %s" % row.values_at( 'pid', 'state', 'application_name', 'query' ) + end + end + end + end + + + # Retrieve the names of the column types of a given result set. + def result_typenames(res) + @conn.exec_params( "SELECT " + res.nfields.times.map{|i| "format_type($#{i*2+1},$#{i*2+2})"}.join(","), + res.nfields.times.flat_map{|i| [res.ftype(i), res.fmod(i)] } ). + values[0] + end + + + # A matcher for checking the status of a PG::Connection to ensure it's still + # usable. + class ConnStillUsableMatcher + + def initialize + @conn = nil + @problem = nil + end + + def matches?( conn ) + @conn = conn + @problem = self.check_for_problems + return @problem.nil? + end + + def check_for_problems + return "is finished" if @conn.finished? + return "has bad status" unless @conn.status == PG::CONNECTION_OK + return "has bad transaction status (%d)" % [ @conn.transaction_status ] unless + @conn.transaction_status.between?( PG::PQTRANS_IDLE, PG::PQTRANS_INTRANS ) + return "is not usable." unless self.can_exec_query? + return nil + end + + def can_exec_query? + @conn.send_query( "VALUES (1)" ) + @conn.get_last_result.values == [["1"]] + end + + def failure_message + return "expected %p to be usable, but it %s" % [ @conn, @problem ] + end + + def failure_message_when_negated + "expected %p not to be usable, but it still is" % [ @conn ] + end + + end + + + ### Return a ConnStillUsableMatcher to be used like: + ### + ### expect( pg_conn ).to still_be_usable + ### + def still_be_usable + return ConnStillUsableMatcher.new + end + + def wait_for_polling_ok(conn, meth = :connect_poll) + status = conn.send(meth) + + while status != PG::PGRES_POLLING_OK + if status == PG::PGRES_POLLING_READING + select( [conn.socket_io], [], [], 5.0 ) or + raise "Asynchronous connection timed out!" + + elsif status == PG::PGRES_POLLING_WRITING + select( [], [conn.socket_io], [], 5.0 ) or + raise "Asynchronous connection timed out!" + end + status = conn.send(meth) + end + end + + def wait_for_query_result(conn) + result = nil + loop do + # Buffer any incoming data on the socket until a full result is ready. + conn.consume_input + while conn.is_busy + select( [conn.socket_io], nil, nil, 5.0 ) or + raise "Timeout waiting for query response." + conn.consume_input + end + + # Fetch the next result. If there isn't one, the query is finished + result = conn.get_result || break + end + result + end + +end + + +RSpec.configure do |config| + config.include( PG::TestingHelpers ) + + config.run_all_when_everything_filtered = true + config.filter_run :focus + config.order = 'random' + config.mock_with( :rspec ) do |mock| + mock.syntax = :expect + end + + if RUBY_PLATFORM =~ /mingw|mswin/ + config.filter_run_excluding :unix + else + config.filter_run_excluding :windows + end + + config.filter_run_excluding( :postgresql_93 ) if PG.library_version < 90300 + config.filter_run_excluding( :postgresql_94 ) if PG.library_version < 90400 + config.filter_run_excluding( :postgresql_95 ) if PG.library_version < 90500 + config.filter_run_excluding( :postgresql_96 ) if PG.library_version < 90600 + config.filter_run_excluding( :postgresql_10 ) if PG.library_version < 100000 + config.filter_run_excluding( :postgresql_12 ) if PG.library_version < 120000 +end diff --git a/pkg/testutils/lint/lint_test.go b/pkg/testutils/lint/lint_test.go index 60f9bedd0508..1bb9f7439d50 100644 --- a/pkg/testutils/lint/lint_test.go +++ b/pkg/testutils/lint/lint_test.go @@ -1282,6 +1282,7 @@ func TestLint(t *testing.T) { stream.GrepNot(`^sql/logictest/testdata/logic_test/pg_extension$`), stream.GrepNot(`^sql/opt/testutils/opttester/testfixtures/.*`), stream.GrepNot(`^util/timeutil/lowercase_timezones_generated.go$`), + stream.GrepNot(`^cmd/roachtest/ruby_pg_blocklist.go$`), stream.Map(func(s string) string { return filepath.Join(pkgDir, s) }),