Merge cockroachdb#97677 cockroachdb#98778 cockroachdb#98830 cockroach…

…db#99100 cockroachdb#99106

97677: tsearch: add stemming and stopword elimination for several languages r=jordanlewis a=jordanlewis

Updates: cockroachdb#41288
Epic: CRDB-22357
First commit is cockroachdb#92966.

    This commit adds stopword elimination for text search. The languages
    supported are the same ones that Postgres does. The stopword lists were
    copied from Postgres commit e757080e041214cf6983e3e77ef01e83f1371d72.

    Also, add snowball stemming provided by the blevesearch snowball
    stemming library.

    Release note (sql change): add stemming and stopword eliminating text
    search configurations for English, Danish, Dutch, Finnish, French,
    German, Hungarian, Italian, Norwegian, Portuguese, Russian, Spanish,
    Swedish, and Turkish.

98778: cli: unskip test tenant zip test r=dhartunian a=aadityasondhi

This patch unskips and re-records the datadriven `TestTenantZip` as it was fixed in cockroachdb#96553, but was not unskipped or recorded. The test was run locally using `--stress` and did not flake:
```
101 runs so far, 0 failures, over 5m0s
```

Fixes cockroachdb#87141

Release note: None

98830: sqlinstance: add `binary_version` column to instances table r=knz,JeffSwenson a=healthy-pod

This code change adds a `binary_version` column to the instances table.

This is done by adding the column to the bootstrap schema for system.sql_instances,
and piggy-backing on the existing code for the V23_1_SystemRbrReadNew migration
that overwrites the live schema for this table by the bootstrap copy.

This redefinition of the meaning of the V23_1_SystemRbrReadNew is backward-incompatible
and is only possible because this commit is merged before the v23.1 branch is cut.

Release note: None
Epic: [CRDB-20860](https://cockroachlabs.atlassian.net/browse/CRDB-20860)

99100: kvserver: skip `TestReplicateQueueExpirationLeasesOnly` under deadlock r=erikgrinaker a=erikgrinaker

I give up.

Resolves cockroachdb#99015.

Epic: none
Release note: None

99106: server: avoid some log spam r=erikgrinaker a=knz

This change removes the following log spam:
```
could not run claimed job 102: no resumer is available for AUTO CONFIG RUNNER
```

Epic: CRDB-23559
Release note: None

Co-authored-by: Jordan Lewis <jordanthelewis@gmail.com>
Co-authored-by: Aaditya Sondhi <20070511+aadityasondhi@users.noreply.github.com>
Co-authored-by: healthy-pod <ahmad@cockroachlabs.com>
Co-authored-by: Erik Grinaker <grinaker@cockroachlabs.com>
Co-authored-by: Raphael 'kena' Poss <knz@thaumogen.net>

DEPS.bzl

@@ -944,6 +944,16 @@ def go_deps():
             "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/blang/semver/com_github_blang_semver-v3.5.1+incompatible.zip",
         ],
     )
+    go_repository(
+        name = "com_github_blevesearch_snowballstem",
+        build_file_proto_mode = "disable_global",
+        importpath = "github.com/blevesearch/snowballstem",
+        sha256 = "6640a408ddcec84810873cc678570717c02d5b7b932f37672c44caea33469506",
+        strip_prefix = "github.com/blevesearch/snowballstem@v0.9.0",
+        urls = [
+            "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/blevesearch/snowballstem/com_github_blevesearch_snowballstem-v0.9.0.zip",
+        ],
+    )
     go_repository(
         name = "com_github_bmizerany_assert",
         build_file_proto_mode = "disable_global",

build/bazelutil/distdir_files.bzl

@@ -258,6 +258,7 @@ DISTDIR_FILES = {
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/bkaradzic/go-lz4/com_github_bkaradzic_go_lz4-v1.0.0.zip": "525f5633a4d9c8a32b5b5763c4e423ad061e773cf8cfeb21737f491feb531666",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/bketelsen/crypt/com_github_bketelsen_crypt-v0.0.4.zip": "ab24f8c0386cc7fce86f4e6680c32214e1e597980bd80127ac84e71ace6763da",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/blang/semver/com_github_blang_semver-v3.5.1+incompatible.zip": "8d032399cf835b93f7cf641b5477a31a002059eed7888a775f97bd3e9677ad3c",
+    "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/blevesearch/snowballstem/com_github_blevesearch_snowballstem-v0.9.0.zip": "6640a408ddcec84810873cc678570717c02d5b7b932f37672c44caea33469506",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/bmizerany/assert/com_github_bmizerany_assert-v0.0.0-20160611221934-b7ed37b82869.zip": "2532a167df77ade7e8012f07c0e3db4d4c15abdb7ffa7b05e1d961408da9a539",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/bmizerany/pat/com_github_bmizerany_pat-v0.0.0-20170815010413-6226ea591a40.zip": "ed04bed4d193e25371ebc6524984da4af9ece5c107fcc82d5aa4914b726706d2",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/bmizerany/perks/com_github_bmizerany_perks-v0.0.0-20141205001514-d9a9656a3a4b.zip": "b78e7083e73b6c2d63a30d073515b2a03dbe3115171601009211208ee0c6046e",

go.mod

@@ -99,6 +99,7 @@ require (
 	github.com/axiomhq/hyperloglog v0.0.0-20181223111420-4b99d0c2c99e
 	github.com/bazelbuild/rules_go v0.26.0
 	github.com/biogo/store v0.0.0-20160505134755-913427a1d5e8
+	github.com/blevesearch/snowballstem v0.9.0
 	github.com/buchgr/bazel-remote v1.3.3
 	github.com/bufbuild/buf v0.56.0
 	github.com/cenkalti/backoff v2.2.1+incompatible

go.sum

@@ -384,6 +384,8 @@ github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqO
 github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
 github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
 github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
+github.com/blevesearch/snowballstem v0.9.0 h1:lMQ189YspGP6sXvZQ4WZ+MLawfV8wOmPoD/iWeNXm8s=
+github.com/blevesearch/snowballstem v0.9.0/go.mod h1:PivSj3JMc8WuaFkTSRDW2SlrulNWPl4ABg1tC/hlgLs=
 github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
 github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c=
 github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b/go.mod h1:ac9efd0D1fsDb3EJvhqgXRbFx7bs2wqZ10HQPeU8U/Q=

pkg/cli/testdata/zip/testzip

@@ -78,7 +78,7 @@ debug zip --concurrency=1 --cpu-profile-duration=1s /dev/null
 [cluster] requesting tenant ranges: creating error output: debug/tenant_ranges.err.txt... done
 [cluster] requesting CPU profiles
 [cluster] profiles generated
-[cluster] profile for node 1... writing binary output: debug//nodes/1/cpu.pprof... done
+[cluster] profile for node 1... writing binary output: debug/nodes/1/cpu.pprof... done
 [node 1] node status... converting to JSON... writing binary output: debug/nodes/1/status.json... done
 [node 1] using SQL connection URL: postgresql://...
 [node 1] retrieving SQL data for crdb_internal.active_range_feeds... writing output: debug/nodes/1/crdb_internal.active_range_feeds.txt... done

pkg/cli/testdata/zip/testzip_tenant

@@ -1,19 +1,13 @@
 zip
 ----
 debug zip --concurrency=1 --cpu-profile-duration=1s /dev/null
+[cluster] discovering tenants on cluster... done
+[cluster] creating output file /dev/null... done
 [cluster] establishing RPC connection to ...
-[cluster] retrieving the node status to get the SQL address... done
 [cluster] using SQL address: ...
-[cluster] creating output file /dev/null... done
-[cluster] requesting data for debug/events... received response...
-[cluster] requesting data for debug/events: last request failed: rpc error: ...
-[cluster] requesting data for debug/events: creating error output: debug/events.json.err.txt... done
-[cluster] requesting data for debug/rangelog... received response...
-[cluster] requesting data for debug/rangelog: last request failed: rpc error: ...
-[cluster] requesting data for debug/rangelog: creating error output: debug/rangelog.json.err.txt... done
-[cluster] requesting data for debug/settings... received response...
-[cluster] requesting data for debug/settings: last request failed: rpc error: ...
-[cluster] requesting data for debug/settings: creating error output: debug/settings.json.err.txt... done
+[cluster] requesting data for debug/events... received response... converting to JSON... writing binary output: debug/events.json... done
+[cluster] requesting data for debug/rangelog... received response... converting to JSON... writing binary output: debug/rangelog.json... done
+[cluster] requesting data for debug/settings... received response... converting to JSON... writing binary output: debug/settings.json... done
 [cluster] requesting data for debug/reports/problemranges... received response...
 [cluster] requesting data for debug/reports/problemranges: last request failed: rpc error: ...
 [cluster] requesting data for debug/reports/problemranges: creating error output: debug/reports/problemranges.json.err.txt... done
@@ -30,6 +24,7 @@ debug zip --concurrency=1 --cpu-profile-duration=1s /dev/null
 [cluster] retrieving SQL data for crdb_internal.cluster_sessions... writing output: debug/crdb_internal.cluster_sessions.txt... done
 [cluster] retrieving SQL data for crdb_internal.cluster_settings... writing output: debug/crdb_internal.cluster_settings.txt... done
 [cluster] retrieving SQL data for crdb_internal.cluster_transactions... writing output: debug/crdb_internal.cluster_transactions.txt... done
+[cluster] retrieving SQL data for crdb_internal.cluster_txn_execution_insights... writing output: debug/crdb_internal.cluster_txn_execution_insights.txt... done
 [cluster] retrieving SQL data for crdb_internal.default_privileges... writing output: debug/crdb_internal.default_privileges.txt... done
 [cluster] retrieving SQL data for crdb_internal.index_usage_statistics... writing output: debug/crdb_internal.index_usage_statistics.txt... done
 [cluster] retrieving SQL data for crdb_internal.invalid_objects... writing output: debug/crdb_internal.invalid_objects.txt... done
@@ -55,7 +50,6 @@ debug zip --concurrency=1 --cpu-profile-duration=1s /dev/null
 [cluster] retrieving SQL data for system.descriptor... writing output: debug/system.descriptor.txt... done
 [cluster] retrieving SQL data for system.eventlog... writing output: debug/system.eventlog.txt... done
 [cluster] retrieving SQL data for system.external_connections... writing output: debug/system.external_connections.txt... done
-[cluster] retrieving SQL data for system.job_info... writing output: debug/system.job_info.txt... done
 [cluster] retrieving SQL data for system.jobs... writing output: debug/system.jobs.txt... done
 [cluster] retrieving SQL data for system.lease... writing output: debug/system.lease.txt... done
 [cluster] retrieving SQL data for system.locations... writing output: debug/system.locations.txt... done
@@ -82,19 +76,25 @@ debug zip --concurrency=1 --cpu-profile-duration=1s /dev/null
 [cluster] retrieving SQL data for system.statement_diagnostics... writing output: debug/system.statement_diagnostics.txt... done
 [cluster] retrieving SQL data for system.statement_diagnostics_requests... writing output: debug/system.statement_diagnostics_requests.txt... done
 [cluster] retrieving SQL data for system.table_statistics... writing output: debug/system.table_statistics.txt... done
+[cluster] retrieving SQL data for system.task_payloads... writing output: debug/system.task_payloads.txt...
+[cluster] retrieving SQL data for system.task_payloads: last request failed: ERROR: relation "system.task_payloads" does not exist (SQLSTATE 42P01)
+[cluster] retrieving SQL data for system.task_payloads: creating error output: debug/system.task_payloads.txt.err.txt... done
 [cluster] retrieving SQL data for system.tenant_settings... writing output: debug/system.tenant_settings.txt...
 [cluster] retrieving SQL data for system.tenant_settings: last request failed: ERROR: relation "system.tenant_settings" does not exist (SQLSTATE 42P01)
 [cluster] retrieving SQL data for system.tenant_settings: creating error output: debug/system.tenant_settings.txt.err.txt... done
+[cluster] retrieving SQL data for system.tenant_tasks... writing output: debug/system.tenant_tasks.txt...
+[cluster] retrieving SQL data for system.tenant_tasks: last request failed: ERROR: relation "system.tenant_tasks" does not exist (SQLSTATE 42P01)
+[cluster] retrieving SQL data for system.tenant_tasks: creating error output: debug/system.tenant_tasks.txt.err.txt... done
 [cluster] retrieving SQL data for system.tenant_usage... writing output: debug/system.tenant_usage.txt...
 [cluster] retrieving SQL data for system.tenant_usage: last request failed: ERROR: relation "system.tenant_usage" does not exist (SQLSTATE 42P01)
 [cluster] retrieving SQL data for system.tenant_usage: creating error output: debug/system.tenant_usage.txt.err.txt... done
 [cluster] retrieving SQL data for system.tenants... writing output: debug/system.tenants.txt...
 [cluster] retrieving SQL data for system.tenants: last request failed: ERROR: relation "system.tenants" does not exist (SQLSTATE 42P01)
 [cluster] retrieving SQL data for system.tenants: creating error output: debug/system.tenants.txt.err.txt... done
-[cluster] requesting nodes... received response... converting to JSON... writing binary output: debug/nodes.json... done
-[cluster] requesting liveness... received response...
-[cluster] requesting liveness: last request failed: rpc error: ...
-[cluster] requesting liveness: creating error output: debug/liveness.json.err.txt... done
+[cluster] requesting nodes... received response...
+[cluster] requesting nodes: last request failed: rpc error: ...
+[cluster] requesting nodes: creating error output: debug/nodes.json.err.txt... done
+[cluster] requesting liveness... received response... converting to JSON... writing binary output: debug/liveness.json... done
 [cluster] requesting tenant ranges... received response...
 [cluster] requesting tenant ranges: last request failed: rpc error: ...
 [cluster] requesting tenant ranges: creating error output: debug/tenant_ranges.err.txt... done
@@ -120,13 +120,18 @@ debug zip --concurrency=1 --cpu-profile-duration=1s /dev/null
 [node 1] retrieving SQL data for crdb_internal.node_distsql_flows... writing output: debug/nodes/1/crdb_internal.node_distsql_flows.txt... done
 [node 1] retrieving SQL data for crdb_internal.node_execution_insights... writing output: debug/nodes/1/crdb_internal.node_execution_insights.txt... done
 [node 1] retrieving SQL data for crdb_internal.node_inflight_trace_spans... writing output: debug/nodes/1/crdb_internal.node_inflight_trace_spans.txt... done
+[node 1] retrieving SQL data for crdb_internal.node_memory_monitors... writing output: debug/nodes/1/crdb_internal.node_memory_monitors.txt... done
 [node 1] retrieving SQL data for crdb_internal.node_metrics... writing output: debug/nodes/1/crdb_internal.node_metrics.txt... done
 [node 1] retrieving SQL data for crdb_internal.node_queries... writing output: debug/nodes/1/crdb_internal.node_queries.txt... done
 [node 1] retrieving SQL data for crdb_internal.node_runtime_info... writing output: debug/nodes/1/crdb_internal.node_runtime_info.txt... done
 [node 1] retrieving SQL data for crdb_internal.node_sessions... writing output: debug/nodes/1/crdb_internal.node_sessions.txt... done
 [node 1] retrieving SQL data for crdb_internal.node_statement_statistics... writing output: debug/nodes/1/crdb_internal.node_statement_statistics.txt... done
+[node 1] retrieving SQL data for crdb_internal.node_tenant_capabilities_cache... writing output: debug/nodes/1/crdb_internal.node_tenant_capabilities_cache.txt...
+[node 1] retrieving SQL data for crdb_internal.node_tenant_capabilities_cache: last request failed: ERROR: operation node_tenant_capabilities_cache supported only by system tenant (SQLSTATE XXUUU)
+[node 1] retrieving SQL data for crdb_internal.node_tenant_capabilities_cache: creating error output: debug/nodes/1/crdb_internal.node_tenant_capabilities_cache.txt.err.txt... done
 [node 1] retrieving SQL data for crdb_internal.node_transaction_statistics... writing output: debug/nodes/1/crdb_internal.node_transaction_statistics.txt... done
 [node 1] retrieving SQL data for crdb_internal.node_transactions... writing output: debug/nodes/1/crdb_internal.node_transactions.txt... done
+[node 1] retrieving SQL data for crdb_internal.node_txn_execution_insights... writing output: debug/nodes/1/crdb_internal.node_txn_execution_insights.txt... done
 [node 1] retrieving SQL data for crdb_internal.node_txn_stats... writing output: debug/nodes/1/crdb_internal.node_txn_stats.txt... done
 [node 1] requesting data for debug/nodes/1/details... received response... converting to JSON... writing binary output: debug/nodes/1/details.json... done
 [node 1] requesting data for debug/nodes/1/gossip... received response...
@@ -141,10 +146,9 @@ debug zip --concurrency=1 --cpu-profile-duration=1s /dev/null
 [node 1] requesting heap file list... received response... done
 [node ?] ? heap profiles found
 [node 1] requesting goroutine dump list... received response... done
-[node 1] 0 goroutine dumps found
-[node 1] requesting log file ...
-[node 1] requesting log file ...
+[node ?] ? goroutine dumps found
 [node 1] requesting log file ...
+[node 1] 0 log file ...
 [node 1] requesting ranges... received response...
 [node 1] requesting ranges: last request failed: rpc error: ...
 [node 1] requesting ranges: creating error output: debug/nodes/1/ranges.err.txt... done

pkg/cli/zip_per_node.go

@@ -133,7 +133,7 @@ func (zc *debugZipContext) collectCPUProfiles(
 			continue // skipped node
 		}
 		nodeID := nodeList[i].NodeID
-		prefix := fmt.Sprintf("%s/%s/%s", zc.prefix, nodesPrefix, fmt.Sprintf("%d", nodeID))
+		prefix := fmt.Sprintf("%s%s/%s", zc.prefix, nodesPrefix, fmt.Sprintf("%d", nodeID))
 		s := zc.clusterPrinter.start("profile for node %d", nodeID)
 		if err := zc.z.createRawOrError(s, prefix+"/cpu.pprof", pd.data, pd.err); err != nil {
 			return err

pkg/cli/zip_tenant_test.go

@@ -31,7 +31,6 @@ var _ = kvtenantccl.Connector{}
 // TestTenantZip tests the operation of zip for a tenant server.
 func TestTenantZip(t *testing.T) {
 	defer leaktest.AfterTest(t)()
-	skip.WithIssue(t, 89192, "flaky test")
 
 	skip.UnderRace(t, "test too slow under race")
 	tenantDir, tenantDirCleanupFn := testutils.TempDir(t)

pkg/cli/zip_test.go

@@ -364,8 +364,9 @@ func eraseNonDeterministicZipOutput(out string) string {
 	out = re.ReplaceAllString(out, ``)
 	re = regexp.MustCompile(`(?m)^\[node \d+\] writing dump.*$` + "\n")
 	out = re.ReplaceAllString(out, ``)
+	re = regexp.MustCompile(`(?m)^\[node \d+\] retrieving goroutine_dump.*$` + "\n")
+	out = re.ReplaceAllString(out, ``)
 
-	//out = strings.ReplaceAll(out, "\n\n", "\n")
 	return out
 }
 

pkg/kv/kvserver/replicate_queue_test.go

@@ -2363,6 +2363,7 @@ func TestReplicateQueueExpirationLeasesOnly(t *testing.T) {
 	defer log.Scope(t).Close(t)
 
 	skip.UnderRace(t) // too slow under stressrace
+	skip.UnderDeadlock(t)
 	skip.UnderShort(t)
 
 	ctx := context.Background()

pkg/server/BUILD.bazel

@@ -159,6 +159,7 @@ go_library(
         "//pkg/security/password",
         "//pkg/security/securityassets",
         "//pkg/security/username",
+        "//pkg/server/autoconfig",
         "//pkg/server/debug",
         "//pkg/server/diagnostics",
         "//pkg/server/diagnostics/diagnosticspb",

pkg/server/server_sql.go

@@ -52,6 +52,9 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/scheduledjobs"
 	"github.com/cockroachdb/cockroach/pkg/security/clientsecopts"
 	"github.com/cockroachdb/cockroach/pkg/security/username"
+	// Ensure the auto config runner job is registered to avoid log spam.
+	// Pending merge of https://github.com/cockroachdb/cockroach/pull/98466.
+	_ "github.com/cockroachdb/cockroach/pkg/server/autoconfig"
 	"github.com/cockroachdb/cockroach/pkg/server/diagnostics"
 	"github.com/cockroachdb/cockroach/pkg/server/pgurl"
 	"github.com/cockroachdb/cockroach/pkg/server/serverpb"
@@ -482,7 +485,14 @@ func (r *refreshInstanceSessionListener) OnSessionDeleted(
 				continue
 			}
 			if _, err := r.cfg.sqlInstanceStorage.CreateNodeInstance(
-				ctx, s.ID(), s.Expiration(), r.cfg.AdvertiseAddr, r.cfg.SQLAdvertiseAddr, r.cfg.Locality, nodeID,
+				ctx,
+				s.ID(),
+				s.Expiration(),
+				r.cfg.AdvertiseAddr,
+				r.cfg.SQLAdvertiseAddr,
+				r.cfg.Locality,
+				r.cfg.Settings.Version.BinaryVersion(),
+				nodeID,
 			); err != nil {
 				log.Warningf(ctx, "failed to update instance with new session ID: %v", err)
 				continue
@@ -1432,14 +1442,27 @@ func (s *SQLServer) preStart(
 	if hasNodeID {
 		// Write/acquire our instance row.
 		instance, err = s.sqlInstanceStorage.CreateNodeInstance(
-			ctx, session.ID(), session.Expiration(), s.cfg.AdvertiseAddr, s.cfg.SQLAdvertiseAddr, s.distSQLServer.Locality, nodeID,
+			ctx,
+			session.ID(),
+			session.Expiration(),
+			s.cfg.AdvertiseAddr,
+			s.cfg.SQLAdvertiseAddr,
+			s.distSQLServer.Locality,
+			s.execCfg.Settings.Version.BinaryVersion(),
+			nodeID,
 		)
 		if err != nil {
 			return err
 		}
 	} else {
 		instance, err = s.sqlInstanceStorage.CreateInstance(
-			ctx, session.ID(), session.Expiration(), s.cfg.AdvertiseAddr, s.cfg.SQLAdvertiseAddr, s.distSQLServer.Locality,
+			ctx,
+			session.ID(),
+			session.Expiration(),
+			s.cfg.AdvertiseAddr,
+			s.cfg.SQLAdvertiseAddr,
+			s.distSQLServer.Locality,
+			s.execCfg.Settings.Version.BinaryVersion(),
 		)
 		if err != nil {
 			return err