server: hook up tenant capabilities subsystem on startup

This patch wires up the tenant capabilities subsystem during server
startup. This includes both starting the subsystem and adding a handle
to the Authorizer so that it can be used by GRPC interceptors to perform
authorization checks.

While working through this patch, I realized we can't instantiate an
Authorizer with a handle to the tenant capability state
(tenantcapabilities.Reader). This is because the afformentioned GRPC
setup happens early on during the Server startup process, and at that
point we do not have access to the dependencies required to setup the
capabilities Watcher (which is what provides the Reader interface to
the Authorizer). To break this dependency cycle, we end up with an
approach to lazily bind the Reader to the Authorizer.

With the Authorizer wired up, we can now start using it to perform
capability checks for incoming tenant requests. Currently, this is
limited to batch requests. Note that the Authorizer is only responsible
for performing capability checks -- other authorization checks, such as
bounds checks, continue to happen outside of Authorizer.

This patch also adds a datadriven framework to test tenant capabilities
end to end. The nice thing about it is it hides the asynchronous nature
of capability checks from test writers. The hope is that we'll be able
to extend this as we add more capabilities.

Informs #94643

Release note: None

pkg/BUILD.bazel

@@ -56,6 +56,7 @@ ALL_TESTS = [
     "//pkg/ccl/logictestccl/tests/multiregion-9node-3region-3azs-vec-off:multiregion-9node-3region-3azs-vec-off_test",
     "//pkg/ccl/logictestccl/tests/multiregion-9node-3region-3azs:multiregion-9node-3region-3azs_test",
     "//pkg/ccl/multiregionccl:multiregionccl_test",
+    "//pkg/ccl/multitenantccl/tenantcapabilitiesccl:tenantcapabilitiesccl_test",
     "//pkg/ccl/multitenantccl/tenantcostclient:tenantcostclient_test",
     "//pkg/ccl/multitenantccl/tenantcostserver/tenanttokenbucket:tenanttokenbucket_test",
     "//pkg/ccl/multitenantccl/tenantcostserver:tenantcostserver_test",
@@ -773,6 +774,7 @@ GO_TARGETS = [
     "//pkg/ccl/multiregionccl/multiregionccltestutils:multiregionccltestutils",
     "//pkg/ccl/multiregionccl:multiregionccl",
     "//pkg/ccl/multiregionccl:multiregionccl_test",
+    "//pkg/ccl/multitenantccl/tenantcapabilitiesccl:tenantcapabilitiesccl_test",
     "//pkg/ccl/multitenantccl/tenantcostclient:tenantcostclient",
     "//pkg/ccl/multitenantccl/tenantcostclient:tenantcostclient_test",
     "//pkg/ccl/multitenantccl/tenantcostserver/tenanttokenbucket:tenanttokenbucket",
@@ -2359,6 +2361,7 @@ GET_X_DATA_TARGETS = [
     "//pkg/ccl/multiregionccl:get_x_data",
     "//pkg/ccl/multiregionccl/multiregionccltestutils:get_x_data",
     "//pkg/ccl/multitenantccl:get_x_data",
+    "//pkg/ccl/multitenantccl/tenantcapabilitiesccl:get_x_data",
     "//pkg/ccl/multitenantccl/tenantcostclient:get_x_data",
     "//pkg/ccl/multitenantccl/tenantcostserver:get_x_data",
     "//pkg/ccl/multitenantccl/tenantcostserver/tenanttokenbucket:get_x_data",

pkg/base/testing_knobs.go

@@ -19,40 +19,41 @@ type ModuleTestingKnobs interface {
 // TestingKnobs contains facilities for controlling various parts of the
 // system for testing.
 type TestingKnobs struct {
-	Store                        ModuleTestingKnobs
-	KVClient                     ModuleTestingKnobs
-	RangeFeed                    ModuleTestingKnobs
-	SQLExecutor                  ModuleTestingKnobs
-	SQLLeaseManager              ModuleTestingKnobs
-	SQLSchemaChanger             ModuleTestingKnobs
-	SQLDeclarativeSchemaChanger  ModuleTestingKnobs
-	SQLTypeSchemaChanger         ModuleTestingKnobs
-	GCJob                        ModuleTestingKnobs
-	PGWireTestingKnobs           ModuleTestingKnobs
-	DistSQL                      ModuleTestingKnobs
-	SQLEvalContext               ModuleTestingKnobs
-	NodeLiveness                 ModuleTestingKnobs
-	Server                       ModuleTestingKnobs
-	TenantTestingKnobs           ModuleTestingKnobs
-	JobsTestingKnobs             ModuleTestingKnobs
-	BackupRestore                ModuleTestingKnobs
-	TTL                          ModuleTestingKnobs
-	SchemaTelemetry              ModuleTestingKnobs
-	Streaming                    ModuleTestingKnobs
-	UpgradeManager               ModuleTestingKnobs
-	IndexUsageStatsKnobs         ModuleTestingKnobs
-	SQLStatsKnobs                ModuleTestingKnobs
-	SpanConfig                   ModuleTestingKnobs
-	SQLLivenessKnobs             ModuleTestingKnobs
-	TelemetryLoggingKnobs        ModuleTestingKnobs
-	DialerKnobs                  ModuleTestingKnobs
-	ProtectedTS                  ModuleTestingKnobs
-	CapturedIndexUsageStatsKnobs ModuleTestingKnobs
-	AdmissionControl             ModuleTestingKnobs
-	UnusedIndexRecommendKnobs    ModuleTestingKnobs
-	ExternalConnection           ModuleTestingKnobs
-	EventExporter                ModuleTestingKnobs
-	EventLog                     ModuleTestingKnobs
-	LOQRecovery                  ModuleTestingKnobs
-	KeyVisualizer                ModuleTestingKnobs
+	Store                          ModuleTestingKnobs
+	KVClient                       ModuleTestingKnobs
+	RangeFeed                      ModuleTestingKnobs
+	SQLExecutor                    ModuleTestingKnobs
+	SQLLeaseManager                ModuleTestingKnobs
+	SQLSchemaChanger               ModuleTestingKnobs
+	SQLDeclarativeSchemaChanger    ModuleTestingKnobs
+	SQLTypeSchemaChanger           ModuleTestingKnobs
+	GCJob                          ModuleTestingKnobs
+	PGWireTestingKnobs             ModuleTestingKnobs
+	DistSQL                        ModuleTestingKnobs
+	SQLEvalContext                 ModuleTestingKnobs
+	NodeLiveness                   ModuleTestingKnobs
+	Server                         ModuleTestingKnobs
+	TenantTestingKnobs             ModuleTestingKnobs
+	JobsTestingKnobs               ModuleTestingKnobs
+	BackupRestore                  ModuleTestingKnobs
+	TTL                            ModuleTestingKnobs
+	SchemaTelemetry                ModuleTestingKnobs
+	Streaming                      ModuleTestingKnobs
+	UpgradeManager                 ModuleTestingKnobs
+	IndexUsageStatsKnobs           ModuleTestingKnobs
+	SQLStatsKnobs                  ModuleTestingKnobs
+	SpanConfig                     ModuleTestingKnobs
+	SQLLivenessKnobs               ModuleTestingKnobs
+	TelemetryLoggingKnobs          ModuleTestingKnobs
+	DialerKnobs                    ModuleTestingKnobs
+	ProtectedTS                    ModuleTestingKnobs
+	CapturedIndexUsageStatsKnobs   ModuleTestingKnobs
+	AdmissionControl               ModuleTestingKnobs
+	UnusedIndexRecommendKnobs      ModuleTestingKnobs
+	ExternalConnection             ModuleTestingKnobs
+	EventExporter                  ModuleTestingKnobs
+	EventLog                       ModuleTestingKnobs
+	LOQRecovery                    ModuleTestingKnobs
+	KeyVisualizer                  ModuleTestingKnobs
+	TenantCapabilitiesTestingKnobs ModuleTestingKnobs
 }

pkg/ccl/backupccl/BUILD.bazel

@@ -233,6 +233,7 @@ go_test(
         "//pkg/kv/kvserver/protectedts/ptpb",
         "//pkg/kv/kvserver/protectedts/ptutil",
         "//pkg/multitenant/mtinfopb",
+        "//pkg/multitenant/tenantcapabilities",
         "//pkg/roachpb",
         "//pkg/scheduledjobs",
         "//pkg/scheduledjobs/schedulebase",

pkg/ccl/backupccl/backup_test.go

@@ -65,6 +65,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts"
 	"github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts/ptutil"
 	"github.com/cockroachdb/cockroach/pkg/multitenant/mtinfopb"
+	"github.com/cockroachdb/cockroach/pkg/multitenant/tenantcapabilities"
 	"github.com/cockroachdb/cockroach/pkg/roachpb"
 	"github.com/cockroachdb/cockroach/pkg/security"
 	"github.com/cockroachdb/cockroach/pkg/security/username"
@@ -2812,7 +2813,16 @@ func TestBackupRestoreCrossTableReferences(t *testing.T) {
 
 	_, origDB, dir, cleanupFn := backupRestoreTestSetup(t, singleNode, numAccounts, InitManualReplication)
 	defer cleanupFn()
-	args := base.TestServerArgs{ExternalIODir: dir}
+	args := base.TestServerArgs{
+		ExternalIODir: dir,
+		Knobs: base.TestingKnobs{
+			TenantCapabilitiesTestingKnobs: &tenantcapabilities.TestingKnobs{
+				// TODO(arul): This can be removed once
+				// https://github.com/cockroachdb/cockroach/issues/96736  is fixed.
+				AuthorizerSkipAdminSplitCapabilityChecks: true,
+			},
+		},
+	}
 
 	// Generate some testdata and back it up.
 	{
@@ -5150,7 +5160,16 @@ func TestBackupRestoreSequence(t *testing.T) {
 	const numAccounts = 1
 	_, origDB, dir, cleanupFn := backupRestoreTestSetup(t, singleNode, numAccounts, InitManualReplication)
 	defer cleanupFn()
-	args := base.TestServerArgs{ExternalIODir: dir}
+	args := base.TestServerArgs{
+		ExternalIODir: dir,
+		Knobs: base.TestingKnobs{
+			TenantCapabilitiesTestingKnobs: &tenantcapabilities.TestingKnobs{
+				// TODO(arul): This can be removed once
+				// https://github.com/cockroachdb/cockroach/issues/96736  is fixed.
+				AuthorizerSkipAdminSplitCapabilityChecks: true,
+			},
+		},
+	}
 
 	backupLoc := localFoo
 

pkg/ccl/backupccl/datadriven_test.go

@@ -30,6 +30,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/kv"
 	"github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord"
 	"github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts"
+	"github.com/cockroachdb/cockroach/pkg/multitenant/tenantcapabilities"
 	"github.com/cockroachdb/cockroach/pkg/roachpb"
 	"github.com/cockroachdb/cockroach/pkg/server"
 	"github.com/cockroachdb/cockroach/pkg/settings/cluster"
@@ -152,6 +153,14 @@ func (d *datadrivenTestState) addCluster(t *testing.T, cfg clusterCfg) error {
 	params.ServerArgs.Knobs = base.TestingKnobs{
 		JobsTestingKnobs: jobs.NewTestingKnobsWithShortIntervals(),
 	}
+	// Backups issue splits underneath the hood, and as such, will fail capability
+	// checks for tests that run as secondary tenants. Skip these checks at a
+	// global level using a testing knob.
+	params.ServerArgs.Knobs.TenantCapabilitiesTestingKnobs = &tenantcapabilities.TestingKnobs{
+		// TODO(arul): This can be removed once
+		// https://github.com/cockroachdb/cockroach/issues/96736  is fixed.
+		AuthorizerSkipAdminSplitCapabilityChecks: true,
+	}
 
 	settings := cluster.MakeTestingClusterSettings()
 

pkg/ccl/backupccl/utils_test.go

@@ -35,6 +35,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/keyvisualizer"
 	"github.com/cockroachdb/cockroach/pkg/kv"
 	"github.com/cockroachdb/cockroach/pkg/kv/kvserver"
+	"github.com/cockroachdb/cockroach/pkg/multitenant/tenantcapabilities"
 	"github.com/cockroachdb/cockroach/pkg/roachpb"
 	"github.com/cockroachdb/cockroach/pkg/sql/catalog/desctestutils"
 	"github.com/cockroachdb/cockroach/pkg/sql/sem/tree"
@@ -95,6 +96,11 @@ func backupRestoreTestSetupWithParams(
 		}
 		params.ServerArgs.Knobs.Store.(*kvserver.StoreTestingKnobs).SmallEngineBlocks = true
 	}
+	params.ServerArgs.Knobs.TenantCapabilitiesTestingKnobs = &tenantcapabilities.TestingKnobs{
+		// TODO(arul): This can be removed once
+		// https://github.com/cockroachdb/cockroach/issues/96736  is fixed.
+		AuthorizerSkipAdminSplitCapabilityChecks: true,
+	}
 
 	params.ServerArgs.Knobs.KeyVisualizer = &keyvisualizer.TestingKnobs{
 		SkipJobBootstrap:        true,
@@ -145,7 +151,17 @@ func backupRestoreTestSetup(
 ) (tc *testcluster.TestCluster, sqlDB *sqlutils.SQLRunner, tempDir string, cleanup func()) {
 	// TODO (msbutler): DisableDefaultTestTenant should be disabled by the caller of this function
 	return backupRestoreTestSetupWithParams(t, clusterSize, numAccounts, init,
-		base.TestClusterArgs{ServerArgs: base.TestServerArgs{DisableDefaultTestTenant: true}})
+		base.TestClusterArgs{
+			ServerArgs: base.TestServerArgs{
+				DisableDefaultTestTenant: true,
+				Knobs: base.TestingKnobs{
+					TenantCapabilitiesTestingKnobs: &tenantcapabilities.TestingKnobs{
+						// TODO(arul): This can be removed once
+						// https://github.com/cockroachdb/cockroach/issues/96736  is fixed.
+						AuthorizerSkipAdminSplitCapabilityChecks: true,
+					},
+				},
+			}})
 }
 
 func backupRestoreTestSetupEmpty(
@@ -157,6 +173,11 @@ func backupRestoreTestSetupEmpty(
 ) (tc *testcluster.TestCluster, sqlDB *sqlutils.SQLRunner, cleanup func()) {
 	// TODO (msbutler): this should be disabled by callers of this function
 	params.ServerArgs.DisableDefaultTestTenant = true
+	params.ServerArgs.Knobs.TenantCapabilitiesTestingKnobs = &tenantcapabilities.TestingKnobs{
+		// TODO(arul): This can be removed once
+		// https://github.com/cockroachdb/cockroach/issues/96736  is fixed.
+		AuthorizerSkipAdminSplitCapabilityChecks: true,
+	}
 	return backupRestoreTestSetupEmptyWithParams(t, clusterSize, tempDir, init, params)
 }
 
@@ -184,6 +205,11 @@ func backupRestoreTestSetupEmptyWithParams(
 		}
 		params.ServerArgs.Knobs.Store.(*kvserver.StoreTestingKnobs).SmallEngineBlocks = true
 	}
+	params.ServerArgs.Knobs.TenantCapabilitiesTestingKnobs = &tenantcapabilities.TestingKnobs{
+		// TODO(arul): This can be removed once
+		// https://github.com/cockroachdb/cockroach/issues/96736  is fixed.
+		AuthorizerSkipAdminSplitCapabilityChecks: true,
+	}
 
 	tc = testcluster.StartTestCluster(t, clusterSize, params)
 	init(tc)
@@ -209,6 +235,11 @@ func createEmptyCluster(
 	params.ServerArgs.Knobs.Store = &kvserver.StoreTestingKnobs{
 		SmallEngineBlocks: smallEngineBlocks,
 	}
+	params.ServerArgs.Knobs.TenantCapabilitiesTestingKnobs = &tenantcapabilities.TestingKnobs{
+		// TODO(arul): This can be removed once
+		// https://github.com/cockroachdb/cockroach/issues/96736  is fixed.
+		AuthorizerSkipAdminSplitCapabilityChecks: true,
+	}
 	tc := testcluster.StartTestCluster(t, clusterSize, params)
 
 	sqlDB = sqlutils.MakeSQLRunner(tc.Conns[0])

pkg/ccl/logictestccl/testdata/logic_test/multi_region_backup

@@ -1,5 +1,6 @@
 # tenant-cluster-setting-override-opt: sql.zone_configs.allow_for_secondary_tenant.enabled=true sql.multi_region.allow_abstractions_for_secondary_tenants.enabled=true
 # LogicTest: multiregion-9node-3region-3azs multiregion-9node-3region-3azs-no-los
+# cluster-opt: can-admin-split
 
 # Tests in this file assume no multi-region tenant setup as tenants have no
 # access to nodelocal.

pkg/ccl/logictestccl/testdata/logic_test/partitioning_hash_sharded_index_mr

@@ -1,4 +1,5 @@
 # tenant-cluster-setting-override-opt: sql.multi_region.allow_abstractions_for_secondary_tenants.enabled=true
+# cluster-opt: can-admin-split
 # LogicTest: multiregion-9node-3region-3azs multiregion-9node-3region-3azs-tenant multiregion-9node-3region-3azs-no-los
 
 statement ok

pkg/ccl/logictestccl/testdata/logic_test/partitioning_hash_sharded_index_query_plan

@@ -1,3 +1,4 @@
+# cluster-opt: can-admin-split
 # LogicTest: 5node !metamorphic-batch-sizes
 
 statement ok

pkg/ccl/logictestccl/testdata/logic_test/regional_by_row_hash_sharded_index

@@ -1,4 +1,5 @@
 # tenant-cluster-setting-override-opt: sql.multi_region.allow_abstractions_for_secondary_tenants.enabled=true
+# cluster-opt: can-admin-split
 # LogicTest: multiregion-9node-3region-3azs multiregion-9node-3region-3azs-no-los
 
 statement ok

pkg/ccl/multitenantccl/tenantcapabilitiesccl/BUILD.bazel

@@ -0,0 +1,39 @@
+load("//build/bazelutil/unused_checker:unused.bzl", "get_x_data")
+load("@io_bazel_rules_go//go:def.bzl", "go_test")
+
+go_test(
+    name = "tenantcapabilitiesccl_test",
+    srcs = [
+        "capabilities_test.go",
+        "main_test.go",
+    ],
+    args = ["-test.timeout=295s"],
+    data = glob(["testdata/**"]),
+    deps = [
+        "//pkg/base",
+        "//pkg/ccl",
+        "//pkg/kv/kvclient/rangefeed/rangefeedcache",
+        "//pkg/multitenant/tenantcapabilities",
+        "//pkg/multitenant/tenantcapabilities/tenantcapabilitieswatcher",
+        "//pkg/security/securityassets",
+        "//pkg/security/securitytest",
+        "//pkg/security/username",
+        "//pkg/server",
+        "//pkg/settings/cluster",
+        "//pkg/sql",
+        "//pkg/testutils",
+        "//pkg/testutils/datapathutils",
+        "//pkg/testutils/serverutils",
+        "//pkg/testutils/sqlutils",
+        "//pkg/testutils/testcluster",
+        "//pkg/util/hlc",
+        "//pkg/util/leaktest",
+        "//pkg/util/randutil",
+        "//pkg/util/syncutil",
+        "@com_github_cockroachdb_datadriven//:datadriven",
+        "@com_github_cockroachdb_errors//:errors",
+        "@com_github_stretchr_testify//require",
+    ],
+)
+
+get_x_data(name = "get_x_data")