tenantcapabilities: gate tenant access to node metadata and tsdb

pkg/ccl/backupccl/testdata/backup-restore/restore-tenants

@@ -21,9 +21,9 @@ ALTER TENANT [5] STOP SERVICE; DROP TENANT [5]
 query-sql
 SELECT id,name,data_state,service_mode,active,crdb_internal.pb_to_json('cockroach.multitenant.ProtoInfo', info, true) FROM system.tenants;
 ----
-1 system 1 2 true {"capabilities": {"canAdminSplit": false}, "deprecatedDataState": "READY", "deprecatedId": "1", "droppedName": "", "tenantReplicationJobId": "0"}
-5 <nil> 2 0 false {"capabilities": {"canAdminSplit": false}, "deprecatedDataState": "DROP", "deprecatedId": "5", "droppedName": "tenant-5", "tenantReplicationJobId": "0"}
-6 tenant-6 1 1 true {"capabilities": {"canAdminSplit": false}, "deprecatedDataState": "READY", "deprecatedId": "6", "droppedName": "", "tenantReplicationJobId": "0"}
+1 system 1 2 true {"capabilities": {"canAdminSplit": false, "canViewNodeInfo": false, "canViewTsdbMetrics": false}, "deprecatedDataState": "READY", "deprecatedId": "1", "droppedName": "", "tenantReplicationJobId": "0"}
+5 <nil> 2 0 false {"capabilities": {"canAdminSplit": false, "canViewNodeInfo": false, "canViewTsdbMetrics": false}, "deprecatedDataState": "DROP", "deprecatedId": "5", "droppedName": "tenant-5", "tenantReplicationJobId": "0"}
+6 tenant-6 1 1 true {"capabilities": {"canAdminSplit": false, "canViewNodeInfo": false, "canViewTsdbMetrics": false}, "deprecatedDataState": "READY", "deprecatedId": "6", "droppedName": "", "tenantReplicationJobId": "0"}
 
 exec-sql
 BACKUP INTO 'nodelocal://1/cluster'
@@ -59,8 +59,8 @@ job paused at pausepoint
 query-sql
 SELECT id,active,crdb_internal.pb_to_json('cockroach.multitenant.ProtoInfo', info, true) FROM system.tenants;
 ----
-1 true {"capabilities": {"canAdminSplit": false}, "deprecatedDataState": "READY", "deprecatedId": "1", "droppedName": "", "tenantReplicationJobId": "0"}
-6 false {"capabilities": {"canAdminSplit": false}, "deprecatedDataState": "ADD", "deprecatedId": "6", "droppedName": "", "tenantReplicationJobId": "0"}
+1 true {"capabilities": {"canAdminSplit": false, "canViewNodeInfo": false, "canViewTsdbMetrics": false}, "deprecatedDataState": "READY", "deprecatedId": "1", "droppedName": "", "tenantReplicationJobId": "0"}
+6 false {"capabilities": {"canAdminSplit": false, "canViewNodeInfo": false, "canViewTsdbMetrics": false}, "deprecatedDataState": "ADD", "deprecatedId": "6", "droppedName": "", "tenantReplicationJobId": "0"}
 
 exec-sql
 SET CLUSTER SETTING jobs.debug.pausepoints = ''
@@ -80,8 +80,8 @@ USE defaultdb;
 query-sql
 SELECT id,name,data_state,service_mode,active,crdb_internal.pb_to_json('cockroach.multitenant.ProtoInfo', info, true) FROM system.tenants;
 ----
-1 system 1 2 true {"capabilities": {"canAdminSplit": false}, "deprecatedDataState": "READY", "deprecatedId": "1", "droppedName": "", "tenantReplicationJobId": "0"}
-6 tenant-6 1 1 true {"capabilities": {"canAdminSplit": false}, "deprecatedDataState": "READY", "deprecatedId": "6", "droppedName": "", "tenantReplicationJobId": "0"}
+1 system 1 2 true {"capabilities": {"canAdminSplit": false, "canViewNodeInfo": false, "canViewTsdbMetrics": false}, "deprecatedDataState": "READY", "deprecatedId": "1", "droppedName": "", "tenantReplicationJobId": "0"}
+6 tenant-6 1 1 true {"capabilities": {"canAdminSplit": false, "canViewNodeInfo": false, "canViewTsdbMetrics": false}, "deprecatedDataState": "READY", "deprecatedId": "6", "droppedName": "", "tenantReplicationJobId": "0"}
 
 exec-sql expect-error-regex=(tenant 6 already exists)
 RESTORE TENANT 6 FROM LATEST IN 'nodelocal://1/tenant6';
@@ -105,9 +105,9 @@ RESTORE TENANT 6 FROM LATEST IN 'nodelocal://1/tenant6' WITH tenant_name = 'newn
 query-sql
 SELECT id,name,data_state,service_mode,active,crdb_internal.pb_to_json('cockroach.multitenant.ProtoInfo', info, true) FROM system.tenants;
 ----
-1 system 1 2 true {"capabilities": {"canAdminSplit": false}, "deprecatedDataState": "READY", "deprecatedId": "1", "droppedName": "", "tenantReplicationJobId": "0"}
-2 newname 1 1 true {"capabilities": {"canAdminSplit": false}, "deprecatedDataState": "READY", "deprecatedId": "2", "droppedName": "", "tenantReplicationJobId": "0"}
-6 tenant-6 1 1 true {"capabilities": {"canAdminSplit": false}, "deprecatedDataState": "READY", "deprecatedId": "6", "droppedName": "", "tenantReplicationJobId": "0"}
+1 system 1 2 true {"capabilities": {"canAdminSplit": false, "canViewNodeInfo": false, "canViewTsdbMetrics": false}, "deprecatedDataState": "READY", "deprecatedId": "1", "droppedName": "", "tenantReplicationJobId": "0"}
+2 newname 1 1 true {"capabilities": {"canAdminSplit": false, "canViewNodeInfo": false, "canViewTsdbMetrics": false}, "deprecatedDataState": "READY", "deprecatedId": "2", "droppedName": "", "tenantReplicationJobId": "0"}
+6 tenant-6 1 1 true {"capabilities": {"canAdminSplit": false, "canViewNodeInfo": false, "canViewTsdbMetrics": false}, "deprecatedDataState": "READY", "deprecatedId": "6", "droppedName": "", "tenantReplicationJobId": "0"}
 
 # Check that another service mode is also preserved.
 exec-sql

pkg/ccl/kvccl/kvtenantccl/connector.go

@@ -464,12 +464,12 @@ func (c *Connector) RangeLookup(
 	return nil, nil, errors.Wrap(ctx.Err(), "range lookup")
 }
 
-// NodesUI implements the serverpb.TenantStatusServer interface
-func (c *Connector) NodesUI(
+// Nodes implements the serverpb.TenantStatusServer interface
+func (c *Connector) Nodes(
 	ctx context.Context, req *serverpb.NodesRequest,
-) (resp *serverpb.NodesResponseExternal, retErr error) {
+) (resp *serverpb.NodesResponse, retErr error) {
 	retErr = c.withClient(ctx, func(ctx context.Context, client *client) (err error) {
-		resp, err = client.NodesUI(ctx, req)
+		resp, err = client.Nodes(ctx, req)
 		return
 	})
 	return

pkg/ccl/logictestccl/testdata/logic_test/tenant_capability

@@ -24,6 +24,8 @@ SHOW TENANT 'five' WITH CAPABILITIES
 id  name  data_state  service_mode  capability_name    capability_value
 5   five  ready       none          can_admin_split    false
 5   five  ready       none          can_admin_unsplit  false
+5   five  ready       none          can_view_node_info     false
+5   five  ready       none          can_view_tsdb_metrics  false
 
 statement ok
 ALTER TENANT [5] GRANT CAPABILITY can_admin_split=true
@@ -38,6 +40,8 @@ SHOW TENANT 'five' WITH CAPABILITIES
 id  name  data_state  service_mode  capability_name    capability_value
 5   five  ready       none          can_admin_split    true
 5   five  ready       none          can_admin_unsplit  false
+5   five  ready       none          can_view_node_info     false
+5   five  ready       none          can_view_tsdb_metrics  false
 
 statement ok
 ALTER TENANT [5] REVOKE CAPABILITY can_admin_split
@@ -48,3 +52,5 @@ SHOW TENANT 'five' WITH CAPABILITIES
 id  name  data_state  service_mode  capability_name    capability_value
 5   five  ready       none          can_admin_split    false
 5   five  ready       none          can_admin_unsplit  false
+5   five  ready       none          can_view_node_info     false
+5   five  ready       none          can_view_tsdb_metrics  false

pkg/ccl/multitenantccl/tenantcapabilitiesccl/testdata/can_admin_split

@@ -3,6 +3,8 @@ SHOW TENANT [10] WITH CAPABILITIES
 ----
 10 tenant-10 ready none can_admin_split false
 10 tenant-10 ready none can_admin_unsplit false
+10 tenant-10 ready none can_view_node_info false
+10 tenant-10 ready none can_view_tsdb_metrics false
 
 exec-sql-tenant
 CREATE TABLE t(a INT)

pkg/ccl/serverccl/adminccl/BUILD.bazel

@@ -17,12 +17,15 @@ go_test(
         "//pkg/server/serverpb",
         "//pkg/spanconfig",
         "//pkg/sql/tests",
+        "//pkg/testutils",
         "//pkg/testutils/serverutils",
         "//pkg/testutils/skip",
         "//pkg/testutils/testcluster",
+        "//pkg/ts/tspb",
         "//pkg/util/leaktest",
         "//pkg/util/log",
         "//pkg/util/randutil",
+        "//pkg/util/timeutil",
         "@com_github_stretchr_testify//require",
     ],
 )

pkg/ccl/serverccl/adminccl/tenant_admin_test.go

@@ -10,16 +10,20 @@ package adminccl
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"testing"
 
 	"github.com/cockroachdb/cockroach/pkg/ccl/serverccl"
 	"github.com/cockroachdb/cockroach/pkg/server/serverpb"
 	"github.com/cockroachdb/cockroach/pkg/spanconfig"
 	"github.com/cockroachdb/cockroach/pkg/sql/tests"
+	"github.com/cockroachdb/cockroach/pkg/testutils"
 	"github.com/cockroachdb/cockroach/pkg/testutils/skip"
+	"github.com/cockroachdb/cockroach/pkg/ts/tspb"
 	"github.com/cockroachdb/cockroach/pkg/util/leaktest"
 	"github.com/cockroachdb/cockroach/pkg/util/log"
+	"github.com/cockroachdb/cockroach/pkg/util/timeutil"
 	"github.com/stretchr/testify/require"
 )
 
@@ -52,6 +56,46 @@ func TestTenantAdminAPI(t *testing.T) {
 	t.Run("tenant_metricmetadata", func(t *testing.T) {
 		testMetricMetadataRPC(ctx, t, testHelper)
 	})
+
+	t.Run("tenant_metrics_capability", func(t *testing.T) {
+		testTenantMetricsCapabilityRPC(ctx, t, testHelper)
+	})
+}
+
+func testTenantMetricsCapabilityRPC(
+	ctx context.Context, t *testing.T, helper serverccl.TenantTestHelper,
+) {
+	http := helper.TestCluster().TenantAdminHTTPClient(t, 1)
+	defer http.Close()
+
+	query := tspb.TimeSeriesQueryRequest{
+		StartNanos: 0,
+		EndNanos:   timeutil.Now().UnixNano(),
+		Queries: []tspb.Query{
+			{
+				Name: "cr.node.sql.select.count",
+			},
+		},
+		SampleNanos: 0,
+	}
+	queryResp := tspb.TimeSeriesQueryResponse{}
+	err := http.PostJSONChecked("/ts/query", &query, &queryResp)
+	require.Error(t, err)
+
+	db := helper.HostCluster().ServerConn(0)
+	_, err = db.Exec("ALTER TENANT [10] GRANT CAPABILITY can_view_tsdb_metrics=true\n")
+	require.NoError(t, err)
+
+	testutils.SucceedsSoon(t, func() error {
+		err := http.PostJSONChecked("/ts/query", &query, &queryResp)
+		if err != nil {
+			return err
+		}
+		if len(queryResp.Results) == 0 {
+			return errors.New("missing metrics data")
+		}
+		return nil
+	})
 }
 
 func testMetricMetadataRPC(ctx context.Context, t *testing.T, helper serverccl.TenantTestHelper) {

pkg/ccl/serverccl/statusccl/tenant_status_test.go

@@ -83,6 +83,12 @@ func TestTenantStatusAPI(t *testing.T) {
 
 	defer testHelper.Cleanup(ctx, t)
 
+	// Speed up propagation of tenant capability changes.
+	db := testHelper.HostCluster().ServerConn(0)
+	tdb := sqlutils.MakeSQLRunner(db)
+	tdb.Exec(t, "SET CLUSTER SETTING kv.closed_timestamp.target_duration = '10ms'")
+	tdb.Exec(t, "SET CLUSTER SETTING kv.closed_timestamp.side_transport_interval = '10 ms'")
+
 	t.Run("reset_sql_stats", func(t *testing.T) {
 		testResetSQLStatsRPCForTenant(ctx, t, testHelper)
 	})
@@ -146,6 +152,10 @@ func TestTenantStatusAPI(t *testing.T) {
 	t.Run("tenant_span_stats", func(t *testing.T) {
 		testTenantSpanStats(ctx, t, testHelper)
 	})
+
+	t.Run("tenant_nodes_capability", func(t *testing.T) {
+		testTenantNodesCapability(ctx, t, testHelper)
+	})
 }
 
 func testTenantSpanStats(ctx context.Context, t *testing.T, helper serverccl.TenantTestHelper) {
@@ -211,6 +221,31 @@ func testTenantSpanStats(ctx context.Context, t *testing.T, helper serverccl.Ten
 		require.Equal(t, controlStats.RangeCount+1, stats.RangeCount)
 		require.Equal(t, controlStats.TotalStats.LiveCount+int64(len(incKeys)), stats.TotalStats.LiveCount)
 	})
+
+}
+
+func testTenantNodesCapability(
+	ctx context.Context, t *testing.T, helper serverccl.TenantTestHelper,
+) {
+	tenant := helper.TestCluster().TenantStatusSrv(0)
+
+	_, err := tenant.NodesUI(ctx, &serverpb.NodesRequest{})
+	require.Error(t, err)
+
+	db := helper.HostCluster().ServerConn(0)
+	_, err = db.Exec("ALTER TENANT [10] GRANT CAPABILITY can_view_node_info=true\n")
+	require.NoError(t, err)
+
+	testutils.SucceedsSoon(t, func() error {
+		resp, err := tenant.NodesUI(ctx, &serverpb.NodesRequest{})
+		if err != nil {
+			return err
+		}
+		if len(resp.Nodes) == 0 || len(resp.LivenessByNodeID) == 0 {
+			return errors.New("missing nodes or liveness data")
+		}
+		return nil
+	})
 }
 
 func testTenantLogs(ctx context.Context, t *testing.T, helper serverccl.TenantTestHelper) {

pkg/multitenant/tenantcapabilities/capabilities.go

@@ -12,6 +12,7 @@ package tenantcapabilities
 
 import (
 	"context"
+	"fmt"
 
 	"github.com/cockroachdb/cockroach/pkg/multitenant/tenantcapabilities/tenantcapabilitiespb"
 	"github.com/cockroachdb/cockroach/pkg/roachpb"
@@ -59,6 +60,14 @@ type Authorizer interface {
 	// implementation. Binding the Reader late allows us to break this dependency
 	// cycle.
 	BindReader(reader Reader)
+
+	// HasNodeStatusCapability returns an error if a tenant, referenced by its ID,
+	// is not allowed to access cluster-level node metadata and liveness.
+	HasNodeStatusCapability(ctx context.Context, tenID roachpb.TenantID) error
+
+	// HasTSDBQueryCapability returns an error if a tenant, referenced by its ID,
+	// is not allowed to query the TSDB for metrics.
+	HasTSDBQueryCapability(ctx context.Context, tenID roachpb.TenantID) error
 }
 
 // Entry ties together a tenantID with its capabilities.
@@ -72,3 +81,14 @@ type Update struct {
 	Entry
 	Deleted bool // whether the entry was deleted or not
 }
+
+func (u Update) String() string {
+	if u.Deleted {
+		return fmt.Sprintf("delete: ten=%+v", u.Entry.TenantID)
+	}
+	return fmt.Sprintf("update: %+v", u.Entry)
+}
+
+func (u Entry) String() string {
+	return fmt.Sprintf("ten=%s cap=%+v", u.TenantID, u.TenantCapabilities)
+}

pkg/multitenant/tenantcapabilities/tenantcapabilitiesauthorizer/authorizer.go

@@ -85,3 +85,39 @@ func (a *Authorizer) HasCapabilityForBatch(
 func (a *Authorizer) BindReader(reader tenantcapabilities.Reader) {
 	a.capabilitiesReader = reader
 }
+
+func (a *Authorizer) HasNodeStatusCapability(ctx context.Context, tenID roachpb.TenantID) error {
+	if tenID.IsSystem() {
+		return nil // the system tenant is allowed to do as it pleases
+	}
+	cp, found := a.capabilitiesReader.GetCapabilities(tenID)
+	if !found {
+		log.Infof(
+			ctx,
+			"no capability information for tenant %s; requests that require capabilities may be denied",
+			tenID,
+		)
+	}
+	if !cp.CanViewNodeInfo {
+		return errors.Newf("tenant %s does not have capability to query cluster node metadata", tenID)
+	}
+	return nil
+}
+
+func (a *Authorizer) HasTSDBQueryCapability(ctx context.Context, tenID roachpb.TenantID) error {
+	if tenID.IsSystem() {
+		return nil // the system tenant is allowed to do as it pleases
+	}
+	cp, found := a.capabilitiesReader.GetCapabilities(tenID)
+	if !found {
+		log.Infof(
+			ctx,
+			"no capability information for tenant %s; requests that require capabilities may be denied",
+			tenID,
+		)
+	}
+	if !cp.CanViewTsdbMetrics {
+		return errors.Newf("tenant %s does not have capability to query timeseries data", tenID)
+	}
+	return nil
+}