cli,server: start a secondary tenant in-memory

Epic: CRDB-14537

This lays the groundwork for the ability to run tenant servers
in-memory.

The secondary in-memory server is started by default. This can
be manually disabled using `--disable-in-memory-tenant`.

For now, the secondary tenant uses separate network listeners for
SQL/HTTP/RPC. The port number is assigned randomly.  To derive a port
number "close" to the base TCP listeners, the operator can pass
e.g. `--secondary-tenant-port-offset=1` (for 1 past the base port
number). NB: This mechanism will be superseded when
#84604 is addressed.

Release note: None

pkg/base/config.go

@@ -229,6 +229,15 @@ type Config struct {
 	// low.
 	RPCHeartbeatIntervalAndHalfTimeout time.Duration
 
+	// SecondaryTenantPortOffset is the increment to add to the various
+	// addresses to generate the network configuration for the in-memory
+	// secondary tenant. If set to zero (the default), ports are
+	// auto-allocated randomly.
+	// TODO(knz): Remove this mechanism altogether in favor of a single
+	// network listener with protocol routing.
+	// See: https://github.com/cockroachdb/cockroach/issues/84585
+	SecondaryTenantPortOffset int
+
 	// Enables the use of an PTP hardware clock user space API for HLC current time.
 	// This contains the path to the device to be used (i.e. /dev/ptp0)
 	ClockDevicePath string
@@ -281,6 +290,7 @@ func (cfg *Config) InitDefaults() {
 	cfg.DisableClusterNameVerification = false
 	cfg.ClockDevicePath = ""
 	cfg.AcceptSQLWithoutTLS = false
+	cfg.SecondaryTenantPortOffset = 0
 }
 
 // HTTPRequestScheme returns "http" or "https" based on the value of

pkg/ccl/sqlproxyccl/tenant/directory_cache_test.go

@@ -433,7 +433,7 @@ func startTenant(
 		})
 	if err != nil {
 		// Remap tenant "not found" error to GRPC NotFound error.
-		if err.Error() == "not found" {
+		if testutils.IsError(err, `tenant \d+ not found`) {
 			return nil, status.Errorf(codes.NotFound, "tenant %d not found", id)
 		}
 		return nil, err

pkg/cli/BUILD.bazel

@@ -176,6 +176,7 @@ go_library(
         "//pkg/sql/sem/catconstants",
         "//pkg/sql/sem/eval",
         "//pkg/sql/sem/tree",
+        "//pkg/sql/sessiondata",
         "//pkg/sql/sqlstats",
         "//pkg/startupmigrations",
         "//pkg/storage",

pkg/cli/cliflags/flags.go

@@ -137,6 +137,18 @@ control is not returned to the shell until the server is ready to
 accept requests.`,
 	}
 
+	DisableInMemoryTenant = FlagInfo{
+		Name:        "disable-in-memory-tenant",
+		Description: `Do not start a secondary tenant in-memory.`,
+	}
+
+	// TODO(knz): Remove this once https://github.com/cockroachdb/cockroach/issues/84604
+	// is addressed.
+	SecondaryTenantPortOffset = FlagInfo{
+		Name:        "secondary-tenant-port-offset",
+		Description: "TCP port number offset to use for the secondary in-memory tenant.",
+	}
+
 	SQLMem = FlagInfo{
 		Name: "max-sql-memory",
 		Description: `

pkg/cli/context.go

@@ -494,6 +494,10 @@ var startCtx struct {
 
 	// geoLibsDir is used to specify locations of the GEOS library.
 	geoLibsDir string
+
+	// disableInMemoryTenant stops the instantiation of an in-memory
+	// SQL server for a secondary tenant.
+	disableInMemoryTenant bool
 }
 
 // setStartContextDefaults set the default values in startCtx.  This
@@ -514,6 +518,7 @@ func setStartContextDefaults() {
 	startCtx.pidFile = ""
 	startCtx.inBackground = false
 	startCtx.geoLibsDir = "/usr/local/lib/cockroach"
+	startCtx.disableInMemoryTenant = false
 }
 
 // drainCtx captures the command-line parameters of the `node drain`

pkg/cli/democluster/demo_cluster.go

@@ -461,7 +461,7 @@ func (c *transientCluster) Start(ctx context.Context) (err error) {
 		server := c.firstServer.Server
 		ctx = server.AnnotateCtx(ctx)
 
-		if err := server.RunInitialSQL(ctx, c.demoCtx.NumNodes < 3, demoUsername, demoPassword); err != nil {
+		if err := server.RunInitialSQL(ctx, c.demoCtx.NumNodes < 3, true /* createAppTenant */, demoUsername, demoPassword); err != nil {
 			return err
 		}
 		if c.demoCtx.Insecure {

pkg/cli/flags.go

@@ -537,6 +537,12 @@ func init() {
 		if backgroundFlagDefined {
 			cliflagcfg.BoolFlag(f, &startBackground, cliflags.Background)
 		}
+
+		// TODO(knz): publicize this machinery.
+		cliflagcfg.BoolFlag(f, &startCtx.disableInMemoryTenant, cliflags.DisableInMemoryTenant)
+		_ = f.MarkHidden(cliflags.DisableInMemoryTenant.Name)
+		cliflagcfg.IntFlag(f, &baseCfg.SecondaryTenantPortOffset, cliflags.SecondaryTenantPortOffset)
+		_ = f.MarkHidden(cliflags.SecondaryTenantPortOffset.Name)
 	}
 
 	// Multi-tenancy start-sql command flags.

pkg/cli/start.go

@@ -22,7 +22,6 @@ import (
 	"path/filepath"
 	"runtime"
 	"strings"
-	"text/tabwriter"
 	"time"
 
 	"github.com/cockroachdb/cockroach/pkg/base"
@@ -40,10 +39,10 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/server"
 	"github.com/cockroachdb/cockroach/pkg/server/status"
 	"github.com/cockroachdb/cockroach/pkg/settings/cluster"
-	"github.com/cockroachdb/cockroach/pkg/sql/catalog/catalogkeys"
 	"github.com/cockroachdb/cockroach/pkg/storage"
 	"github.com/cockroachdb/cockroach/pkg/storage/enginepb"
 	"github.com/cockroachdb/cockroach/pkg/storage/fs"
+	"github.com/cockroachdb/cockroach/pkg/util"
 	"github.com/cockroachdb/cockroach/pkg/util/cgroups"
 	"github.com/cockroachdb/cockroach/pkg/util/envutil"
 	"github.com/cockroachdb/cockroach/pkg/util/grpcutil"
@@ -350,7 +349,9 @@ type serverStartupInterface interface {
 	// The arguments are:
 	// - startSingleNode is used by 'demo' and 'start-single-node'.
 	// - adminUser/adminPassword is used for 'demo'.
-	RunInitialSQL(ctx context.Context, startSingleNode bool, adminUser, adminPassword string) error
+	// - createAppTenant is temporary until moved to a long-running migration.
+	//   (https://github.com/cockroachdb/cockroach/issues/91051)
+	RunInitialSQL(ctx context.Context, startSingleNode, createAppTenant bool, adminUser, adminPassword string) error
 }
 
 var errCannotUseJoin = errors.New("cannot use --join with 'cockroach start-single-node' -- use 'cockroach start' instead")
@@ -713,7 +714,9 @@ func createAndStartServerAsync(
 			}
 
 			// Run one-off cluster initialization.
-			if err := s.RunInitialSQL(ctx, startSingleNode, "" /* adminUser */, "" /* adminPassword */); err != nil {
+			if err := s.RunInitialSQL(ctx, startSingleNode,
+				!startCtx.disableInMemoryTenant, /* createAppTenant */
+				"" /* adminUser */, "" /* adminPassword */); err != nil {
 				return err
 			}
 
@@ -722,6 +725,15 @@ func createAndStartServerAsync(
 				return err
 			}
 
+			if !startCtx.disableInMemoryTenant {
+				// Also start the SQL services for tenants, when defined.
+				// TODO(knz): Do this dynamically upon first use instead.
+				// See: https://github.com/cockroachdb/cockroach/issues/84604
+				if err := s.(*server.Server).StartInMemoryAppTenant(ctx); err != nil {
+					return err
+				}
+			}
+
 			// Now inform the user that the server is running and tell the
 			// user about its run-time derived parameters.
 			return reportServerInfo(ctx, tBegin, serverCfg, s.ClusterSettings(),
@@ -1037,24 +1049,6 @@ func startShutdownAsync(
 	}()
 }
 
-// makeServerOptionsForURL creates the input for MakeURLForServer().
-// Beware of not calling this too early; the server address
-// is finalized late in the network initialization sequence.
-func makeServerOptionsForURL(
-	serverCfg *server.Config,
-) (clientsecopts.ClientSecurityOptions, clientsecopts.ServerParameters) {
-	clientConnOptions := clientsecopts.ClientSecurityOptions{
-		Insecure: serverCfg.Config.Insecure,
-		CertsDir: serverCfg.Config.SSLCertsDir,
-	}
-	serverParams := clientsecopts.ServerParameters{
-		ServerAddr:      serverCfg.Config.SQLAdvertiseAddr,
-		DefaultPort:     base.DefaultPort,
-		DefaultDatabase: catalogkeys.DefaultDatabaseName,
-	}
-	return clientConnOptions, serverParams
-}
-
 // reportServerInfo prints out the server version and network details
 // in a standardized format.
 func reportServerInfo(
@@ -1076,7 +1070,7 @@ func reportServerInfo(
 	// (Re-)compute the client connection URL. We cannot do this
 	// earlier (e.g. above, in the runStart function) because
 	// at this time the address and port have not been resolved yet.
-	clientConnOptions, serverParams := makeServerOptionsForURL(serverCfg)
+	clientConnOptions, serverParams := server.MakeServerOptionsForURL(serverCfg.Config)
 	pgURL, err := clientsecopts.MakeURLForServer(clientConnOptions, serverParams, url.User(username.RootUser))
 	if err != nil {
 		log.Ops.Errorf(ctx, "failed computing the URL: %v", err)
@@ -1161,7 +1155,7 @@ func reportServerInfo(
 	}
 
 	// Collect the formatted string and show it to the user.
-	msg, err := expandTabsInRedactableBytes(buf.RedactableBytes())
+	msg, err := util.ExpandTabsInRedactableBytes(buf.RedactableBytes())
 	if err != nil {
 		return err
 	}
@@ -1174,22 +1168,6 @@ func reportServerInfo(
 	return nil
 }
 
-// expandTabsInRedactableBytes expands tabs in the redactable byte
-// slice, so that columns are aligned. The correctness of this
-// function depends on the assumption that the `tabwriter` does not
-// replace characters.
-func expandTabsInRedactableBytes(s redact.RedactableBytes) (redact.RedactableBytes, error) {
-	var buf bytes.Buffer
-	tw := tabwriter.NewWriter(&buf, 2, 1, 2, ' ', 0)
-	if _, err := tw.Write([]byte(s)); err != nil {
-		return nil, err
-	}
-	if err := tw.Flush(); err != nil {
-		return nil, err
-	}
-	return redact.RedactableBytes(buf.Bytes()), nil
-}
-
 func hintServerCmdFlags(ctx context.Context, cmd *cobra.Command) {
 	pf := cliflagcfg.FlagSetForCmd(cmd)
 
@@ -1431,7 +1409,7 @@ func reportReadinessExternally(ctx context.Context, cmd *cobra.Command, waitForI
 		// (Re-)compute the client connection URL. We cannot do this
 		// earlier (e.g. above, in the runStart function) because
 		// at this time the address and port have not been resolved yet.
-		clientConnOptions, serverParams := makeServerOptionsForURL(&serverCfg)
+		clientConnOptions, serverParams := server.MakeServerOptionsForURL(serverCfg.Config)
 		pgURL, err := clientsecopts.MakeURLForServer(clientConnOptions, serverParams, url.User(username.RootUser))
 		if err != nil {
 			log.Errorf(ctx, "failed computing the URL: %v", err)

pkg/server/BUILD.bazel

@@ -65,6 +65,7 @@ go_library(
         "tcp_keepalive_manager.go",
         "tenant.go",
         "tenant_admin.go",
+        "tenant_server_controller.go",
         "tenant_status.go",
         "testing_knobs.go",
         "testserver.go",

pkg/server/initial_sql.go

@@ -28,7 +28,7 @@ import (
 // If adminUser is non-empty, an admin user with that name is
 // created upon initialization. Its password is then also returned.
 func (s *Server) RunInitialSQL(
-	ctx context.Context, startSingleNode bool, adminUser, adminPassword string,
+	ctx context.Context, startSingleNode, createAppTenant bool, adminUser, adminPassword string,
 ) error {
 	newCluster := s.InitialStart() && s.NodeID() == kvserver.FirstNodeID
 	if !newCluster {
@@ -54,14 +54,41 @@ func (s *Server) RunInitialSQL(
 		}
 	}
 
+	if createAppTenant {
+		if err := s.createAppTenant(ctx); err != nil {
+			return err
+		}
+	}
+
 	return nil
 }
 
 // RunInitialSQL implements cli.serverStartupInterface.
-func (s *SQLServerWrapper) RunInitialSQL(context.Context, bool, string, string) error {
+func (s *SQLServerWrapper) RunInitialSQL(context.Context, bool, bool, string, string) error {
 	return nil
 }
 
+// createAppTenant creates an application tenant if it does
+// not exist yet.
+// TODO(knz): Replace this by a long-running migration.
+// See: https://github.com/cockroachdb/cockroach/issues/91051
+func (s *Server) createAppTenant(ctx context.Context) error {
+	ie := s.sqlServer.internalExecutor
+	_, err := ie.Exec(
+		ctx, "create-app-tenant", nil, /* txn */
+		// This query picks up the first available ID automatically.
+		`
+WITH tid AS (
+  SELECT id+1 AS newid
+    FROM (VALUES (1) UNION ALL SELECT id FROM system.tenants) AS u(id)
+   WHERE NOT EXISTS (SELECT 1 FROM system.tenants t WHERE t.id=u.id+1)
+     AND NOT EXISTS (SELECT 1 FROM system.tenants t WHERE t.name='app')
+   LIMIT 1
+) SELECT "".crdb_internal.create_tenant(newid, 'app') FROM tid
+`)
+	return err
+}
+
 // createAdminUser creates an admin user with the given name.
 func (s *Server) createAdminUser(ctx context.Context, adminUser, adminPassword string) error {
 	ie := s.sqlServer.internalExecutor

pkg/server/server.go

@@ -49,6 +49,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/roachpb"
 	"github.com/cockroachdb/cockroach/pkg/rpc"
 	"github.com/cockroachdb/cockroach/pkg/rpc/nodedialer"
+	"github.com/cockroachdb/cockroach/pkg/security/clientsecopts"
 	"github.com/cockroachdb/cockroach/pkg/security/username"
 	"github.com/cockroachdb/cockroach/pkg/server/debug"
 	"github.com/cockroachdb/cockroach/pkg/server/diagnostics"
@@ -65,6 +66,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigkvsubscriber"
 	"github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigptsreader"
 	"github.com/cockroachdb/cockroach/pkg/sql"
+	"github.com/cockroachdb/cockroach/pkg/sql/catalog/catalogkeys"
 	_ "github.com/cockroachdb/cockroach/pkg/sql/catalog/schematelemetry" // register schedules declared outside of pkg/sql
 	"github.com/cockroachdb/cockroach/pkg/sql/catalog/systemschema"
 	"github.com/cockroachdb/cockroach/pkg/sql/flowinfra"
@@ -1839,3 +1841,21 @@ func (s *Server) Drain(
 ) (remaining uint64, info redact.RedactableString, err error) {
 	return s.drain.runDrain(ctx, verbose)
 }
+
+// MakeServerOptionsForURL creates the input for MakeURLForServer().
+// Beware of not calling this too early; the server address
+// is finalized late in the network initialization sequence.
+func MakeServerOptionsForURL(
+	baseCfg *base.Config,
+) (clientsecopts.ClientSecurityOptions, clientsecopts.ServerParameters) {
+	clientConnOptions := clientsecopts.ClientSecurityOptions{
+		Insecure: baseCfg.Insecure,
+		CertsDir: baseCfg.SSLCertsDir,
+	}
+	serverParams := clientsecopts.ServerParameters{
+		ServerAddr:      baseCfg.SQLAdvertiseAddr,
+		DefaultPort:     base.DefaultPort,
+		DefaultDatabase: catalogkeys.DefaultDatabaseName,
+	}
+	return clientConnOptions, serverParams
+}