server: enabling instantiating secondary tenant servers in-memory

Epic: CRDB-14537

This commit lays the groundwork for the ability to run tenant servers
in-memory. It introduces a new `serverController` with two roles:

- it maps tenant names to server instances.
- it only starts servers the first time the tenant name is referenced,
  and only if that tenant is marked active.

As of this commit, no subsystem inside CockroachDB refers to this
new server controller; this means there is no signal hooked up
to start these servers automatically yet.

For testing, a debug HTTP endpoint has been added:
`/debug/tickle?name=<tenantname>`

Example use:

1. start a server. At this point no secondary tenant server is created yet.
2. create a test tenant, e.g. via `select crdb_internal.create_tenant(123, 'hello');`.
   At this point, the secondary tenant server is still not running.
3. Perform a HTTP request to the debug endpoint, e.g. to `/debug/tickle?name=hello`
4. Observe (e.g. in logs): the secondary server has been started.

One can also observe that the controller also serves the name `system`
to refer to the system tenant.

For now, the secondary servers created this way use separate network
listeners for SQL/HTTP/RPC. NB: This mechanism will be superseded when
cockroachdb#84604 is addressed.

The port number is assigned randomly. To derive a predictable port
number for testing (until issue cockroachdb#84604 is addressed), the operator can
pass e.g. `--secondary-tenant-port-offset=100` to the start
command (for 100 past the base port number).

Release note: None

pkg/base/config.go

@@ -229,6 +229,15 @@ type Config struct {
 	// low.
 	RPCHeartbeatIntervalAndHalfTimeout time.Duration
 
+	// SecondaryTenantPortOffset is the increment to add to the various
+	// addresses to generate the network configuration for the in-memory
+	// secondary tenant. If set to zero (the default), ports are
+	// auto-allocated randomly.
+	// TODO(knz): Remove this mechanism altogether in favor of a single
+	// network listener with protocol routing.
+	// See: https://github.com/cockroachdb/cockroach/issues/84585
+	SecondaryTenantPortOffset int
+
 	// Enables the use of an PTP hardware clock user space API for HLC current time.
 	// This contains the path to the device to be used (i.e. /dev/ptp0)
 	ClockDevicePath string
@@ -281,6 +290,7 @@ func (cfg *Config) InitDefaults() {
 	cfg.DisableClusterNameVerification = false
 	cfg.ClockDevicePath = ""
 	cfg.AcceptSQLWithoutTLS = false
+	cfg.SecondaryTenantPortOffset = 0
 }
 
 // HTTPRequestScheme returns "http" or "https" based on the value of

pkg/cli/cliflags/flags.go

@@ -137,6 +137,18 @@ control is not returned to the shell until the server is ready to
 accept requests.`,
 	}
 
+	DisableInMemoryTenant = FlagInfo{
+		Name:        "disable-in-memory-tenant",
+		Description: `Do not start a secondary tenant in-memory.`,
+	}
+
+	// TODO(knz): Remove this once https://github.com/cockroachdb/cockroach/issues/84604
+	// is addressed.
+	SecondaryTenantPortOffset = FlagInfo{
+		Name:        "secondary-tenant-port-offset",
+		Description: "TCP port number offset to use for the secondary in-memory tenant.",
+	}
+
 	SQLMem = FlagInfo{
 		Name: "max-sql-memory",
 		Description: `

pkg/cli/flags.go

@@ -537,6 +537,11 @@ func init() {
 		if backgroundFlagDefined {
 			cliflagcfg.BoolFlag(f, &startBackground, cliflags.Background)
 		}
+
+		// TODO(knz): Remove this port offset mechanism once we implement
+		// a shared listener. See: https://github.com/cockroachdb/cockroach/issues/84585
+		cliflagcfg.IntFlag(f, &baseCfg.SecondaryTenantPortOffset, cliflags.SecondaryTenantPortOffset)
+		_ = f.MarkHidden(cliflags.SecondaryTenantPortOffset.Name)
 	}
 
 	// Multi-tenancy start-sql command flags.

pkg/cli/start.go

@@ -22,7 +22,6 @@ import (
 	"path/filepath"
 	"runtime"
 	"strings"
-	"text/tabwriter"
 	"time"
 
 	"github.com/cockroachdb/cockroach/pkg/base"
@@ -40,10 +39,10 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/server"
 	"github.com/cockroachdb/cockroach/pkg/server/status"
 	"github.com/cockroachdb/cockroach/pkg/settings/cluster"
-	"github.com/cockroachdb/cockroach/pkg/sql/catalog/catalogkeys"
 	"github.com/cockroachdb/cockroach/pkg/storage"
 	"github.com/cockroachdb/cockroach/pkg/storage/enginepb"
 	"github.com/cockroachdb/cockroach/pkg/storage/fs"
+	"github.com/cockroachdb/cockroach/pkg/util"
 	"github.com/cockroachdb/cockroach/pkg/util/cgroups"
 	"github.com/cockroachdb/cockroach/pkg/util/envutil"
 	"github.com/cockroachdb/cockroach/pkg/util/grpcutil"
@@ -1033,24 +1032,6 @@ func startShutdownAsync(
 	}()
 }
 
-// makeServerOptionsForURL creates the input for MakeURLForServer().
-// Beware of not calling this too early; the server address
-// is finalized late in the network initialization sequence.
-func makeServerOptionsForURL(
-	serverCfg *server.Config,
-) (clientsecopts.ClientSecurityOptions, clientsecopts.ServerParameters) {
-	clientConnOptions := clientsecopts.ClientSecurityOptions{
-		Insecure: serverCfg.Config.Insecure,
-		CertsDir: serverCfg.Config.SSLCertsDir,
-	}
-	serverParams := clientsecopts.ServerParameters{
-		ServerAddr:      serverCfg.Config.SQLAdvertiseAddr,
-		DefaultPort:     base.DefaultPort,
-		DefaultDatabase: catalogkeys.DefaultDatabaseName,
-	}
-	return clientConnOptions, serverParams
-}
-
 // reportServerInfo prints out the server version and network details
 // in a standardized format.
 func reportServerInfo(
@@ -1072,7 +1053,7 @@ func reportServerInfo(
 	// (Re-)compute the client connection URL. We cannot do this
 	// earlier (e.g. above, in the runStart function) because
 	// at this time the address and port have not been resolved yet.
-	clientConnOptions, serverParams := makeServerOptionsForURL(serverCfg)
+	clientConnOptions, serverParams := server.MakeServerOptionsForURL(serverCfg.Config)
 	pgURL, err := clientsecopts.MakeURLForServer(clientConnOptions, serverParams, url.User(username.RootUser))
 	if err != nil {
 		log.Ops.Errorf(ctx, "failed computing the URL: %v", err)
@@ -1157,7 +1138,7 @@ func reportServerInfo(
 	}
 
 	// Collect the formatted string and show it to the user.
-	msg, err := expandTabsInRedactableBytes(buf.RedactableBytes())
+	msg, err := util.ExpandTabsInRedactableBytes(buf.RedactableBytes())
 	if err != nil {
 		return err
 	}
@@ -1170,22 +1151,6 @@ func reportServerInfo(
 	return nil
 }
 
-// expandTabsInRedactableBytes expands tabs in the redactable byte
-// slice, so that columns are aligned. The correctness of this
-// function depends on the assumption that the `tabwriter` does not
-// replace characters.
-func expandTabsInRedactableBytes(s redact.RedactableBytes) (redact.RedactableBytes, error) {
-	var buf bytes.Buffer
-	tw := tabwriter.NewWriter(&buf, 2, 1, 2, ' ', 0)
-	if _, err := tw.Write([]byte(s)); err != nil {
-		return nil, err
-	}
-	if err := tw.Flush(); err != nil {
-		return nil, err
-	}
-	return redact.RedactableBytes(buf.Bytes()), nil
-}
-
 func hintServerCmdFlags(ctx context.Context, cmd *cobra.Command) {
 	pf := cliflagcfg.FlagSetForCmd(cmd)
 
@@ -1428,7 +1393,7 @@ func reportReadinessExternally(ctx context.Context, cmd *cobra.Command, waitForI
 		// (Re-)compute the client connection URL. We cannot do this
 		// earlier (e.g. above, in the runStart function) because
 		// at this time the address and port have not been resolved yet.
-		clientConnOptions, serverParams := makeServerOptionsForURL(&serverCfg)
+		clientConnOptions, serverParams := server.MakeServerOptionsForURL(serverCfg.Config)
 		pgURL, err := clientsecopts.MakeURLForServer(clientConnOptions, serverParams, url.User(username.RootUser))
 		if err != nil {
 			log.Errorf(ctx, "failed computing the URL: %v", err)

pkg/server/BUILD.bazel

@@ -49,6 +49,7 @@ go_library(
         "rlimit_darwin.go",
         "rlimit_unix.go",
         "server.go",
+        "server_controller.go",
         "server_http.go",
         "server_sql.go",
         "server_systemlog_gc.go",
@@ -378,6 +379,7 @@ go_test(
         "pagination_test.go",
         "purge_auth_session_test.go",
         "servemode_test.go",
+        "server_controller_test.go",
         "server_http_test.go",
         "server_import_ts_test.go",
         "server_internal_executor_factory_test.go",

pkg/server/debug/server.go

@@ -66,12 +66,15 @@ type Server struct {
 	spy        logSpy
 }
 
+type serverTickleFn = func(ctx context.Context, name string) error
+
 // NewServer sets up a debug server.
 func NewServer(
 	ambientContext log.AmbientContext,
 	st *cluster.Settings,
 	hbaConfDebugFn http.HandlerFunc,
 	profiler pprofui.Profiler,
+	serverTickleFn serverTickleFn,
 ) *Server {
 	mux := http.NewServeMux()
 
@@ -110,6 +113,15 @@ func NewServer(
 	// Register the stopper endpoint, which lists all active tasks.
 	mux.HandleFunc("/debug/stopper", stop.HandleDebug)
 
+	if serverTickleFn != nil {
+		// Register the server tickling function.
+		//
+		// TODO(knz): This can be removed once
+		// https://github.com/cockroachdb/cockroach/issues/84585 is
+		// implemented.
+		mux.Handle("/debug/tickle", handleTickle(serverTickleFn))
+	}
+
 	// Set up the vmodule endpoint.
 	vsrv := &vmoduleServer{}
 	mux.HandleFunc("/debug/vmodule", vsrv.vmoduleHandleDebug)
@@ -266,3 +278,25 @@ If you are not redirected automatically, follow this <a href='/#/debug'>link</a>
 </html>
 `)
 }
+
+type handleTickle serverTickleFn
+
+func (h handleTickle) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	opts := r.URL.Query()
+	var name string
+	if n := opts["name"]; len(n) > 0 {
+		name = n[0]
+	}
+	w.Header().Add("Content-type", "text/plain")
+	if name == "" {
+		fmt.Fprint(w, "no name specified")
+		return
+	}
+	ctx := r.Context()
+	err := serverTickleFn(h)(ctx, name)
+	if err != nil {
+		fmt.Fprint(w, err)
+		return
+	}
+	fmt.Fprintf(w, "server for tenant %q was tickled", name)
+}

pkg/server/server.go

@@ -50,6 +50,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/roachpb"
 	"github.com/cockroachdb/cockroach/pkg/rpc"
 	"github.com/cockroachdb/cockroach/pkg/rpc/nodedialer"
+	"github.com/cockroachdb/cockroach/pkg/security/clientsecopts"
 	"github.com/cockroachdb/cockroach/pkg/security/username"
 	"github.com/cockroachdb/cockroach/pkg/server/debug"
 	"github.com/cockroachdb/cockroach/pkg/server/diagnostics"
@@ -66,6 +67,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigkvsubscriber"
 	"github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigptsreader"
 	"github.com/cockroachdb/cockroach/pkg/sql"
+	"github.com/cockroachdb/cockroach/pkg/sql/catalog/catalogkeys"
 	_ "github.com/cockroachdb/cockroach/pkg/sql/catalog/schematelemetry" // register schedules declared outside of pkg/sql
 	"github.com/cockroachdb/cockroach/pkg/sql/catalog/systemschema"
 	"github.com/cockroachdb/cockroach/pkg/sql/flowinfra"
@@ -154,8 +156,13 @@ type Server struct {
 
 	spanConfigSubscriber spanconfig.KVSubscriber
 
+	// TODO(knz): pull this down under the serverController.
 	sqlServer *SQLServer
 
+	// serverController is responsible for on-demand instantiation
+	// of services.
+	serverController *serverController
+
 	// Created in NewServer but initialized (made usable) in `(*Server).PreStart`.
 	externalStorageBuilder *externalStorageBuilder
 
@@ -919,12 +926,26 @@ func NewServer(cfg Config, stopper *stop.Stopper) (*Server, error) {
 	sStatus.setStmtDiagnosticsRequester(sqlServer.execCfg.StmtDiagnosticsRecorder)
 	sStatus.baseStatusServer.sqlServer = sqlServer
 
+	// Create a server controller.
+	sc := newServerController(ctx, stopper,
+		lateBoundServer.newServerForTenant, &systemServerWrapper{server: lateBoundServer})
+
 	// Create the debug API server.
 	debugServer := debug.NewServer(
 		cfg.BaseConfig.AmbientCtx,
 		st,
 		sqlServer.pgServer.HBADebugFn(),
 		sqlServer.execCfg.SQLStatusServer,
+		// TODO(knz): Remove this once
+		// https://github.com/cockroachdb/cockroach/issues/84585 is
+		// implemented.
+		func(ctx context.Context, name string) error {
+			d, err := sc.getOrCreateServer(ctx, name)
+			if err != nil {
+				return err
+			}
+			return errors.Newf("server found with type %T", d)
+		},
 	)
 
 	// Create a drain server.
@@ -973,6 +994,7 @@ func NewServer(cfg Config, stopper *stop.Stopper) (*Server, error) {
 		protectedtsProvider:    protectedtsProvider,
 		spanConfigSubscriber:   spanConfig.subscriber,
 		sqlServer:              sqlServer,
+		serverController:       sc,
 		externalStorageBuilder: externalStorageBuilder,
 		storeGrantCoords:       gcoords.Stores,
 		kvMemoryMonitor:        kvMemoryMonitor,
@@ -1839,3 +1861,21 @@ func (s *Server) Drain(
 ) (remaining uint64, info redact.RedactableString, err error) {
 	return s.drain.runDrain(ctx, verbose)
 }
+
+// MakeServerOptionsForURL creates the input for MakeURLForServer().
+// Beware of not calling this too early; the server address
+// is finalized late in the network initialization sequence.
+func MakeServerOptionsForURL(
+	baseCfg *base.Config,
+) (clientsecopts.ClientSecurityOptions, clientsecopts.ServerParameters) {
+	clientConnOptions := clientsecopts.ClientSecurityOptions{
+		Insecure: baseCfg.Insecure,
+		CertsDir: baseCfg.SSLCertsDir,
+	}
+	serverParams := clientsecopts.ServerParameters{
+		ServerAddr:      baseCfg.SQLAdvertiseAddr,
+		DefaultPort:     base.DefaultPort,
+		DefaultDatabase: catalogkeys.DefaultDatabaseName,
+	}
+	return clientConnOptions, serverParams
+}