Merge cockroachdb#76076

76076: roachprod: Make it possible to specify file descriptor limit. r=miretskiy a=miretskiy

Add a `--num-files-limit` flag to roachprod start command
to change the default file descriptor limit for cockroach process.

Release Notes: None

Co-authored-by: Yevgeniy Miretskiy <yevgeniy@cockroachlabs.com>

pkg/cmd/roachprod/flags.go

@@ -256,6 +256,8 @@ func initFlags() {
 	for _, cmd := range []*cobra.Command{startCmd, startTenantCmd} {
 		cmd.Flags().BoolVar(&startOpts.Sequential,
 			"sequential", startOpts.Sequential, "start nodes sequentially so node IDs match hostnames")
+		cmd.Flags().Int64Var(&startOpts.NumFilesLimit, "num-files-limit", startOpts.NumFilesLimit,
+			"limit the number of files that can be created by the cockroach process")
 	}
 
 	for _, cmd := range []*cobra.Command{

pkg/roachprod/config/config.go

@@ -71,6 +71,10 @@ const (
 	// DefaultAdminUIPort is the default port on which the cockroach process is
 	// listening for HTTP connections for the Admin UI.
 	DefaultAdminUIPort = 26258
+
+	// DefaultNumFilesLimit is the default limit on the number of files that can
+	// be opened by the process.
+	DefaultNumFilesLimit = 65 << 13
 )
 
 // IsLocalClusterName returns true if the given name is a valid name for a local

pkg/roachprod/install/cockroach.go

@@ -107,6 +107,9 @@ type StartOpts struct {
 	Sequential bool
 	ExtraArgs  []string
 
+	// systemd limits on resources.
+	NumFilesLimit int64
+
 	// -- Options that apply only to StartDefault target --
 
 	SkipInit        bool
@@ -420,21 +423,23 @@ func (c *SyncedCluster) generateStartCmd(
 			"GOTRACEBACK=crash",
 			"COCKROACH_SKIP_ENABLING_DIAGNOSTIC_REPORTING=1",
 		}, c.Env...), getEnvVars()...),
-		Binary:    cockroachNodeBinary(c, node),
-		Args:      args,
-		MemoryMax: config.MemoryMax,
-		Local:     c.IsLocal(),
+		Binary:        cockroachNodeBinary(c, node),
+		Args:          args,
+		MemoryMax:     config.MemoryMax,
+		NumFilesLimit: startOpts.NumFilesLimit,
+		Local:         c.IsLocal(),
 	})
 }
 
 type startTemplateData struct {
-	Local     bool
-	LogDir    string
-	Binary    string
-	KeyCmd    string
-	MemoryMax string
-	Args      []string
-	EnvVars   []string
+	Local         bool
+	LogDir        string
+	Binary        string
+	KeyCmd        string
+	MemoryMax     string
+	NumFilesLimit int64
+	Args          []string
+	EnvVars       []string
 }
 
 func execStartTemplate(data startTemplateData) (string, error) {

pkg/roachprod/install/scripts/start.sh

@@ -19,6 +19,7 @@ LOG_DIR=#{shesc .LogDir#}
 BINARY=#{shesc .Binary#}
 KEY_CMD=#{.KeyCmd#}
 MEMORY_MAX=#{.MemoryMax#}
+NUM_FILES_LIMIT=#{.NumFilesLimit#}
 ARGS=(
 #{range .Args -#}
 #{shesc .#}
@@ -93,5 +94,5 @@ sudo systemd-run --unit cockroach \
   --service-type=notify -p NotifyAccess=all \
   -p "MemoryMax=${MEMORY_MAX}" \
   -p LimitCORE=infinity \
-  -p LimitNOFILE=65536 \
+  -p "LimitNOFILE=${NUM_FILES_LIMIT}" \
   bash "${0}" run

pkg/roachprod/install/testdata/start/start.txt

@@ -23,6 +23,7 @@ BINARY=./cockroach
 KEY_CMD=echo foo && \
 echo bar $HOME
 MEMORY_MAX=81%
+NUM_FILES_LIMIT=0
 ARGS=(
 start
 --log
@@ -97,7 +98,7 @@ sudo systemd-run --unit cockroach \
   --service-type=notify -p NotifyAccess=all \
   -p "MemoryMax=${MEMORY_MAX}" \
   -p LimitCORE=infinity \
-  -p LimitNOFILE=65536 \
+  -p "LimitNOFILE=${NUM_FILES_LIMIT}" \
   bash "${0}" run
 ----
 ----

pkg/roachprod/roachprod.go

@@ -635,6 +635,7 @@ func DefaultStartOpts() install.StartOpts {
 	return install.StartOpts{
 		Sequential:      true,
 		EncryptedStores: false,
+		NumFilesLimit:   config.DefaultNumFilesLimit,
 		SkipInit:        false,
 		StoreCount:      1,
 		TenantID:        2,