cp 12973 14007 14113 (vitessio#3255)

* metrics: change vtbackup_duration_by_phase to binary-valued vtbackup_phase (vitessio#12973)

Signed-off-by: Max Englander <max@planetscale.com>
Signed-off-by: Andrew Mason <andrew@planetscale.com>

* [CLI] cobra lots of things (vitessio#14007)

Signed-off-by: Andrew Mason <andrew@planetscale.com>

* go/cmd/vtbackup: wait for plugins to finish initializing (vitessio#14113)

Signed-off-by: Max Englander <max@planetscale.com>

---------

Signed-off-by: Max Englander <max@planetscale.com>
Signed-off-by: Andrew Mason <andrew@planetscale.com>
Co-authored-by: Max Englander <max@planetscale.com>

changelog/18.0/18.0.0/summary.md

@@ -23,6 +23,7 @@
     - [Deprecated VDiff v1](#deprecated-vdiff-v1)
   - **[New Stats](#new-stats)**
     - [VTGate Vindex unknown parameters](#vtgate-vindex-unknown-parameters)
+    - [VTBackup stat `Phase`](#vtbackup-stat-phase)
     - [VTBackup stat `PhaseStatus`](#vtbackup-stat-phase-status)
     - [Backup and restore metrics for AWS S3](#backup-restore-metrics-aws-s3)
     - [VTCtld and VTOrc reparenting stats](#vtctld-and-vtorc-reparenting-stats)
@@ -157,11 +158,27 @@ VTBackup stat `DurationByPhase` is deprecated. Use the binary-valued `Phase` sta
 
 The VTGate stat `VindexUnknownParameters` gauges unknown Vindex parameters found in the latest VSchema pulled from the topology.
 
+#### <a id="vtbackup-stat-phase"/>VTBackup `Phase` stat
+
+In v17, the `vtbackup` stat `DurationByPhase` stat was added measuring the time spent by `vtbackup` in each phase. This stat turned out to be awkward to use in production, and has been replaced in v18 by a binary-valued `Phase` stat.
+
+`Phase` reports a 1 (active) or a 0 (inactive) for each of the following phases:
+
+ * `CatchupReplication`
+ * `InitialBackup`
+ * `RestoreLastBackup`
+ * `TakeNewBackup`
+
+To calculate how long `vtbackup` has spent in a given phase, sum the 1-valued data points over time and multiply by the data collection or reporting interval. For example, in Prometheus:
+
+```
+sum_over_time(vtbackup_phase{phase="TakeNewBackup"}) * <interval>
+```
 #### <a id="vtbackup-stat-phase-status"/>VTBackup `PhaseStatus` stat
 
 `PhaseStatus` reports a 1 (active) or a 0 (inactive) for each of the following phases and statuses:
 
- * `CatchUpReplication` phase has statuses `Stalled` and `Stopped`.
+ * `CatchupReplication` phase has statuses `Stalled` and `Stopped`.
     * `Stalled` is set to `1` when replication stops advancing.
     * `Stopped` is set to `1` when replication stops before `vtbackup` catches up with the primary.
 
@@ -220,4 +237,4 @@ removing Vitess support.
 
 #### <a id="new-durability-policies"/>New Durability Policies
 
-2 new inbuilt durability policies have been added to Vitess in this release namely `semi_sync_with_rdonly_ack` and `cross_cell_with_rdonly_ack`. These policies are exactly like `semi_sync` and `cross_cell` respectively, and differ just in the part where the rdonly tablets can also send semi-sync ACKs. 
+2 new inbuilt durability policies have been added to Vitess in this release namely `semi_sync_with_rdonly_ack` and `cross_cell_with_rdonly_ack`. These policies are exactly like `semi_sync` and `cross_cell` respectively, and differ just in the part where the rdonly tablets can also send semi-sync ACKs. 

go/cmd/vtaclcheck/cli/vtactlcheck.go

@@ -0,0 +1,68 @@
+/*
+Copyright 2023 The Vitess Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cli
+
+import (
+	"github.com/spf13/cobra"
+
+	"vitess.io/vitess/go/acl"
+	"vitess.io/vitess/go/vt/logutil"
+	"vitess.io/vitess/go/vt/servenv"
+	"vitess.io/vitess/go/vt/vtaclcheck"
+)
+
+var (
+	aclFile        string
+	staticAuthFile string
+
+	Main = &cobra.Command{
+		Use:     "vtaclcheck",
+		Short:   "vtaclcheck checks that the access-control list (ACL) rules in a given file are valid.",
+		Args:    cobra.NoArgs,
+		Version: servenv.AppVersion.String(),
+		PreRunE: servenv.CobraPreRunE,
+		PostRun: func(cmd *cobra.Command, args []string) {
+			logutil.Flush()
+		},
+		RunE: run,
+	}
+)
+
+func run(cmd *cobra.Command, args []string) error {
+	servenv.Init()
+	defer servenv.Close()
+
+	opts := &vtaclcheck.Options{
+		ACLFile:        aclFile,
+		StaticAuthFile: staticAuthFile,
+	}
+
+	if err := vtaclcheck.Init(opts); err != nil {
+		return err
+	}
+
+	return vtaclcheck.Run()
+}
+
+func init() {
+	servenv.MoveFlagsToCobraCommand(Main)
+
+	Main.Flags().StringVar(&aclFile, "acl-file", aclFile, "The path of the JSON ACL file to check")
+	Main.Flags().StringVar(&staticAuthFile, "static-auth-file", staticAuthFile, "The path of the auth_server_static JSON file to check")
+
+	acl.RegisterFlags(Main.Flags())
+}

go/cmd/vtaclcheck/docgen/main.go

@@ -0,0 +1,37 @@
+/*
+Copyright 2023 The Vitess Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"github.com/spf13/cobra"
+
+	"vitess.io/vitess/go/cmd/internal/docgen"
+	"vitess.io/vitess/go/cmd/vtaclcheck/cli"
+)
+
+func main() {
+	var dir string
+	cmd := cobra.Command{
+		Use: "docgen [-d <dir>]",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return docgen.GenerateMarkdownTree(cli.Main, dir)
+		},
+	}
+
+	cmd.Flags().StringVarP(&dir, "dir", "d", "doc", "output directory to write documentation")
+	_ = cmd.Execute()
+}

go/cmd/vtaclcheck/vtaclcheck.go

@@ -19,52 +19,21 @@ package main
 import (
 	"fmt"
 
-	"github.com/spf13/pflag"
-
-	"vitess.io/vitess/go/acl"
+	"vitess.io/vitess/go/cmd/vtaclcheck/cli"
 	"vitess.io/vitess/go/exit"
 	"vitess.io/vitess/go/vt/logutil"
-	"vitess.io/vitess/go/vt/servenv"
-	"vitess.io/vitess/go/vt/vtaclcheck"
 )
 
-var aclFile, staticAuthFile string
-
 func init() {
 	logger := logutil.NewConsoleLogger()
-	servenv.OnParse(func(fs *pflag.FlagSet) {
-		fs.StringVar(&aclFile, "acl-file", aclFile, "The path of the JSON ACL file to check")
-		fs.StringVar(&staticAuthFile, "static-auth-file", staticAuthFile, "The path of the auth_server_static JSON file to check")
-
-		acl.RegisterFlags(fs)
-
-		fs.SetOutput(logutil.NewLoggerWriter(logger))
-	})
+	cli.Main.SetOutput(logutil.NewLoggerWriter(logger))
 }
 
 func main() {
 	defer exit.RecoverAll()
-	defer logutil.Flush()
-
-	servenv.ParseFlags("vtaclcheck")
-	servenv.Init()
 
-	err := run()
-	if err != nil {
+	if err := cli.Main.Execute(); err != nil {
 		fmt.Printf("ERROR: %s\n", err)
 		exit.Return(1)
 	}
 }
-
-func run() error {
-	opts := &vtaclcheck.Options{
-		ACLFile:        aclFile,
-		StaticAuthFile: staticAuthFile,
-	}
-
-	if err := vtaclcheck.Init(opts); err != nil {
-		return err
-	}
-
-	return vtaclcheck.Run()
-}

go/cmd/vtbackup/plugin_azblobbackupstorage.go → go/cmd/vtbackup/cli/plugin_azblobbackupstorage.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package main
+package cli
 
 import (
 	_ "vitess.io/vitess/go/vt/mysqlctl/azblobbackupstorage"

go/cmd/vtbackup/plugin_cephbackupstorage.go → go/cmd/vtbackup/cli/plugin_cephbackupstorage.go

@@ -13,7 +13,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */
-package main
+package cli
 
 import (
 	_ "vitess.io/vitess/go/vt/mysqlctl/cephbackupstorage"

go/cmd/vtbackup/plugin_consultopo.go → go/cmd/vtbackup/cli/plugin_consultopo.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package main
+package cli
 
 import (
 	_ "vitess.io/vitess/go/vt/topo/consultopo"

go/cmd/vtbackup/plugin_etcd2topo.go → go/cmd/vtbackup/cli/plugin_etcd2topo.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package main
+package cli
 
 import (
 	_ "vitess.io/vitess/go/vt/topo/etcd2topo"

go/cmd/vtbackup/plugin_filebackupstorage.go → go/cmd/vtbackup/cli/plugin_filebackupstorage.go

@@ -13,7 +13,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */
-package main
+package cli
 
 import (
 	_ "vitess.io/vitess/go/vt/mysqlctl/filebackupstorage"

go/cmd/vtbackup/plugin_gcsbackupstorage.go → go/cmd/vtbackup/cli/plugin_gcsbackupstorage.go

@@ -13,7 +13,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */
-package main
+package cli
 
 import (
 	_ "vitess.io/vitess/go/vt/mysqlctl/gcsbackupstorage"

go/cmd/vtbackup/cli/plugin_opentsdb.go

@@ -0,0 +1,25 @@
+/*
+Copyright 2023 The Vitess Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cli
+
+import "vitess.io/vitess/go/stats/opentsdb"
+
+// This plugin imports opentsdb to register the opentsdb stats backend.
+
+func init() {
+	opentsdb.Init("vtbackup")
+}

go/cmd/vtbackup/plugin_prometheusbackend.go → go/cmd/vtbackup/cli/plugin_prometheusbackend.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package main
+package cli
 
 // This plugin imports Prometheus to allow for instrumentation
 // with the Prometheus client library

go/cmd/vtbackup/plugin_s3backupstorage.go → go/cmd/vtbackup/cli/plugin_s3backupstorage.go

@@ -13,7 +13,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */
-package main
+package cli
 
 import (
 	_ "vitess.io/vitess/go/vt/mysqlctl/s3backupstorage"

go/cmd/vtbackup/plugin_zk2topo.go → go/cmd/vtbackup/cli/plugin_zk2topo.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package main
+package cli
 
 import (
 	_ "vitess.io/vitess/go/vt/topo/zk2topo"