Merge branch 'master' into fix-nightly-version

components/dm/spec/topology_dm.go

@@ -89,8 +89,9 @@ type (
 type (
 	// DMServerConfigs represents the server runtime configuration
 	DMServerConfigs struct {
-		Master map[string]interface{} `yaml:"master"`
-		Worker map[string]interface{} `yaml:"worker"`
+		Master  map[string]interface{} `yaml:"master"`
+		Worker  map[string]interface{} `yaml:"worker"`
+		Grafana map[string]string      `yaml:"grafana"`
 	}
 
 	// Specification represents the specification of topology.yaml
@@ -838,3 +839,8 @@ func getPort(v reflect.Value) string {
 	}
 	return ""
 }
+
+// GetGrafanaConfig returns global grafana configurations
+func (s *Specification) GetGrafanaConfig() map[string]string {
+	return s.ServerConfigs.Grafana
+}

components/playground/playground.go

@@ -1041,7 +1041,7 @@ func (p *Playground) bootMonitor(ctx context.Context, env *environment.Environme
 	}
 
 	p.instanceWaiter.Go(func() error {
-		err := p.monitor.wait()
+		err := monitor.wait()
 		if err != nil && atomic.LoadInt32(&p.curSig) == 0 {
 			fmt.Printf("Prometheus quit: %v\n", err)
 		} else {
@@ -1077,7 +1077,7 @@ func (p *Playground) bootNGMonitoring(ctx context.Context, env *environment.Envi
 	}
 
 	p.instanceWaiter.Go(func() error {
-		err := p.ngmonitoring.wait()
+		err := ngm.wait()
 		if err != nil && atomic.LoadInt32(&p.curSig) == 0 {
 			fmt.Printf("ng-monitoring quit: %v\n", err)
 		} else {
@@ -1146,7 +1146,7 @@ func (p *Playground) bootGrafana(ctx context.Context, env *environment.Environme
 	}
 
 	p.instanceWaiter.Go(func() error {
-		err := p.grafana.wait()
+		err := grafana.wait()
 		if err != nil && atomic.LoadInt32(&p.curSig) == 0 {
 			fmt.Printf("Grafana quit: %v\n", err)
 		} else {

doc/user/mirrors.md

@@ -1,55 +1,44 @@
 # Build a private mirror
 
-When building a private cloud, it is common to use an isolated network environment where the official mirror of TiUP is not accessible, so we provide a solution for building a private mirror, which is mainly implemented by the mirros component, which can also be used for offline deployment.
+When building a private cloud, it is common to use an isolated network environment where the official mirror of TiUP is not accessible, so we provide a solution for building a private mirror, which is mainly implemented by the mirror component, which can also be used for offline deployment.
 
 ## Mirrors component introduction
 
-First, let's look at the `mirrors' help file.
+First, let's look at the `mirror' help file.
 
 ```bash
-$ tiup mirrors --help
-Starting component `mirrors`: /Users/joshua/.tiup/components/mirrors/v0.0.1/mirrors
-Build a local mirrors and download all selected components
+$ tiup mirror --help
+The 'mirror' command is used to manage a component repository for TiUP, you can use
+it to create a private repository, or to add new component to an existing repository.
+The repository can be used either online or offline.
+It also provides some useful utilities to help managing keys, users and versions
+of components or the repository itself.
 
 Usage:
-  tiup mirrors <target-dir> [global-version] [flags]
-
-Examples:
-  tiup mirrors local-path --arch amd64,arm --os linux,darwin    # Specify the architectures and OSs
-  tiup mirrors local-path --full                                # Build a full local mirrors
-  tiup mirrors local-path --tikv v4                             # Specify the version via prefix
-  tiup mirrors local-path --tidb all --pd all                   # Download all version for specific component
-
-Flags:
-      --overwrite                   Overwrite the exists tarball
-  -f, --full                        Build a full mirrors repository
-  -a, --arch strings                Specify the downloading architecture (default [amd64])
-  -o, --os strings                  Specify the downloading os (default [linux,darwin])
-      --tidb strings                Specify the versions for component tidb
-      --tikv strings                Specify the versions for component tikv
-      --pd strings                  Specify the versions for component pd
-      --playground strings          Specify the versions for component playground
-      --client strings              Specify the versions for component client
-      --prometheus strings          Specify the versions for component prometheus
-      --package strings             Specify the versions for component package
-      --grafana strings             Specify the versions for component grafana
-      --alertmanager strings        Specify the versions for component alertmanager
-      --blackbox_exporter strings   Specify the versions for component blackbox_exporter
-      --node_exporter strings       Specify the versions for component node_exporter
-      --pushgateway strings         Specify the versions for component pushgateway
-      --tiflash strings             Specify the versions for component tiflash
-      --drainer strings             Specify the versions for component drainer
-      --pump strings                Specify the versions for component pump
-      --cluster strings             Specify the versions for component cluster
-      --mirrors strings             Specify the versions for component mirrors
-      --bench strings               Specify the versions for component bench
-      --insight strings             Specify the versions for component insight
-      --doc strings                 Specify the versions for component doc
-      --ctl strings                 Specify the versions for component ctl
-  -h, --help                        help for tiup
+  tiup mirror <command> [flags]
+
+Available Commands:
+  init        Initialize an empty repository
+  sign        Add signatures to a manifest file
+  genkey      Generate a new key pair
+  clone       Clone a local mirror from remote mirror and download all selected components
+  merge       Merge two or more offline mirror
+  publish     Publish a component
+  show        Show mirror address
+  set         Set mirror address
+  modify      Modify published component
+  renew       Renew the manifest of a published component.
+  grant       grant a new owner
+  rotate      Rotate root.json
+
+Global Flags:
+      --help                 Help for this command
+      --skip-version-check   Skip the strict version check, by default a version must be a valid SemVer string
+
+Use "tiup mirror [command] --help" for more information about a command.
 ```
 
-Its basic use is `tiup mirrors <target-dir> [global-version] [flags]`, the target-dir is the directory in which the cloned data needs to be placed. global-version is used to quickly set a common version for all components.
+Its basic use is `tiup mirror clone <target-dir> [global-version] [flags]`, the target-dir is the directory in which the cloned data needs to be placed. global-version is used to quickly set a common version for all components.
 
 Then this order has very scary dozens of flags and even more later. But there is no need to be intimidated by the number of these flags, which are in fact of four types.
 
@@ -68,16 +57,16 @@ If `--full` is specified, the official image will be cloned intact.
 ### 3. Platform limitation
 
 If you only want to clone packages for a particular platform, you can use `-os` and `-arch` to qualify:
-- `tiup mirros <target-dir> ---os=linux`
-- Just want to clone amd64 architecture: `tiup mirros <target-dir> --arch=amd64`
-- Just want to clone linux/amd64: `tiup mirros <target-dir> --os=linux --arch=amd64`
+- `tiup mirror clone <target-dir> ---os=linux`
+- Just want to clone amd64 architecture: `tiup mirror clone <target-dir> --arch=amd64`
+- Just want to clone linux/amd64: `tiup mirror clone <target-dir> --os=linux --arch=amd64`
 
 ### 4. Component version limited
 
 If you want to clone only one version of a component and not all versions, use `--<component>=<version>` to qualify, for example "
-- Just want to clone the v4 version of tidb: `tiup mirrors <target-dir> --tidb v4`
-- Just want to clone the v4 version of tidb, and all versions of tikv: `tiup mirros <target-dir> --tidb v4 --tikv all` 
-- Clone specific versions of all components that start a cluster: `tiup mirrors <target-dir> v4.0.0-rc`
+- Just want to clone the v4 version of tidb: `tiup mirror clone <target-dir> --tidb v4`
+- Just want to clone the v4 version of tidb, and all versions of tikv: `tiup mirror clone <target-dir> --tidb v4 --tikv all` 
+- Clone specific versions of all components that start a cluster: `tiup mirror clone <target-dir> v4.0.0-rc`
 
 ## The real thing
 
@@ -86,7 +75,7 @@ If you want to clone only one version of a component and not all versions, use `
 For example, if we want to install a v4.0.0-rc TiDB cluster in an isolated environment, we can execute the following command on a machine connected to the extranet to pull the required components:
 
 ```bash
-tiup mirrors package --os=linux v4.0.0-rc
+tiup mirror package --os=linux v4.0.0-rc
 ```
 
 This command creates a directory called `package` in the current directory that contains the package of components necessary to start a cluster, which is then packaged by the tar command and sent to a central control unit in an isolated environment:
@@ -110,7 +99,7 @@ export TIUP_MIRRORS=/path/to/mirror
 tiup cluster xxx
 ```
 
-`/path/to/mirror` is the location of <target-dir> in `tiup mirrors <target-dir>`, or if in /tmp/package:
+`/path/to/mirror` is the location of <target-dir> in `tiup mirror clone <target-dir>`, or if in /tmp/package:
 ```bash
 export TIUP_MIRRORS=/tmp/package
 ```

embed/examples/cluster/minimal.yaml

@@ -224,6 +224,8 @@ monitoring_servers:
     # ssh_port: 22
     # # Prometheus Service communication port.
     # port: 9090
+    # # ng-monitoring servive communication port
+    # ng_port: 12020
     # # Prometheus deployment file, startup script, configuration file storage directory.
     # deploy_dir: "/tidb-deploy/prometheus-8249"
     # # Prometheus data storage directory.

embed/examples/cluster/multi-dc.yaml

@@ -276,6 +276,8 @@ monitoring_servers:
     # ssh_port: 22
     # # Prometheus Service communication port.
     # port: 9090
+    # # ng-monitoring servive communication port
+    # ng_port: 12020
     # # Prometheus deployment file, startup script, configuration file storage directory.
     # deploy_dir: "/tidb-deploy/prometheus-8249"
     # # Prometheus data storage directory.

embed/examples/cluster/topology.example.yaml

@@ -286,6 +286,8 @@ monitoring_servers:
     # ssh_port: 22
     # # Prometheus Service communication port.
     # port: 9090
+    # # ng-monitoring servive communication port
+    # ng_port: 12020
     # # Prometheus deployment file, startup script, configuration file storage directory.
     # deploy_dir: "/tidb-deploy/prometheus-8249"
     # # Prometheus data storage directory.

embed/templates/scripts/run_drainer.sh.tpl

@@ -29,5 +29,4 @@ exec bin/drainer \
     --pd-urls="{{template "PDList" .Endpoints}}" \
     --data-dir="{{.DataDir}}" \
     --log-file="{{.LogDir}}/drainer.log" \
-    --config=conf/drainer.toml \
-    --initial-commit-ts={{.CommitTs}} 2>> "{{.LogDir}}/drainer_stderr.log"
+    --config=conf/drainer.toml 2>> "{{.LogDir}}/drainer_stderr.log"

pkg/cluster/ansible/import_test.go

@@ -129,6 +129,7 @@ server_configs:
     pump: {}
     drainer: {}
     cdc: {}
+    grafana: {}
 tidb_servers: []
 tikv_servers: []
 tiflash_servers: []

pkg/cluster/ansible/service.go

@@ -196,7 +196,8 @@ func parseDirs(ctx context.Context, user string, ins spec.InstanceSpec, sshTimeo
 			if strings.Contains(line, "--initial-commit-ts=") {
 				tsArg := strings.Split(line, " ")[4] // 4 whitespaces ahead
 				tmpTs, _ := strconv.Atoi(strings.TrimPrefix(tsArg, "--initial-commit-ts="))
-				newIns.CommitTS = int64(tmpTs)
+				newIns.Config = make(map[string]interface{})
+				newIns.Config["initial_commit_ts"] = int64(tmpTs)
 			}
 		}
 		return newIns, nil

pkg/cluster/ansible/test-data/meta.yaml

@@ -24,6 +24,7 @@ topology:
     pump: {}
     drainer: {}
     cdc: {}
+    grafana: {}
   tidb_servers:
     - host: 172.16.1.218
       ssh_port: 9999

pkg/cluster/manager/deploy.go

@@ -411,7 +411,7 @@ func (m *Manager) Deploy(
 		return err
 	}
 
-	hint := color.New(color.Bold).Sprintf("%s start %s", tui.OsArgs0(), name)
+	hint := color.New(color.Bold).Sprintf("%s start %s --init", tui.OsArgs0(), name)
 	m.logger.Infof("Cluster `%s` deployed successfully, you can start it with command: `%s`", name, hint)
 	return nil
 }

pkg/cluster/manager/upgrade.go

@@ -53,6 +53,11 @@ func (m *Manager) Upgrade(name string, clusterVersion string, opt operator.Optio
 	topo := metadata.GetTopology()
 	base := metadata.GetBaseMeta()
 
+	// Adjust topo by new version
+	if clusterTopo, ok := topo.(*spec.Specification); ok {
+		clusterTopo.AdjustByVersion(clusterVersion)
+	}
+
 	var (
 		downloadCompTasks []task.Task // tasks which are used to download components
 		copyCompTasks     []task.Task // tasks which are used to copy components to remote host

pkg/cluster/spec/drainer.go

@@ -38,7 +38,7 @@ type DrainerSpec struct {
 	DeployDir       string                 `yaml:"deploy_dir,omitempty"`
 	DataDir         string                 `yaml:"data_dir,omitempty"`
 	LogDir          string                 `yaml:"log_dir,omitempty"`
-	CommitTS        int64                  `yaml:"commit_ts" default:"-1" validate:"commit_ts:editable"`
+	CommitTS        *int64                 `yaml:"commit_ts,omitempty" validate:"commit_ts:editable"` // do not use it anymore, exist for compatibility
 	Offline         bool                   `yaml:"offline,omitempty"`
 	NumaNode        string                 `yaml:"numa_node,omitempty" validate:"numa_node:editable"`
 	Config          map[string]interface{} `yaml:"config,omitempty" validate:"config:ignore"`
@@ -168,8 +168,6 @@ func (i *DrainerInstance) InitConfig(
 		paths.Log,
 	).WithPort(spec.Port).WithNumaNode(spec.NumaNode).AppendEndpoints(topo.Endpoints(deployUser)...)
 
-	cfg.WithCommitTs(spec.CommitTS)
-
 	fp := filepath.Join(paths.Cache, fmt.Sprintf("run_drainer_%s_%d.sh", i.GetHost(), i.GetPort()))
 
 	if err := cfg.ConfigToFile(fp); err != nil {

pkg/cluster/spec/grafana.go

@@ -17,6 +17,7 @@ import (
 	"context"
 	"crypto/tls"
 	"fmt"
+	"os"
 	"path/filepath"
 	"reflect"
 	"strings"
@@ -27,6 +28,7 @@ import (
 	"github.com/pingcap/tiup/pkg/cluster/template/config"
 	"github.com/pingcap/tiup/pkg/cluster/template/scripts"
 	"github.com/pingcap/tiup/pkg/meta"
+	"gopkg.in/ini.v1"
 )
 
 // GrafanaSpec represents the Grafana topology specification in topology.yaml
@@ -38,6 +40,7 @@ type GrafanaSpec struct {
 	IgnoreExporter  bool                 `yaml:"ignore_exporter,omitempty"`
 	Port            int                  `yaml:"port" default:"3000"`
 	DeployDir       string               `yaml:"deploy_dir,omitempty"`
+	Config          map[string]string    `yaml:"config,omitempty" validate:"config:ignore"`
 	ResourceControl meta.ResourceControl `yaml:"resource_control,omitempty" validate:"resource_control:editable"`
 	Arch            string               `yaml:"arch,omitempty"`
 	OS              string               `yaml:"os,omitempty"`
@@ -181,11 +184,22 @@ func (i *GrafanaInstance) InitConfig(
 		return err
 	}
 
+	userConfig := i.topo.GetGrafanaConfig()
+	if userConfig == nil {
+		userConfig = make(map[string]string)
+	}
+	for k, v := range spec.Config {
+		userConfig[k] = v
+	}
+	err := mergeAdditionalGrafanaConf(fp, userConfig)
+	if err != nil {
+		return err
+	}
+
 	dst = filepath.Join(paths.Deploy, "conf", "grafana.ini")
 	if err := e.Transfer(ctx, fp, dst, false, 0, false); err != nil {
 		return err
 	}
-
 	if err := i.installDashboards(ctx, e, paths.Deploy, clusterName, clusterVersion); err != nil {
 		return errors.Annotate(err, "install dashboards")
 	}
@@ -330,3 +344,24 @@ func (i *GrafanaInstance) ScaleConfig(
 	i.topo = topo.Merge(i.topo)
 	return i.InitConfig(ctx, e, clusterName, clusterVersion, deployUser, paths)
 }
+
+func mergeAdditionalGrafanaConf(source string, addition map[string]string) error {
+	bytes, err := os.ReadFile(source)
+	if err != nil {
+		return err
+	}
+	result, err := ini.Load(bytes)
+	if err != nil {
+		return err
+	}
+	for k, v := range addition {
+		// convert "log.file.level to [log.file] level"
+		for i := len(k) - 1; i >= 0; i-- {
+			if k[i] == '.' {
+				result.Section(k[:i]).Key(k[i+1:]).SetValue(v)
+				break
+			}
+		}
+	}
+	return result.SaveTo(source)
+}