Fix dependences version binding when cloning mirror

cmd/mirror.go

@@ -23,6 +23,7 @@ import (
 	"time"
 
 	"github.com/pingcap/errors"
+	"github.com/pingcap/tiup/pkg/cluster/spec"
 	"github.com/pingcap/tiup/pkg/environment"
 	"github.com/pingcap/tiup/pkg/localdata"
 	"github.com/pingcap/tiup/pkg/repository"
@@ -669,7 +670,11 @@ func newMirrorCloneCmd() *cobra.Command {
 			}
 			defer repo.Mirror().Close()
 
-			return repository.CloneMirror(repo, components, args[0], args[1:], options)
+			var versionMapper = func(ver string) string {
+				return spec.TiDBComponentVersion(ver, "")
+			}
+
+			return repository.CloneMirror(repo, components, versionMapper, args[0], args[1:], options)
 		},
 	}
 

go.mod

@@ -40,6 +40,7 @@ require (
 	github.com/mattn/go-colorable v0.1.6 // indirect
 	github.com/mattn/go-runewidth v0.0.7
 	github.com/otiai10/copy v1.2.0
+	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/pingcap/check v0.0.0-20200212061837-5e12011dc712
 	github.com/pingcap/dm v1.1.0-alpha.0.20200521025928-83063141c5fd
 	github.com/pingcap/errors v0.11.5-0.20200820035142-66eb5bf1d1cd

go.sum

@@ -576,6 +576,8 @@ github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6
 github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo=
 github.com/otiai10/mint v1.3.1 h1:BCmzIS3n71sGfHB5NMNDB3lHYPz8fWSkCAErHed//qc=
 github.com/otiai10/mint v1.3.1/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc=
+github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
+github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
 github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pelletier/go-toml v1.3.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo=

pkg/cluster/prepare.go

@@ -26,20 +26,23 @@ type InstanceIter interface {
 }
 
 // BuildDownloadCompTasks build download component tasks
-func BuildDownloadCompTasks(version string, instanceIter InstanceIter, bindVersion spec.BindVersion) []*task.StepDisplay {
+func BuildDownloadCompTasks(clusterVersion string, instanceIter InstanceIter, bindVersion spec.BindVersion) []*task.StepDisplay {
 	var tasks []*task.StepDisplay
 	uniqueTaskList := make(map[string]struct{}) // map["comp-os-arch"]{}
 	instanceIter.IterInstance(func(inst spec.Instance) {
 		key := fmt.Sprintf("%s-%s-%s", inst.ComponentName(), inst.OS(), inst.Arch())
 		if _, found := uniqueTaskList[key]; !found {
 			uniqueTaskList[key] = struct{}{}
 
-			// download spark as dependency of tispark
-			if inst.ComponentName() == spec.ComponentTiSpark {
+			// we don't set version for tispark, so the lastest tispark will be used
+			var version string
+			if inst.ComponentName() != spec.ComponentTiSpark {
+				version = bindVersion(inst.ComponentName(), clusterVersion)
+			} else {
+				// download spark as dependency of tispark
 				tasks = append(tasks, buildDownloadSparkTask(version, inst, bindVersion))
 			}
 
-			version := bindVersion(inst.ComponentName(), version)
 			t := task.NewBuilder().
 				Download(inst.ComponentName(), inst.OS(), inst.Arch(), version).
 				BuildAsStep(fmt.Sprintf("  - Download %s:%s (%s/%s)",

pkg/cluster/spec/bindversion.go

@@ -31,8 +31,6 @@ func TiDBComponentVersion(comp, version string) string {
 		return "v0.7.0"
 	case ComponentCheckCollector:
 		return "v0.3.1"
-	case ComponentTiSpark:
-		return "v2.3.1"
 	case ComponentSpark:
 		return "v2.4.3"
 	default:

pkg/cluster/task/builder.go

@@ -370,7 +370,7 @@ func (b *Builder) DeploySpark(inst spec.Instance, version, srcPath, deployDir st
 		inst.ComponentName(),
 		inst.OS(),
 		inst.Arch(),
-		version,
+		"", // use the latest stable version
 		srcPath,
 		inst.GetHost(),
 		deployDir,

pkg/cluster/task/copy_component.go

@@ -17,6 +17,7 @@ import (
 	"fmt"
 
 	"github.com/pingcap/tiup/pkg/cluster/spec"
+	"github.com/pingcap/tiup/pkg/environment"
 )
 
 // CopyComponent is used to copy all files related the specific version a component
@@ -39,6 +40,18 @@ func PackagePath(comp string, version string, os string, arch string) string {
 
 // Execute implements the Task interface
 func (c *CopyComponent) Execute(ctx *Context) error {
+	// If the version is not specified, the last stable one will be used
+	if c.version == "" {
+		env := environment.GlobalEnv()
+		env.V1Repository().Lock()
+		defer env.V1Repository().Unlock()
+		ver, _, err := env.V1Repository().LatestStableVersion(c.component, false)
+		if err != nil {
+			return err
+		}
+		c.version = string(ver)
+	}
+
 	// Copy to remote server
 	srcPath := c.srcPath
 	if srcPath == "" {

pkg/cluster/task/download.go

@@ -17,6 +17,7 @@ import (
 	"fmt"
 
 	operator "github.com/pingcap/tiup/pkg/cluster/operation"
+	"github.com/pingcap/tiup/pkg/environment"
 )
 
 // Downloader is used to download the specific version of a component from
@@ -40,6 +41,17 @@ func NewDownloader(component string, os string, arch string, version string) *Do
 
 // Execute implements the Task interface
 func (d *Downloader) Execute(_ *Context) error {
+	// If the version is not specified, the last stable one will be used
+	if d.version == "" {
+		env := environment.GlobalEnv()
+		env.V1Repository().Lock()
+		defer env.V1Repository().Unlock()
+		ver, _, err := env.V1Repository().LatestStableVersion(d.component, false)
+		if err != nil {
+			return err
+		}
+		d.version = string(ver)
+	}
 	return operator.Download(d.component, d.os, d.arch, d.version)
 }
 

pkg/repository/clone_mirror.go

@@ -44,7 +44,13 @@ type CloneOptions struct {
 }
 
 // CloneMirror clones a local mirror from the remote repository
-func CloneMirror(repo *V1Repository, components []string, targetDir string, selectedVersions []string, options CloneOptions) error {
+func CloneMirror(repo *V1Repository,
+	components []string,
+	tidbClusterVersionMapper func(string) string,
+	targetDir string,
+	selectedVersions []string,
+	options CloneOptions) error {
+
 	fmt.Printf("Start to clone mirror, targetDir is %s, selectedVersions are [%s]\n", targetDir, strings.Join(selectedVersions, ","))
 	fmt.Println("If this does not meet expectations, please abort this process, read `tiup mirror clone --help` and run again")
 
@@ -146,7 +152,7 @@ func CloneMirror(repo *V1Repository, components []string, targetDir string, sele
 	snapshot := v1manifest.NewSnapshot(initTime)
 	snapshot.SetExpiresAt(expirsAt)
 
-	componentManifests, err := cloneComponents(repo, components, selectedVersions, targetDir, tmpDir, options)
+	componentManifests, err := cloneComponents(repo, components, selectedVersions, tidbClusterVersionMapper, targetDir, tmpDir, options)
 	if err != nil {
 		return err
 	}
@@ -243,16 +249,18 @@ func CloneMirror(repo *V1Repository, components []string, targetDir string, sele
 
 func cloneComponents(repo *V1Repository,
 	components, selectedVersions []string,
+	tidbClusterVersionMapper func(string) string,
 	targetDir, tmpDir string,
 	options CloneOptions) (map[string]*v1manifest.Component, error) {
+
 	compManifests := map[string]*v1manifest.Component{}
 	for _, name := range components {
 		manifest, err := repo.FetchComponentManifest(name, true)
 		if err != nil {
 			return nil, errors.Annotatef(err, "fetch component '%s' manifest failed", name)
 		}
 
-		vs := combineVersions(options.Components[name], manifest, options.OSs, options.Archs, selectedVersions)
+		vs := combineVersions(options.Components[name], tidbClusterVersionMapper, manifest, options.OSs, options.Archs, selectedVersions)
 		var newManifest *v1manifest.Component
 		if options.Full {
 			newManifest = manifest
@@ -391,20 +399,17 @@ func checkVersion(options CloneOptions, versions set.StringSet, version string)
 	return false
 }
 
-func combineVersions(versions *[]string, manifest *v1manifest.Component, oss, archs, selectedVersions []string) set.StringSet {
+func combineVersions(versions *[]string,
+	tidbClusterVersionMapper func(string) string,
+	manifest *v1manifest.Component, oss, archs,
+	selectedVersions []string) set.StringSet {
+
 	if (versions == nil || len(*versions) < 1) && len(selectedVersions) < 1 {
 		return nil
 	}
 
-	switch manifest.ID {
-	case "alertmanager":
-		return set.NewStringSet("v0.17.0")
-	case "blackbox_exporter":
-		return set.NewStringSet("v0.12.0")
-	case "node_exporter":
-		return set.NewStringSet("v0.17.0")
-	case "pushgateway":
-		return set.NewStringSet("v0.7.0")
+	if bindver := tidbClusterVersionMapper(manifest.ID); bindver != "" {
+		return set.NewStringSet(bindver)
 	}
 
 	result := set.NewStringSet()

pkg/repository/v1_repository.go

@@ -24,10 +24,12 @@ import (
 	"runtime"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/fatih/color"
 	cjson "github.com/gibson042/canonicaljson-go"
+	"github.com/patrickmn/go-cache"
 	"github.com/pingcap/errors"
 	"github.com/pingcap/tiup/pkg/repository/v0manifest"
 	"github.com/pingcap/tiup/pkg/repository/v1manifest"
@@ -40,11 +42,17 @@ import (
 // errUnknownComponent represents the specific component cannot be found in index.json
 var errUnknownComponent = stderrors.New("unknown component")
 
+const (
+	componentManifestCacheKey = "component-manifest"
+)
+
 // V1Repository represents a remote repository viewed with the v1 manifest design.
 type V1Repository struct {
 	Options
+	sync.Mutex
 	mirror Mirror
 	local  v1manifest.LocalManifests
+	cache  *cache.Cache
 }
 
 // ComponentSpec describes a component a user would like to have or use.
@@ -76,6 +84,7 @@ func NewV1Repo(mirror Mirror, opts Options, local v1manifest.LocalManifests) *V1
 		Options: opts,
 		mirror:  mirror,
 		local:   local,
+		cache:   cache.New(cache.NoExpiration, 0),
 	}
 
 	return repo
@@ -657,12 +666,23 @@ func (r *V1Repository) UpdateComponentManifests() error {
 
 // FetchComponentManifest fetch the component manifest.
 func (r *V1Repository) FetchComponentManifest(id string, withYanked bool) (com *v1manifest.Component, err error) {
+	// cache is used to avoid fetching the same component manifests multiple time
+	cacheKey := fmt.Sprintf("%s-%s-%t", componentManifestCacheKey, id, withYanked)
+	if com, ok := r.cache.Get(cacheKey); ok {
+		return com.(*v1manifest.Component), nil
+	}
+
 	err = r.ensureManifests()
 	if err != nil {
 		return nil, errors.AddStack(err)
 	}
 
-	return r.updateComponentManifest(id, withYanked)
+	com, err = r.updateComponentManifest(id, withYanked)
+	if err != nil {
+		return nil, err
+	}
+	r.cache.Set(cacheKey, com, cache.DefaultExpiration)
+	return
 }
 
 // ComponentVersion returns version item of a component