From bdbc15f86cf70a9397a2d0858d999ebeb4309fbd Mon Sep 17 00:00:00 2001 From: lucklove Date: Sun, 13 Sep 2020 21:42:26 +0800 Subject: [PATCH 1/9] Fix dependences version binding when cloning mirror Signed-off-by: lucklove --- cmd/mirror.go | 7 ++++++- pkg/repository/clone_mirror.go | 31 ++++++++++++++++++------------- 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/cmd/mirror.go b/cmd/mirror.go index abfedaa024..eb024441fe 100644 --- a/cmd/mirror.go +++ b/cmd/mirror.go @@ -23,6 +23,7 @@ import ( "time" "github.com/pingcap/errors" + "github.com/pingcap/tiup/pkg/cluster/spec" "github.com/pingcap/tiup/pkg/environment" "github.com/pingcap/tiup/pkg/localdata" "github.com/pingcap/tiup/pkg/repository" @@ -669,7 +670,11 @@ func newMirrorCloneCmd() *cobra.Command { } defer repo.Mirror().Close() - return repository.CloneMirror(repo, components, args[0], args[1:], options) + var versionMapper = func(ver string) string { + return spec.TiDBComponentVersion(ver, "") + } + + return repository.CloneMirror(repo, components, versionMapper, args[0], args[1:], options) }, } diff --git a/pkg/repository/clone_mirror.go b/pkg/repository/clone_mirror.go index bf9eb0bb08..a6a5d6b02e 100644 --- a/pkg/repository/clone_mirror.go +++ b/pkg/repository/clone_mirror.go @@ -44,7 +44,13 @@ type CloneOptions struct { } // CloneMirror clones a local mirror from the remote repository -func CloneMirror(repo *V1Repository, components []string, targetDir string, selectedVersions []string, options CloneOptions) error { +func CloneMirror(repo *V1Repository, + components []string, + tidbClusterVersionMapper func(string) string, + targetDir string, + selectedVersions []string, + options CloneOptions) error { + fmt.Printf("Start to clone mirror, targetDir is %s, selectedVersions are [%s]\n", targetDir, strings.Join(selectedVersions, ",")) fmt.Println("If this does not meet expectations, please abort this process, read `tiup mirror clone --help` and run again") @@ -146,7 +152,7 @@ func CloneMirror(repo *V1Repository, components []string, targetDir string, sele snapshot := v1manifest.NewSnapshot(initTime) snapshot.SetExpiresAt(expirsAt) - componentManifests, err := cloneComponents(repo, components, selectedVersions, targetDir, tmpDir, options) + componentManifests, err := cloneComponents(repo, components, selectedVersions, tidbClusterVersionMapper, targetDir, tmpDir, options) if err != nil { return err } @@ -243,8 +249,10 @@ func CloneMirror(repo *V1Repository, components []string, targetDir string, sele func cloneComponents(repo *V1Repository, components, selectedVersions []string, + tidbClusterVersionMapper func(string) string, targetDir, tmpDir string, options CloneOptions) (map[string]*v1manifest.Component, error) { + compManifests := map[string]*v1manifest.Component{} for _, name := range components { manifest, err := repo.FetchComponentManifest(name, true) @@ -252,7 +260,7 @@ func cloneComponents(repo *V1Repository, return nil, errors.Annotatef(err, "fetch component '%s' manifest failed", name) } - vs := combineVersions(options.Components[name], manifest, options.OSs, options.Archs, selectedVersions) + vs := combineVersions(options.Components[name], tidbClusterVersionMapper, manifest, options.OSs, options.Archs, selectedVersions) var newManifest *v1manifest.Component if options.Full { newManifest = manifest @@ -391,20 +399,17 @@ func checkVersion(options CloneOptions, versions set.StringSet, version string) return false } -func combineVersions(versions *[]string, manifest *v1manifest.Component, oss, archs, selectedVersions []string) set.StringSet { +func combineVersions(versions *[]string, + tidbClusterVersionMapper func(string) string, + manifest *v1manifest.Component, oss, archs, + selectedVersions []string) set.StringSet { + if (versions == nil || len(*versions) < 1) && len(selectedVersions) < 1 { return nil } - switch manifest.ID { - case "alertmanager": - return set.NewStringSet("v0.17.0") - case "blackbox_exporter": - return set.NewStringSet("v0.12.0") - case "node_exporter": - return set.NewStringSet("v0.17.0") - case "pushgateway": - return set.NewStringSet("v0.7.0") + if bindver := tidbClusterVersionMapper(manifest.ID); bindver != "" { + return set.NewStringSet(bindver) } result := set.NewStringSet() From 2e533e6f7464ca278171d260814ab973a3a5be13 Mon Sep 17 00:00:00 2001 From: lucklove Date: Mon, 14 Sep 2020 21:07:31 +0800 Subject: [PATCH 2/9] Use latest stable tispark version Signed-off-by: lucklove --- go.mod | 1 + go.sum | 3 +++ pkg/cluster/prepare.go | 9 ++++++--- pkg/cluster/spec/bindversion.go | 2 -- pkg/cluster/task/builder.go | 2 +- pkg/cluster/task/copy_component.go | 13 +++++++++++++ pkg/cluster/task/download.go | 12 ++++++++++++ pkg/repository/v1_repository.go | 22 +++++++++++++++++++++- 8 files changed, 57 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 3a619b1192..74954870b6 100644 --- a/go.mod +++ b/go.mod @@ -40,6 +40,7 @@ require ( github.com/mattn/go-colorable v0.1.6 // indirect github.com/mattn/go-runewidth v0.0.7 github.com/otiai10/copy v1.2.0 + github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pingcap/check v0.0.0-20200212061837-5e12011dc712 github.com/pingcap/dm v1.1.0-alpha.0.20200521025928-83063141c5fd github.com/pingcap/errors v0.11.5-0.20200820035142-66eb5bf1d1cd diff --git a/go.sum b/go.sum index aa095ea147..582a8e0331 100644 --- a/go.sum +++ b/go.sum @@ -576,6 +576,9 @@ github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6 github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo= github.com/otiai10/mint v1.3.1 h1:BCmzIS3n71sGfHB5NMNDB3lHYPz8fWSkCAErHed//qc= github.com/otiai10/mint v1.3.1/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc= +github.com/patrickmn/go-cache v1.0.0 h1:3gD5McaYs9CxjyK5AXGcq8gdeCARtd/9gJDUvVeaZ0Y= +github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= +github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.3.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= diff --git a/pkg/cluster/prepare.go b/pkg/cluster/prepare.go index eb1232d497..eef559588b 100644 --- a/pkg/cluster/prepare.go +++ b/pkg/cluster/prepare.go @@ -34,12 +34,15 @@ func BuildDownloadCompTasks(version string, instanceIter InstanceIter, bindVersi if _, found := uniqueTaskList[key]; !found { uniqueTaskList[key] = struct{}{} - // download spark as dependency of tispark - if inst.ComponentName() == spec.ComponentTiSpark { + // we don't set version for tispark, so the lastest tispark will be used + var version string + if inst.ComponentName() != spec.ComponentTiSpark { + version = bindVersion(inst.ComponentName(), version) + } else { + // download spark as dependency of tispark tasks = append(tasks, buildDownloadSparkTask(version, inst, bindVersion)) } - version := bindVersion(inst.ComponentName(), version) t := task.NewBuilder(). Download(inst.ComponentName(), inst.OS(), inst.Arch(), version). BuildAsStep(fmt.Sprintf(" - Download %s:%s (%s/%s)", diff --git a/pkg/cluster/spec/bindversion.go b/pkg/cluster/spec/bindversion.go index 8369faf779..f485343072 100644 --- a/pkg/cluster/spec/bindversion.go +++ b/pkg/cluster/spec/bindversion.go @@ -31,8 +31,6 @@ func TiDBComponentVersion(comp, version string) string { return "v0.7.0" case ComponentCheckCollector: return "v0.3.1" - case ComponentTiSpark: - return "v2.3.1" case ComponentSpark: return "v2.4.3" default: diff --git a/pkg/cluster/task/builder.go b/pkg/cluster/task/builder.go index 989e4061b2..fc7b08dc7a 100644 --- a/pkg/cluster/task/builder.go +++ b/pkg/cluster/task/builder.go @@ -370,7 +370,7 @@ func (b *Builder) DeploySpark(inst spec.Instance, version, srcPath, deployDir st inst.ComponentName(), inst.OS(), inst.Arch(), - version, + "", // use the latest stable version srcPath, inst.GetHost(), deployDir, diff --git a/pkg/cluster/task/copy_component.go b/pkg/cluster/task/copy_component.go index 50dbd51920..c2ce8bd2af 100644 --- a/pkg/cluster/task/copy_component.go +++ b/pkg/cluster/task/copy_component.go @@ -17,6 +17,7 @@ import ( "fmt" "github.com/pingcap/tiup/pkg/cluster/spec" + "github.com/pingcap/tiup/pkg/environment" ) // CopyComponent is used to copy all files related the specific version a component @@ -39,6 +40,18 @@ func PackagePath(comp string, version string, os string, arch string) string { // Execute implements the Task interface func (c *CopyComponent) Execute(ctx *Context) error { + // If the version is not specified, the last stable one will be used + if c.version == "" { + env := environment.GlobalEnv() + env.V1Repository().Lock() + defer env.V1Repository().Unlock() + ver, _, err := env.V1Repository().LatestStableVersion(c.component, false) + if err != nil { + return err + } + c.version = string(ver) + } + // Copy to remote server srcPath := c.srcPath if srcPath == "" { diff --git a/pkg/cluster/task/download.go b/pkg/cluster/task/download.go index ee3cbef0a7..73e0f4ff8c 100644 --- a/pkg/cluster/task/download.go +++ b/pkg/cluster/task/download.go @@ -17,6 +17,7 @@ import ( "fmt" operator "github.com/pingcap/tiup/pkg/cluster/operation" + "github.com/pingcap/tiup/pkg/environment" ) // Downloader is used to download the specific version of a component from @@ -40,6 +41,17 @@ func NewDownloader(component string, os string, arch string, version string) *Do // Execute implements the Task interface func (d *Downloader) Execute(_ *Context) error { + // If the version is not specified, the last stable one will be used + if d.version == "" { + env := environment.GlobalEnv() + env.V1Repository().Lock() + defer env.V1Repository().Unlock() + ver, _, err := env.V1Repository().LatestStableVersion(d.component, false) + if err != nil { + return err + } + d.version = string(ver) + } return operator.Download(d.component, d.os, d.arch, d.version) } diff --git a/pkg/repository/v1_repository.go b/pkg/repository/v1_repository.go index f56a3166c4..6789f2bf93 100644 --- a/pkg/repository/v1_repository.go +++ b/pkg/repository/v1_repository.go @@ -24,10 +24,12 @@ import ( "runtime" "strconv" "strings" + "sync" "time" "github.com/fatih/color" cjson "github.com/gibson042/canonicaljson-go" + "github.com/patrickmn/go-cache" "github.com/pingcap/errors" "github.com/pingcap/tiup/pkg/repository/v0manifest" "github.com/pingcap/tiup/pkg/repository/v1manifest" @@ -40,11 +42,17 @@ import ( // errUnknownComponent represents the specific component cannot be found in index.json var errUnknownComponent = stderrors.New("unknown component") +const ( + componentManifestCacheKey = "component-manifest" +) + // V1Repository represents a remote repository viewed with the v1 manifest design. type V1Repository struct { Options + sync.Mutex mirror Mirror local v1manifest.LocalManifests + cache *cache.Cache } // ComponentSpec describes a component a user would like to have or use. @@ -76,6 +84,7 @@ func NewV1Repo(mirror Mirror, opts Options, local v1manifest.LocalManifests) *V1 Options: opts, mirror: mirror, local: local, + cache: cache.New(cache.NoExpiration, 0), } return repo @@ -657,12 +666,23 @@ func (r *V1Repository) UpdateComponentManifests() error { // FetchComponentManifest fetch the component manifest. func (r *V1Repository) FetchComponentManifest(id string, withYanked bool) (com *v1manifest.Component, err error) { + // cache is used to avoid fetching the same component manifests multiple time + cacheKey := fmt.Sprintf("%s-%s-%t", componentManifestCacheKey, id, withYanked) + if com, ok := r.cache.Get(cacheKey); ok { + return com.(*v1manifest.Component), nil + } + err = r.ensureManifests() if err != nil { return nil, errors.AddStack(err) } - return r.updateComponentManifest(id, withYanked) + com, err = r.updateComponentManifest(id, withYanked) + if err != nil { + return nil, err + } + r.cache.Set(cacheKey, com, cache.DefaultExpiration) + return } // ComponentVersion returns version item of a component From eb937815e97a0242474ffa99e0fe6be054077d8f Mon Sep 17 00:00:00 2001 From: lucklove Date: Tue, 15 Sep 2020 17:06:27 +0800 Subject: [PATCH 3/9] Fix tests Signed-off-by: lucklove --- pkg/cluster/prepare.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/cluster/prepare.go b/pkg/cluster/prepare.go index eef559588b..f50f3c1227 100644 --- a/pkg/cluster/prepare.go +++ b/pkg/cluster/prepare.go @@ -26,7 +26,7 @@ type InstanceIter interface { } // BuildDownloadCompTasks build download component tasks -func BuildDownloadCompTasks(version string, instanceIter InstanceIter, bindVersion spec.BindVersion) []*task.StepDisplay { +func BuildDownloadCompTasks(clusterVersion string, instanceIter InstanceIter, bindVersion spec.BindVersion) []*task.StepDisplay { var tasks []*task.StepDisplay uniqueTaskList := make(map[string]struct{}) // map["comp-os-arch"]{} instanceIter.IterInstance(func(inst spec.Instance) { @@ -37,7 +37,7 @@ func BuildDownloadCompTasks(version string, instanceIter InstanceIter, bindVersi // we don't set version for tispark, so the lastest tispark will be used var version string if inst.ComponentName() != spec.ComponentTiSpark { - version = bindVersion(inst.ComponentName(), version) + version = bindVersion(inst.ComponentName(), clusterVersion) } else { // download spark as dependency of tispark tasks = append(tasks, buildDownloadSparkTask(version, inst, bindVersion)) From 00bce89cd14c8f34914d2b6796870e732cfba8cc Mon Sep 17 00:00:00 2001 From: lucklove Date: Tue, 15 Sep 2020 17:16:50 +0800 Subject: [PATCH 4/9] make tidy Signed-off-by: lucklove --- go.sum | 1 - 1 file changed, 1 deletion(-) diff --git a/go.sum b/go.sum index 582a8e0331..b8b890aeed 100644 --- a/go.sum +++ b/go.sum @@ -576,7 +576,6 @@ github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6 github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo= github.com/otiai10/mint v1.3.1 h1:BCmzIS3n71sGfHB5NMNDB3lHYPz8fWSkCAErHed//qc= github.com/otiai10/mint v1.3.1/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc= -github.com/patrickmn/go-cache v1.0.0 h1:3gD5McaYs9CxjyK5AXGcq8gdeCARtd/9gJDUvVeaZ0Y= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE= From e8afba3cb00d9a890cc8ce1577316fbd63f4ff4c Mon Sep 17 00:00:00 2001 From: lucklove Date: Wed, 16 Sep 2020 11:28:30 +0800 Subject: [PATCH 5/9] Remove cache Signed-off-by: lucklove --- go.mod | 1 - go.sum | 2 -- pkg/cluster/prepare.go | 11 +++++------ pkg/cluster/spec/bindversion.go | 2 -- pkg/repository/v1_repository.go | 16 +--------------- 5 files changed, 6 insertions(+), 26 deletions(-) diff --git a/go.mod b/go.mod index 74954870b6..3a619b1192 100644 --- a/go.mod +++ b/go.mod @@ -40,7 +40,6 @@ require ( github.com/mattn/go-colorable v0.1.6 // indirect github.com/mattn/go-runewidth v0.0.7 github.com/otiai10/copy v1.2.0 - github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pingcap/check v0.0.0-20200212061837-5e12011dc712 github.com/pingcap/dm v1.1.0-alpha.0.20200521025928-83063141c5fd github.com/pingcap/errors v0.11.5-0.20200820035142-66eb5bf1d1cd diff --git a/go.sum b/go.sum index b8b890aeed..aa095ea147 100644 --- a/go.sum +++ b/go.sum @@ -576,8 +576,6 @@ github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6 github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo= github.com/otiai10/mint v1.3.1 h1:BCmzIS3n71sGfHB5NMNDB3lHYPz8fWSkCAErHed//qc= github.com/otiai10/mint v1.3.1/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc= -github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= -github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.3.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= diff --git a/pkg/cluster/prepare.go b/pkg/cluster/prepare.go index f50f3c1227..0c704b3ee3 100644 --- a/pkg/cluster/prepare.go +++ b/pkg/cluster/prepare.go @@ -40,7 +40,7 @@ func BuildDownloadCompTasks(clusterVersion string, instanceIter InstanceIter, bi version = bindVersion(inst.ComponentName(), clusterVersion) } else { // download spark as dependency of tispark - tasks = append(tasks, buildDownloadSparkTask(version, inst, bindVersion)) + tasks = append(tasks, buildDownloadSparkTask(inst)) } t := task.NewBuilder(). @@ -55,10 +55,9 @@ func BuildDownloadCompTasks(clusterVersion string, instanceIter InstanceIter, bi // buildDownloadSparkTask build download task for spark, which is a dependency of tispark // FIXME: this is a hack and should be replaced by dependency handling in manifest processing -func buildDownloadSparkTask(version string, inst spec.Instance, bindVersion spec.BindVersion) *task.StepDisplay { - ver := bindVersion(spec.ComponentSpark, version) +func buildDownloadSparkTask(inst spec.Instance) *task.StepDisplay { return task.NewBuilder(). - Download(spec.ComponentSpark, inst.OS(), inst.Arch(), ver). - BuildAsStep(fmt.Sprintf(" - Download %s:%s (%s/%s)", - spec.ComponentSpark, version, inst.OS(), inst.Arch())) + Download(spec.ComponentSpark, inst.OS(), inst.Arch(), ""). + BuildAsStep(fmt.Sprintf(" - Download %s: (%s/%s)", + spec.ComponentSpark, inst.OS(), inst.Arch())) } diff --git a/pkg/cluster/spec/bindversion.go b/pkg/cluster/spec/bindversion.go index f485343072..058b502d45 100644 --- a/pkg/cluster/spec/bindversion.go +++ b/pkg/cluster/spec/bindversion.go @@ -31,8 +31,6 @@ func TiDBComponentVersion(comp, version string) string { return "v0.7.0" case ComponentCheckCollector: return "v0.3.1" - case ComponentSpark: - return "v2.4.3" default: return version } diff --git a/pkg/repository/v1_repository.go b/pkg/repository/v1_repository.go index 6789f2bf93..4af0f341f8 100644 --- a/pkg/repository/v1_repository.go +++ b/pkg/repository/v1_repository.go @@ -29,7 +29,6 @@ import ( "github.com/fatih/color" cjson "github.com/gibson042/canonicaljson-go" - "github.com/patrickmn/go-cache" "github.com/pingcap/errors" "github.com/pingcap/tiup/pkg/repository/v0manifest" "github.com/pingcap/tiup/pkg/repository/v1manifest" @@ -52,7 +51,6 @@ type V1Repository struct { sync.Mutex mirror Mirror local v1manifest.LocalManifests - cache *cache.Cache } // ComponentSpec describes a component a user would like to have or use. @@ -84,7 +82,6 @@ func NewV1Repo(mirror Mirror, opts Options, local v1manifest.LocalManifests) *V1 Options: opts, mirror: mirror, local: local, - cache: cache.New(cache.NoExpiration, 0), } return repo @@ -666,23 +663,12 @@ func (r *V1Repository) UpdateComponentManifests() error { // FetchComponentManifest fetch the component manifest. func (r *V1Repository) FetchComponentManifest(id string, withYanked bool) (com *v1manifest.Component, err error) { - // cache is used to avoid fetching the same component manifests multiple time - cacheKey := fmt.Sprintf("%s-%s-%t", componentManifestCacheKey, id, withYanked) - if com, ok := r.cache.Get(cacheKey); ok { - return com.(*v1manifest.Component), nil - } - err = r.ensureManifests() if err != nil { return nil, errors.AddStack(err) } - com, err = r.updateComponentManifest(id, withYanked) - if err != nil { - return nil, err - } - r.cache.Set(cacheKey, com, cache.DefaultExpiration) - return + return r.updateComponentManifest(id, withYanked) } // ComponentVersion returns version item of a component From b0f4e360a460c8d52076c4c0e5af88ef1f620dcb Mon Sep 17 00:00:00 2001 From: lucklove Date: Wed, 16 Sep 2020 17:29:09 +0800 Subject: [PATCH 6/9] Address comment Signed-off-by: lucklove --- pkg/cluster/manager.go | 16 +++++++++-- pkg/cluster/prepare.go | 6 ++-- pkg/cluster/task/builder.go | 7 ++--- pkg/cluster/task/copy_component.go | 2 -- pkg/cluster/task/download.go | 2 -- pkg/repository/v1_repository.go | 2 -- pkg/repository/v1manifest/key_store.go | 30 +++++++++++++------- pkg/repository/v1manifest/local_manifests.go | 17 +++++------ 8 files changed, 48 insertions(+), 34 deletions(-) diff --git a/pkg/cluster/manager.go b/pkg/cluster/manager.go index 3c56f820c7..a9dc6fa917 100644 --- a/pkg/cluster/manager.go +++ b/pkg/cluster/manager.go @@ -39,10 +39,12 @@ import ( "github.com/pingcap/tiup/pkg/cluster/spec" "github.com/pingcap/tiup/pkg/cluster/task" "github.com/pingcap/tiup/pkg/crypto" + "github.com/pingcap/tiup/pkg/environment" "github.com/pingcap/tiup/pkg/errutil" "github.com/pingcap/tiup/pkg/file" "github.com/pingcap/tiup/pkg/logger/log" "github.com/pingcap/tiup/pkg/meta" + "github.com/pingcap/tiup/pkg/repository/v0manifest" "github.com/pingcap/tiup/pkg/set" "github.com/pingcap/tiup/pkg/utils" "github.com/pingcap/tiup/pkg/version" @@ -846,7 +848,12 @@ func (m *Manager) Upgrade(clusterName string, clusterVersion string, opt operato // copy dependency component if needed switch inst.ComponentName() { case spec.ComponentTiSpark: - tb = tb.DeploySpark(inst, version, "" /* default srcPath */, deployDir, m.bindVersion) + env := environment.GlobalEnv() + sparkVer, _, err := env.V1Repository().LatestStableVersion(spec.ComponentSpark, false) + if err != nil { + return err + } + tb = tb.DeploySpark(inst, sparkVer.String(), "" /* default srcPath */, deployDir, m.bindVersion) default: tb = tb.CopyComponent( inst.ComponentName(), @@ -1946,7 +1953,12 @@ func buildScaleOutTask( // copy dependency component if needed switch inst.ComponentName() { case spec.ComponentTiSpark: - tb = tb.DeploySpark(inst, version, srcPath, deployDir, m.bindVersion) + env := environment.GlobalEnv() + var sparkVer v0manifest.Version + if sparkVer, _, iterErr = env.V1Repository().LatestStableVersion(spec.ComponentSpark, false); iterErr != nil { + return + } + tb = tb.DeploySpark(inst, sparkVer.String(), srcPath, deployDir, m.bindVersion) default: tb.CopyComponent( inst.ComponentName(), diff --git a/pkg/cluster/prepare.go b/pkg/cluster/prepare.go index 0c704b3ee3..f0dfc04ca2 100644 --- a/pkg/cluster/prepare.go +++ b/pkg/cluster/prepare.go @@ -36,11 +36,11 @@ func BuildDownloadCompTasks(clusterVersion string, instanceIter InstanceIter, bi // we don't set version for tispark, so the lastest tispark will be used var version string - if inst.ComponentName() != spec.ComponentTiSpark { - version = bindVersion(inst.ComponentName(), clusterVersion) - } else { + if inst.ComponentName() == spec.ComponentTiSpark { // download spark as dependency of tispark tasks = append(tasks, buildDownloadSparkTask(inst)) + } else { + version = bindVersion(inst.ComponentName(), clusterVersion) } t := task.NewBuilder(). diff --git a/pkg/cluster/task/builder.go b/pkg/cluster/task/builder.go index fc7b08dc7a..25a2a9eb3b 100644 --- a/pkg/cluster/task/builder.go +++ b/pkg/cluster/task/builder.go @@ -346,14 +346,13 @@ func (b *Builder) CheckSys(host, dataDir, checkType string, topo *spec.Specifica } // DeploySpark deployes spark as dependency of TiSpark -func (b *Builder) DeploySpark(inst spec.Instance, version, srcPath, deployDir string, bindVersion spec.BindVersion) *Builder { - sparkSubPath := spec.ComponentSubDir(spec.ComponentSpark, - bindVersion(spec.ComponentSpark, version)) +func (b *Builder) DeploySpark(inst spec.Instance, sparkVersion, srcPath, deployDir string, bindVersion spec.BindVersion) *Builder { + sparkSubPath := spec.ComponentSubDir(spec.ComponentSpark, sparkVersion) return b.CopyComponent( spec.ComponentSpark, inst.OS(), inst.Arch(), - bindVersion(spec.ComponentSpark, version), + sparkVersion, srcPath, inst.GetHost(), deployDir, diff --git a/pkg/cluster/task/copy_component.go b/pkg/cluster/task/copy_component.go index c2ce8bd2af..f7fe9c687b 100644 --- a/pkg/cluster/task/copy_component.go +++ b/pkg/cluster/task/copy_component.go @@ -43,8 +43,6 @@ func (c *CopyComponent) Execute(ctx *Context) error { // If the version is not specified, the last stable one will be used if c.version == "" { env := environment.GlobalEnv() - env.V1Repository().Lock() - defer env.V1Repository().Unlock() ver, _, err := env.V1Repository().LatestStableVersion(c.component, false) if err != nil { return err diff --git a/pkg/cluster/task/download.go b/pkg/cluster/task/download.go index 73e0f4ff8c..75c61fb522 100644 --- a/pkg/cluster/task/download.go +++ b/pkg/cluster/task/download.go @@ -44,8 +44,6 @@ func (d *Downloader) Execute(_ *Context) error { // If the version is not specified, the last stable one will be used if d.version == "" { env := environment.GlobalEnv() - env.V1Repository().Lock() - defer env.V1Repository().Unlock() ver, _, err := env.V1Repository().LatestStableVersion(d.component, false) if err != nil { return err diff --git a/pkg/repository/v1_repository.go b/pkg/repository/v1_repository.go index 4af0f341f8..86f0505d9e 100644 --- a/pkg/repository/v1_repository.go +++ b/pkg/repository/v1_repository.go @@ -24,7 +24,6 @@ import ( "runtime" "strconv" "strings" - "sync" "time" "github.com/fatih/color" @@ -48,7 +47,6 @@ const ( // V1Repository represents a remote repository viewed with the v1 manifest design. type V1Repository struct { Options - sync.Mutex mirror Mirror local v1manifest.LocalManifests } diff --git a/pkg/repository/v1manifest/key_store.go b/pkg/repository/v1manifest/key_store.go index d477a1f5e3..4136232879 100644 --- a/pkg/repository/v1manifest/key_store.go +++ b/pkg/repository/v1manifest/key_store.go @@ -15,19 +15,22 @@ package v1manifest import ( "fmt" + "sync" "github.com/pingcap/errors" "github.com/pingcap/tiup/pkg/crypto" ) -// KeyStore tracks roles, keys, etc. and verifies signatures against this metadata. -type KeyStore map[string]roleKeys +// KeyStore tracks roles, keys, etc. and verifies signatures against this metadata. (map[string]roleKeys) +type KeyStore struct { + sync.Map +} type roleKeys struct { threshold uint expiry string - // key id -> public key - keys map[string]crypto.PubKey + // key id -> public key (map[string]crypto.PubKey) + keys sync.Map } // NewKeyStore return a KeyStore @@ -41,15 +44,17 @@ func (s *KeyStore) AddKeys(role string, threshold uint, expiry string, keys map[ return errors.Errorf("invalid threshold (0)") } - (*s)[role] = roleKeys{threshold: threshold, expiry: expiry, keys: map[string]crypto.PubKey{}} + rk := roleKeys{threshold: threshold, expiry: expiry} for id, info := range keys { pub, err := info.publicKey() if err != nil { return err } - (*s)[role].keys[id] = pub + + rk.keys.Store(id, pub) } + s.Store(role, rk) return nil } @@ -80,7 +85,7 @@ func (s *SignatureError) Error() string { // transitionRoot checks that signed is verified by signatures using newThreshold, and if so, updates the keys for the root // role in the key store. func (s *KeyStore) transitionRoot(signed []byte, newThreshold uint, expiry string, signatures []Signature, newKeys map[string]*KeyInfo) error { - oldKeys := (*s)[ManifestTypeRoot] + oldKeys, hasOldKeys := s.Load(ManifestTypeRoot) err := s.AddKeys(ManifestTypeRoot, newThreshold, expiry, newKeys) if err != nil { @@ -90,7 +95,9 @@ func (s *KeyStore) transitionRoot(signed []byte, newThreshold uint, expiry strin err = s.verifySignature(signed, ManifestTypeRoot, signatures, ManifestFilenameRoot) if err != nil { // Restore the old root keys. - (*s)[ManifestTypeRoot] = oldKeys + if hasOldKeys { + s.Store(ManifestTypeRoot, oldKeys) + } return err } @@ -114,18 +121,19 @@ func (s *KeyStore) verifySignature(signed []byte, role string, signatures []Sign has[sig.KeyID] = struct{}{} } - keys, ok := (*s)[role] + ks, ok := s.Load(role) if !ok { return errors.Errorf("Unknown role %s", role) } + keys := ks.(roleKeys) var validSigs uint = 0 for _, sig := range signatures { - key, ok := keys.keys[sig.KeyID] + key, ok := keys.keys.Load(sig.KeyID) if !ok { continue } - err := key.VerifySignature(signed, sig.Sig) + err := key.(crypto.PubKey).VerifySignature(signed, sig.Sig) if err != nil { return newSignatureError(filename, err) } diff --git a/pkg/repository/v1manifest/local_manifests.go b/pkg/repository/v1manifest/local_manifests.go index 5eb1944b65..5d40937655 100644 --- a/pkg/repository/v1manifest/local_manifests.go +++ b/pkg/repository/v1manifest/local_manifests.go @@ -21,6 +21,7 @@ import ( "os" "path/filepath" "strings" + "sync" cjson "github.com/gibson042/canonicaljson-go" "github.com/pingcap/errors" @@ -56,7 +57,7 @@ type LocalManifests interface { type FsManifests struct { profile *localdata.Profile keys *KeyStore - cache map[string]string + cache sync.Map // map[string]string } // FIXME implement garbage collection of old manifests @@ -64,7 +65,7 @@ type FsManifests struct { // NewManifests creates a new FsManifests with local store at root. // There must exist a trusted root.json. func NewManifests(profile *localdata.Profile) (*FsManifests, error) { - result := &FsManifests{profile: profile, keys: NewKeyStore(), cache: make(map[string]string)} + result := &FsManifests{profile: profile, keys: NewKeyStore()} // Load the root manifest. manifest, err := result.load(ManifestFilenameRoot) @@ -91,7 +92,7 @@ func NewManifests(profile *localdata.Profile) (*FsManifests, error) { return nil, errors.AddStack(err) } - result.cache[ManifestFilenameRoot] = manifest + result.cache.Store(ManifestFilenameRoot, manifest) return result, nil } @@ -129,7 +130,7 @@ func (ms *FsManifests) save(manifest *Manifest, filename string) error { return err } - ms.cache[filename] = string(bytes) + ms.cache.Store(filename, string(bytes)) return nil } @@ -154,7 +155,7 @@ func (ms *FsManifests) LoadManifest(role ValidManifest) (*Manifest, bool, error) return m, true, err } - ms.cache[filename] = manifest + ms.cache.Store(filename, manifest) return m, true, loadKeys(role, ms.keys) } @@ -179,16 +180,16 @@ func (ms *FsManifests) LoadComponentManifest(item *ComponentItem, filename strin return nil, err } - ms.cache[filename] = manifest + ms.cache.Store(filename, manifest) return component, nil } // load return the file for the manifest from disk. // The returned string is empty if the file does not exist. func (ms *FsManifests) load(filename string) (string, error) { - str, cached := ms.cache[filename] + str, cached := ms.cache.Load(filename) if cached { - return str, nil + return str.(string), nil } fullPath := filepath.Join(ms.profile.Root(), localdata.ManifestParentDir, filename) From c7ac32c6fa218174743e21f1029d562cf03ff119 Mon Sep 17 00:00:00 2001 From: lucklove Date: Wed, 16 Sep 2020 17:41:06 +0800 Subject: [PATCH 7/9] Fix test Signed-off-by: lucklove --- pkg/cluster/manager.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/pkg/cluster/manager.go b/pkg/cluster/manager.go index a9dc6fa917..6ea65c15a9 100644 --- a/pkg/cluster/manager.go +++ b/pkg/cluster/manager.go @@ -1202,7 +1202,12 @@ func (m *Manager) Deploy( // copy dependency component if needed switch inst.ComponentName() { case spec.ComponentTiSpark: - t = t.DeploySpark(inst, version, "" /* default srcPath */, deployDir, m.bindVersion) + env := environment.GlobalEnv() + var sparkVer v0manifest.Version + if sparkVer, _, iterErr = env.V1Repository().LatestStableVersion(spec.ComponentSpark, false); iterErr != nil { + return + } + t = t.DeploySpark(inst, sparkVer.String(), "" /* default srcPath */, deployDir, m.bindVersion) default: t = t.CopyComponent( inst.ComponentName(), @@ -1245,6 +1250,10 @@ func (m *Manager) Deploy( ) }) + if iterErr != nil { + return iterErr + } + // Deploy monitor relevant components to remote dlTasks, dpTasks := buildMonitoredDeployTask( m.bindVersion, From d4e67a11c0c00899ab54b27f0466bb26d766f337 Mon Sep 17 00:00:00 2001 From: lucklove Date: Wed, 16 Sep 2020 17:52:44 +0800 Subject: [PATCH 8/9] Fix tidy Signed-off-by: lucklove --- pkg/repository/v1_repository.go | 4 ---- pkg/repository/v1manifest/key_store.go | 4 ++-- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/pkg/repository/v1_repository.go b/pkg/repository/v1_repository.go index 86f0505d9e..f56a3166c4 100644 --- a/pkg/repository/v1_repository.go +++ b/pkg/repository/v1_repository.go @@ -40,10 +40,6 @@ import ( // errUnknownComponent represents the specific component cannot be found in index.json var errUnknownComponent = stderrors.New("unknown component") -const ( - componentManifestCacheKey = "component-manifest" -) - // V1Repository represents a remote repository viewed with the v1 manifest design. type V1Repository struct { Options diff --git a/pkg/repository/v1manifest/key_store.go b/pkg/repository/v1manifest/key_store.go index 4136232879..5bfd69d551 100644 --- a/pkg/repository/v1manifest/key_store.go +++ b/pkg/repository/v1manifest/key_store.go @@ -30,7 +30,7 @@ type roleKeys struct { threshold uint expiry string // key id -> public key (map[string]crypto.PubKey) - keys sync.Map + keys *sync.Map } // NewKeyStore return a KeyStore @@ -44,7 +44,7 @@ func (s *KeyStore) AddKeys(role string, threshold uint, expiry string, keys map[ return errors.Errorf("invalid threshold (0)") } - rk := roleKeys{threshold: threshold, expiry: expiry} + rk := roleKeys{threshold: threshold, expiry: expiry, keys: &sync.Map{}} for id, info := range keys { pub, err := info.publicKey() From 1cc4a8f07448ddb3227f0178b01a91de9e079564 Mon Sep 17 00:00:00 2001 From: lucklove Date: Wed, 16 Sep 2020 18:08:34 +0800 Subject: [PATCH 9/9] Remove useless params Signed-off-by: lucklove --- pkg/cluster/manager.go | 6 +++--- pkg/cluster/task/builder.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/cluster/manager.go b/pkg/cluster/manager.go index 6ea65c15a9..ba4b57b060 100644 --- a/pkg/cluster/manager.go +++ b/pkg/cluster/manager.go @@ -853,7 +853,7 @@ func (m *Manager) Upgrade(clusterName string, clusterVersion string, opt operato if err != nil { return err } - tb = tb.DeploySpark(inst, sparkVer.String(), "" /* default srcPath */, deployDir, m.bindVersion) + tb = tb.DeploySpark(inst, sparkVer.String(), "" /* default srcPath */, deployDir) default: tb = tb.CopyComponent( inst.ComponentName(), @@ -1207,7 +1207,7 @@ func (m *Manager) Deploy( if sparkVer, _, iterErr = env.V1Repository().LatestStableVersion(spec.ComponentSpark, false); iterErr != nil { return } - t = t.DeploySpark(inst, sparkVer.String(), "" /* default srcPath */, deployDir, m.bindVersion) + t = t.DeploySpark(inst, sparkVer.String(), "" /* default srcPath */, deployDir) default: t = t.CopyComponent( inst.ComponentName(), @@ -1967,7 +1967,7 @@ func buildScaleOutTask( if sparkVer, _, iterErr = env.V1Repository().LatestStableVersion(spec.ComponentSpark, false); iterErr != nil { return } - tb = tb.DeploySpark(inst, sparkVer.String(), srcPath, deployDir, m.bindVersion) + tb = tb.DeploySpark(inst, sparkVer.String(), srcPath, deployDir) default: tb.CopyComponent( inst.ComponentName(), diff --git a/pkg/cluster/task/builder.go b/pkg/cluster/task/builder.go index 25a2a9eb3b..547d166ecc 100644 --- a/pkg/cluster/task/builder.go +++ b/pkg/cluster/task/builder.go @@ -346,7 +346,7 @@ func (b *Builder) CheckSys(host, dataDir, checkType string, topo *spec.Specifica } // DeploySpark deployes spark as dependency of TiSpark -func (b *Builder) DeploySpark(inst spec.Instance, sparkVersion, srcPath, deployDir string, bindVersion spec.BindVersion) *Builder { +func (b *Builder) DeploySpark(inst spec.Instance, sparkVersion, srcPath, deployDir string) *Builder { sparkSubPath := spec.ComponentSubDir(spec.ComponentSpark, sparkVersion) return b.CopyComponent( spec.ComponentSpark,