Clone with yanked version

cmd/list.go

@@ -118,20 +118,17 @@ func showComponentList(env *environment.Environment, opt listOptions) (*listResu
 
 	localComponents := set.NewStringSet(installed...)
 	compIDs := []string{}
-	for id := range index.Components {
+	components := index.ComponentList()
+	for id := range components {
 		compIDs = append(compIDs, id)
 	}
 	sort.Strings(compIDs)
 	for _, id := range compIDs {
-		comp := index.Components[id]
+		comp := components[id]
 		if opt.installedOnly && !localComponents.Exist(id) {
 			continue
 		}
 
-		if comp.Yanked {
-			continue
-		}
-
 		if (!opt.installedOnly && !opt.showAll) && comp.Hidden {
 			continue
 		}
@@ -198,7 +195,8 @@ func showComponentVersions(env *environment.Environment, component string, opt l
 	platforms := make(map[string][]string)
 	released := make(map[string]string)
 
-	for plat, versions := range comp.Platforms {
+	for plat := range comp.Platforms {
+		versions := comp.VersionList(plat)
 		for ver, verinfo := range versions {
 			if v0manifest.Version(ver).IsNightly() && ver == comp.Nightly {
 				platforms[version.NightlyVersion] = append(platforms[version.NightlyVersion], plat)

pkg/cluster/clusterutil/cluster.go

@@ -54,7 +54,7 @@ func NewRepository(os, arch string) (Repository, error) {
 }
 
 func (r *repositoryT) DownloadComponent(comp, version, target string) error {
-	versionItem, err := r.repo.ComponentVersion(comp, version)
+	versionItem, err := r.repo.ComponentVersion(comp, version, false)
 	if err != nil {
 		return err
 	}
@@ -75,7 +75,7 @@ func (r *repositoryT) DownloadComponent(comp, version, target string) error {
 }
 
 func (r *repositoryT) VerifyComponent(comp, version, target string) error {
-	versionItem, err := r.repo.ComponentVersion(comp, version)
+	versionItem, err := r.repo.ComponentVersion(comp, version, true)
 	if err != nil {
 		return err
 	}
@@ -90,7 +90,7 @@ func (r *repositoryT) VerifyComponent(comp, version, target string) error {
 }
 
 func (r *repositoryT) ComponentBinEntry(comp, version string) (string, error) {
-	versionItem, err := r.repo.ComponentVersion(comp, version)
+	versionItem, err := r.repo.ComponentVersion(comp, version, true)
 	if err != nil {
 		return "", err
 	}

pkg/repository/clone_mirror.go

@@ -276,21 +276,26 @@ func cloneComponents(repo *V1Repository,
 		for _, goos := range options.OSs {
 			for _, goarch := range options.Archs {
 				platform := PlatformString(goos, goarch)
-				versions, found := manifest.Platforms[platform]
-				if !found {
+				versions := manifest.VersionListWithYanked(platform)
+				if versions == nil {
 					fmt.Printf("The component '%s' donesn't have %s/%s, skipped\n", name, goos, goarch)
 				}
 				for v, versionItem := range versions {
-					if !checkVersion(options, vs, v) {
-						continue
-					}
 					if !options.Full {
-						newVersions, found := newManifest.Platforms[platform]
-						if !found {
+						newVersions := newManifest.VersionListWithYanked(platform)
+						if newVersions == nil {
 							newVersions = map[string]v1manifest.VersionItem{}
 							newManifest.Platforms[platform] = newVersions
 						}
 						newVersions[v] = versionItem
+						if !checkVersion(options, vs, v) {
+							versionItem.Yanked = true
+							newVersions[v] = versionItem
+							continue
+						}
+					}
+					if versionItem.Yanked {
+						continue
 					}
 					if err := download(targetDir, tmpDir, repo, &versionItem); err != nil {
 						return nil, errors.Annotatef(err, "download resource: %s", name)
@@ -413,8 +418,8 @@ func combineVersions(versions *[]string, manifest *v1manifest.Component, oss, ar
 	for _, os := range oss {
 		for _, arch := range archs {
 			platform := PlatformString(os, arch)
-			versions, found := manifest.Platforms[platform]
-			if !found {
+			versions := manifest.VersionList(platform)
+			if versions == nil {
 				continue
 			}
 			for _, selectedVersion := range selectedVersions {

pkg/repository/v1_repository.go

@@ -130,12 +130,10 @@ func (r *V1Repository) UpdateComponents(specs []ComponentSpec) error {
 		}
 
 		platform := r.PlatformString()
-		versions, ok := manifest.Platforms[platform]
-		if !ok {
-			if versions, ok = manifest.Platforms[v1manifest.AnyPlatform]; !ok {
-				errs = append(errs, fmt.Sprintf("platform %s not supported by component %s", platform, spec.ID))
-				continue
-			}
+		versions := manifest.VersionList(platform)
+		if versions == nil {
+			errs = append(errs, fmt.Sprintf("platform %s not supported by component %s", platform, spec.ID))
+			continue
 		}
 
 		version, versionItem, err := r.selectVersion(spec.ID, versions, specVersion)
@@ -661,15 +659,15 @@ func (r *V1Repository) FetchComponentManifest(id string) (com *v1manifest.Compon
 }
 
 // ComponentVersion returns version item of a component
-func (r *V1Repository) ComponentVersion(id, version string) (*v1manifest.VersionItem, error) {
+func (r *V1Repository) ComponentVersion(id, version string, includeYanked bool) (*v1manifest.VersionItem, error) {
 	manifest, err := r.FetchComponentManifest(id)
 	if err != nil {
 		return nil, err
 	}
 	if v0manifest.Version(version).IsNightly() && manifest.Nightly != "" {
 		version = manifest.Nightly
 	}
-	vi := manifest.VersionItem(r.PlatformString(), version)
+	vi := manifest.VersionItem(r.PlatformString(), version, includeYanked)
 	if vi == nil {
 		return nil, fmt.Errorf("version %s on %s for component %s not found", version, r.PlatformString(), id)
 	}
@@ -683,12 +681,9 @@ func (r *V1Repository) LatestStableVersion(id string) (v0manifest.Version, *v1ma
 		return "", nil, err
 	}
 
-	versions := com.Platforms[r.PlatformString()]
+	versions := com.VersionList(r.PlatformString())
 	if versions == nil {
-		versions = com.Platforms[v1manifest.AnyPlatform]
-		if versions == nil {
-			return "", nil, fmt.Errorf("component %s doesn't support platform %s", id, r.PlatformString())
-		}
+		return "", nil, fmt.Errorf("component %s doesn't support platform %s", id, r.PlatformString())
 	}
 
 	var last string
@@ -706,7 +701,7 @@ func (r *V1Repository) LatestStableVersion(id string) (v0manifest.Version, *v1ma
 		return "", nil, fmt.Errorf("component %s doesn't has a stable version", id)
 	}
 
-	return v0manifest.Version(last), com.VersionItem(r.PlatformString(), last), nil
+	return v0manifest.Version(last), com.VersionItem(r.PlatformString(), last, false), nil
 }
 
 // BinaryPath return the binary path of the component.
@@ -738,11 +733,10 @@ func (r *V1Repository) BinaryPath(installPath string, componentID string, versio
 		specVersion = component.Nightly
 	}
 
-	versionItem, ok := component.Platforms[r.PlatformString()][specVersion]
+	// We need yanked version because we may install that version before it was yanked
+	versionItem, ok := component.VersionListWithYanked(r.PlatformString())[specVersion]
 	if !ok {
-		if versionItem, ok = component.Platforms[v1manifest.AnyPlatform][specVersion]; !ok {
-			return "", errors.Errorf("no version: %s", version)
-		}
+		return "", errors.Errorf("no version: %s", version)
 	}
 
 	entry := versionItem.Entry

pkg/repository/v1manifest/manifest.go

@@ -23,7 +23,6 @@ import (
 
 	cjson "github.com/gibson042/canonicaljson-go"
 	"github.com/pingcap/errors"
-	"github.com/pingcap/tiup/pkg/set"
 )
 
 // Names of manifest ManifestsConfig
@@ -274,31 +273,17 @@ func (manifest *Component) isValid() error {
 	return nil
 }
 
-// ListVersion returns version list of the component
-func (manifest *Component) ListVersion() []string {
-	s := set.NewStringSet()
-	for _, vi := range manifest.Platforms {
-		for v := range vi {
-			s.Insert(v)
-		}
-	}
-	list := []string{}
-	for v := range s {
-		list = append(list, v)
-	}
-	return list
-}
-
 // VersionItem returns VersionItem by platform and version
-func (manifest *Component) VersionItem(plat, ver string) *VersionItem {
-	p := manifest.Platforms[plat]
-	if p == nil {
-		if p = manifest.Platforms[AnyPlatform]; p == nil {
-			return nil
-		}
+func (manifest *Component) VersionItem(plat, ver string, includeYanked bool) *VersionItem {
+	var v VersionItem
+	var ok bool
+
+	if includeYanked {
+		v, ok = manifest.VersionListWithYanked(plat)[ver]
+	} else {
+		v, ok = manifest.VersionList(plat)[ver]
 	}
-	v := p[ver]
-	if v.Entry == "" {
+	if !ok || v.Entry == "" {
 		return nil
 	}
 	return &v

pkg/repository/v1manifest/manifest_test.go

@@ -27,6 +27,7 @@ func TestVersionItem(t *testing.T) {
 		Platforms: map[string]map[string]VersionItem{
 			"linux/amd64": {
 				"v1.0.0": {Entry: "test"},
+				"v1.1.1": {Entry: "test", Yanked: true},
 			},
 			"any/any": {
 				"v1.0.0": {Entry: "test"},
@@ -41,11 +42,13 @@ func TestVersionItem(t *testing.T) {
 		},
 	}
 
-	assert.NotNil(t, manifest.VersionItem("linux/amd64", "v1.0.0"))
-	assert.NotNil(t, manifest.VersionItem("windows/386", "v1.0.0"))
-	assert.NotNil(t, manifest.VersionItem("any/any", "v1.0.0"))
-	assert.Nil(t, manifest.VersionItem("darwin/any", "v1.0.0"))
-	assert.Nil(t, manifest.VersionItem("any/arm64", "v1.0.0"))
+	assert.NotNil(t, manifest.VersionItem("linux/amd64", "v1.0.0", false))
+	assert.Nil(t, manifest.VersionItem("linux/amd64", "v1.1.1", false))
+	assert.NotNil(t, manifest.VersionItem("linux/amd64", "v1.1.1", true))
+	assert.NotNil(t, manifest.VersionItem("windows/386", "v1.0.0", false))
+	assert.NotNil(t, manifest.VersionItem("any/any", "v1.0.0", false))
+	assert.Nil(t, manifest.VersionItem("darwin/any", "v1.0.0", false))
+	assert.Nil(t, manifest.VersionItem("any/arm64", "v1.0.0", false))
 
 	manifest = &Component{
 		Platforms: map[string]map[string]VersionItem{
@@ -55,9 +58,9 @@ func TestVersionItem(t *testing.T) {
 		},
 	}
 
-	assert.NotNil(t, manifest.VersionItem("linux/amd64", "v1.0.0"))
-	assert.Nil(t, manifest.VersionItem("windows/386", "v1.0.0"))
-	assert.Nil(t, manifest.VersionItem("any/any", "v1.0.0"))
-	assert.Nil(t, manifest.VersionItem("darwin/any", "v1.0.0"))
-	assert.Nil(t, manifest.VersionItem("any/arm64", "v1.0.0"))
+	assert.NotNil(t, manifest.VersionItem("linux/amd64", "v1.0.0", false))
+	assert.Nil(t, manifest.VersionItem("windows/386", "v1.0.0", false))
+	assert.Nil(t, manifest.VersionItem("any/any", "v1.0.0", false))
+	assert.Nil(t, manifest.VersionItem("darwin/any", "v1.0.0", false))
+	assert.Nil(t, manifest.VersionItem("any/arm64", "v1.0.0", false))
 }

pkg/repository/v1manifest/types.go

@@ -189,6 +189,23 @@ func (manifest *Index) Filename() string {
 	return ManifestFilenameIndex
 }
 
+// ComponentList returns non-yanked components
+func (manifest *Index) ComponentList() map[string]ComponentItem {
+	components := make(map[string]ComponentItem)
+	for n, c := range manifest.Components {
+		if c.Yanked {
+			continue
+		}
+		components[n] = c
+	}
+	return components
+}
+
+// ComponentListWithYanked return all components
+func (manifest *Index) ComponentListWithYanked() map[string]ComponentItem {
+	return manifest.Components
+}
+
 // Filename implements ValidManifest
 func (manifest *Component) Filename() string {
 	return manifest.ID + ".json"
@@ -210,6 +227,42 @@ func (manifest *Component) HasNightly(platform string) bool {
 		return false
 	}
 
-	_, ok := manifest.Platforms[platform][manifest.Nightly]
-	return ok
+	v, ok := manifest.Platforms[platform][manifest.Nightly]
+	if !ok {
+		return false
+	}
+
+	return !v.Yanked
+}
+
+// VersionList return all versions exclude yanked versions
+func (manifest *Component) VersionList(platform string) map[string]VersionItem {
+	versions := make(map[string]VersionItem)
+
+	vs := manifest.VersionListWithYanked(platform)
+	if vs == nil {
+		return nil
+	}
+
+	for v, vi := range vs {
+		if vi.Yanked {
+			continue
+		}
+		versions[v] = vi
+	}
+
+	return versions
+}
+
+// VersionListWithYanked return all versions include yanked versions
+func (manifest *Component) VersionListWithYanked(platform string) map[string]VersionItem {
+	vs, ok := manifest.Platforms[platform]
+	if !ok {
+		vs, ok = manifest.Platforms[AnyPlatform]
+		if !ok {
+			return nil
+		}
+	}
+
+	return vs
 }