support ZETA_TRANSPORT_MAX_ENTRIES and ZETA_TRANSPORT_LARGE_SIZE

Makefile

@@ -5,7 +5,7 @@ SOURCE_DIR    := $(abspath $(dir $(lastword ${MAKEFILE_LIST})))
 BUILD_DIR     := ${SOURCE_DIR}/_build
 BUILD_TIME    := $(shell date +'%Y-%m-%dT%H:%M:%S%z')
 BUILD_COMMIT  := $(shell git rev-parse --short HEAD 2>/dev/null || echo 'none')
-BUILD_VERSION := $(shell cat VERSION || echo '0.12.14')
+BUILD_VERSION := $(shell cat VERSION || echo '0.13.0')
 GO_PACKAGES   := $(shell go list ./... | grep -v '^${PKG}/mock/' | grep -v '^${PKG}/proto/')
 GO_LDFLAGS    := -ldflags '-X ${PKG}/pkg/version.version=${BUILD_VERSION} -X ${PKG}/pkg/version.buildTime=${BUILD_TIME} -X ${PKG}/pkg/version.buildCommit=${BUILD_COMMIT}'
 

VERSION

@@ -1 +1 @@
-0.12.14
+0.13.0

bali.toml

@@ -3,7 +3,7 @@ name = "zeta"
 summary = "HugeSCM - A next generation cloud-based version control system"
 description = "HugeSCM - A next generation cloud-based version control system"
 package-name = "alipay-linkc-zeta"
-version = "0.12.14"
+version = "0.13.0"
 license = "MIT"
 prefix = "/usr/local"
 packager = "江二"

cmd/zeta-mc/crate.toml

@@ -1,7 +1,7 @@
 name = "zeta-mc"
 description = "zeta-mc - Migrate Git repository to zeta"
 destination = "bin"
-version = "0.12.14"
+version = "0.13.0"
 goflags = [
     "-ldflags",
     "-X github.com/antgroup/hugescm/pkg/version.version=$BUILD_VERSION -X github.com/antgroup/hugescm/pkg/version.buildTime=$BUILD_TIME -X github.com/antgroup/hugescm/pkg/version.buildCommit=$BUILD_COMMIT",

cmd/zeta/crate.toml

@@ -1,7 +1,7 @@
 name = "zeta"
 description = "HugeSCM - A next generation cloud-based version control system"
 destination = "bin"
-version = "0.12.14"
+version = "0.13.0"
 goflags = [
     "-ldflags",
     "-X github.com/antgroup/hugescm/pkg/version.version=$BUILD_VERSION -X github.com/antgroup/hugescm/pkg/version.buildTime=$BUILD_TIME -X github.com/antgroup/hugescm/pkg/version.buildCommit=$BUILD_COMMIT",

docs/env.md

@@ -6,6 +6,8 @@
 +  `ZETA_TRACE` 开启 zeta 调试模式
 +  `ZETA_EXTENSION_DRAGONFLY_GET` 设置 `dfget` 路径。
 +  `ZETA_SHARING_ROOT` 设置 `core.shardingRoot` 目录，该配置存在时，zeta 会使用该变量设置的目录作为 BLOB 的存储目录。
++  `ZETA_TRANSPORT_MAX_ENTRIES` 指定批量下载对象一次性下载数量，默认 32000，用户可以修改。
++  `ZETA_TRANSPORT_LARGE_SIZE` 将指定大小的文件试为大文件，使用单个文件下载接口，默认为 10M，用户可以指定 `ZETA_TRANSPORT_LARGE_SIZE=512k`。
 
 ## 注入配置
 

modules/env/constant.go

@@ -12,8 +12,9 @@ type K string
 
 // VALUE
 const (
-	ZETA_TERMINAL_PROMPT K      = "ZETA_TERMINAL_PROMPT"
-	StandardSeparator    string = ";"
+	ZETA_TERMINAL_PROMPT  K      = "ZETA_TERMINAL_PROMPT"
+	ZETA_NO_SSH_AUTH_SOCK K      = "ZETA_NO_SSH_AUTH_SOCK"
+	StandardSeparator     string = ";"
 )
 
 func (k K) With(s string) string {
@@ -57,12 +58,18 @@ func (k K) StrSplit(sep string) []string {
 
 // SimpleAtob Obtain the boolean variable from the environment variable, if it does not exist, return the default value
 func (k K) SimpleAtob(dv bool) bool {
-	v := os.Getenv(string(k))
-	return strengthen.SimpleAtob(v, dv)
+	s, ok := os.LookupEnv(string(k))
+	if !ok {
+		return dv
+	}
+	return strengthen.SimpleAtob(s, dv)
 }
 
 func (k K) SimpleAtoi(dv int64) int64 {
-	s := os.Getenv(string(k))
+	s, ok := os.LookupEnv(string(k))
+	if !ok {
+		return dv
+	}
 	if i, err := strconv.ParseInt(s, 10, 64); err == nil {
 		return i
 	}

modules/strengthen/strings.go

@@ -114,3 +114,56 @@ func SimpleAtob(s string, dv bool) bool {
 	}
 	return dv
 }
+
+const (
+	Byte int64 = 1 << (iota * 10)
+	KiByte
+	MiByte
+	GiByte
+	TiByte
+	PiByte
+	EiByte
+)
+
+func toLower(c byte) byte {
+	if 'A' <= c && c <= 'Z' {
+		c += 'a' - 'A'
+	}
+	return c
+}
+
+var (
+	ErrSyntaxSize = errors.New("size synatx error")
+)
+
+func ParseSize(data string) (int64, error) {
+	var ratio int64 = Byte
+	if len(data) == 0 {
+		return 0, ErrSyntaxSize
+	}
+	switch toLower(data[len(data)-1]) {
+	case 'k':
+		ratio = KiByte
+		data = data[0 : len(data)-1]
+	case 'm':
+		ratio = MiByte
+		data = data[0 : len(data)-1]
+	case 'g':
+		ratio = GiByte
+		data = data[0 : len(data)-1]
+	case 't':
+		ratio = GiByte
+		data = data[0 : len(data)-1]
+	case 'p':
+		ratio = PiByte
+		data = data[0 : len(data)-1]
+	case 'e':
+		ratio = EiByte
+		data = data[0 : len(data)-1]
+	}
+	sz, err := strconv.ParseInt(strings.TrimSpace(data), 10, 64)
+	if err != nil {
+		return 0, ErrSyntaxSize
+	}
+	return sz * ratio, nil
+}

modules/zeta/config/config.go

@@ -6,12 +6,14 @@ package config
 import (
 	"errors"
 	"fmt"
+
+	"github.com/antgroup/hugescm/modules/strengthen"
 )
 
 const (
 	NoZetaDir               = ""
-	FragmentThreshold int64 = 1 * GiByte //1G
-	FragmentSize      int64 = 1 * GiByte //1G
+	FragmentThreshold int64 = 1 * strengthen.GiByte //1G
+	FragmentSize      int64 = 1 * strengthen.GiByte //1G
 )
 
 // ErrNotExist commit not exist error
@@ -103,23 +105,23 @@ type Fragment struct {
 }
 
 func (f *Fragment) Overwrite(o *Fragment) {
-	if o.ThresholdRaw.Size != 0 {
+	if o.ThresholdRaw.Size > 0 {
 		f.ThresholdRaw.Size = o.ThresholdRaw.Size
 	}
-	if o.SizeRaw.Size != 0 {
+	if o.SizeRaw.Size > 0 {
 		f.SizeRaw.Size = o.SizeRaw.Size
 	}
 }
 
 func (f Fragment) Threshold() int64 {
-	if f.ThresholdRaw.Size < MiByte {
+	if f.ThresholdRaw.Size < strengthen.MiByte {
 		return FragmentThreshold
 	}
 	return f.ThresholdRaw.Size
 }
 
 func (f Fragment) Size() int64 {
-	if f.SizeRaw.Size < MiByte {
+	if f.SizeRaw.Size < strengthen.MiByte {
 		return FragmentSize
 	}
 	return f.SizeRaw.Size
@@ -137,11 +139,38 @@ func (h *HTTP) Overwrite(o *HTTP) {
 	h.SSLVerify.Merge(&o.SSLVerify)
 }
 
+type Transport struct {
+	MaxEntries   int  `toml:"maxEntries,omitempty"`
+	LargeSizeRaw Size `toml:"largeSize,omitempty"`
+}
+
+const (
+	minLargSize = 512 << 10 // 512K
+	largeSize   = 10 << 20  // 10M
+)
+
+func (t Transport) LargeSize() int64 {
+	if t.LargeSizeRaw.Size < minLargSize {
+		return largeSize
+	}
+	return t.LargeSizeRaw.Size
+}
+
+func (t *Transport) Overwrite(o *Transport) {
+	if o.LargeSizeRaw.Size >= minLargSize {
+		t.LargeSizeRaw.Size = o.LargeSizeRaw.Size
+	}
+	if o.MaxEntries > 0 {
+		t.MaxEntries = o.MaxEntries
+	}
+}
+
 type Config struct {
-	Core     Core     `toml:"core,omitempty"`
-	User     User     `toml:"user,omitempty"`
-	Fragment Fragment `toml:"fragment,omitempty"`
-	HTTP     HTTP     `toml:"http,omitempty"`
+	Core      Core      `toml:"core,omitempty"`
+	User      User      `toml:"user,omitempty"`
+	Fragment  Fragment  `toml:"fragment,omitempty"`
+	HTTP      HTTP      `toml:"http,omitempty"`
+	Transport Transport `toml:"transport,omitempty"`
 }
 
 // Overwrite: use local config overwrite config
@@ -150,4 +179,5 @@ func (c *Config) Overwrite(co *Config) {
 	c.User.Overwrite(&co.User)
 	c.Fragment.Overwrite(&co.Fragment)
 	c.HTTP.Overwrite(&co.HTTP)
+	c.Transport.Overwrite(&co.Transport)
 }

modules/zeta/config/type.go

@@ -5,21 +5,11 @@ package config
 
 import (
 	"bytes"
-	"errors"
 	"fmt"
 	"reflect"
-	"strconv"
 	"strings"
-)
 
-const (
-	Byte int64 = 1 << (iota * 10)
-	KiByte
-	MiByte
-	GiByte
-	TiByte
-	PiByte
-	EiByte
+	"github.com/antgroup/hugescm/modules/strengthen"
 )
 
 const (
@@ -135,51 +125,12 @@ type Size struct {
 	Size int64
 }
 
-func toLower(c byte) byte {
-	if 'A' <= c && c <= 'Z' {
-		c += 'a' - 'A'
-	}
-	return c
-}
-
-var (
-	ErrSyntaxSize = errors.New("size synatx error")
-)
-
-func (s *Size) UnmarshalText(text []byte) error {
+func (s *Size) UnmarshalText(text []byte) (err error) {
 	if bytes.HasSuffix(text, []byte("b")) || bytes.HasSuffix(text, []byte("B")) {
 		text = text[0 : len(text)-1]
 	}
-	if len(text) == 0 {
-		return ErrSyntaxSize
-	}
-	var ratio int64 = Byte
-	switch toLower(text[len(text)-1]) {
-	case 'k':
-		ratio = KiByte
-		text = text[0 : len(text)-1]
-	case 'm':
-		ratio = MiByte
-		text = text[0 : len(text)-1]
-	case 'g':
-		ratio = GiByte
-		text = text[0 : len(text)-1]
-	case 't':
-		ratio = GiByte
-		text = text[0 : len(text)-1]
-	case 'p':
-		ratio = PiByte
-		text = text[0 : len(text)-1]
-	case 'e':
-		ratio = EiByte
-		text = text[0 : len(text)-1]
-	}
-	sz, err := strconv.ParseInt(strings.TrimSpace(string(text)), 10, 64)
-	if err != nil {
-		return ErrSyntaxSize
-	}
-	s.Size = sz * ratio
-	return nil
+	s.Size, err = strengthen.ParseSize(string(text))
+	return
 }
 
 type Accelerator string
@@ -342,27 +293,17 @@ func valuesToInt64Array(o any) []int64 {
 	return nil
 }
 
-func simpleAtob(s string, dv bool) bool {
-	switch strings.ToLower(s) {
-	case "true", "yes", "on", "1":
-		return true
-	case "false", "no", "off", "0":
-		return false
-	}
-	return dv
-}
-
 func valuesToBoolArray(o any) []bool {
 	switch v := o.(type) {
 	case string:
-		return []bool{simpleAtob(v, false)}
+		return []bool{strengthen.SimpleAtob(v, false)}
 	case bool:
 		return []bool{v}
 	case []any:
 		values := make([]bool, 0, len(v)+1)
 		for _, e := range v {
 			if s, ok := e.(string); ok {
-				values = append(values, simpleAtob(s, false))
+				values = append(values, strengthen.SimpleAtob(s, false))
 				continue
 			}
 			if i, ok := e.(bool); ok {

pkg/transport/ssh/auth.go

@@ -13,7 +13,6 @@ import (
 	"strings"
 
 	"github.com/antgroup/hugescm/modules/env"
-	"github.com/antgroup/hugescm/modules/strengthen"
 	"github.com/antgroup/hugescm/modules/survey"
 	"golang.org/x/crypto/ssh"
 	"golang.org/x/crypto/ssh/agent"
@@ -167,7 +166,7 @@ func (c *client) openPrivateKey(name string) (ssh.Signer, error) {
 }
 
 func (c *client) sshAuthSigners() ([]ssh.Signer, error) {
-	if strengthen.SimpleAtob("ZETA_NO_SSH_AUTH_SOCK", false) {
+	if env.ZETA_NO_SSH_AUTH_SOCK.SimpleAtob(false) {
 		return nil, nil
 	}
 	sock, ok := os.LookupEnv("SSH_AUTH_SOCK")

pkg/zeta/misc.go

@@ -35,6 +35,8 @@ const (
 	ENV_ZETA_MERGE_TEXT_DRIVER         = "ZETA_MERGE_TEXT_DRIVER"
 	ENV_ZETA_EDITOR                    = "ZETA_EDITOR"
 	ENV_ZETA_SSL_NO_VERIFY             = "ZETA_SSL_NO_VERIFY"
+	ENV_ZETA_TRANSPORT_MAX_ENTRIES     = "ZETA_TRANSPORT_MAX_ENTRIES"
+	ENV_ZETA_TRANSPORT_LARGE_SIZE      = "ZETA_TRANSPORT_LARGE_SIZE"
 )
 
 var (