From 90a691a73c5e76c0004f74f6e45b0c0bfd79a646 Mon Sep 17 00:00:00 2001 From: Ruphane <24724395+LNSSPsd@users.noreply.github.com> Date: Sun, 15 Jan 2023 21:24:39 -0800 Subject: [PATCH 1/4] Fix #200 --- fastbuilder/bdump/command/pool.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastbuilder/bdump/command/pool.go b/fastbuilder/bdump/command/pool.go index 2dc214ae6..5885ccb2a 100644 --- a/fastbuilder/bdump/command/pool.go +++ b/fastbuilder/bdump/command/pool.go @@ -8,7 +8,7 @@ var BDumpCommandPool map[uint16]func()Command = map[uint16]func()Command { 8: func()Command { return &AddZValue0{} }, 9: func()Command { return &NoOperation{} }, 12: func()Command { return &AddInt32ZValue0{} }, - 13: func()Command { return &PlaceBlockWithBlockStates{} }, + 13: func()Command { return &PlaceBlockWithBlockStatesDeprecated{} }, 14: func()Command { return &AddXValue{} }, 15: func()Command { return &SubtractXValue{} }, 16: func()Command { return &AddYValue{} }, From 3827ded1954fc282ecd2978890f47d9e38ba0916 Mon Sep 17 00:00:00 2001 From: HDTianRu <103573273+HDTianRu@users.noreply.github.com> Date: Mon, 16 Jan 2023 13:27:24 +0800 Subject: [PATCH 2/4] Fix BDump Strict Statement (#197) --- fastbuilder/builder/bdump.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastbuilder/builder/bdump.go b/fastbuilder/builder/bdump.go index 460d55004..16861930f 100644 --- a/fastbuilder/builder/bdump.go +++ b/fastbuilder/builder/bdump.go @@ -49,7 +49,7 @@ func BDump(config *types.MainConfig, blc chan *types.Module) error { br := brotli.NewReader(file) signed, corrupted, signer_username, err:=bdump.VerifyStreamBDX(br) if !signed { - if !config.Strict { + if config.Strict { return fmt.Errorf("%s.", I18n.T(I18n.BDump_FileNotSigned)) }else{ types.ForwardedBrokSender <- fmt.Sprintf("%s!", I18n.T(I18n.BDump_FileNotSigned)) From 91e39c2512fc33ec407ebc9ccebd60773005d8f3 Mon Sep 17 00:00:00 2001 From: Ruphane <24724395+LNSSPsd@users.noreply.github.com> Date: Sun, 15 Jan 2023 21:29:27 -0800 Subject: [PATCH 3/4] Update version --- omega/utils/structure/bdx.go | 2 ++ version | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/omega/utils/structure/bdx.go b/omega/utils/structure/bdx.go index 970e99af7..eab5eb647 100644 --- a/omega/utils/structure/bdx.go +++ b/omega/utils/structure/bdx.go @@ -32,6 +32,8 @@ func ReadBrString(src io.Reader) (string, error) { } func handleBDXCMD(br io.Reader, infoSender func(string)) (author string, blockChan chan *IOBlockForDecoder, cancelFn func(), err error) { + err = fmt.Errorf("Please use command `bdump' instead.") + return legacyRunTimeIDRemapper := NewDoubleValueLegacyBlockToRuntimeIDMapper() blockChan = make(chan *IOBlockForDecoder, 10240) { diff --git a/version b/version index 0062ac971..32f3eaad0 100644 --- a/version +++ b/version @@ -1 +1 @@ -5.0.0 +5.0.1 \ No newline at end of file From bbc98d2663a4d6e7bfe9177a97e34b3e228f696c Mon Sep 17 00:00:00 2001 From: CMA2401PT <97827666+CMA2401PT@users.noreply.github.com> Date: Mon, 16 Jan 2023 15:46:05 +0800 Subject: [PATCH 4/4] add cos release repo --- .github/workflows/build.yml | 36 ++++++++++++++ tools/compressor/main.go | 90 +++++++++++++++++++++++++++++++++ tools/cos_uploader/go.mod | 11 +++++ tools/cos_uploader/go.sum | 23 +++++++++ tools/cos_uploader/main.go | 99 +++++++++++++++++++++++++++++++++++++ 5 files changed, 259 insertions(+) create mode 100644 tools/compressor/main.go create mode 100644 tools/cos_uploader/go.mod create mode 100644 tools/cos_uploader/go.sum create mode 100644 tools/cos_uploader/main.go diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8d97ad7a8..256e0f46d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -242,6 +242,42 @@ jobs: files: | fb-upload.tar.gz build/* + - name: Compress And Upload to COS + env: + COS_ACCESSURL: ${{ secrets.COS_ACCESSURL }} + COS_SERVICEURL: ${{ secrets.COS_SERVICEURL }} + COS_SECRETID: ${{ secrets.COS_SECRETID }} + COS_SECRETKEY: ${{ secrets.COS_SECRETKEY }} + run: | + cd tools/cos_uploader && go build -o ../../cos_uploader -ldflags "-X main.AccessUrl=${COS_ACCESSURL} -X main.ServiceUrl=${COS_SERVICEURL} -X main.SecretID=${COS_SECRETID} -X main.SecretKey=${COS_SECRETKEY}" main.go && cd ../../ + ./cos_uploader -l build -r fastbuilder_official + mkdir -p build_compressed + go run tools/compressor/main.go -in "\ + build/phoenixbuilder, + build/phoenixbuilder-aarch64, + build/phoenixbuilder-android-executable-arm64, + build/phoenixbuilder-android-executable-armv7, + build/phoenixbuilder-ios-executable, + build/phoenixbuilder-macos, + build/phoenixbuilder-macos-arm64, + build/phoenixbuilder-macos-x86_64, + build/phoenixbuilder-windows-executable-x86.exe, + build/phoenixbuilder-windows-executable-x86_64.exe + " -out "\ + build_compressed/phoenixbuilder, + build_compressed/phoenixbuilder-aarch64, + build_compressed/phoenixbuilder-android-executable-arm64, + build_compressed/phoenixbuilder-android-executable-armv7, + build_compressed/phoenixbuilder-ios-executable, + build_compressed/phoenixbuilder-macos, + build_compressed/phoenixbuilder-macos-arm64, + build_compressed/phoenixbuilder-macos-x86_64, + build_compressed/phoenixbuilder-windows-executable-x86.exe, + build_compressed/phoenixbuilder-windows-executable-x86_64.exe + " + cp build/hashes.json build_compressed + cp build/version build_compressed + ./cos_uploader -l build_compressed -r fastbuilder_official_compressed #- name: Generate index files for storage over github # run: node index_generator.js build diff --git a/tools/compressor/main.go b/tools/compressor/main.go new file mode 100644 index 000000000..de13cf554 --- /dev/null +++ b/tools/compressor/main.go @@ -0,0 +1,90 @@ +package main + +import ( + "bytes" + "flag" + "fmt" + "io/ioutil" + "os" + "strings" + "sync" + + "github.com/andybalholm/brotli" +) + +func GetFileData(fname string) ([]byte, error) { + fp, err := os.OpenFile(fname, os.O_CREATE|os.O_RDONLY, 0755) + if err != nil { + return nil, err + } + defer fp.Close() + buf, err := ioutil.ReadAll(fp) + if err != nil { + return nil, err + } + return buf, err +} + +func WriteFileData(fname string, data []byte) error { + fp, err := os.OpenFile(fname, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0755) + if err != nil { + return err + } + defer fp.Close() + if _, err := fp.Write(data); err != nil { + return err + } + return nil +} + +func CompressSingleFile(in, out string) { + var origData []byte + var err error + in = strings.TrimSpace(in) + out = strings.TrimSpace(out) + origData, err = GetFileData(in) + if err != nil { + panic(fmt.Sprintf("read %v fail: %v", in, err)) + } else if len(origData) == 0 { + panic(fmt.Sprintf("read %v fail: data length = 0", in)) + } + + buf := bytes.NewBuffer([]byte{}) + compressor := brotli.NewWriterLevel(buf, brotli.DefaultCompression) + compressor.Write(origData) + compressor.Close() + newData := buf.Bytes() + + if err := WriteFileData(out, newData); err != nil { + panic(err) + } + fmt.Printf("comprerss: %v -> %v compress %.3f\n", in, out, float32(len(newData))/float32(len(origData))) +} + +func main() { + _inFile := flag.String("in", "", "input") + _outFile := flag.String("out", "", "outfile") + flag.Parse() + inFile := strings.TrimSpace(*_inFile) + outFile := strings.TrimSpace(*_outFile) + fmt.Println(inFile, outFile) + if strings.Contains(inFile, ",") { + ins := strings.Split(inFile, ",") + outs := strings.Split(outFile, ",") + if len(ins) != len(outs) { + panic(fmt.Errorf("%v!->%v :input/outputs mismatch", ins, outs)) + } + var wg sync.WaitGroup + for i := range ins { + wg.Add(1) + in, out := strings.TrimSpace(ins[i]), strings.TrimSpace(outs[i]) + go func() { + CompressSingleFile(in, out) + wg.Done() + }() + } + wg.Wait() + } else { + CompressSingleFile(inFile, outFile) + } +} diff --git a/tools/cos_uploader/go.mod b/tools/cos_uploader/go.mod new file mode 100644 index 000000000..5923cfcbd --- /dev/null +++ b/tools/cos_uploader/go.mod @@ -0,0 +1,11 @@ +module cos_uploader + +go 1.18 + +require ( + github.com/clbanning/mxj v1.8.4 // indirect + github.com/google/go-querystring v1.1.0 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/mozillazg/go-httpheader v0.3.1 // indirect + github.com/tencentyun/cos-go-sdk-v5 v0.7.41 // indirect +) diff --git a/tools/cos_uploader/go.sum b/tools/cos_uploader/go.sum new file mode 100644 index 000000000..06d08e4a5 --- /dev/null +++ b/tools/cos_uploader/go.sum @@ -0,0 +1,23 @@ +github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409/go.mod h1:1pk82RBxDY/JZnPQrtqHlUFfCctgdorsd9M06fMynOM= +github.com/clbanning/mxj v1.8.4 h1:HuhwZtbyvyOw+3Z1AowPkU87JkJUSv751ELWaiTpj8I= +github.com/clbanning/mxj v1.8.4/go.mod h1:BVjHeAH+rl9rs6f+QIpeRl0tfu10SXn1pUSa5PVGJng= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= +github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= +github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mozillazg/go-httpheader v0.2.1/go.mod h1:jJ8xECTlalr6ValeXYdOF8fFUISeBAdw6E61aqQma60= +github.com/mozillazg/go-httpheader v0.3.1 h1:IRP+HFrMX2SlwY9riuio7raffXUpzAosHtZu25BSJok= +github.com/mozillazg/go-httpheader v0.3.1/go.mod h1:PuT8h0pw6efvp8ZeUec1Rs7dwjK08bt6gKSReGMqtdA= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.194/go.mod h1:7sCQWVkxcsR38nffDW057DRGk8mUjK1Ing/EFOK8s8Y= +github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/kms v1.0.194/go.mod h1:yrBKWhChnDqNz1xuXdSbWXG56XawEq0G5j1lg4VwBD4= +github.com/tencentyun/cos-go-sdk-v5 v0.7.41 h1:iU0Li/Np78H4SBna0ECQoF3mpgi6ImLXU+doGzPFXGc= +github.com/tencentyun/cos-go-sdk-v5 v0.7.41/go.mod h1:4dCEtLHGh8QPxHEkgq+nFaky7yZxQuYwgSJM87icDaw= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/tools/cos_uploader/main.go b/tools/cos_uploader/main.go new file mode 100644 index 000000000..fa47b466b --- /dev/null +++ b/tools/cos_uploader/main.go @@ -0,0 +1,99 @@ +package main + +import ( + "context" + "flag" + "fmt" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" + + "github.com/tencentyun/cos-go-sdk-v5" +) + +var ( + AccessUrl = "https://data-?????.cos.ap-shanghai.myqcloud.com" + ServiceUrl = "https://cos.ap-??????.myqcloud.com" + SecretID = "AKxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + SecretKey = "????????????????????????????????" +) + +func AcquireCosClient() *cos.Client { + fmt.Printf("AccessUrl: %v... ServiceUrl: %v... SecretID: %v... SecretKey: %v... \n", AccessUrl[:16], ServiceUrl[:16], SecretID[:4], SecretKey[:1]) + u, _ := url.Parse(AccessUrl) + su, _ := url.Parse(ServiceUrl) + b := &cos.BaseURL{BucketURL: u, ServiceURL: su} + client := cos.NewClient(b, &http.Client{ + Transport: &cos.AuthorizationTransport{ + SecretID: SecretID, + SecretKey: SecretKey, + }, + }) + return client +} + +var localDir = flag.String("l", ".", "local dir to upload") +var remoteDir = flag.String("r", "", "remote dir to upload to") + +func main() { + flag.Parse() + client := AcquireCosClient() + fmt.Println(client) + + s, _, err := client.Service.Get(context.Background()) + if err != nil { + panic(err) + } + if len(s.Buckets) == 0 { + _, err := client.Bucket.Put(context.Background(), nil) + if err != nil { + panic(err) + } + } + for _, b := range s.Buckets { + fmt.Printf("Bucket: %#v\n", b) + } + localDirName := *localDir + remoteDirName := *remoteDir + + localDirName, err = filepath.Abs(localDirName) + if err != nil { + panic(err) + } + fmt.Printf("dir to upload: %v -> %v \n", localDirName, remoteDirName) + + if err = filepath.Walk(localDirName, func(fullPath string, info os.FileInfo, err error) error { + i := 0 + // have to do so to avoid error remote path + for i = 0; i < len(localDirName) && i < len(fullPath); i++ { + if localDirName[i] != fullPath[i] { + break + } + } + fileSubPath := fullPath[i:] + if len(fileSubPath) > 0 && fileSubPath[0] == '/' { + fileSubPath = fileSubPath[1:] + } + // no need (also cannot) to upload dir + if info.IsDir() { + return nil + } + targetFilePath := fileSubPath + if len(remoteDirName) > 0 && remoteDirName[len(remoteDirName)-1] != '/' { + remoteDirName = strings.TrimRight(remoteDirName, "/") + targetFilePath = remoteDirName + "/" + fileSubPath + } + fmt.Printf("uploading %v -> %v\n", fullPath, targetFilePath) + _, err = client.Object.PutFromFile(context.Background(), targetFilePath, fullPath, nil) + if err != nil { + fmt.Printf("on uploading %v -> %v, error happen %v", fullPath, targetFilePath, err) + return err + } + return nil + }); err != nil { + panic(err) + } + fmt.Println("Done") +}