From 31e98f5aa719522e1565ff7481d46a188b8c8caf Mon Sep 17 00:00:00 2001 From: Jason Song Date: Fri, 2 Aug 2024 18:26:03 +0800 Subject: [PATCH 1/6] fix: specify const types to support the 386 arch --- pkg/encoder.go | 13 ++++++------- pkg/seekable.go | 8 +++++--- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/pkg/encoder.go b/pkg/encoder.go index 89ad4ef..1f9d98c 100644 --- a/pkg/encoder.go +++ b/pkg/encoder.go @@ -2,7 +2,6 @@ package seekable import ( "fmt" - "math" "github.com/cespare/xxhash/v2" "go.uber.org/zap" @@ -27,9 +26,9 @@ func NewEncoder(encoder ZSTDEncoder, opts ...wOption) (Encoder, error) { } func (s *writerImpl) Encode(src []byte) ([]byte, error) { - if len(src) > math.MaxUint32 { + if int64(len(src)) > maxChunkSize { return nil, fmt.Errorf("chunk size too big for seekable format: %d > %d", - len(src), math.MaxUint32) + len(src), maxChunkSize) } if len(src) == 0 { @@ -38,9 +37,9 @@ func (s *writerImpl) Encode(src []byte) ([]byte, error) { dst := s.enc.EncodeAll(src, nil) - if len(dst) > math.MaxUint32 { + if int64(len(dst)) > maxChunkSize { return nil, fmt.Errorf("result size too big for seekable format: %d > %d", - len(src), math.MaxUint32) + len(src), maxChunkSize) } entry := seekTableEntry{ @@ -56,9 +55,9 @@ func (s *writerImpl) Encode(src []byte) ([]byte, error) { } func (s *writerImpl) EndStream() ([]byte, error) { - if len(s.frameEntries) > math.MaxUint32 { + if int64(len(s.frameEntries)) > maxChunkSize { return nil, fmt.Errorf("number of frames for seekable format: %d > %d", - len(s.frameEntries), math.MaxUint32) + len(s.frameEntries), maxChunkSize) } seekTable := make([]byte, len(s.frameEntries)*12+9) diff --git a/pkg/seekable.go b/pkg/seekable.go index b5c6658..e4b7074 100644 --- a/pkg/seekable.go +++ b/pkg/seekable.go @@ -36,9 +36,9 @@ const ( https://github.com/facebook/zstd/blob/dev/contrib/seekable_format/zstd_seekable_compression_format.md */ - skippableFrameMagic = 0x184D2A50 + skippableFrameMagic uint32 = 0x184D2A50 - seekableMagicNumber = 0x8F92EAB1 + seekableMagicNumber uint32 = 0x8F92EAB1 seekTableFooterOffset = 9 @@ -49,6 +49,8 @@ const ( maxDecoderFrameSize = 128 << 20 seekableTag = 0xE + + maxChunkSize int64 = math.MaxUint32 ) /* @@ -234,7 +236,7 @@ func createSkippableFrame(tag uint32, payload []byte) ([]byte, error) { return nil, fmt.Errorf("requested tag (%d) > 0xf", tag) } - if len(payload) > math.MaxUint32 { + if int64(len(payload)) > maxChunkSize { return nil, fmt.Errorf("requested skippable frame size (%d) > max uint32", len(payload)) } From a1b5b9371142b4397de883a9d074a0265e513b13 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Fri, 2 Aug 2024 18:45:02 +0800 Subject: [PATCH 2/6] chore: improve workflow to test building on more os arch --- .github/workflows/go.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 0836ca9..c396e70 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -29,7 +29,12 @@ jobs: go get . - name: Build (${{ matrix.dir }}) working-directory: ./${{ matrix.dir }} - run: go build -v ./... + run: | + for OSARCH in $(go tool dist list); do + IFS="/" read -r OS ARCH <<< "$OSARCH" + echo "Building for $OS $ARCH" + GOOS=$OS GOARCH=$ARCH go build ./... + done - name: Test (${{ matrix.dir }}) working-directory: ./${{ matrix.dir }} run: go test -v ./... From 6193e68365ba07580bdc7b757b26715a0430b804 Mon Sep 17 00:00:00 2001 From: Alexey Ivanov Date: Fri, 2 Aug 2024 07:11:53 -0700 Subject: [PATCH 3/6] seekable: add maxNumberOfFrames constant --- pkg/encoder.go | 4 ++-- pkg/seekable.go | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/pkg/encoder.go b/pkg/encoder.go index 1f9d98c..7b343bd 100644 --- a/pkg/encoder.go +++ b/pkg/encoder.go @@ -55,9 +55,9 @@ func (s *writerImpl) Encode(src []byte) ([]byte, error) { } func (s *writerImpl) EndStream() ([]byte, error) { - if int64(len(s.frameEntries)) > maxChunkSize { + if int64(len(s.frameEntries)) > maxNumberOfFrames { return nil, fmt.Errorf("number of frames for seekable format: %d > %d", - len(s.frameEntries), maxChunkSize) + len(s.frameEntries), maxNumberOfFrames) } seekTable := make([]byte, len(s.frameEntries)*12+9) diff --git a/pkg/seekable.go b/pkg/seekable.go index e4b7074..795a5e9 100644 --- a/pkg/seekable.go +++ b/pkg/seekable.go @@ -50,7 +50,11 @@ const ( seekableTag = 0xE + // maximum size of a single frame maxChunkSize int64 = math.MaxUint32 + + // maximum number of frames in a seekable stream + maxNumberOfFrames int64 = math.MaxUint32 ) /* From 114ee75b5b057a395021c35c9b449ad9fc374ab1 Mon Sep 17 00:00:00 2001 From: Alexey Ivanov Date: Fri, 2 Aug 2024 07:15:05 -0700 Subject: [PATCH 4/6] go: enable cgo for multi-arch builds --- .github/workflows/go.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index c396e70..b02349e 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -33,7 +33,7 @@ jobs: for OSARCH in $(go tool dist list); do IFS="/" read -r OS ARCH <<< "$OSARCH" echo "Building for $OS $ARCH" - GOOS=$OS GOARCH=$ARCH go build ./... + CGO_ENABLED=1 GOOS=$OS GOARCH=$ARCH go build ./... done - name: Test (${{ matrix.dir }}) working-directory: ./${{ matrix.dir }} From 0d60cdaf0123690fcd1861bfd230ac049635b22c Mon Sep 17 00:00:00 2001 From: Alexey Ivanov Date: Fri, 2 Aug 2024 07:40:52 -0700 Subject: [PATCH 5/6] github: switch to go workspace, allowlist a subset of OS/arch --- .github/workflows/go.yml | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index b02349e..d8e1f14 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -27,13 +27,31 @@ jobs: working-directory: ./${{ matrix.dir }} run: | go get . + - name: go work + run: | + if [ "${{ matrix.dir }} " == "pkg" ]; then + echo "Skipping go workspace for ${{ matrix.dir }}" + rm -f go.work* + exit 0 + fi + go work init + go work use pkg + go work use ${{ matrix.dir }} - name: Build (${{ matrix.dir }}) working-directory: ./${{ matrix.dir }} run: | for OSARCH in $(go tool dist list); do + case $OSARCH in + freebsd*|darwin*|linux*|windows*|wasip*) ;; + *) continue ;; + esac + case $OSARCH in + *arm64|*arm|*amd64|*386|*riscv64|*wasm) ;; + *) continue ;; + esac IFS="/" read -r OS ARCH <<< "$OSARCH" echo "Building for $OS $ARCH" - CGO_ENABLED=1 GOOS=$OS GOARCH=$ARCH go build ./... + GOOS=$OS GOARCH=$ARCH go build ./... done - name: Test (${{ matrix.dir }}) working-directory: ./${{ matrix.dir }} From 301c94bd5a131b88a75042ae694d5a451a963e9a Mon Sep 17 00:00:00 2001 From: Alexey Ivanov Date: Fri, 2 Aug 2024 07:44:44 -0700 Subject: [PATCH 6/6] . --- .github/workflows/go.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index d8e1f14..5eee8b7 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -29,7 +29,7 @@ jobs: go get . - name: go work run: | - if [ "${{ matrix.dir }} " == "pkg" ]; then + if [ "${{ matrix.dir }}" == "pkg" ]; then echo "Skipping go workspace for ${{ matrix.dir }}" rm -f go.work* exit 0