From df2029166058c81b6e7d2768f9441431a04e70cd Mon Sep 17 00:00:00 2001 From: sesky4 Date: Thu, 10 Jun 2021 16:35:52 +0800 Subject: [PATCH 1/2] lz4: update lz4 version to v4.1.7 to avoid possibly panic --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 7ebe7f59258d..01aee64c4450 100644 --- a/go.mod +++ b/go.mod @@ -48,7 +48,7 @@ require ( github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f github.com/opentracing/opentracing-go v1.2.0 // github.com/pierrec/lz4 v2.0.5+incompatible - github.com/pierrec/lz4/v4 v4.1.1 + github.com/pierrec/lz4/v4 v4.1.7 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.10.0 github.com/prometheus/client_model v0.2.0 diff --git a/go.sum b/go.sum index 3aafe839b863..4d2d0f95490b 100644 --- a/go.sum +++ b/go.sum @@ -1348,8 +1348,8 @@ github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0 github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4 v2.5.2+incompatible h1:WCjObylUIOlKy/+7Abdn34TLIkXiA4UWUMhxq9m9ZXI= github.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= -github.com/pierrec/lz4/v4 v4.1.1 h1:cS6aGkNLJr4u+UwaA21yp+gbWN3WJWtKo1axmPDObMA= -github.com/pierrec/lz4/v4 v4.1.1/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pierrec/lz4/v4 v4.1.7 h1:UDV9geJWhFIufAliH7HQlz9wP3JA0t748w+RwbWMLow= +github.com/pierrec/lz4/v4 v4.1.7/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= From 863952dab19a33a389d40a487c174aed16bebc99 Mon Sep 17 00:00:00 2001 From: sesky4 Date: Thu, 10 Jun 2021 16:56:11 +0800 Subject: [PATCH 2/2] lz4: add lz4 v4.1.7 to vendor --- vendor/github.com/pierrec/lz4/v4/.gitignore | 4 +- vendor/github.com/pierrec/lz4/v4/.travis.yml | 19 --- vendor/github.com/pierrec/lz4/v4/README.md | 8 +- vendor/github.com/pierrec/lz4/v4/go.sum | 3 - .../pierrec/lz4/v4/internal/lz4block/block.go | 7 +- .../{decode_asm.go => decode_amd64.go} | 3 +- .../lz4/v4/internal/lz4block/decode_amd64.s | 130 +++++++++++++++--- .../lz4/v4/internal/lz4block/decode_arm.go | 15 ++ .../lz4/v4/internal/lz4block/decode_arm.s | 34 ++--- .../lz4/v4/internal/lz4block/decode_noasm.go | 7 + .../lz4/v4/internal/lz4block/decode_other.go | 51 ++++++- .../lz4/v4/internal/lz4stream/block.go | 29 +++- .../lz4/v4/internal/lz4stream/frame.go | 18 ++- vendor/github.com/pierrec/lz4/v4/lz4.go | 12 +- vendor/github.com/pierrec/lz4/v4/options.go | 5 +- vendor/github.com/pierrec/lz4/v4/reader.go | 22 ++- vendor/github.com/pierrec/lz4/v4/writer.go | 2 +- vendor/modules.txt | 2 +- 18 files changed, 278 insertions(+), 93 deletions(-) delete mode 100644 vendor/github.com/pierrec/lz4/v4/.travis.yml rename vendor/github.com/pierrec/lz4/v4/internal/lz4block/{decode_asm.go => decode_amd64.go} (59%) create mode 100644 vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_arm.go create mode 100644 vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_noasm.go diff --git a/vendor/github.com/pierrec/lz4/v4/.gitignore b/vendor/github.com/pierrec/lz4/v4/.gitignore index 5e987350471d..5d7e88de0a3d 100644 --- a/vendor/github.com/pierrec/lz4/v4/.gitignore +++ b/vendor/github.com/pierrec/lz4/v4/.gitignore @@ -31,4 +31,6 @@ Temporary Items # End of https://www.gitignore.io/api/macos cmd/*/*exe -.idea \ No newline at end of file +.idea + +fuzz/*.zip diff --git a/vendor/github.com/pierrec/lz4/v4/.travis.yml b/vendor/github.com/pierrec/lz4/v4/.travis.yml deleted file mode 100644 index 4a9819e03a13..000000000000 --- a/vendor/github.com/pierrec/lz4/v4/.travis.yml +++ /dev/null @@ -1,19 +0,0 @@ -language: go - -env: - - GO111MODULE=off - -go: - - 1.13.x - - 1.14.x - -matrix: - fast_finish: true - -sudo: false - -script: - - go test -v -cpu=2 - - go test -v -cpu=2 -race - - go test -v -cpu=2 -tags noasm - - go test -v -cpu=2 -race -tags noasm diff --git a/vendor/github.com/pierrec/lz4/v4/README.md b/vendor/github.com/pierrec/lz4/v4/README.md index 4ee388e81bfb..df027e2c3018 100644 --- a/vendor/github.com/pierrec/lz4/v4/README.md +++ b/vendor/github.com/pierrec/lz4/v4/README.md @@ -1,7 +1,7 @@ # lz4 : LZ4 compression in pure Go -[![GoDoc](https://godoc.org/github.com/pierrec/lz4?status.svg)](https://godoc.org/github.com/pierrec/lz4) -[![Build Status](https://travis-ci.org/pierrec/lz4.svg?branch=master)](https://travis-ci.org/pierrec/lz4) +[![Go Reference](https://pkg.go.dev/badge/github.com/pierrec/lz4/v4.svg)](https://pkg.go.dev/github.com/pierrec/lz4/v4) +[![CI](https://github.com/pierrec/lz4/workflows/ci/badge.svg)](https://github.com/pierrec/lz4/actions) [![Go Report Card](https://goreportcard.com/badge/github.com/pierrec/lz4)](https://goreportcard.com/report/github.com/pierrec/lz4) [![GitHub tag (latest SemVer)](https://img.shields.io/github/tag/pierrec/lz4.svg?style=social)](https://github.com/pierrec/lz4/tags) @@ -15,13 +15,13 @@ The implementation is based on the reference C [one](https://github.com/lz4/lz4) Assuming you have the go toolchain installed: ``` -go get github.com/pierrec/lz4 +go get github.com/pierrec/lz4/v4 ``` There is a command line interface tool to compress and decompress LZ4 files. ``` -go install github.com/pierrec/lz4/cmd/lz4c +go install github.com/pierrec/lz4/v4/cmd/lz4c ``` Usage diff --git a/vendor/github.com/pierrec/lz4/v4/go.sum b/vendor/github.com/pierrec/lz4/v4/go.sum index 6973bd668a8f..e69de29bb2d1 100644 --- a/vendor/github.com/pierrec/lz4/v4/go.sum +++ b/vendor/github.com/pierrec/lz4/v4/go.sum @@ -1,3 +0,0 @@ -github.com/pierrec/lz4 v1.0.1 h1:w6GMGWSsCI04fTM8wQRdnW74MuJISakuUU0onU0TYB4= -github.com/pierrec/lz4 v2.6.0+incompatible h1:Ix9yFKn1nSPBLFl/yZknTp8TU5G4Ps0JDmguYK6iH1A= -github.com/pierrec/lz4 v2.6.0+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= diff --git a/vendor/github.com/pierrec/lz4/v4/internal/lz4block/block.go b/vendor/github.com/pierrec/lz4/v4/internal/lz4block/block.go index f38264943084..88319105a6c8 100644 --- a/vendor/github.com/pierrec/lz4/v4/internal/lz4block/block.go +++ b/vendor/github.com/pierrec/lz4/v4/internal/lz4block/block.go @@ -41,11 +41,11 @@ func CompressBlockBound(n int) int { return n + n/255 + 16 } -func UncompressBlock(src, dst []byte) (int, error) { +func UncompressBlock(src, dst, dict []byte) (int, error) { if len(src) == 0 { return 0, nil } - if di := decodeBlock(dst, src); di >= 0 { + if di := decodeBlock(dst, src, dict); di >= 0 { return di, nil } return 0, lz4errors.ErrInvalidSourceShortBuffer @@ -187,6 +187,9 @@ func (c *Compressor) CompressBlock(src, dst []byte) (int, error) { } mLen = si - mLen + if di >= len(dst) { + return 0, lz4errors.ErrInvalidSourceShortBuffer + } if mLen < 0xF { dst[di] = byte(mLen) } else { diff --git a/vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_asm.go b/vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_amd64.go similarity index 59% rename from vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_asm.go rename to vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_amd64.go index e26f8cd613ed..e16bd57937d2 100644 --- a/vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_asm.go +++ b/vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_amd64.go @@ -1,4 +1,3 @@ -// +build amd64 arm // +build !appengine // +build gc // +build !noasm @@ -6,4 +5,4 @@ package lz4block //go:noescape -func decodeBlock(dst, src []byte) int +func decodeBlock(dst, src, dict []byte) int diff --git a/vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_amd64.s b/vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_amd64.s index be79faa3fe85..dfcca57291a4 100644 --- a/vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_amd64.s +++ b/vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_amd64.s @@ -16,9 +16,11 @@ // R11 &dst // R12 short output end // R13 short input end -// func decodeBlock(dst, src []byte) int -// using 50 bytes of stack currently -TEXT ·decodeBlock(SB), NOSPLIT, $64-56 +// R14 &dict +// R15 &dict + len(dict) + +// func decodeBlock(dst, src, dict []byte) int +TEXT ·decodeBlock(SB), NOSPLIT, $48-80 MOVQ dst_base+0(FP), DI MOVQ DI, R11 MOVQ dst_len+8(FP), R8 @@ -30,6 +32,10 @@ TEXT ·decodeBlock(SB), NOSPLIT, $64-56 JE err_corrupt ADDQ SI, R9 + MOVQ dict_base+48(FP), R14 + MOVQ dict_len+56(FP), R15 + ADDQ R14, R15 + // shortcut ends // short output end MOVQ R8, R12 @@ -96,6 +102,8 @@ loop: // match length, we already have the offset. CMPQ CX, $0xF JEQ match_len_loop_pre + CMPQ DX, R11 + JLT match_len_loop_pre CMPQ DX, $8 JLT match_len_loop_pre CMPQ AX, R11 @@ -174,6 +182,9 @@ copy_literal: MOVOU (SI), X0 MOVOU X0, (DI) + ADDQ CX, SI + ADDQ CX, DI + JMP finish_lit_copy memmove_lit: @@ -181,18 +192,20 @@ memmove_lit: MOVQ DI, 0(SP) MOVQ SI, 8(SP) MOVQ CX, 16(SP) - // spill + + // Spill registers. Increment SI, DI now so we don't need to save CX. + ADDQ CX, DI + ADDQ CX, SI MOVQ DI, 24(SP) MOVQ SI, 32(SP) - MOVQ CX, 40(SP) // need len to inc SI, DI after - MOVB DX, 48(SP) + MOVL DX, 40(SP) + CALL runtime·memmove(SB) // restore registers MOVQ 24(SP), DI MOVQ 32(SP), SI - MOVQ 40(SP), CX - MOVB 48(SP), DX + MOVL 40(SP), DX // recalc initial values MOVQ dst_base+0(FP), R8 @@ -206,9 +219,6 @@ memmove_lit: SUBQ $16, R13 finish_lit_copy: - ADDQ CX, SI - ADDQ CX, DI - CMPQ SI, R9 JGE end @@ -278,7 +288,7 @@ copy_match: // check BX is within dst // if BX < &dst CMPQ BX, R11 - JLT err_short_buf + JLT copy_match_from_dict // if offset + match_len < di LEAQ (BX)(CX*1), AX @@ -325,21 +335,49 @@ copy_interior_match: ADDQ CX, DI JMP loop -memmove_match: +copy_match_from_dict: + // CX = match_len + // BX = &dst + (di - offset) + + // AX = offset - di = dict_bytes_available => count of bytes potentially covered by the dictionary + MOVQ R11, AX + SUBQ BX, AX + + // BX = &dict_end - dict_bytes_available + MOVQ R15, BX + SUBQ AX, BX + + // check BX is within dict + // if BX < &dict + CMPQ BX, R14 + JLT err_short_dict + + // if match_len > dict_bytes_available, match fits entirely within external dictionary : just copy + CMPQ CX, AX + JLT memmove_match + + // The match stretches over the dictionary and our block + // 1) copy what comes from the dictionary + // AX = dict_bytes_available = copy_size + // BX = &dict_end - copy_size + // CX = match_len + // memmove(to, from, len) MOVQ DI, 0(SP) MOVQ BX, 8(SP) - MOVQ CX, 16(SP) + MOVQ AX, 16(SP) + // store extra stuff we want to recover // spill MOVQ DI, 24(SP) MOVQ SI, 32(SP) - MOVQ CX, 40(SP) // need len to inc SI, DI after + MOVQ CX, 40(SP) CALL runtime·memmove(SB) // restore registers + MOVQ 16(SP), AX // copy_size MOVQ 24(SP), DI MOVQ 32(SP), SI - MOVQ 40(SP), CX + MOVQ 40(SP), CX // match_len // recalc initial values MOVQ dst_base+0(FP), R8 @@ -347,23 +385,77 @@ memmove_match: ADDQ dst_len+8(FP), R8 MOVQ src_base+24(FP), R9 ADDQ src_len+32(FP), R9 + MOVQ dict_base+48(FP), R14 + MOVQ dict_len+56(FP), R15 + ADDQ R14, R15 MOVQ R8, R12 SUBQ $32, R12 MOVQ R9, R13 SUBQ $16, R13 + // di+=copy_size + ADDQ AX, DI + + // 2) copy the rest from the current block + // CX = match_len - copy_size = rest_size + SUBQ AX, CX + MOVQ R11, BX + + // check if we have a copy overlap + // AX = &dst + rest_size + MOVQ CX, AX + ADDQ BX, AX + // if &dst + rest_size > di, copy byte by byte + CMPQ AX, DI + + JGT copy_match_loop + +memmove_match: + // memmove(to, from, len) + MOVQ DI, 0(SP) + MOVQ BX, 8(SP) + MOVQ CX, 16(SP) + + // Spill registers. Increment DI now so we don't need to save CX. ADDQ CX, DI + MOVQ DI, 24(SP) + MOVQ SI, 32(SP) + + CALL runtime·memmove(SB) + + // restore registers + MOVQ 24(SP), DI + MOVQ 32(SP), SI + + // recalc initial values + MOVQ dst_base+0(FP), R8 + MOVQ R8, R11 // TODO: make these sensible numbers + ADDQ dst_len+8(FP), R8 + MOVQ src_base+24(FP), R9 + ADDQ src_len+32(FP), R9 + MOVQ R8, R12 + SUBQ $32, R12 + MOVQ R9, R13 + SUBQ $16, R13 + MOVQ dict_base+48(FP), R14 + MOVQ dict_len+56(FP), R15 + ADDQ R14, R15 + JMP loop err_corrupt: - MOVQ $-1, ret+48(FP) + MOVQ $-1, ret+72(FP) RET err_short_buf: - MOVQ $-2, ret+48(FP) + MOVQ $-2, ret+72(FP) + RET + +err_short_dict: + MOVQ $-3, ret+72(FP) RET end: SUBQ R11, DI - MOVQ DI, ret+48(FP) + MOVQ DI, ret+72(FP) RET diff --git a/vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_arm.go b/vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_arm.go new file mode 100644 index 000000000000..c35acda0550f --- /dev/null +++ b/vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_arm.go @@ -0,0 +1,15 @@ +// +build gc,!noasm + +package lz4block + +func decodeBlock(dst, src, dict []byte) int { + if len(dict) == 0 { + return decodeBlockNodict(dst, src) + } + return decodeBlockGo(dst, src, dict) +} + +// Assembler version of decodeBlock, without linked block support. + +//go:noescape +func decodeBlockNodict(dst, src []byte) int diff --git a/vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_arm.s b/vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_arm.s index ec94b7b3c316..5ae321468509 100644 --- a/vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_arm.s +++ b/vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_arm.s @@ -19,12 +19,12 @@ #define minMatch $4 -// func decodeBlock(dst, src []byte) int -TEXT ·decodeBlock(SB), NOFRAME|NOSPLIT, $-4-28 - MOVW dst_base +0(FP), dst - MOVW dst_len +4(FP), dstend - MOVW src_base+12(FP), src - MOVW src_len +16(FP), srcend +// func decodeBlockNodict(dst, src []byte) int +TEXT ·decodeBlockNodict(SB), NOFRAME+NOSPLIT, $-4-28 + MOVW dst_base +0(FP), dst + MOVW dst_len +4(FP), dstend + MOVW src_base +12(FP), src + MOVW src_len +16(FP), srcend CMP $0, srcend BEQ shortSrc @@ -140,25 +140,22 @@ readMatchlenLoop: BEQ readMatchlenLoop readMatchlenDone: - ADD minMatch, len - - // Bounds check dst+len and match = dst-offset. + // Bounds check dst+len+minMatch and match = dst-offset. ADD dst, len, tmp1 + ADD minMatch, tmp1 CMP dstend, tmp1 //BHI shortDst // Uncomment for distinct error codes. SUB offset, dst, match CMP.LS match, dstorig BHI corrupt - // If the offset is at least four (len is, because of minMatch), - // do a four-way unrolled byte copy loop. Using MOVD instead of four - // byte loads is much faster, but to remain portable we'd have to - // align match first, which in turn is too expensive. - CMP $4, offset - BLO copyMatch - - SUB $4, len + // Since len+minMatch is at least four, we can do a 4× unrolled + // byte copy loop. Using MOVW instead of four byte loads is faster, + // but to remain portable we'd have to align match first, which is + // too expensive. By alternating loads and stores, we also handle + // the case offset < 4. copyMatch4: + SUB.S $4, len MOVBU.P 4(match), tmp1 MOVB.P tmp1, 4(dst) MOVBU -3(match), tmp2 @@ -167,7 +164,6 @@ copyMatch4: MOVB tmp3, -2(dst) MOVBU -1(match), tmp1 MOVB tmp1, -1(dst) - SUB.S $4, len BPL copyMatch4 // Restore len, which is now negative. @@ -175,7 +171,7 @@ copyMatch4: BEQ copyMatchDone copyMatch: - // Simple byte-at-a-time copy. + // Finish with a byte-at-a-time copy. SUB.S $1, len MOVBU.P 1(match), tmp2 MOVB.P tmp2, 1(dst) diff --git a/vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_noasm.go b/vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_noasm.go new file mode 100644 index 000000000000..1cc703834ec9 --- /dev/null +++ b/vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_noasm.go @@ -0,0 +1,7 @@ +// +build !amd64,!arm appengine !gc noasm + +package lz4block + +func decodeBlock(dst, src, dict []byte) int { + return decodeBlockGo(dst, src, dict) +} diff --git a/vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_other.go b/vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_other.go index 9065653a9f28..4b580d4a6ff5 100644 --- a/vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_other.go +++ b/vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_other.go @@ -1,8 +1,15 @@ -// +build !amd64,!arm appengine !gc noasm - package lz4block -func decodeBlock(dst, src []byte) (ret int) { +import ( + "encoding/binary" +) + +func decodeBlockGo(dst, src, dict []byte) (ret int) { + // Restrict capacities so we don't read or write out of bounds. + dst = dst[:len(dst):len(dst)] + src = src[:len(src):len(src)] + dictLen := uint(len(dict)) + const hasError = -2 defer func() { if recover() != nil { @@ -32,7 +39,7 @@ func decodeBlock(dst, src []byte) (ret int) { // if the match length (4..18) fits within the literals, then copy // all 18 bytes, even if not all are part of the literals. mLen += 4 - if offset := uint(src[si]) | uint(src[si+1])<<8; mLen <= offset { + if offset := u16(src[si:]); mLen <= offset && offset < di { i := di - offset end := i + 18 if end > uint(len(dst)) { @@ -66,7 +73,7 @@ func decodeBlock(dst, src []byte) (ret int) { return hasError } - offset := uint(src[si]) | uint(src[si+1])<<8 + offset := u16(src[si:]) if offset == 0 { return hasError } @@ -85,6 +92,38 @@ func decodeBlock(dst, src []byte) (ret int) { mLen += minMatch // Copy the match. + if di < offset { + // The match is beyond our block, meaning in the dictionary + if offset-di > mLen { + // The match is entirely contained in the dictionary. Just copy! + copy(dst[di:di+mLen], dict[dictLen+di-offset:dictLen+di-offset+mLen]) + di = di + mLen + } else { + // The match stretches over the dictionary and our block + copySize := offset - di + restSize := mLen - copySize + + copy(dst[di:di+copySize], dict[dictLen-copySize:]) + di = di + copySize + + if di < restSize { + // Overlap - we want to copy more than what we have available, + // so copy byte per byte. + copyFrom := 0 + endOfMatch := di + restSize + for di < endOfMatch { + dst[di] = dst[copyFrom] + di = di + 1 + copyFrom = copyFrom + 1 + } + } else { + copy(dst[di:di+restSize], dst[0:restSize]) + di = di + restSize + } + } + continue + } + expanded := dst[di-offset:] if mLen > offset { // Efficiently copy the match dst[di-offset:di] into the dst slice. @@ -98,3 +137,5 @@ func decodeBlock(dst, src []byte) (ret int) { di += uint(copy(dst[di:di+mLen], expanded[:mLen])) } } + +func u16(p []byte) uint { return uint(binary.LittleEndian.Uint16(p)) } diff --git a/vendor/github.com/pierrec/lz4/v4/internal/lz4stream/block.go b/vendor/github.com/pierrec/lz4/v4/internal/lz4stream/block.go index 279a8cc49330..e96465460c5d 100644 --- a/vendor/github.com/pierrec/lz4/v4/internal/lz4stream/block.go +++ b/vendor/github.com/pierrec/lz4/v4/internal/lz4stream/block.go @@ -67,8 +67,9 @@ func (b *Blocks) close(f *Frame, num int) error { return err } if b.Blocks == nil { - // Not initialized yet. - return nil + err := b.err + b.err = nil + return err } c := make(chan *FrameDataBlock) b.Blocks <- c @@ -114,18 +115,23 @@ func (b *Blocks) initR(f *Frame, num int, src io.Reader) (chan []byte, error) { block := NewFrameDataBlock(f) cumx, err = block.Read(f, src, 0) if err != nil { + block.Close(f) break } // Recheck for an error as reading may be slow and uncompressing is expensive. if b.ErrorR() != nil { + block.Close(f) break } c := make(chan []byte) blocks <- c go func() { - data, err := block.Uncompress(f, size.Get(), false) + defer block.Close(f) + data, err := block.Uncompress(f, size.Get(), nil, false) if err != nil { b.closeR(err) + // Close the block channel to indicate an error. + close(c) } else { c <- data } @@ -146,13 +152,24 @@ func (b *Blocks) initR(f *Frame, num int, src io.Reader) (chan []byte, error) { // on the returned channel. go func(leg bool) { defer close(blocks) + skipBlocks := false for c := range blocks { - buf := <-c + buf, ok := <-c + if !ok { + // A closed channel indicates an error. + // All remaining channels should be discarded. + skipBlocks = true + continue + } if buf == nil { // Signal to end the loop. close(c) return } + if skipBlocks { + // A previous error has occurred, skipping remaining channels. + continue + } // Perform checksum now as the blocks are received in order. if f.Descriptor.Flags.ContentChecksum() { _, _ = f.checksum.Write(buf) @@ -299,12 +316,12 @@ func (b *FrameDataBlock) Read(f *Frame, src io.Reader, cum uint32) (uint32, erro return x, nil } -func (b *FrameDataBlock) Uncompress(f *Frame, dst []byte, sum bool) ([]byte, error) { +func (b *FrameDataBlock) Uncompress(f *Frame, dst, dict []byte, sum bool) ([]byte, error) { if b.Size.Uncompressed() { n := copy(dst, b.data) dst = dst[:n] } else { - n, err := lz4block.UncompressBlock(b.data, dst) + n, err := lz4block.UncompressBlock(b.data, dst, dict) if err != nil { return nil, err } diff --git a/vendor/github.com/pierrec/lz4/v4/internal/lz4stream/frame.go b/vendor/github.com/pierrec/lz4/v4/internal/lz4stream/frame.go index cfbd5674d9d1..18192a9433d5 100644 --- a/vendor/github.com/pierrec/lz4/v4/internal/lz4stream/frame.go +++ b/vendor/github.com/pierrec/lz4/v4/internal/lz4stream/frame.go @@ -77,16 +77,16 @@ func (f *Frame) isLegacy() bool { return f.Magic == frameMagicLegacy } -func (f *Frame) InitR(src io.Reader, num int) (chan []byte, error) { +func (f *Frame) ParseHeaders(src io.Reader) error { if f.Magic > 0 { // Header already read. - return nil, nil + return nil } newFrame: var err error if f.Magic, err = f.readUint32(src); err != nil { - return nil, err + return err } switch m := f.Magic; { case m == frameMagic || m == frameMagicLegacy: @@ -94,19 +94,23 @@ newFrame: case m>>8 == frameSkipMagic>>8: skip, err := f.readUint32(src) if err != nil { - return nil, err + return err } if _, err := io.CopyN(ioutil.Discard, src, int64(skip)); err != nil { - return nil, err + return err } goto newFrame default: - return nil, lz4errors.ErrInvalidFrame + return lz4errors.ErrInvalidFrame } if err := f.Descriptor.initR(f, src); err != nil { - return nil, err + return err } f.checksum.Reset() + return nil +} + +func (f *Frame) InitR(src io.Reader, num int) (chan []byte, error) { return f.Blocks.initR(f, num, src) } diff --git a/vendor/github.com/pierrec/lz4/v4/lz4.go b/vendor/github.com/pierrec/lz4/v4/lz4.go index c585d4064f5c..a62022e08839 100644 --- a/vendor/github.com/pierrec/lz4/v4/lz4.go +++ b/vendor/github.com/pierrec/lz4/v4/lz4.go @@ -35,7 +35,17 @@ func CompressBlockBound(n int) int { // // An error is returned if the source data is invalid or the destination buffer is too small. func UncompressBlock(src, dst []byte) (int, error) { - return lz4block.UncompressBlock(src, dst) + return lz4block.UncompressBlock(src, dst, nil) +} + +// UncompressBlockWithDict uncompresses the source buffer into the destination one using a +// dictionary, and returns the uncompressed size. +// +// The destination buffer must be sized appropriately. +// +// An error is returned if the source data is invalid or the destination buffer is too small. +func UncompressBlockWithDict(src, dst, dict []byte) (int, error) { + return lz4block.UncompressBlock(src, dst, dict) } // A Compressor compresses data into the LZ4 block format. diff --git a/vendor/github.com/pierrec/lz4/v4/options.go b/vendor/github.com/pierrec/lz4/v4/options.go index 4e1b6703b57d..46a87380313f 100644 --- a/vendor/github.com/pierrec/lz4/v4/options.go +++ b/vendor/github.com/pierrec/lz4/v4/options.go @@ -2,10 +2,11 @@ package lz4 import ( "fmt" - "github.com/pierrec/lz4/v4/internal/lz4block" - "github.com/pierrec/lz4/v4/internal/lz4errors" "reflect" "runtime" + + "github.com/pierrec/lz4/v4/internal/lz4block" + "github.com/pierrec/lz4/v4/internal/lz4errors" ) //go:generate go run golang.org/x/tools/cmd/stringer -type=BlockSize,CompressionLevel -output options_gen.go diff --git a/vendor/github.com/pierrec/lz4/v4/reader.go b/vendor/github.com/pierrec/lz4/v4/reader.go index 403aaf697a3b..f8458807c4e7 100644 --- a/vendor/github.com/pierrec/lz4/v4/reader.go +++ b/vendor/github.com/pierrec/lz4/v4/reader.go @@ -40,6 +40,7 @@ type Reader struct { idx int // size of pending data handler func(int) cum uint32 + dict []byte } func (*Reader) private() {} @@ -77,6 +78,15 @@ func (r *Reader) isNotConcurrent() bool { } func (r *Reader) init() error { + err := r.frame.ParseHeaders(r.src) + if err != nil { + return err + } + if !r.frame.Descriptor.Flags.BlockIndependence() { + // We can't decompress dependent blocks concurrently. + // Instead of throwing an error to the user, silently drop concurrency + r.num = 1 + } data, err := r.frame.InitR(r.src, r.num) if err != nil { return err @@ -162,10 +172,20 @@ func (r *Reader) read(buf []byte) (int, error) { direct = true dst = buf } - dst, err = block.Uncompress(r.frame, dst, true) + dst, err = block.Uncompress(r.frame, dst, r.dict, true) if err != nil { return 0, err } + if !r.frame.Descriptor.Flags.BlockIndependence() { + if len(r.dict)+len(dst) > 128*1024 { + preserveSize := 64*1024 - len(dst) + if preserveSize < 0 { + preserveSize = 0 + } + r.dict = r.dict[len(r.dict)-preserveSize:] + } + r.dict = append(r.dict, dst...) + } r.cum += uint32(len(dst)) if direct { return len(dst), nil diff --git a/vendor/github.com/pierrec/lz4/v4/writer.go b/vendor/github.com/pierrec/lz4/v4/writer.go index 44a43d251b0d..05a6aa14a871 100644 --- a/vendor/github.com/pierrec/lz4/v4/writer.go +++ b/vendor/github.com/pierrec/lz4/v4/writer.go @@ -89,7 +89,7 @@ func (w *Writer) Write(buf []byte) (n int, err error) { zn := len(w.data) for len(buf) > 0 { - if w.idx == 0 && len(buf) >= zn { + if w.isNotConcurrent() && w.idx == 0 && len(buf) >= zn { // Avoid a copy as there is enough data for a block. if err = w.write(buf[:zn], false); err != nil { return diff --git a/vendor/modules.txt b/vendor/modules.txt index 56c125885447..af109839fe2f 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -717,7 +717,7 @@ github.com/opentracing-contrib/go-stdlib/nethttp github.com/opentracing/opentracing-go github.com/opentracing/opentracing-go/ext github.com/opentracing/opentracing-go/log -# github.com/pierrec/lz4/v4 v4.1.1 +# github.com/pierrec/lz4/v4 v4.1.7 ## explicit github.com/pierrec/lz4/v4 github.com/pierrec/lz4/v4/internal/lz4block