From 9fa7d4b42d1ceff5756e1907b414ddaa849d24e0 Mon Sep 17 00:00:00 2001 From: Klaus Post Date: Tue, 21 Dec 2021 17:27:11 +0100 Subject: [PATCH 1/5] flate: Use predefined tables for huffman coding Even though they are rarely a benefit check for size of predefined tables. Minor tweaks, biggest being bigger number of tokens. --- flate/deflate.go | 14 +-- flate/flate_test.go | 2 +- flate/huffman_bit_writer.go | 92 ++++++++++++++---- flate/inflate.go | 9 ++ .../huffman-rand-1k.dyn.expect-noinput | Bin 1069 -> 1057 bytes flate/testdata/huffman-rand-limit.dyn.expect | Bin 252 -> 186 bytes .../huffman-rand-limit.dyn.expect-noinput | Bin 226 -> 186 bytes flate/testdata/huffman-rand-limit.sync.expect | Bin 229 -> 186 bytes .../huffman-rand-limit.sync.expect-noinput | Bin 229 -> 186 bytes flate/testdata/huffman-shifts.dyn.expect | Bin 59 -> 51 bytes .../huffman-shifts.dyn.expect-noinput | Bin 59 -> 51 bytes flate/testdata/huffman-text-shift.dyn.expect | Bin 258 -> 242 bytes .../huffman-text-shift.dyn.expect-noinput | Bin 255 -> 242 bytes flate/testdata/huffman-text.dyn.expect | Bin 304 -> 273 bytes flate/testdata/huffman-zero.dyn.expect | Bin 313 -> 6 bytes .../testdata/huffman-zero.dyn.expect-noinput | Bin 34 -> 6 bytes flate/testdata/huffman-zero.sync.expect | Bin 17 -> 6 bytes .../testdata/huffman-zero.sync.expect-noinput | Bin 17 -> 6 bytes 18 files changed, 93 insertions(+), 24 deletions(-) diff --git a/flate/deflate.go b/flate/deflate.go index 18ec6bdd4e..b54e67acd1 100644 --- a/flate/deflate.go +++ b/flate/deflate.go @@ -41,9 +41,10 @@ const ( maxMatchLength = 258 // The longest match for the compressor minOffsetSize = 1 // The shortest offset that makes any sense - // The maximum number of tokens we put into a single flat block, just too - // stop things from getting too large. - maxFlateBlockTokens = 1 << 14 + // The maximum number of tokens we will encode at the time. + // Smaller sizes usually creates more less optimal blocks. + // Bigger can make context switching slow. + maxFlateBlockTokens = 1 << 15 maxStoreBlockSize = 65535 hashBits = 17 // After 17 performance degrades hashSize = 1 << hashBits @@ -175,7 +176,8 @@ func (d *compressor) writeBlock(tok *tokens, index int, eof bool) error { window = d.window[d.blockStart:index] } d.blockStart = index - d.w.writeBlock(tok, eof, window) + //d.w.writeBlock(tok, eof, window) + d.w.writeBlockDynamic(tok, eof, window, d.sync) return d.w.err } return nil @@ -301,7 +303,7 @@ func (d *compressor) findMatch(pos int, prevHead int, lookahead, bpb int) (lengt if wEnd == win[i+length] { n := matchLen(win[i:i+minMatchLook], wPos) if n > length { - newGain := n*bpb - bits.Len32(uint32(pos-i)) - 1 + newGain := n*bpb - bits.Len32(uint32(pos-i)) if newGain > cGain { length = n offset = pos - i @@ -703,7 +705,7 @@ func (d *compressor) init(w io.Writer, level int) (err error) { d.fill = (*compressor).fillBlock d.step = (*compressor).storeFast case 7 <= level && level <= 9: - d.w.logNewTablePenalty = 10 + d.w.logNewTablePenalty = 8 d.state = &advancedState{} d.compressionLevel = levels[level] d.initDeflate() diff --git a/flate/flate_test.go b/flate/flate_test.go index 648d9796b8..182909ab0e 100644 --- a/flate/flate_test.go +++ b/flate/flate_test.go @@ -163,7 +163,7 @@ func TestRegressions(t *testing.T) { t.Error(err) } if !bytes.Equal(data1, data2) { - fmt.Printf("want:%x\ngot: %x\n", data1, data2) + //fmt.Printf("want:%x\ngot: %x\n", data1, data2) t.Error("not equal") } }) diff --git a/flate/huffman_bit_writer.go b/flate/huffman_bit_writer.go index fda85c2571..9b163d7e3b 100644 --- a/flate/huffman_bit_writer.go +++ b/flate/huffman_bit_writer.go @@ -566,7 +566,7 @@ func (w *huffmanBitWriter) writeBlock(tokens *tokens, eof bool, input []byte) { w.lastHeader = 0 } numLiterals, numOffsets := w.indexTokens(tokens, false) - w.generate(tokens) + w.generate() var extraBits int storedSize, storable := w.storedSize(input) if storable { @@ -595,7 +595,7 @@ func (w *huffmanBitWriter) writeBlock(tokens *tokens, eof bool, input []byte) { } // Stored bytes? - if storable && storedSize < size { + if storable && storedSize <= size { w.writeStoredHeader(len(input), eof) w.writeBytes(input) return @@ -634,10 +634,14 @@ func (w *huffmanBitWriter) writeBlockDynamic(tokens *tokens, eof bool, input []b w.lastHeader = 0 w.lastHuffMan = false } - if !sync { - tokens.Fill() - } + numLiterals, numOffsets := w.indexTokens(tokens, !sync) + extraBits := 0 + ssize, storable := w.storedSize(input) + const usePrefs = true + if storable || w.lastHeader > 0 { + extraBits = w.extraBitSize() + } var size int // Check if we should reuse. @@ -645,11 +649,11 @@ func (w *huffmanBitWriter) writeBlockDynamic(tokens *tokens, eof bool, input []b // Estimate size for using a new table. // Use the previous header size as the best estimate. newSize := w.lastHeader + tokens.EstimatedBits() - newSize += newSize >> w.logNewTablePenalty + newSize += int(w.literalEncoding.codes[endBlockMarker].len) + newSize>>w.logNewTablePenalty // The estimated size is calculated as an optimal table. // We add a penalty to make it more realistic and re-use a bit more. - reuseSize := w.dynamicReuseSize(w.literalEncoding, w.offsetEncoding) + w.extraBitSize() + reuseSize := w.dynamicReuseSize(w.literalEncoding, w.offsetEncoding) + extraBits // Check if a new table is better. if newSize < reuseSize { @@ -660,35 +664,76 @@ func (w *huffmanBitWriter) writeBlockDynamic(tokens *tokens, eof bool, input []b } else { size = reuseSize } + + if preSize := w.fixedSize(extraBits) + 7; usePrefs && preSize < size { + // Check if we get a reasonable size decrease. + if storable && ssize <= size { + w.writeStoredHeader(len(input), eof) + w.writeBytes(input) + return + } + w.writeFixedHeader(eof) + if !sync { + tokens.AddEOB() + } + w.writeTokens(tokens.Slice(), fixedLiteralEncoding.codes, fixedOffsetEncoding.codes) + return + } // Check if we get a reasonable size decrease. - if ssize, storable := w.storedSize(input); storable && ssize < (size+size>>4) { + if storable && ssize <= size { w.writeStoredHeader(len(input), eof) w.writeBytes(input) - w.lastHeader = 0 return } } // We want a new block/table if w.lastHeader == 0 { - w.generate(tokens) + if !sync { + w.fillTokens() + numLiterals, numOffsets = maxNumLit, maxNumDist + } + w.generate() // Generate codegen and codegenFrequencies, which indicates how to encode // the literalEncoding and the offsetEncoding. w.generateCodegen(numLiterals, numOffsets, w.literalEncoding, w.offsetEncoding) w.codegenEncoding.generate(w.codegenFreq[:], 7) + var numCodegens int - size, numCodegens = w.dynamicSize(w.literalEncoding, w.offsetEncoding, w.extraBitSize()) - // Store bytes, if we don't get a reasonable improvement. - if ssize, storable := w.storedSize(input); storable && ssize < (size+size>>4) { + if !sync { + // Reindex for accurate size... + w.indexTokens(tokens, true) + } + size, numCodegens = w.dynamicSize(w.literalEncoding, w.offsetEncoding, extraBits) + + // Store predefined, if we don't get a reasonable improvement. + if preSize := w.fixedSize(extraBits); usePrefs && preSize <= size { + // Store bytes, if we don't get an improvement. + if storable && ssize <= preSize { + w.writeStoredHeader(len(input), eof) + w.writeBytes(input) + return + } + w.writeFixedHeader(eof) + if !sync { + tokens.AddEOB() + } + w.writeTokens(tokens.Slice(), fixedLiteralEncoding.codes, fixedOffsetEncoding.codes) + return + } + + if storable && ssize <= size { + // Store bytes, if we don't get an improvement. w.writeStoredHeader(len(input), eof) w.writeBytes(input) - w.lastHeader = 0 return } // Write Huffman table. w.writeDynamicHeader(numLiterals, numOffsets, numCodegens, eof) - w.lastHeader, _ = w.headerSize() + if !sync { + w.lastHeader, _ = w.headerSize() + } w.lastHuffMan = false } @@ -699,6 +744,19 @@ func (w *huffmanBitWriter) writeBlockDynamic(tokens *tokens, eof bool, input []b w.writeTokens(tokens.Slice(), w.literalEncoding.codes, w.offsetEncoding.codes) } +func (w *huffmanBitWriter) fillTokens() { + for i, v := range w.literalFreq[:literalCount] { + if v == 0 { + w.literalFreq[i] = 1 + } + } + for i, v := range w.offsetFreq[:offsetCodeCount] { + if v == 0 { + w.offsetFreq[i] = 1 + } + } +} + // indexTokens indexes a slice of tokens, and updates // literalFreq and offsetFreq, and generates literalEncoding // and offsetEncoding. @@ -733,7 +791,7 @@ func (w *huffmanBitWriter) indexTokens(t *tokens, filled bool) (numLiterals, num return } -func (w *huffmanBitWriter) generate(t *tokens) { +func (w *huffmanBitWriter) generate() { w.literalEncoding.generate(w.literalFreq[:literalCount], 15) w.offsetEncoding.generate(w.offsetFreq[:offsetCodeCount], 15) } @@ -867,7 +925,7 @@ func (w *huffmanBitWriter) writeTokens(tokens []token, leCodes, oeCodes []hcode) offsetComb := offsetCombined[offsetCode] if offsetComb > 1<<16 { //w.writeBits(extraOffset, extraOffsetBits) - bits |= uint64(offset&matchOffsetOnlyMask-(offsetComb&0xffff)) << (nbits & 63) + bits |= uint64(offset-(offsetComb&0xffff)) << (nbits & 63) nbits += uint16(offsetComb >> 16) if nbits >= 48 { binary.LittleEndian.PutUint64(w.bytes[nbytes:], bits) diff --git a/flate/inflate.go b/flate/inflate.go index d1edb356c4..d5f62f6a2c 100644 --- a/flate/inflate.go +++ b/flate/inflate.go @@ -328,11 +328,17 @@ func (f *decompressor) nextBlock() { switch typ { case 0: f.dataBlock() + if debugDecode { + fmt.Println("stored block") + } case 1: // compressed, fixed Huffman tables f.hl = &fixedHuffmanDecoder f.hd = nil f.huffmanBlockDecoder()() + if debugDecode { + fmt.Println("predefinied huffman block") + } case 2: // compressed, dynamic Huffman tables if f.err = f.readHuffman(); f.err != nil { @@ -341,6 +347,9 @@ func (f *decompressor) nextBlock() { f.hl = &f.h1 f.hd = &f.h2 f.huffmanBlockDecoder()() + if debugDecode { + fmt.Println("dynamic huffman block") + } default: // 3 is reserved. if debugDecode { diff --git a/flate/testdata/huffman-rand-1k.dyn.expect-noinput b/flate/testdata/huffman-rand-1k.dyn.expect-noinput index 5162399686d8b32a892409ff10092f29975928a3..c3b1888028bdd1bc8c58ee681bed8aa000c87016 100644 GIT binary patch literal 1057 zcmV++1m64l(LJrqgST~xYQy?N|9W32ycTaex&7!pwpX z+&C|&*fKV2Rd8oFPOxbQ)>6c^slqt_a&vbUd`r8;eMd3x%d0?W}DylH?Q+w;vPEl zp8m8v8E2pNM>RL&eQI7zV7_fI^_mio`wrjiy()sMb*}HrtGdRqc*c#pqFKKt8!)QM zE-GpB&YJq-p&wt_PS2I`3zj9@)YT~n`|nPhv_o&nFOSpfcG|4JEdTIouT$=aKRrAQ z_Z(JtyHEF9alY^NvTg;=ma4R;Gc=i|KC>`NV17JHF62dWgU6ib$;NpLQk#Dr+g0UU z$oe%Svvtlc*R$E&k{5T^7qm{gxFw}&s$ly>VchrSdFuYCDn5u3>B*yYn0t{pxT{?;-}^=#RzT~?nK zPK|w9cz#aOysTeW4KJ=pGcaBiC@$PI^Mpvkd|#v7Tas#uck1+1(~C~=PdfYU+UeP3!@}nrz9zzIvvK|86J6o;!N2rZ<)tlIeVT|M>p?t^!?cCApKIX&~BO;Zn7jBa#?or!6ixm)GKRg%9~-4o^H zl?wmAmH)GLRFzlSs#M&%*evFsO1uI$-7`h)-vuq^xMUOf<;UD7GdfSCTV9Pb ztx$hq)3C0x@Y3Wjs{(~&yrrM!iRM4q9XQ){{?8*lTEC_4hhFxWtj?%*C_6NF(eIY- zlk+qVy_ZsXGke|MgqNbG@0^wxNB`pA_MLGVlh3IM5mwV2pP%OJx!vl!qVx0CzQ3i> z{%`NuzqVMsZ$t5)9Upo{b~V0FfAY+R=hPCe%FW^3 bU8OT5ob)vpH@%n3Mj6P+o?vCg_ShmPSrMST~nIEi=HI4(`R^@1141 zY;kSu@4B#tmajUuHWSxtmPFp6&-_b8bl7VQjK&z-7a^>O)07IR$?3@Nlq>YYm!SWuB*SY++Z!OFE4P z1T_1g>BothtHk?RRAj14VcVsi0xDQN<~({R&2)A0u(hSH>R`r`?lQkf$1JHI44?CI zw4#xF9zj_9=yX7A7T2K_w_d-!CgMyiy`AG9RVq1k2Y%3qzRS?%0C`>-p9{px=&Cpk zY0(~S5t?Eu3LWsL4Z3$dn(~j|Bj?EpIz=P>xoWyB?>AFcbvN8aP1s zz^s!3zVd$mq=;K+lAan+506UJII+{~qTOz*xMvTB4ixJz6^lXYS_Y_c{tYZN180l+h&t)^!+v{tNG_ zVeIkD>B_)anNMknw5Nt#zR&`=O$vW!(J%s@VA0`e`(WWp#f5DqBkUNeIOMugSQ%h8 zZW%mHkD1x8Nx5l*Hz#)CDIsRPzDCS-ViGl^Y1xl7x7Y7FETLg~VJD8m2%(;UarlLj z^oq;ZrhuI|(h{05HiDB@ub9Z-k?c@HN0PkQ?f-^{Hx1!I!IB!cOo!~n z+lgjN27Ml#j!7me+j%&TpxmVxYBOU8Z^~fXcqY5ncLZcLX`cu4QUELFRSH%+@h^MY zrW}yAs3-QsIu|B9EWe~AdAUxA_l;RI>uwMqSGmtSK{4jmx$b&gz^z#u!E*kAMl_oH n+7w1I4A+&C|&*fKV2Rd8oFPOxbQ)>6c^slqt_a&vbUd`qL0Dk3ZG5`Po literal 252 zcmZSh&cN|Mk$QmZNB6Wc53bfJstxN8{p)?9@LIg>oXRI~IhW&XxGJNu5- o$p8QV diff --git a/flate/testdata/huffman-rand-limit.dyn.expect-noinput b/flate/testdata/huffman-rand-limit.dyn.expect-noinput index 008b9afee933f30f6429fe5a1e2c426a195cafd4..881e59c9ab9bb356c5f1b8f2e188818bd42dbcf0 100644 GIT binary patch literal 186 zcmV;r07d^wq#oe<(LJrqgR6ClYQy?N|9W32ycTaex&7!pwpX+&C|&*fKV2Rd8oFPOxbQ)>6c^slqt_a&vbUd`qL0Dk3ZG5`Po literal 226 zcmV<803HAA9Rq-xVRUHvXM43Zv)K&|%|>hXR%>j{Yes9J&$jX2g_a{g2I=oZB=GL{EFz7OOBO<+$G>M9*>n4go=+eN-~%|kJnVh0tKb$H zG;!C+&C|&*fKV2Rd8oFPOxbQ)>6c^slqt_a&vbUd`qL0Dk3ZG5`Po literal 229 zcmV+&C|&*fKV2Rd8oFPOxbQ)>6c^slqt_a&vbUd`qL0Dk3ZG5`Po literal 229 zcmVf1w#y6vcB~mT diff --git a/flate/testdata/huffman-shifts.dyn.expect-noinput b/flate/testdata/huffman-shifts.dyn.expect-noinput index 2f4fd17add552d86b47dcb321a4f50988aac86cc..8d2d93d5e2eac5a83bd6785ac77d629792736537 100644 GIT binary patch literal 51 ncmXpqk!W39Akudz(1OeTp@s~CqiCwAKYijyWFesZO92J|+^-Tx literal 59 zcmaEJmpS2qfe6Ee&AGR?<<16^Kn4iitBcRIKKV790S0FH%{ITYgA>f1w#y6vcB~mT diff --git a/flate/testdata/huffman-text-shift.dyn.expect b/flate/testdata/huffman-text-shift.dyn.expect index 3a4dcc4cab48cb09c9db911a39299334b04be2e5..cb9e0b9c8a73eb5cf9788ea4e179b8050c52be01 100644 GIT binary patch literal 242 zcmV6)(sRto0Sgxum8IzMa_Q@b=BJfp zWagI^B`4>nWTxkr73HNQRXPQ`=$2IGfE46pCg+u8lqBaRrRA6ArDW!nWcYYGyZQyY zre)^n@pAEU6(nb;=Vs=Cn3=f+`9&p4`AQ&8T4`Q#Zf2f_W;HJtXIW8_ZEm(kbZk;( ziH@16sj;ahFBfN;PP~qm#3XtDC!rrnub+QFU{G*KXjnu=WtF0m@}VWFYWns{01bq9AOHXW literal 258 zcmW-Z?MlNi7)FQxEc7CbC>^ZLP%0=4>AJd2=ltlVf{2imG+}Ap5|Xki-i=q{eTk;~ zcb>!H>HYTh{xD9VZnY@6IT;SmS9}VIH5_!5NV(HmH$~)o7_WgDF2HI^p%5iBS{0-n z=abO1vJ0f3%5Xy!VPI7u+*r)hY?)_FNO$3THQkhzP#ub@25k_aZZNB;^Fja9_x2l2 zM%5;GeIu6DTgECCDl27JZ6l%3u85wsFadG=<$K9Eon%+j f+5Gxual5>`e|XGS>!+7(`*DB{f4+~7@g@2PbNEq% diff --git a/flate/testdata/huffman-text-shift.dyn.expect-noinput b/flate/testdata/huffman-text-shift.dyn.expect-noinput index 29788aa0a8ec587e76ecf8a55d49ab54b381bf79..cb9e0b9c8a73eb5cf9788ea4e179b8050c52be01 100644 GIT binary patch literal 242 zcmV6)(sRto0Sgxum8IzMa_Q@b=BJfp zWagI^B`4>nWTxkr73HNQRXPQ`=$2IGfE46pCg+u8lqBaRrRA6ArDW!nWcYYGyZQyY zre)^n@pAEU6(nb;=Vs=Cn3=f+`9&p4`AQ&8T4`Q#Zf2f_W;HJtXIW8_ZEm(kbZk;( ziH@16sj;ahFBfN;PP~qm#3XtDC!rrnub+QFU{G*KXjnu=WtF0m@}VWFYWns{01bq9AOHXW literal 255 zcmV-Ewx>6eYjn29UI FJOMauajpOW diff --git a/flate/testdata/huffman-text.dyn.expect b/flate/testdata/huffman-text.dyn.expect index 1fb84b322ca8eb90e65f72d22896e461de3348eb..6ef6dd44dd48997edd6fc17040716e9a434f1c71 100644 GIT binary patch literal 273 zcmV+s0q*|n9R*<1O7wqxf#9x>VjZmMkm3VI8ScY9O)pJYnk%_Wp}4!dySux)ySux) z%Zj@@=kMOR`S5KocOAnBs$C4xc^H5L)XOBD|yvPpiMEZyve{Syr0TlzfGZI0i} ze9YP9`QQI)g;Ecb5pCyiwO-%iwIe)gY`yEWu{pLx6~X z-qeIzH-vz#9?2+bP)%UC-#=v5O_F9!ikehVUB?`1rce>M_NswUAtXFMmAjYy4qQkI zPjNJ``oLO3&xAU$=m%{a5^a+ca>*K^5fGEeh&Bf_8x@Eb=*!%AHu diff --git a/flate/testdata/huffman-zero.dyn.expect b/flate/testdata/huffman-zero.dyn.expect index 230433ca0c4dc164e29ff3722e5e24ba4a30ad90..dbe401c54c4b6f45f3169376185a476dcf00dde9 100644 GIT binary patch literal 6 NcmXq#U{zse0006o0CxZY literal 313 RcmZQDVLbiMU=)xX0suzCJHG$` diff --git a/flate/testdata/huffman-zero.dyn.expect-noinput b/flate/testdata/huffman-zero.dyn.expect-noinput index cefc1d3f6629324dca5d7efcd9d20c3e8cd9b208..dbe401c54c4b6f45f3169376185a476dcf00dde9 100644 GIT binary patch literal 6 NcmXq#U{zse0006o0CxZY literal 34 gcmaDO%iJL0kg+ZI_OshS0)Z~xH2YP=k=-T;07eMUV%O diff --git a/flate/testdata/huffman-zero.sync.expect-noinput b/flate/testdata/huffman-zero.sync.expect-noinput index 830348a79ad9ab38d0edc449e8335c056f7d185f..dbe401c54c4b6f45f3169376185a476dcf00dde9 100644 GIT binary patch literal 6 NcmXq#U{zse0006o0CxZY literal 17 XcmaEJU?T$%G#D)X^D^m0zK$>eMUV%O From 94fbdd870faf14834b51dd053daabdd01646d9db Mon Sep 17 00:00:00 2001 From: Klaus Post Date: Wed, 22 Dec 2021 16:01:57 +0100 Subject: [PATCH 2/5] Disable filling and use bigger blocks. --- flate/deflate.go | 7 ++- flate/huffman_bit_writer.go | 58 +++++++++++------- flate/testdata/huffman-null-max.dyn.expect | Bin 101 -> 79 bytes .../huffman-null-max.dyn.expect-noinput | Bin 101 -> 79 bytes flate/testdata/huffman-pi.dyn.expect | Bin 1751 -> 1698 bytes flate/testdata/huffman-pi.dyn.expect-noinput | Bin 1751 -> 1698 bytes .../huffman-rand-1k.dyn.expect-noinput | Bin 1057 -> 1052 bytes flate/testdata/huffman-shifts.dyn.expect | Bin 51 -> 33 bytes .../huffman-shifts.dyn.expect-noinput | Bin 51 -> 33 bytes flate/testdata/huffman-text-shift.dyn.expect | Bin 242 -> 232 bytes .../huffman-text-shift.dyn.expect-noinput | Bin 242 -> 232 bytes flate/testdata/huffman-text.dyn.expect | 7 ++- .../testdata/huffman-text.dyn.expect-noinput | 7 ++- .../null-long-match.dyn.expect-noinput | Bin 227 -> 208 bytes 14 files changed, 47 insertions(+), 32 deletions(-) diff --git a/flate/deflate.go b/flate/deflate.go index b54e67acd1..e064b01ee5 100644 --- a/flate/deflate.go +++ b/flate/deflate.go @@ -42,9 +42,10 @@ const ( minOffsetSize = 1 // The shortest offset that makes any sense // The maximum number of tokens we will encode at the time. - // Smaller sizes usually creates more less optimal blocks. + // Smaller sizes usually creates less optimal blocks. // Bigger can make context switching slow. - maxFlateBlockTokens = 1 << 15 + // We use this for levels 7-9, so we make it as big as we can. + maxFlateBlockTokens = maxStoreBlockSize maxStoreBlockSize = 65535 hashBits = 17 // After 17 performance degrades hashSize = 1 << hashBits @@ -699,7 +700,7 @@ func (d *compressor) init(w io.Writer, level int) (err error) { level = 5 fallthrough case level >= 1 && level <= 6: - d.w.logNewTablePenalty = 8 + d.w.logNewTablePenalty = 7 d.fast = newFastEnc(level) d.window = make([]byte, maxStoreBlockSize) d.fill = (*compressor).fillBlock diff --git a/flate/huffman_bit_writer.go b/flate/huffman_bit_writer.go index 9b163d7e3b..fb1701eecc 100644 --- a/flate/huffman_bit_writer.go +++ b/flate/huffman_bit_writer.go @@ -155,37 +155,33 @@ func (w *huffmanBitWriter) reset(writer io.Writer) { w.lastHuffMan = false } -func (w *huffmanBitWriter) canReuse(t *tokens) (offsets, lits bool) { - offsets, lits = true, true +func (w *huffmanBitWriter) canReuse(t *tokens) (ok bool) { a := t.offHist[:offsetCodeCount] - b := w.offsetFreq[:len(a)] - for i := range a { - if b[i] == 0 && a[i] != 0 { - offsets = false - break + b := w.offsetEncoding.codes + b = b[:len(a)] + for i, v := range a { + if v != 0 && b[i].len == 0 { + return false } } a = t.extraHist[:literalCount-256] - b = w.literalFreq[256:literalCount] + b = w.literalEncoding.codes[256:literalCount] b = b[:len(a)] - for i := range a { - if b[i] == 0 && a[i] != 0 { - lits = false - break + for i, v := range a { + if v != 0 && b[i].len == 0 { + return false } } - if lits { - a = t.litHist[:] - b = w.literalFreq[:len(a)] - for i := range a { - if b[i] == 0 && a[i] != 0 { - lits = false - break - } + + a = t.litHist[:256] + b = w.literalEncoding.codes[:len(a)] + for i, v := range a { + if v != 0 && b[i].len == 0 { + return false } } - return + return true } func (w *huffmanBitWriter) flush() { @@ -635,15 +631,28 @@ func (w *huffmanBitWriter) writeBlockDynamic(tokens *tokens, eof bool, input []b w.lastHuffMan = false } + // fillReuse enables filling of empty values. + // This will make encodings always reusable without testing. + // However, this does not appear to benefit on most cases. + const fillReuse = false + + // Check if we can reuse... + if !fillReuse && w.lastHeader > 0 && !w.canReuse(tokens) { + w.writeCode(w.literalEncoding.codes[endBlockMarker]) + w.lastHeader = 0 + } + numLiterals, numOffsets := w.indexTokens(tokens, !sync) extraBits := 0 ssize, storable := w.storedSize(input) + const usePrefs = true if storable || w.lastHeader > 0 { extraBits = w.extraBitSize() } var size int + // Check if we should reuse. if w.lastHeader > 0 { // Estimate size for using a new table. @@ -689,10 +698,13 @@ func (w *huffmanBitWriter) writeBlockDynamic(tokens *tokens, eof bool, input []b // We want a new block/table if w.lastHeader == 0 { - if !sync { + if fillReuse && !sync { w.fillTokens() numLiterals, numOffsets = maxNumLit, maxNumDist + } else { + w.literalFreq[endBlockMarker] = 1 } + w.generate() // Generate codegen and codegenFrequencies, which indicates how to encode // the literalEncoding and the offsetEncoding. @@ -700,7 +712,7 @@ func (w *huffmanBitWriter) writeBlockDynamic(tokens *tokens, eof bool, input []b w.codegenEncoding.generate(w.codegenFreq[:], 7) var numCodegens int - if !sync { + if fillReuse && !sync { // Reindex for accurate size... w.indexTokens(tokens, true) } diff --git a/flate/testdata/huffman-null-max.dyn.expect b/flate/testdata/huffman-null-max.dyn.expect index 0a3c71ceb3bee4671bb61c42561246285bd50ec1..f4e27a8146cf720a410fc6b835858f1d60b137c6 100644 GIT binary patch literal 79 acmaEJw~+w|8W#LrDXuvQ#H0W$U;_ZIF$ReM literal 101 ncmaEJmwCYrD+Y!G-*azoJ9~2*kc6Z1@Y{Kl7$^iB92Q&vuC^3| diff --git a/flate/testdata/huffman-null-max.dyn.expect-noinput b/flate/testdata/huffman-null-max.dyn.expect-noinput index 0a3c71ceb3bee4671bb61c42561246285bd50ec1..f4e27a8146cf720a410fc6b835858f1d60b137c6 100644 GIT binary patch literal 79 acmaEJw~+w|8W#LrDXuvQ#H0W$U;_ZIF$ReM literal 101 ncmaEJmwCYrD+Y!G-*azoJ9~2*kc6Z1@Y{Kl7$^iB92Q&vuC^3| diff --git a/flate/testdata/huffman-pi.dyn.expect b/flate/testdata/huffman-pi.dyn.expect index 11756feafbe613e8979b814493a5fd41d0483395..66c76ceb237b0f0b35a4e304f096fff4e72ed98d 100644 GIT binary patch literal 1698 zcmV;T23`5=o!!wZVF-dd%7}_e|BYpMw~ljjGWj<9gZRMS)ej9ShsD#XvaTnmtFCgz z$h(FXqmpWg;ALoA-BecWVLB#nGETb5i|R{OC$KTAOZ3GJQF&Q0(;KSmo)8(l%)%y% z>Nw=}0oeBC8<87O9TNzWn(oO|pV+FXgTKl3R^trdDJihAp!=)5I+ZVRC3t*W~$*52~% zpDZ0L!}m z;=XE4CZ$H%RIN>{AFq0a0zqycH#?gR#>o0zidDeeU~|uVqgaNl#1jHs8tZ8xeQ0|;U{vG{F11Z^W45gtgRSOyit-sb9`+_;Wo0WYMI|t-Zrx_&TARq4 zwq4i~pfecS+ORlNBLgLeHu>Hp39mMm+p~EJ?e0FNM1<+TVZMzwy#?dJzD)FP76XlT zh7`xV!k`A=7L$fxwa<++d#=s-sw~Bp8eLV*-koe)D2dY*M}6dENC{;ou5)dl^sUd` z&SDilYdjfdyyth5n+`YZO&r(PtY`k!&K&IC2(j)Q9OaaXxrpgZBAI6ImeRg7boyo1 zO^%wLazn#BhVRVFiJS8H`iYuRGK-rwZ`K+XV<2)I91CIdtoTk3ms_#|b#jnib|s z1}Wj_Q)g2wQC%>%*I&)WHSO~)*8}nThIJ0QNk8(0jID_aBD?O|()n|vybn9M=M7rW zY%WT>5l9(1r*7DHbEMDTG3A<*v6&Y3d7uwGyTIeO^=!;iGjsOC(iz1GTRpvp?3pne zvvkuOLkv@Vo^m5tw-uf7qMB-F>vmjw6{Ed znx6)Z8vs^$b1pNty)T|G+tusrtwk#|OP%wsh8FGMlRHISo90r+?``JrE}RIolR7tg zmvlX-)ij!5n*Lc`(`@6tyI1s&&B#}C8b)P#37Cb~ATR@07HY(rEVTKbeXUyD0)gfZ z+X)-FQgugdXT-Wc?XeV-&~v7@=z8BuUsa-;3!Bif+r@BEGH6&3qOT7Ttc`Jbd31gW z43l-e8+6zk22(HG+l}k3)7$!cw0u28o`c{zbLd&ZS%EBllTs{hTAc+AI3BjcOLR># zX{^meom-;y_7YASEp(ZfI@Rv2TZ|uEtBl<;*Atf<=lhIft@FjtIOe*25L+2JL3h@- zEfD4508xGe@h2blMF24-`|BXCN_?*nM=ZM5Dd4Mt9LAkDSEr+NS?77W sx|PK?KHTB%y#-cU2YL(2qm`2><{9 literal 1751 zcmV;|1}ORLoddun!LEd#ZQEulNd?cgZQHoEZQHhO+qP{RHG6G)dG_Bq(>+P$tLkO^ z2ohW`24{Q2W3s$5V#0fAK9kr+ZR|!9#&O%u&E$Gm4=SyE%58S1d@ORDkq)#~?>KU7 ziU=PMtID>G$t+vaIPQUV1~+ZslLMer%O}j+urYMNNN#18`{pcj!(yNh`3!8ERosZo z(tVnh;}Jd86DWGa4Qq?kPqAt}vb5!bB{RJA7DQ~6wU>D=65i3CoHnbn3Xn7CoB`d| zZOg~knYL&!Dx-SVxfW#xL>R-EGj3T$M@%AXiA@kF+l(-_w?(nJTr9)ZF$J`qG+eJX zU<8Ia)+$HzZh7ajI*1uIy*+2)-W!7n12z}F6Hy&eo@vFZ44X1XY;IO8-Ls>y*5>G) zV`pvNZH#p^x_kp&%|kMs1ASMuY3j3?u?F5qdIihIyNBI~^k(;LsZvqV=xImRWZD|s za~T~rJ2oPlFusuDQ zpw+5=z&dGVX|~cDs)1Ou9ZgLdd^k}k3s$ssmBCUa zA8h&_c+m4oi#^dPP>qw(hZ4L-m#1Z`qbX`>8YeZVbz(ju5Y=ryI}5s&prhPaTOxvS z#2pGu1f+!V46`7oE6uZWD?7YG_}F!^B)#^8id$_>(NlETNVK{I*QKfUfEMOc7~f8k zZPlr&(U_YYb#Zk^mYUrT>s4jLRElt*7@0fm=B?V8N2TP#+y!<7hE#1BtcqFz#fuVo zPdN#VHkzH)K5N_EeK5t$ME^GNtu*Pm5FVsUM|OMAkVtt*u*54Aq5yW#DF{aD-WaiG zQk;+QSnJqCMnp2Z%Tp3^Vs?VHIb^x*I@aAW=`H!Vwq7$`jb77T?I=T)=6t5SUFL?K zvSDk@IKijH9t=A62|7i_MNz1!7i4I^mVv<;bTjd(>VH%IfM$>sdnE zsR`A~xoi_?#$7^oc|%&aS7WnBlQCX~?Ao;z=nPS@nIZj!sS0k9TTV}+HMLtr2GVzI zu~uA{1lt|8l_fH0X$psVP|^)rx+unE7v2Jt*})#MtH_ds&smt2+c#6_h^C|6q|#)-DdC=a%M9j z-PMq~e_DlNj$6B{u!jb6z>x~neo0Tlj7_>%pTN=x#STf`HHXZq5*s~sDp`UUDtKs0 ziJ+BMc7&IWij=o@hNsvAT9??>n}n_2Zkr~Xn(1xWY$)*5B5JB#+EV)|ehM^70O<9p zQJ9&|KJ0ls6}?8r3Ky_HCU zF8X_9R5FG4%u49~zsh@+EM-Ex_VWBuP$ZA4f0gpY=skO#bZhBw~EHzu^wNy+JX;hoC z7^OtjTSHhbv`}H9Vm7*`m16YZ+w0aH;<{s;C4aAyt)(CKQ?j{CUBp~BFQYQ+G30K@ z71D;*Au?B$sVSb3rhS;ab}AIBWb$H#iq78h8kP!38!?j8Tf1^Zb}AT7nlcNL3z>17 zL}O6ShwIQ`n&DVQWtF5s4Q#p{k$)B8tu;Y%W#v^JF&qk5PKdn zVkePD_E=ONMz=?>R<>z(O$ZfIRKbO5^EzA5$Iwzt!xf4qTr&re*ftnjYNw=}0oeBC8<87O9TNzWn(oO|pV+FXgTKl3R^trdDJihAp!=)5I+ZVRC3t*W~$*52~% zpDZ0L!}m z;=XE4CZ$H%RIN>{AFq0a0zqycH#?gR#>o0zidDeeU~|uVqgaNl#1jHs8tZ8xeQ0|;U{vG{F11Z^W45gtgRSOyit-sb9`+_;Wo0WYMI|t-Zrx_&TARq4 zwq4i~pfecS+ORlNBLgLeHu>Hp39mMm+p~EJ?e0FNM1<+TVZMzwy#?dJzD)FP76XlT zh7`xV!k`A=7L$fxwa<++d#=s-sw~Bp8eLV*-koe)D2dY*M}6dENC{;ou5)dl^sUd` z&SDilYdjfdyyth5n+`YZO&r(PtY`k!&K&IC2(j)Q9OaaXxrpgZBAI6ImeRg7boyo1 zO^%wLazn#BhVRVFiJS8H`iYuRGK-rwZ`K+XV<2)I91CIdtoTk3ms_#|b#jnib|s z1}Wj_Q)g2wQC%>%*I&)WHSO~)*8}nThIJ0QNk8(0jID_aBD?O|()n|vybn9M=M7rW zY%WT>5l9(1r*7DHbEMDTG3A<*v6&Y3d7uwGyTIeO^=!;iGjsOC(iz1GTRpvp?3pne zvvkuOLkv@Vo^m5tw-uf7qMB-F>vmjw6{Ed znx6)Z8vs^$b1pNty)T|G+tusrtwk#|OP%wsh8FGMlRHISo90r+?``JrE}RIolR7tg zmvlX-)ij!5n*Lc`(`@6tyI1s&&B#}C8b)P#37Cb~ATR@07HY(rEVTKbeXUyD0)gfZ z+X)-FQgugdXT-Wc?XeV-&~v7@=z8BuUsa-;3!Bif+r@BEGH6&3qOT7Ttc`Jbd31gW z43l-e8+6zk22(HG+l}k3)7$!cw0u28o`c{zbLd&ZS%EBllTs{hTAc+AI3BjcOLR># zX{^meom-;y_7YASEp(ZfI@Rv2TZ|uEtBl<;*Atf<=lhIft@FjtIOe*25L+2JL3h@- zEfD4508xGe@h2blMF24-`|BXCN_?*nM=ZM5Dd4Mt9LAkDSEr+NS?77W sx|PK?KHTB%y#-cU2YL(2qm`2><{9 literal 1751 zcmV;|1}ORLoddun!LEd#ZQEulNd?cgZQHoEZQHhO+qP{RHG6G)dG_Bq(>+P$tLkO^ z2ohW`24{Q2W3s$5V#0fAK9kr+ZR|!9#&O%u&E$Gm4=SyE%58S1d@ORDkq)#~?>KU7 ziU=PMtID>G$t+vaIPQUV1~+ZslLMer%O}j+urYMNNN#18`{pcj!(yNh`3!8ERosZo z(tVnh;}Jd86DWGa4Qq?kPqAt}vb5!bB{RJA7DQ~6wU>D=65i3CoHnbn3Xn7CoB`d| zZOg~knYL&!Dx-SVxfW#xL>R-EGj3T$M@%AXiA@kF+l(-_w?(nJTr9)ZF$J`qG+eJX zU<8Ia)+$HzZh7ajI*1uIy*+2)-W!7n12z}F6Hy&eo@vFZ44X1XY;IO8-Ls>y*5>G) zV`pvNZH#p^x_kp&%|kMs1ASMuY3j3?u?F5qdIihIyNBI~^k(;LsZvqV=xImRWZD|s za~T~rJ2oPlFusuDQ zpw+5=z&dGVX|~cDs)1Ou9ZgLdd^k}k3s$ssmBCUa zA8h&_c+m4oi#^dPP>qw(hZ4L-m#1Z`qbX`>8YeZVbz(ju5Y=ryI}5s&prhPaTOxvS z#2pGu1f+!V46`7oE6uZWD?7YG_}F!^B)#^8id$_>(NlETNVK{I*QKfUfEMOc7~f8k zZPlr&(U_YYb#Zk^mYUrT>s4jLRElt*7@0fm=B?V8N2TP#+y!<7hE#1BtcqFz#fuVo zPdN#VHkzH)K5N_EeK5t$ME^GNtu*Pm5FVsUM|OMAkVtt*u*54Aq5yW#DF{aD-WaiG zQk;+QSnJqCMnp2Z%Tp3^Vs?VHIb^x*I@aAW=`H!Vwq7$`jb77T?I=T)=6t5SUFL?K zvSDk@IKijH9t=A62|7i_MNz1!7i4I^mVv<;bTjd(>VH%IfM$>sdnE zsR`A~xoi_?#$7^oc|%&aS7WnBlQCX~?Ao;z=nPS@nIZj!sS0k9TTV}+HMLtr2GVzI zu~uA{1lt|8l_fH0X$psVP|^)rx+unE7v2Jt*})#MtH_ds&smt2+c#6_h^C|6q|#)-DdC=a%M9j z-PMq~e_DlNj$6B{u!jb6z>x~neo0Tlj7_>%pTN=x#STf`HHXZq5*s~sDp`UUDtKs0 ziJ+BMc7&IWij=o@hNsvAT9??>n}n_2Zkr~Xn(1xWY$)*5B5JB#+EV)|ehM^70O<9p zQJ9&|KJ0ls6}?8r3Ky_HCU zF8X_9R5FG4%u49~zsh@+EM-Ex_VWBuP$ZA4f0gpY=skO#bZhBw~EHzu^wNy+JX;hoC z7^OtjTSHhbv`}H9Vm7*`m16YZ+w0aH;<{s;C4aAyt)(CKQ?j{CUBp~BFQYQ+G30K@ z71D;*Au?B$sVSb3rhS;ab}AIBWb$H#iq78h8kP!38!?j8Tf1^Zb}AT7nlcNL3z>17 zL}O6ShwIQ`n&DVQWtF5s4Q#p{k$)B8tu;Y%W#v^JF&qk5PKdn zVkePD_E=ONMz=?>R<>z(O$ZfIRKbO5^EzA5$Iwzt!xf4qTr&re*ftnjY2R$KdB zk1BP(AeujoC5qok<7nsf@1!G*JV7sIj6JI$u?;79c|lzp%6k%=8mg-k4{6qVWe({V zZj#Q$R?I6C&?{TQHU>T`NHCodR8G!E*p0M8tt-J-Je)B2_m3Dp*L;T-rXQc9HK^{N zufoll6_IZ#Igui;)QE-iL)cr^(p6$f&T?BN!s4KwBwDM|h`zW0G3`5$X2Fl=+u6SL z$bbs5Whvee;g$C>RT%D-Dl1f-g_g`**F<)Ek@PCVoPL;K9`bu?lgzjHb%kI(!e^ty=Z#3MZLmvl-&bwoibV!(A?s7z7%Ml5xcOc z$TFX%@u#zmRIqw>3VONR^lZtBopprjQsp=PvZxHtYH1Vpq6Ns6Wx0o{&SI;6!J(Nh06EJ&fp9hCZ(318H6z5HF*v;xweqj@lwL#Z;8M7)@L9 zpZ>fZo#!a$$q72ms|$5%x~$N@O3%y;zC&%=V~xj>*0>IP&r3p=3FFm~46H(6HPM8I z!dm<;>tt38>qgXgTgQp?Z(f^AvoOXy;sfbp`!l0u6j&fJ;n~&Kw;)W--K@dk`{aj9 zncN=KeN^?}jP7}OTWirU1(kkJVcCL5XDMJ4*H~8woK%tWI33Ii6Y!NE7Pds(CX)OrL(={ zj>A#F8&cXoM~3O}e9Or|hmVKPIv?LKS!HKa#fdMJ!mb#z4eR(FdiJi1Cgqhq-hw!d zrzD!qMHn$Jh)LAM;gcxRzEji-SpMM7p*fDjNTi0qIQ%9_MQ7pjZQ#@|e-9ZHr~VKi zcRyt`JQ%8#nGMk5ZM1v(eS~>s#BnDGTVN2z4#lb-UDx_bh>QW<-ujsdn1uVu5C4>u z=MQ7-1!1}0}Zk-Ww4Ys0%&mhqr?Nv~g(d(F|C0<&X-`G6_UA`_LHJX}Cr z-ElT`gmH_vZE$0wirp8n3bNgDtA_c7vqq%= z+&C|&*fKV2Rd8oFPOxbQ)>6c^slqt_a&vbUd`r8;eMd3x%d0?W}DylH?Q+w;vPEl zp8m8v8E2pNM>RL&eQI7zV7_fI^_mio`wrjiy()sMb*}HrtGdRqc*c#pqFKKt8!)QM zE-GpB&YJq-p&wt_PS2I`3zj9@)YT~n`|nPhv_o&nFOSpfcG|4JEdTIouT$=aKRrAQ z_Z(JtyHEF9alY^NvTg;=ma4R;Gc=i|KC>`NV17JHF62dWgU6ib$;NpLQk#Dr+g0UU z$oe%Svvtlc*R$E&k{5T^7qm{gxFw}&s$ly>VchrSdFuYCDn5u3>B*yYn0t{pxT{?;-}^=#RzT~?nK zPK|w9cz#aOysTeW4KJ=pGcaBiC@$PI^Mpvkd|#v7Tas#uck1+1(~C~=PdfYU+UeP3!@}nrz9zzIvvK|86J6o;!N2rZ<)tlIeVT|M>p?t^!?cCApKIX&~BO;Zn7jBa#?or!6ixm)GKRg%9~-4o^H zl?wmAmH)GLRFzlSs#M&%*evFsO1uI$-7`h)-vuq^xMUOf<;UD7GdfSCTV9Pb ztx$hq)3C0x@Y3Wjs{(~&yrrM!iRM4q9XQ){{?8*lTEC_4hhFxWtj?%*C_6NF(eIY- zlk+qVy_ZsXGke|MgqNbG@0^wxNB`pA_MLGVlh3IM5mwV2pP%OJx!vl!qVx0CzQ3i> z{%`NuzqVMsZ$t5)9Upo{b~V0FfAY+R=hPCe%FW^3 bU8OT5ob)vpH@%+{UeCQBetd7^G}B90VP;@pxbY?4l!4(oF97Fv3~vAc literal 51 ncmXpqk!W39Akudz(1OeTp@s~CqiCwAKYijyWFesZO92J|+^-Tx diff --git a/flate/testdata/huffman-shifts.dyn.expect-noinput b/flate/testdata/huffman-shifts.dyn.expect-noinput index 8d2d93d5e2eac5a83bd6785ac77d629792736537..9ad731f3cf7242cc802c880bffb96b1576cda835 100644 GIT binary patch literal 33 pcmaEJ*OHgPfP>+{UeCQBetd7^G}B90VP;@pxbY?4l!4(oF97Fv3~vAc literal 51 ncmXpqk!W39Akudz(1OeTp@s~CqiCwAKYijyWFesZO92J|+^-Tx diff --git a/flate/testdata/huffman-text-shift.dyn.expect b/flate/testdata/huffman-text-shift.dyn.expect index cb9e0b9c8a73eb5cf9788ea4e179b8050c52be01..486bdf6f69ddb727839412cf030cc4b4f4b5c6e7 100644 GIT binary patch literal 232 zcmV> zcoI)0-BmuHH?Kp)aRyE2b*=DZGCAL<1*8ELc%B+_$lkXVrzDQIAW>-uUI?gk4Xt-o z+0Bai7(3GgqlH6-!j%$&tF%M4T4wpW$fY)%5=t7etF3nZ)V2YS3|N0;aKdUA4ZkQE zw7#6$UG(-|b~rjqlG9{J$v}=iE~daUj}cQrO6$fh7&Eqet{poF{^B)8<&ctZO43<2 i O%bVNP-TMCF@u}EspI`Ug=RfxU_vh$1zGQ#LJ#}~h literal 242 zcmV6)(sRto0Sgxum8IzMa_Q@b=BJfp zWagI^B`4>nWTxkr73HNQRXPQ`=$2IGfE46pCg+u8lqBaRrRA6ArDW!nWcYYGyZQyY zre)^n@pAEU6(nb;=Vs=Cn3=f+`9&p4`AQ&8T4`Q#Zf2f_W;HJtXIW8_ZEm(kbZk;( ziH@16sj;ahFBfN;PP~qm#3XtDC!rrnub+QFU{G*KXjnu=WtF0m@}VWFYWns{01bq9AOHXW diff --git a/flate/testdata/huffman-text-shift.dyn.expect-noinput b/flate/testdata/huffman-text-shift.dyn.expect-noinput index cb9e0b9c8a73eb5cf9788ea4e179b8050c52be01..486bdf6f69ddb727839412cf030cc4b4f4b5c6e7 100644 GIT binary patch literal 232 zcmV> zcoI)0-BmuHH?Kp)aRyE2b*=DZGCAL<1*8ELc%B+_$lkXVrzDQIAW>-uUI?gk4Xt-o z+0Bai7(3GgqlH6-!j%$&tF%M4T4wpW$fY)%5=t7etF3nZ)V2YS3|N0;aKdUA4ZkQE zw7#6$UG(-|b~rjqlG9{J$v}=iE~daUj}cQrO6$fh7&Eqet{poF{^B)8<&ctZO43<2 i O%bVNP-TMCF@u}EspI`Ug=RfxU_vh$1zGQ#LJ#}~h literal 242 zcmV6)(sRto0Sgxum8IzMa_Q@b=BJfp zWagI^B`4>nWTxkr73HNQRXPQ`=$2IGfE46pCg+u8lqBaRrRA6ArDW!nWcYYGyZQyY zre)^n@pAEU6(nb;=Vs=Cn3=f+`9&p4`AQ&8T4`Q#Zf2f_W;HJtXIW8_ZEm(kbZk;( ziH@16sj;ahFBfN;PP~qm#3XtDC!rrnub+QFU{G*KXjnu=WtF0m@}VWFYWns{01bq9AOHXW diff --git a/flate/testdata/huffman-text.dyn.expect b/flate/testdata/huffman-text.dyn.expect index 6ef6dd44dd..b9cc20d0eb 100644 --- a/flate/testdata/huffman-text.dyn.expect +++ b/flate/testdata/huffman-text.dyn.expect @@ -1,3 +1,4 @@ -`J|ஏbF=M/MX+Kˊ;޹`.&;$ -A A :F8T h ͍˘P "PI&@ lG p`7TdxDGA^k, OAU!AVJQV2,ށj(,;]X` -*xqF_2>n^AUm Œ2>T gO U+d5ʕd6_i2 \ No newline at end of file +J0r=`K2Aasē)HIɟb]ky{h0E{66[cdb;"%#u["llB +%* +&HSvah9'B62CICG6tg R]Km!Č*ꚺx5[gQFء?>) +7풳^ w;$d2E^/έ{-x6S.9 \ No newline at end of file diff --git a/flate/testdata/huffman-text.dyn.expect-noinput b/flate/testdata/huffman-text.dyn.expect-noinput index 6ef6dd44dd..b9cc20d0eb 100644 --- a/flate/testdata/huffman-text.dyn.expect-noinput +++ b/flate/testdata/huffman-text.dyn.expect-noinput @@ -1,3 +1,4 @@ -`J|ஏbF=M/MX+Kˊ;޹`.&;$ -A A :F8T h ͍˘P "PI&@ lG p`7TdxDGA^k, OAU!AVJQV2,ށj(,;]X` -*xqF_2>n^AUm Œ2>T gO U+d5ʕd6_i2 \ No newline at end of file +J0r=`K2Aasē)HIɟb]ky{h0E{66[cdb;"%#u["llB +%* +&HSvah9'B62CICG6tg R]Km!Č*ꚺx5[gQFء?>) +7풳^ w;$d2E^/έ{-x6S.9 \ No newline at end of file diff --git a/flate/testdata/null-long-match.dyn.expect-noinput b/flate/testdata/null-long-match.dyn.expect-noinput index 14167a3344bcdc219ee9fd904418ae61298e584d..62d55e6b83a8e55bb699433966794c903d4d4116 100644 GIT binary patch literal 208 ccmaEJ_aFlhFm3#^vU=`OHzN>d=z Date: Wed, 22 Dec 2021 17:18:51 +0100 Subject: [PATCH 3/5] Reset maxtokens --- flate/deflate.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flate/deflate.go b/flate/deflate.go index e064b01ee5..d9a1185276 100644 --- a/flate/deflate.go +++ b/flate/deflate.go @@ -44,8 +44,8 @@ const ( // The maximum number of tokens we will encode at the time. // Smaller sizes usually creates less optimal blocks. // Bigger can make context switching slow. - // We use this for levels 7-9, so we make it as big as we can. - maxFlateBlockTokens = maxStoreBlockSize + // We use this for levels 7-9, so we make it big. + maxFlateBlockTokens = 1 << 15 maxStoreBlockSize = 65535 hashBits = 17 // After 17 performance degrades hashSize = 1 << hashBits From 21cdfe6e125ab20b7583ad2ddf7e2c6fa9bdf371 Mon Sep 17 00:00:00 2001 From: Klaus Post Date: Thu, 23 Dec 2021 10:12:45 +0100 Subject: [PATCH 4/5] Index while skipping. --- flate/deflate.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/flate/deflate.go b/flate/deflate.go index d9a1185276..e1f221f8a1 100644 --- a/flate/deflate.go +++ b/flate/deflate.go @@ -544,13 +544,12 @@ func (d *compressor) deflateLazy() { // If we have a long run of no matches, skip additional bytes // Resets when s.ii overflows after 64KB. - if s.ii > uint16(d.nice) { - n := int(s.ii >> 5) + if n := int(s.ii) - d.chain; n > 0 { + n = 1 + int(n>>6) for j := 0; j < n; j++ { if s.index >= d.windowEnd-1 { break } - d.tokens.AddLiteral(d.window[s.index-1]) if d.tokens.n == maxFlateBlockTokens { if d.err = d.writeBlock(&d.tokens, s.index, false); d.err != nil { @@ -558,6 +557,14 @@ func (d *compressor) deflateLazy() { } d.tokens.Reset() } + // Index... + if s.index < s.maxInsertIndex { + h := hash4(d.window[s.index:]) + ch := s.hashHead[h] + s.chainHead = int(ch) + s.hashPrev[s.index&windowMask] = ch + s.hashHead[h] = uint32(s.index + s.hashOffset) + } s.index++ } // Flush last byte From f6b442e9573b2f65135939f3a6db46b56c9f6bdc Mon Sep 17 00:00:00 2001 From: Klaus Post Date: Thu, 23 Dec 2021 11:26:49 +0100 Subject: [PATCH 5/5] Increase level 7 speed. --- flate/deflate.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flate/deflate.go b/flate/deflate.go index e1f221f8a1..b27f5a93bc 100644 --- a/flate/deflate.go +++ b/flate/deflate.go @@ -76,7 +76,7 @@ var levels = []compressionLevel{ {0, 0, 0, 0, 0, 6}, // Levels 7-9 use increasingly more lazy matching // and increasingly stringent conditions for "good enough". - {8, 12, 24, 24, skipNever, 7}, + {6, 10, 12, 16, skipNever, 7}, {10, 24, 32, 64, skipNever, 8}, {32, 258, 258, 1024, skipNever, 9}, }