Skip to content

Commit

Permalink
Encode 4 bytes between flushing (#88)
Browse files Browse the repository at this point in the history
If tablelog is sufficiently small, encode 4 bytes between each flush.

```
name                     old time/op    new time/op    delta
Compress/gettysburg-8      10.2µs ± 0%    10.5µs ± 1%   +3.17%          (p=0.008 n=5+5)
Compress/digits-8           498µs ± 1%     503µs ± 2%     ~             (p=0.151 n=5+5)
Compress/twain-8           2.12ms ± 0%    2.12ms ± 1%     ~             (p=0.730 n=4+5)
Compress/low-ent-8          215µs ± 3%     214µs ± 2%     ~             (p=1.000 n=5+5)
Compress/superlow-ent-8    58.0µs ± 2%    58.4µs ± 1%     ~             (p=0.310 n=5+5)
Compress/endzerobits-8      775ns ± 2%     642ns ± 3%  -17.13%          (p=0.008 n=5+5)
Compress/pngdata.001-8      281µs ± 1%     282µs ± 2%     ~             (p=1.000 n=5+5)
Compress/normcount2-8      3.04µs ± 2%    3.05µs ± 3%     ~             (p=1.000 n=5+5)

name                     old speed      new speed      delta
Compress/gettysburg-8     152MB/s ± 0%   147MB/s ± 1%   -3.07%          (p=0.008 n=5+5)
Compress/digits-8         201MB/s ± 1%   199MB/s ± 2%     ~             (p=0.151 n=5+5)
Compress/twain-8          183MB/s ± 0%   183MB/s ± 1%     ~             (p=0.730 n=4+5)
Compress/low-ent-8        186MB/s ± 3%   187MB/s ± 2%     ~             (p=1.000 n=5+5)
Compress/superlow-ent-8   181MB/s ± 2%   180MB/s ± 1%     ~             (p=0.310 n=5+5)
Compress/endzerobits-8   6.45MB/s ± 2%  7.78MB/s ± 3%  +20.69%          (p=0.008 n=5+5)
Compress/pngdata.001-8    182MB/s ± 1%   182MB/s ± 2%     ~             (p=1.000 n=5+5)
Compress/normcount2-8    28.6MB/s ± 2%  28.5MB/s ± 3%     ~             (p=1.000 n=5+5)
```
  • Loading branch information
klauspost authored Mar 4, 2018
1 parent 8a8c9b9 commit 2c62357
Show file tree
Hide file tree
Showing 2 changed files with 35 additions and 3 deletions.
9 changes: 8 additions & 1 deletion fse/bitwriter.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,13 @@ func (b *bitWriter) addBits16NC(value uint16, bits uint8) {
b.nBits += bits
}

// addBits16Clean will add up to 16 bits. value may not contain more set bits than indicated.
// It will not check if there is space for them, so the caller must ensure that it has flushed recently.
func (b *bitWriter) addBits16Clean(value uint16, bits uint8) {
b.bitContainer |= uint64(value) << (b.nBits & 63)
b.nBits += bits
}

// addBits16ZeroNC will add up to 16 bits.
// It will not check if there is space for them,
// so the caller must ensure that it has flushed recently.
Expand Down Expand Up @@ -147,7 +154,7 @@ func (b *bitWriter) flushAlign() {
// to the output.
func (b *bitWriter) close() error {
// End mark
b.addBits16NC(1, 1)
b.addBits16Clean(1, 1)
// flush until next byte.
b.flushAlign()
return nil
Expand Down
29 changes: 27 additions & 2 deletions fse/compress.go
Original file line number Diff line number Diff line change
Expand Up @@ -149,7 +149,21 @@ func (s *Scratch) compress(src []byte) error {
}

// Main compression loop.
if !s.zeroBits {
switch {
case !s.zeroBits && s.actualTableLog <= 8:
// We can encode 4 symbols without requiring a flush.
// We do not need to check if any output is 0 bits.
for ip >= 4 {
s.bw.flush32()
v3, v2, v1, v0 := src[ip-4], src[ip-3], src[ip-2], src[ip-1]
c2.encode(tt[v0])
c1.encode(tt[v1])
c2.encode(tt[v2])
c1.encode(tt[v3])
ip -= 4
}
case !s.zeroBits:
// We do not need to check if any output is 0 bits.
for ip >= 4 {
s.bw.flush32()
v3, v2, v1, v0 := src[ip-4], src[ip-3], src[ip-2], src[ip-1]
Expand All @@ -160,7 +174,18 @@ func (s *Scratch) compress(src []byte) error {
c1.encode(tt[v3])
ip -= 4
}
} else {
case s.actualTableLog <= 8:
// We can encode 4 symbols without requiring a flush
for ip >= 4 {
s.bw.flush32()
v3, v2, v1, v0 := src[ip-4], src[ip-3], src[ip-2], src[ip-1]
c2.encodeZero(tt[v0])
c1.encodeZero(tt[v1])
c2.encodeZero(tt[v2])
c1.encodeZero(tt[v3])
ip -= 4
}
default:
for ip >= 4 {
s.bw.flush32()
v3, v2, v1, v0 := src[ip-4], src[ip-3], src[ip-2], src[ip-1]
Expand Down

0 comments on commit 2c62357

Please sign in to comment.