From efb5ca7d9fcba566b00b776af054bac3ad9beb24 Mon Sep 17 00:00:00 2001 From: Ned Loynd Date: Thu, 9 Nov 2023 11:52:33 +1100 Subject: [PATCH] Basic uncompressed block merging --- .../deft4j/deflate/DeflateBlock.java | 2 + .../deft4j/deflate/DeflateBlockHuffman.java | 12 +++ .../deflate/DeflateBlockUncompressed.java | 22 +++++ .../deft4j/deflate/DeflateStream.java | 82 +++++++++++++++++- runTestOpt.sh | 1 + test/deflate-store-2-opt.txt.gz | Bin 0 -> 118 bytes test/deflate-store-2-opt.txt.gz.txt | 4 + 7 files changed, 122 insertions(+), 1 deletion(-) create mode 100644 test/deflate-store-2-opt.txt.gz create mode 100644 test/deflate-store-2-opt.txt.gz.txt diff --git a/deft4j-base/src/main/java/com/github/NeRdTheNed/deft4j/deflate/DeflateBlock.java b/deft4j-base/src/main/java/com/github/NeRdTheNed/deft4j/deflate/DeflateBlock.java index c90d315..b2c3f88 100644 --- a/deft4j-base/src/main/java/com/github/NeRdTheNed/deft4j/deflate/DeflateBlock.java +++ b/deft4j-base/src/main/java/com/github/NeRdTheNed/deft4j/deflate/DeflateBlock.java @@ -24,6 +24,8 @@ static DeflateBlock copy(DeflateBlock old, DeflateBlock newBlock) { } public abstract DeflateBlock copy(); + public abstract boolean canMerge(DeflateBlock append); + public abstract DeflateBlock merge(DeflateBlock append); public abstract long optimise(); public abstract DeflateBlockType getDeflateBlockType(); abstract boolean parse(BitInputStream is) throws IOException; diff --git a/deft4j-base/src/main/java/com/github/NeRdTheNed/deft4j/deflate/DeflateBlockHuffman.java b/deft4j-base/src/main/java/com/github/NeRdTheNed/deft4j/deflate/DeflateBlockHuffman.java index 0e12150..10c2bb6 100644 --- a/deft4j-base/src/main/java/com/github/NeRdTheNed/deft4j/deflate/DeflateBlockHuffman.java +++ b/deft4j-base/src/main/java/com/github/NeRdTheNed/deft4j/deflate/DeflateBlockHuffman.java @@ -1222,4 +1222,16 @@ public void discard() { litlens = null; rlePairs = null; } + + // TODO + @Override + public boolean canMerge(DeflateBlock append) { + return false; + } + + // TODO + @Override + public DeflateBlock merge(DeflateBlock append) { + return null; + } } diff --git a/deft4j-base/src/main/java/com/github/NeRdTheNed/deft4j/deflate/DeflateBlockUncompressed.java b/deft4j-base/src/main/java/com/github/NeRdTheNed/deft4j/deflate/DeflateBlockUncompressed.java index 388317a..16848b1 100644 --- a/deft4j-base/src/main/java/com/github/NeRdTheNed/deft4j/deflate/DeflateBlockUncompressed.java +++ b/deft4j-base/src/main/java/com/github/NeRdTheNed/deft4j/deflate/DeflateBlockUncompressed.java @@ -5,6 +5,7 @@ import com.github.NeRdTheNed.deft4j.io.BitInputStream; import com.github.NeRdTheNed.deft4j.io.BitOutputStream; import com.github.NeRdTheNed.deft4j.util.BitInputStreamUtil; +import com.github.NeRdTheNed.deft4j.util.Util; public class DeflateBlockUncompressed extends DeflateBlock { private byte[] storedData; @@ -94,4 +95,25 @@ public void discard() { storedData = null; } + @Override + public boolean canMerge(DeflateBlock append) { + if (append != null) { + final long totalSize = getUncompressedData().length + append.getUncompressedData().length; + + if (totalSize <= 65535) { + return true; + } + } + + return false; + } + + @Override + public DeflateBlock merge(DeflateBlock append) { + final DeflateBlockUncompressed merged = new DeflateBlockUncompressed(getPrevious()); + merged.setNext(append.getNext()); + merged.fromUncompressed(Util.combine(getUncompressedData(), append.getUncompressedData())); + return merged; + } + } diff --git a/deft4j-base/src/main/java/com/github/NeRdTheNed/deft4j/deflate/DeflateStream.java b/deft4j-base/src/main/java/com/github/NeRdTheNed/deft4j/deflate/DeflateStream.java index a021fde..38fa0cc 100644 --- a/deft4j-base/src/main/java/com/github/NeRdTheNed/deft4j/deflate/DeflateStream.java +++ b/deft4j-base/src/main/java/com/github/NeRdTheNed/deft4j/deflate/DeflateStream.java @@ -556,7 +556,87 @@ public long optimise() { } // TODO Try other types of blocks - // TODO Try merging blocks + // TODO Try merging more block types + // TODO Try merging blocks at different passes + saved += mergeBlocks(); + return saved; + } + + public long mergeBlocks() { + int block = 0; + int pass = 0; + long pos = 0; + long saved = 0; + boolean first = true; + DeflateBlock currentBlock = firstBlock; + + while (currentBlock != null) { + boolean finishPass = true; + boolean didRemove = false; + final DeflateBlock nextBlock = currentBlock.getNext(); + + // Merge block if it's not empty. If it's the only block in a stream, ignore it. + if (first && (nextBlock == null)) { + pos += currentBlock.getSizeBits(pos + 3) + 3; + } else if (currentBlock.getUncompressedData().length > 0) { + pos += 3; + + if ((nextBlock != null) && currentBlock.canMerge(nextBlock)) { + final DeflateBlock merged = currentBlock.merge(nextBlock); + final long currentSizeNoMerge = currentBlock.getSizeBits(pos); + final long nextSizeNoMerge = nextBlock.getSizeBits(pos + currentSizeNoMerge + 3); + final long currentSaved = (currentSizeNoMerge + 3 + nextSizeNoMerge) - merged.getSizeBits(pos); + + if ((merged != currentBlock) && (currentSaved > 0)) { + finishPass = false; + pass++; + saved += currentSaved; + + if (PRINT_OPT_FINE) { + System.out.println("Merge pass " + pass + " saved " + currentSaved + " bits in block " + block); + } + + currentBlock.replace(merged); + currentBlock.discard(); + merged.setNext(nextBlock.getNext()); + currentBlock = merged; + + if (first) { + setFirstBlock(merged); + } + } + } + + pos += currentBlock.getSizeBits(pos); + } else { + final long currentSaved = currentBlock.getSizeBits(pos + 3) + 3; + + if (PRINT_OPT_FINE) { + System.out.println("Removed empty block " + block + ", saved " + currentSaved + " bits"); + } + + saved += currentSaved; + + if (first) { + setFirstBlock(currentBlock.getNext()); + } + + currentBlock.remove(); + didRemove = true; + } + + if (finishPass) { + block++; + currentBlock = currentBlock.getNext(); + + if (first && !didRemove) { + first = false; + } + + pass = 0; + } + } + return saved; } diff --git a/runTestOpt.sh b/runTestOpt.sh index d95a9b1..3209ce8 100755 --- a/runTestOpt.sh +++ b/runTestOpt.sh @@ -1,5 +1,6 @@ gradle build +java -ea -jar ./deft4j-cmd/build/libs/deft4j-cmd-all.jar optimise ./test/deflate-store-2.txt.gz ./test/deflate-store-2-opt.txt.gz | tee ./test/deflate-store-2-opt.txt.gz.txt java -ea -jar ./deft4j-cmd/build/libs/deft4j-cmd-all.jar optimise ./test/asyoulik/asyoulik-gzip.txt.gz ./test/asyoulik/asyoulik-gzip-opt.txt.gz | tee ./test/asyoulik/asyoulik-gzip-opt.txt.gz.txt java -ea -jar ./deft4j-cmd/build/libs/deft4j-cmd-all.jar optimise ./test/asyoulik/asyoulik-zopfli.txt.gz ./test/asyoulik/asyoulik-zopfli-opt.txt.gz | tee ./test/asyoulik/asyoulik-zopfli-opt.txt.gz.txt java -ea -jar ./deft4j-cmd/build/libs/deft4j-cmd-all.jar optimise ./test/text.png ./test/text-opt.png | tee ./test/text-opt.png.txt diff --git a/test/deflate-store-2-opt.txt.gz b/test/deflate-store-2-opt.txt.gz new file mode 100644 index 0000000000000000000000000000000000000000..63cab62c60c0214d49c841655fbd6254762c6ab4 GIT binary patch literal 118 zcmb2|=HOregOt>?oWzn;-Qtq`qEuZYy^@L&21aj&E&oF@GK&>}C{e*ZDziW#Ei)%o zp(G=*M4_}ewOFB~JYRvUG%q