Basic uncompressed block merging

NeRdTheNed · Nov 9, 2023 · efb5ca7 · efb5ca7
1 parent 76d7cf4
commit efb5ca7
Show file tree

Hide file tree

Showing 7 changed files with 122 additions and 1 deletion.
diff --git a/deft4j-base/src/main/java/com/github/NeRdTheNed/deft4j/deflate/DeflateBlock.java b/deft4j-base/src/main/java/com/github/NeRdTheNed/deft4j/deflate/DeflateBlock.java
@@ -24,6 +24,8 @@ static DeflateBlock copy(DeflateBlock old, DeflateBlock newBlock) {
     }
 
     public abstract DeflateBlock copy();
+    public abstract boolean canMerge(DeflateBlock append);
+    public abstract DeflateBlock merge(DeflateBlock append);
     public abstract long optimise();
     public abstract DeflateBlockType getDeflateBlockType();
     abstract boolean parse(BitInputStream is) throws IOException;

diff --git a/deft4j-base/src/main/java/com/github/NeRdTheNed/deft4j/deflate/DeflateBlockHuffman.java b/deft4j-base/src/main/java/com/github/NeRdTheNed/deft4j/deflate/DeflateBlockHuffman.java
@@ -1222,4 +1222,16 @@ public void discard() {
         litlens = null;
         rlePairs = null;
     }
+
+    // TODO
+    @Override
+    public boolean canMerge(DeflateBlock append) {
+        return false;
+    }
+
+    // TODO
+    @Override
+    public DeflateBlock merge(DeflateBlock append) {
+        return null;
+    }
 }
diff --git a/deft4j-base/src/main/java/com/github/NeRdTheNed/deft4j/deflate/DeflateBlockUncompressed.java b/deft4j-base/src/main/java/com/github/NeRdTheNed/deft4j/deflate/DeflateBlockUncompressed.java
@@ -5,6 +5,7 @@
 import com.github.NeRdTheNed.deft4j.io.BitInputStream;
 import com.github.NeRdTheNed.deft4j.io.BitOutputStream;
 import com.github.NeRdTheNed.deft4j.util.BitInputStreamUtil;
+import com.github.NeRdTheNed.deft4j.util.Util;
 
 public class DeflateBlockUncompressed extends DeflateBlock {
     private byte[] storedData;
@@ -94,4 +95,25 @@ public void discard() {
         storedData = null;
     }
 
+    @Override
+    public boolean canMerge(DeflateBlock append) {
+        if (append != null) {
+            final long totalSize = getUncompressedData().length + append.getUncompressedData().length;
+
+            if (totalSize <= 65535) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    @Override
+    public DeflateBlock merge(DeflateBlock append) {
+        final DeflateBlockUncompressed merged = new DeflateBlockUncompressed(getPrevious());
+        merged.setNext(append.getNext());
+        merged.fromUncompressed(Util.combine(getUncompressedData(), append.getUncompressedData()));
+        return merged;
+    }
+
 }
diff --git a/deft4j-base/src/main/java/com/github/NeRdTheNed/deft4j/deflate/DeflateStream.java b/deft4j-base/src/main/java/com/github/NeRdTheNed/deft4j/deflate/DeflateStream.java
@@ -556,7 +556,87 @@ public long optimise() {
         }
 
         // TODO Try other types of blocks
-        // TODO Try merging blocks
+        // TODO Try merging more block types
+        // TODO Try merging blocks at different passes
+        saved += mergeBlocks();
+        return saved;
+    }
+
+    public long mergeBlocks() {
+        int block = 0;
+        int pass = 0;
+        long pos = 0;
+        long saved = 0;
+        boolean first = true;
+        DeflateBlock currentBlock = firstBlock;
+
+        while (currentBlock != null) {
+            boolean finishPass = true;
+            boolean didRemove = false;
+            final DeflateBlock nextBlock = currentBlock.getNext();
+
+            // Merge block if it's not empty. If it's the only block in a stream, ignore it.
+            if (first && (nextBlock == null)) {
+                pos += currentBlock.getSizeBits(pos + 3) + 3;
+            } else if (currentBlock.getUncompressedData().length > 0) {
+                pos += 3;
+
+                if ((nextBlock != null) && currentBlock.canMerge(nextBlock)) {
+                    final DeflateBlock merged = currentBlock.merge(nextBlock);
+                    final long currentSizeNoMerge = currentBlock.getSizeBits(pos);
+                    final long nextSizeNoMerge = nextBlock.getSizeBits(pos + currentSizeNoMerge + 3);
+                    final long currentSaved = (currentSizeNoMerge + 3 + nextSizeNoMerge) - merged.getSizeBits(pos);
+
+                    if ((merged != currentBlock) && (currentSaved > 0)) {
+                        finishPass = false;
+                        pass++;
+                        saved += currentSaved;
+
+                        if (PRINT_OPT_FINE) {
+                            System.out.println("Merge pass " + pass + " saved " + currentSaved + " bits in block " + block);
+                        }
+
+                        currentBlock.replace(merged);
+                        currentBlock.discard();
+                        merged.setNext(nextBlock.getNext());
+                        currentBlock = merged;
+
+                        if (first) {
+                            setFirstBlock(merged);
+                        }
+                    }
+                }
+
+                pos += currentBlock.getSizeBits(pos);
+            } else {
+                final long currentSaved = currentBlock.getSizeBits(pos + 3) + 3;
+
+                if (PRINT_OPT_FINE) {
+                    System.out.println("Removed empty block " + block + ", saved " + currentSaved + " bits");
+                }
+
+                saved += currentSaved;
+
+                if (first) {
+                    setFirstBlock(currentBlock.getNext());
+                }
+
+                currentBlock.remove();
+                didRemove = true;
+            }
+
+            if (finishPass) {
+                block++;
+                currentBlock = currentBlock.getNext();
+
+                if (first && !didRemove) {
+                    first = false;
+                }
+
+                pass = 0;
+            }
+        }
+
         return saved;
     }
 

diff --git a/runTestOpt.sh b/runTestOpt.sh
@@ -1,5 +1,6 @@
 gradle build
 
+java -ea -jar ./deft4j-cmd/build/libs/deft4j-cmd-all.jar optimise ./test/deflate-store-2.txt.gz ./test/deflate-store-2-opt.txt.gz | tee ./test/deflate-store-2-opt.txt.gz.txt
 java -ea -jar ./deft4j-cmd/build/libs/deft4j-cmd-all.jar optimise ./test/asyoulik/asyoulik-gzip.txt.gz ./test/asyoulik/asyoulik-gzip-opt.txt.gz | tee ./test/asyoulik/asyoulik-gzip-opt.txt.gz.txt
 java -ea -jar ./deft4j-cmd/build/libs/deft4j-cmd-all.jar optimise ./test/asyoulik/asyoulik-zopfli.txt.gz ./test/asyoulik/asyoulik-zopfli-opt.txt.gz | tee ./test/asyoulik/asyoulik-zopfli-opt.txt.gz.txt
 java -ea -jar ./deft4j-cmd/build/libs/deft4j-cmd-all.jar optimise ./test/text.png ./test/text-opt.png | tee ./test/text-opt.png.txt

diff --git a/test/deflate-store-2-opt.txt.gz b/test/deflate-store-2-opt.txt.gz
diff --git a/test/deflate-store-2-opt.txt.gz.txt b/test/deflate-store-2-opt.txt.gz.txt
@@ -0,0 +1,4 @@
+File type recognised as GZip
+40 bits saved in stream 0 (deflate-store-2.txt)
+Total bits saved 40
+Saved 40 bits with optimisation