Merge pull request #4290 from facebook/lmax

--max command
facebook · Feb 10, 2025 · e3a9351 · e3a9351
2 parents 071a4a0 + 613901b
commit e3a9351
Show file tree

Hide file tree

Showing 4 changed files with 67 additions and 13 deletions.
diff --git a/programs/zstd.1.md b/programs/zstd.1.md
@@ -161,6 +161,10 @@ the last one takes effect.
 
     Note: If `windowLog` is set to larger than 27, `--long=windowLog` or
     `--memory=windowSize` needs to be passed to the decompressor.
+* `--max`:
+    set advanced parameters to maximum compression.
+    warning: this setting is very slow and uses a lot of resources.
+    It's inappropriate for 32-bit mode and therefore disabled in this mode.
 * `-D DICT`:
     use `DICT` as Dictionary to compress or decompress FILE(s)
 * `--patch-from FILE`:

diff --git a/programs/zstdcli.c b/programs/zstdcli.c
@@ -639,6 +639,22 @@ static unsigned parseCompressionParameters(const char* stringPtr, ZSTD_compressi
     return 1;
 }
 
+static void setMaxCompression(ZSTD_compressionParameters* params)
+{
+    params->windowLog = ZSTD_WINDOWLOG_MAX;
+    params->chainLog = ZSTD_CHAINLOG_MAX;
+    params->hashLog = ZSTD_HASHLOG_MAX;
+    params->searchLog = ZSTD_SEARCHLOG_MAX;
+    params->minMatch = ZSTD_MINMATCH_MIN;
+    params->targetLength = ZSTD_TARGETLENGTH_MAX;
+    params->strategy = ZSTD_STRATEGY_MAX;
+    g_overlapLog = ZSTD_OVERLAPLOG_MAX;
+    g_ldmHashLog = ZSTD_LDM_HASHLOG_MAX;
+    g_ldmHashRateLog = 0; /* automatically derived */
+    g_ldmMinMatch = 16; /* heuristic */
+    g_ldmBucketSizeLog = ZSTD_LDM_BUCKETSIZELOG_MAX;
+}
+
 static void printVersion(void)
 {
     if (g_displayLevel < DISPLAY_LEVEL_DEFAULT) {
@@ -794,22 +810,22 @@ static unsigned default_nbThreads(void) {
             CLEAN_RETURN(1);      \
 }   }   }
 
-#define NEXT_UINT32(val32) {      \
-    const char* __nb;             \
-    NEXT_FIELD(__nb);             \
+#define NEXT_UINT32(val32) {        \
+    const char* __nb;               \
+    NEXT_FIELD(__nb);               \
     val32 = readU32FromChar(&__nb); \
-    if(*__nb != 0) {         \
+    if(*__nb != 0) {                \
         errorOut("error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed"); \
-    }                             \
+    }                               \
 }
 
-#define NEXT_TSIZE(valTsize) {      \
-    const char* __nb;             \
-    NEXT_FIELD(__nb);             \
+#define NEXT_TSIZE(valTsize) {           \
+    const char* __nb;                    \
+    NEXT_FIELD(__nb);                    \
     valTsize = readSizeTFromChar(&__nb); \
-    if(*__nb != 0) {         \
+    if(*__nb != 0) {                     \
         errorOut("error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed"); \
-    }                             \
+    }                                    \
 }
 
 typedef enum { zom_compress, zom_decompress, zom_test, zom_bench, zom_train, zom_list } zstd_operation_mode;
@@ -1007,6 +1023,16 @@ int main(int argCount, const char* argv[])
                 if (!strcmp(argument, "--fake-stderr-is-console")) { UTIL_fakeStderrIsConsole(); continue; }
                 if (!strcmp(argument, "--trace-file-stat")) { UTIL_traceFileStat(); continue; }
 
+                if (!strcmp(argument, "--max")) {
+                    if (sizeof(void*)==4) {
+                        DISPLAYLEVEL(2, "--max is incompatible with 32-bit mode \n");
+                        badUsage(programName, originalArgument);
+                        CLEAN_RETURN(1);
+                    }
+                    ultra=1; ldmFlag = 1; setMaxCompression(&compressionParams);
+                    continue;
+                }
+
                 /* long commands with arguments */
 #ifndef ZSTD_NODICT
                 if (longCommandWArg(&argument, "--train-cover")) {

diff --git a/tests/cli-tests/compression/levels.sh b/tests/cli-tests/compression/levels.sh
@@ -5,14 +5,27 @@ set -v
 
 datagen > file
 
+# Retrieve the program's version information
+# Note: command echoing differs between macos and linux, so it's disabled below
+set +v
+version_info=$(zstd -V)
+set -v
+
 # Compress with various levels and ensure that their sizes are ordered
 zstd --fast=10 file -o file-f10.zst -q
 zstd --fast=1 file -o file-f1.zst -q
 zstd -1 file -o file-1.zst -q
 zstd -19 file -o file-19.zst -q
+if echo "$version_info" | grep -q '32-bit'; then
+    # skip --max test: not enough address space
+    cp file-19.zst file-max.zst
+else
+    zstd --max file -o file-max.zst -q
+fi
 
-zstd -t file-f10.zst file-f1.zst file-1.zst file-19.zst
+zstd -t file-f10.zst file-f1.zst file-1.zst file-19.zst file-max.zst
 
+cmp_size -le file-max.zst file-19.zst
 cmp_size -lt file-19.zst file-1.zst
 cmp_size -lt file-1.zst file-f1.zst
 cmp_size -lt file-f1.zst file-f10.zst

diff --git a/tests/cli-tests/compression/levels.sh.stderr.exact b/tests/cli-tests/compression/levels.sh.stderr.exact
@@ -1,15 +1,26 @@
 
 datagen > file
 
+# Retrieve the program's version information
+# Note: command echoing differs between macos and linux, so it's disabled below
+set +v
+
 # Compress with various levels and ensure that their sizes are ordered
 zstd --fast=10 file -o file-f10.zst -q
 zstd --fast=1 file -o file-f1.zst -q
 zstd -1 file -o file-1.zst -q
 zstd -19 file -o file-19.zst -q
+if echo "$version_info" | grep -q '32-bit'; then
+    # skip --max test: not enough address space
+    cp file-19.zst file-max.zst
+else
+    zstd --max file -o file-max.zst -q
+fi
 
-zstd -t file-f10.zst file-f1.zst file-1.zst file-19.zst
-4 files decompressed : 262148 bytes total 
+zstd -t file-f10.zst file-f1.zst file-1.zst file-19.zst file-max.zst
+5 files decompressed : 327685 bytes total 
 
+cmp_size -le file-max.zst file-19.zst
 cmp_size -lt file-19.zst file-1.zst
 cmp_size -lt file-1.zst file-f1.zst
 cmp_size -lt file-f1.zst file-f10.zst