feat: zstd 1.5.6

Rogdham · May 20, 2024 · d2c3045 · d2c3045
1 parent 3316718
commit d2c3045
Show file tree

Hide file tree

Showing 10 changed files with 72 additions and 2 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,6 +4,7 @@ All notable changes to this project will be documented in this file.
 
 ## Next version (Unreleased)
 
+- Upgrade zstd source code from v1.5.5 to [v1.5.6](https://github.com/facebook/zstd/releases/tag/v1.5.6)
 - Fix pyzstd_pep517 parameter name in `get_requires_for_build_wheel`
 - Deprecate support for Python version before 3.8 and stop building wheels for them
 - Minor fixes in type hints

diff --git a/build_script/pyzstd_build_cffi.py b/build_script/pyzstd_build_cffi.py
@@ -57,6 +57,7 @@
     ZSTD_c_minMatch,
     ZSTD_c_targetLength,
     ZSTD_c_strategy,
+    ZSTD_c_targetCBlockSize,
 
     /* LDM mode parameters */
     ZSTD_c_enableLongDistanceMatching,
@@ -203,6 +204,12 @@
 }
 #endif
 
+#if ZSTD_VERSION_NUMBER < 10506
+typedef enum {
+    ZSTD_c_targetCBlockSize=130
+} PYZSTD_compat_c_targetCBlockSize;
+#endif
+
 #ifdef PYZSTD_STATIC_LINK
 int pyzstd_static_link = 1;
 #else

diff --git a/docs/index.rst b/docs/index.rst
@@ -983,7 +983,7 @@ Advanced parameters
     Attributes of :py:class:`CParameter` class:
 
         - Compression level (:py:attr:`~CParameter.compressionLevel`)
-        - Compress algorithm parameters (:py:attr:`~CParameter.windowLog`, :py:attr:`~CParameter.hashLog`, :py:attr:`~CParameter.chainLog`, :py:attr:`~CParameter.searchLog`, :py:attr:`~CParameter.minMatch`, :py:attr:`~CParameter.targetLength`, :py:attr:`~CParameter.strategy`)
+        - Compress algorithm parameters (:py:attr:`~CParameter.windowLog`, :py:attr:`~CParameter.hashLog`, :py:attr:`~CParameter.chainLog`, :py:attr:`~CParameter.searchLog`, :py:attr:`~CParameter.minMatch`, :py:attr:`~CParameter.targetLength`, :py:attr:`~CParameter.strategy`, :py:attr:`~CParameter.targetCBlockSize`)
         - Long distance matching (:py:attr:`~CParameter.enableLongDistanceMatching`, :py:attr:`~CParameter.ldmHashLog`, :py:attr:`~CParameter.ldmMinMatch`, :py:attr:`~CParameter.ldmBucketSizeLog`, :py:attr:`~CParameter.ldmHashRateLog`)
         - Misc (:py:attr:`~CParameter.contentSizeFlag`, :py:attr:`~CParameter.checksumFlag`, :py:attr:`~CParameter.dictIDFlag`)
         - Multi-threaded compression (:py:attr:`~CParameter.nbWorkers`, :py:attr:`~CParameter.jobSize`, :py:attr:`~CParameter.overlapLog`)
@@ -1126,6 +1126,18 @@ Advanced parameters
 
         Special: value ``0`` means "use default strategy", then the value is dynamically set, see "strat" column in `this table <https://github.com/facebook/zstd/blob/release/lib/compress/clevels.h>`_.
 
+     .. py:attribute:: targetCBlockSize
+
+        Attempts to fit compressed block size into approximatively targetCBlockSize (in bytes). Note that it's not a guarantee, just a convergence target.
+
+        This is helpful in low bandwidth streaming environments to improve end-to-end latency, when a client can make use of partial documents.
+
+        Bound by ZSTD_TARGETCBLOCKSIZE_MIN and ZSTD_TARGETCBLOCKSIZE_MAX. No target when targetCBlockSize == 0.
+
+        Default value is ``0``.
+
+        Only available for zstd v1.5.6+.
+
     .. py:attribute:: enableLongDistanceMatching
 
         Enable long distance matching.

diff --git a/src/__init__.pyi b/src/__init__.pyi
@@ -37,6 +37,7 @@ class CParameter(IntEnum):
     minMatch: int
     targetLength: int
     strategy: int
+    targetCBlockSize: int
 
     enableLongDistanceMatching: int
     ldmHashLog: int

diff --git a/src/bin_ext/pyzstd.c b/src/bin_ext/pyzstd.c
@@ -211,6 +211,9 @@ add_parameters(PyObject *module)
     ADD_INT_PREFIX_MACRO(module, ZSTD_c_minMatch);
     ADD_INT_PREFIX_MACRO(module, ZSTD_c_targetLength);
     ADD_INT_PREFIX_MACRO(module, ZSTD_c_strategy);
+#if ZSTD_VERSION_NUMBER >= 10506
+    ADD_INT_PREFIX_MACRO(module, ZSTD_c_targetCBlockSize);
+#endif
 
     ADD_INT_PREFIX_MACRO(module, ZSTD_c_enableLongDistanceMatching);
     ADD_INT_PREFIX_MACRO(module, ZSTD_c_ldmHashLog);

diff --git a/src/bin_ext/pyzstd.h b/src/bin_ext/pyzstd.h
@@ -251,6 +251,10 @@ static const ParameterInfo cp_list[] =
     {ZSTD_c_targetLength,     "targetLength"},
     {ZSTD_c_strategy,         "strategy"},
 
+#if ZSTD_VERSION_NUMBER >= 10506
+    {ZSTD_c_targetCBlockSize, "targetCBlockSize"},
+#endif
+
     {ZSTD_c_enableLongDistanceMatching, "enableLongDistanceMatching"},
     {ZSTD_c_ldmHashLog,       "ldmHashLog"},
     {ZSTD_c_ldmMinMatch,      "ldmMinMatch"},

diff --git a/src/c/__init__.py b/src/c/__init__.py
@@ -55,6 +55,18 @@ def get_frame_info(frame_buffer):
     return _nt_frame_info(*ret_tuple)
 
 
+class _UnsupportedCParameter:
+    def __set_name__(self, _, name):
+        self.name = name
+
+    def __get__(self, *_, **__):
+        msg = ("%s CParameter only available when the underlying "
+               "zstd library's version is greater than or equal to v1.5.6. "
+               "At pyzstd module's run-time, zstd version is %s.") % \
+               (self.name, zstd_version)
+        raise NotImplementedError(msg)
+
+
 class CParameter(IntEnum):
     """Compression parameters"""
 
@@ -66,6 +78,10 @@ class CParameter(IntEnum):
     minMatch                   = _zstd._ZSTD_c_minMatch
     targetLength               = _zstd._ZSTD_c_targetLength
     strategy                   = _zstd._ZSTD_c_strategy
+    if zstd_version_info >= (1, 5, 6):
+        targetCBlockSize       = _zstd._ZSTD_c_targetCBlockSize
+    else:
+        targetCBlockSize       = _UnsupportedCParameter()
 
     enableLongDistanceMatching = _zstd._ZSTD_c_enableLongDistanceMatching
     ldmHashLog                 = _zstd._ZSTD_c_ldmHashLog

diff --git a/src/cffi/common.py b/src/cffi/common.py
@@ -44,6 +44,19 @@ def _get_param_bounds(is_compress, key):
 
     return (bounds.lowerBound, bounds.upperBound)
 
+
+class _UnsupportedCParameter:
+    def __set_name__(self, _, name):
+        self.name = name
+
+    def __get__(self, *_, **__):
+        msg = ("%s CParameter only available when the underlying "
+               "zstd library's version is greater than or equal to v1.5.6. "
+               "At pyzstd module's run-time, zstd version is %s.") % \
+               (self.name, zstd_version)
+        raise NotImplementedError(msg)
+
+
 class CParameter(IntEnum):
     """Compression parameters"""
 
@@ -55,6 +68,10 @@ class CParameter(IntEnum):
     minMatch                   = m.ZSTD_c_minMatch
     targetLength               = m.ZSTD_c_targetLength
     strategy                   = m.ZSTD_c_strategy
+    if zstd_version_info >= (1, 5, 6):
+        targetCBlockSize       = m.ZSTD_c_targetCBlockSize
+    else:
+        targetCBlockSize       = _UnsupportedCParameter()
 
     enableLongDistanceMatching = m.ZSTD_c_enableLongDistanceMatching
     ldmHashLog                 = m.ZSTD_c_ldmHashLog
@@ -166,6 +183,8 @@ def _set_parameter_error(is_compress, key, value):
      m.ZSTD_c_nbWorkers:        "nbWorkers",
      m.ZSTD_c_jobSize:          "jobSize",
      m.ZSTD_c_overlapLog:       "overlapLog"}
+    if zstd_version_info >= (1, 5, 6):
+        COMPRESS_PARAMETERS[m.ZSTD_c_targetCBlockSize] = "targetCBlockSize"
 
     DECOMPRESS_PARAMETERS = {m.ZSTD_d_windowLogMax: "windowLogMax"}
 

diff --git a/tests/test_zstd.py b/tests/test_zstd.py
@@ -429,6 +429,11 @@ def test_CParameter(self):
         CParameter.minMatch
         CParameter.targetLength
         CParameter.strategy
+        if zstd_version_info >= (1, 5, 6):
+            CParameter.targetCBlockSize
+        else:
+            with self.assertRaises(NotImplementedError):
+                CParameter.targetCBlockSize
 
         CParameter.enableLongDistanceMatching
         CParameter.ldmHashLog
@@ -611,6 +616,8 @@ def test_compress_parameters(self):
              CParameter.jobSize : 5*MB if zstd_support_multithread else 0,
              CParameter.overlapLog : 9 if zstd_support_multithread else 0,
              }
+        if zstd_version_info >= (1, 5, 6):
+            d[CParameter.targetCBlockSize] = 150
         ZstdCompressor(level_or_option=d)
 
         # larger than signed int, ValueError

diff --git a/zstd b/zstd