Add experimental igzip (#182)

.github/workflows/CI.yml

@@ -88,10 +88,18 @@ jobs:
         with:
           target: ${{ matrix.conf.target-triple }}
 
+      - name: Install nasm (Windows / Linux)
+        if: runner.os != 'macOS'
+        uses: ilammy/setup-nasm@v1
+
+      - name: Set MSVC developer prompt
+        if: runner.os == 'Windows'
+        uses: ilammy/msvc-dev-cmd@v1
+
       - name: Setup (Mac)
         if: runner.os == 'macOS'
         run: |
-          brew install ninja
+          brew install ninja automake autoconf coreutils libtool nasm
           echo "MACOSX_DEPLOYMENT_TARGET=10.12" >> $GITHUB_ENV
 
       - name: Setup (Windows)
@@ -105,22 +113,39 @@ jobs:
           TARGET_TRIPLE: ${{ matrix.conf.target-triple }}
           WORKSPACE: ${{ github.workspace }}
           BLOSC2_INSTALL_PREFIX: ${{ github.workspace }}/blosc2
+          ISAL_INSTALL_PREFIX: ${{ github.workspace }}/isal
         run: |
           sudo apt update
           sudo apt install ninja-build -y
 
           echo "BLOSC2_INSTALL_PREFIX=$BLOSC2_INSTALL_PREFIX" >> $GITHUB_ENV
-          echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$BLOSC2_INSTALL_PREFIX/lib:$BLOSC2_INSTALL_PREFIX/lib64" >> $GITHUB_ENV
+          echo "ISAL_INSTALL_PREFIX=$ISAL_INSTALL_PREFIX" >> $GITHUB_ENV
+
+          echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$BLOSC2_INSTALL_PREFIX/lib:$BLOSC2_INSTALL_PREFIX/lib64:$ISAL_INSTALL_PREFIX/lib:$ISAL_INSTALL_PREFIX/lib64" >> $GITHUB_ENV
 
-          # maturin action fails building blosc2 for armv7 and aarch64
           # so we'll just use 'cross' to build and pass it in for the action for all archs
           # nothing special for the revision pin, just deterministic install
           cargo install cross --git https://github.com/cross-rs/cross --rev 6d097fb
+
+          # Build blosc2
           cross build --release --target $TARGET_TRIPLE --package blosc2-sys --target-dir build
           blosc2_sys_dir=$(ls build/$TARGET_TRIPLE/release/build/ | grep blosc2-sys)
           mv $WORKSPACE/build/$TARGET_TRIPLE/release/build/$blosc2_sys_dir/out $BLOSC2_INSTALL_PREFIX
           tree -L 2 $BLOSC2_INSTALL_PREFIX
 
+          # Build isal only on 64-bit systems
+          # At the time of this writing, it technically builds for all unix 32-bit systems
+          # but ISA-L has explicitly stated they're dropping support.
+          if [[ "$TARGET_TRIPLE" == armv7* || "$TARGET_TRIPLE" == i686* ]]; then
+            echo "Not building ISA-L on 32 bit target"
+            mkdir -p $ISAL_INSTALL_PREFIX
+          else
+            cross build --release --target $TARGET_TRIPLE --package isal-sys --target-dir build
+            isal_sys_dir=$(ls build/$TARGET_TRIPLE/release/build/ | grep isal-sys)
+            mv $WORKSPACE/build/$TARGET_TRIPLE/release/build/$isal_sys_dir/out $ISAL_INSTALL_PREFIX
+            tree -L 2 $ISAL_INSTALL_PREFIX
+          fi
+
       - name: Rust Tests
         if: matrix.conf.target == 'x86_64' && !startsWith(matrix.python-version, 'pypy') && matrix.python-version == '3.12'
         run: cargo test
@@ -131,24 +156,22 @@ jobs:
         with:
           target: ${{ matrix.conf.target }}
           manylinux: ${{ matrix.conf.manylinux }}
-          docker-options: -e BLOSC2_INSTALL_PREFIX=${{ github.workspace }}/blosc2 -e LD_LIBRARY_PATH=${{ github.workspace }}/blosc2/lib:${{ github.workspace }}/blosc2/lib64
-          args: -i ${{ matrix.python-version }} --release --out dist --features use-system-blosc2-static
-          before-script-linux: ls -l $BLOSC2_INSTALL_PREFIX
+          docker-options: |
+            -e BLOSC2_INSTALL_PREFIX=${{ github.workspace }}/blosc2
+            -e ISAL_INSTALL_PREFIX=${{ github.workspace }}/isal
+            -e LD_LIBRARY_PATH=${{ github.workspace }}/blosc2/lib:${{ github.workspace }}/blosc2/lib64:${{ github.workspace }}/isal/lib:${{ github.workspace }}/isal/lib64
+          args: -i ${{ matrix.python-version }} --release --out dist --features use-system-blosc2-static --features use-system-isal-static
+          before-script-linux: |
+            ls -l $BLOSC2_INSTALL_PREFIX
+            ls -l $ISAL_INSTALL_PREFIX
 
       - name: Build wheel (Windows)
         if: runner.os == 'Windows'
-        shell: bash
         run: |
           python -m pip install maturin delvewheel
-
-          maturin build \
-            -i python \
-            --release \
-            --out wheels \
-            --target ${{ matrix.conf.target-triple }}
-
-          name=$(ls -1 wheels/ | head -n 1)
-          delvewheel repair -v wheels/$name -w dist
+          maturin build -i python --release --out wheels --target ${{ matrix.conf.target-triple }}
+          $file = Get-ChildItem -Path "wheels" | Select-Object -First 1
+          delvewheel repair -v "wheels\$($file.Name)" -w "dist"
 
       - name: Build wheel (MacOS)
         if: runner.os == 'macOS'

Cargo.toml

@@ -14,7 +14,7 @@ name = "cramjam"
 crate-type = ["cdylib"]
 
 [features]
-default                  = ["extension-module", "snappy", "lz4", "bzip2", "brotli", "xz", "zstd", "gzip", "deflate", "blosc2"]
+default                  = ["extension-module", "snappy", "lz4", "bzip2", "brotli", "xz", "zstd", "gzip", "deflate", "blosc2", "igzip"]
 extension-module         = ["pyo3/extension-module"]
 generate-import-lib      = ["pyo3/generate-import-lib"]  # needed for Windows PyPy builds
 
@@ -28,6 +28,12 @@ xz                       = ["xz-static"]
 xz-static                = ["libcramjam/xz-static"]
 xz-shared                = ["libcramjam/xz-shared"]
 
+igzip                    = ["igzip-static"]
+igzip-static             = ["libcramjam/igzip-static"]
+igzip-shared             = ["libcramjam/igzip-shared"]
+use-system-isal-static   = ["libcramjam/use-system-isal", "libcramjam/igzip-static"]
+use-system-isal-shared   = ["libcramjam/use-system-isal", "libcramjam/igzip-shared"]
+
 gzip                     = ["gzip-static"]
 gzip-static              = ["libcramjam/gzip-static"]
 gzip-shared              = ["libcramjam/gzip-shared"]
@@ -47,7 +53,7 @@ wasm32-compat            = ["libcramjam/wasm32-compat"]
 
 [dependencies]
 pyo3 = { version = "^0.22", default-features = false, features = ["macros"] }
-libcramjam = { version = "0.4.6", default-features = false }
+libcramjam = { version = "0.5.1", default-features = false }
 
 [build-dependencies]
 pyo3-build-config = "^0.22"

Cross.toml

@@ -0,0 +1,50 @@
+[target.x86_64-unknown-linux-musl]
+pre-build = [
+  "apt update",
+  "apt install -y nasm libtool autotools-dev autoconf"
+]
+[target.x86_64-unknown-linux-gnu]
+pre-build = [
+  "apt update",
+  "apt install -y nasm libtool autotools-dev autoconf"
+]
+[target.s390x-unknown-linux-gnu]
+pre-build = [
+  "apt update",
+  "apt install -y nasm libtool autotools-dev autoconf"
+]
+[target.aarch64-unknown-linux-musl]
+pre-build = [
+  "apt update",
+  "apt install -y nasm libtool autotools-dev autoconf"
+]
+[target.aarch64-unknown-linux-gnu]
+pre-build = [
+  "apt update",
+  "apt install -y nasm libtool autotools-dev autoconf"
+]
+[target.powerpc64le-unknown-linux-gnu]
+pre-build = [
+  "apt update",
+  "apt install -y nasm libtool autotools-dev autoconf"
+]
+[target.armv7-unknown-linux-musleabihf]
+pre-build = [
+  "apt update",
+  "apt install -y nasm libtool autotools-dev autoconf"
+]
+[target.armv7-unknown-linux-gnueabihf]
+pre-build = [
+  "apt update",
+  "apt install -y nasm libtool autotools-dev autoconf"
+]
+[target.i686-unknown-linux-musl]
+pre-build = [
+  "apt update",
+  "apt install -y nasm libtool autotools-dev autoconf"
+]
+[target.i686-unknown-linux-gnu]
+pre-build = [
+  "apt update",
+  "apt install -y nasm libtool autotools-dev autoconf"
+]

README.md

@@ -45,6 +45,7 @@ Available algorithms:
 - [X] ZSTD           `cramjam.zstd`
 - [X] XZ / LZMA  `cramjam.xz`
 - [X] Blosc2        `cramjam.experimental.blosc2`
+- [X] IGzip         `cramjam.experimental.igzip` (only on 64-bit targets)
 
 All available for use as: