Merge pull request #87 from Intel-HLS/pv_update_compression_2.12ISAL

Update to ISAL_2.21

src/main/native/compression/IntelDeflater.cc

@@ -84,7 +84,7 @@ JNIEXPORT void JNICALL Java_com_intel_gkl_compression_IntelDeflater_resetNative
   jint level = env->GetIntField(obj, FID_level);
 
 
-  if(level == 1) {
+  if(level == 1 || level ==2) {
     isal_zstream* lz_stream = (isal_zstream*)env->GetLongField(obj, FID_lz_stream);
 
     if (lz_stream == 0) {
@@ -95,19 +95,27 @@ JNIEXPORT void JNICALL Java_com_intel_gkl_compression_IntelDeflater_resetNative
       }
 
       env->SetLongField(obj, FID_lz_stream, (jlong)lz_stream);
+
        isal_deflate_stateless_init(lz_stream);
-       lz_stream->hufftables = 0x0;
+
+    lz_stream->level = level;
+    lz_stream->level_buf = (uint8_t*)malloc(ISAL_DEF_LVL2_DEFAULT);
+    lz_stream->level_buf_size = ISAL_DEF_LVL2_DEFAULT;
+
 
     }
     else {
-     isal_hufftables *temp_huffman_pointer = lz_stream->hufftables;
 
-     DBG("lz_stream = 0x%lx", (long)temp_huffman_pointer);
       isal_deflate_stateless_init(lz_stream);
-      lz_stream->hufftables = temp_huffman_pointer;
 
     }
 
+    uint8_t *level_buf = NULL;
+    lz_stream->level = level;
+    jint level_size =ISAL_DEF_LVL2_DEFAULT;
+    level_buf = (uint8_t*)malloc(level_size);
+    lz_stream->level_buf = level_buf;
+    lz_stream->level_buf_size = ISAL_DEF_LVL2_DEFAULT;
     lz_stream->end_of_stream = 0;
 
    // DBG("lz_stream = 0x%lx", (long)lz_stream);
@@ -190,7 +198,7 @@ JNIEXPORT jint JNICALL Java_com_intel_gkl_compression_IntelDeflater_deflateNativ
   jint level = env->GetIntField(obj, FID_level);
 
 
-  if(level == 1) {
+  if(level == 1 || level ==2 ) {
 
     isal_zstream* lz_stream = (isal_zstream*)env->GetLongField(obj, FID_lz_stream);
 
@@ -211,27 +219,6 @@ JNIEXPORT jint JNICALL Java_com_intel_gkl_compression_IntelDeflater_deflateNativ
     struct timeval  tv1, tv2;
     gettimeofday(&tv1, NULL);
 #endif
-    // compress and update lz_stream state
-    // Generate the dynamic huff tables using the first buffer of the stream
-  if(lz_stream->hufftables == 0x0)
-      {
-        struct isal_huff_histogram histogram;
-        struct isal_hufftables *hufftables_custom;
-
-         hufftables_custom = (isal_hufftables*) malloc(sizeof(isal_hufftables));
-
-         int sixtyfourK = 64*1024;
-         int usable_buffer= (inputBufferLength < sixtyfourK) ? inputBufferLength : sixtyfourK;
-         DBG("lz_stream = 0x%lx", (long)hufftables_custom);
-
-         memset(&histogram, 0, sizeof(histogram));
-         isal_update_histogram((unsigned char*)next_in,usable_buffer, &histogram);
-         isal_create_hufftables(hufftables_custom, &histogram);
-         isal_deflate_set_hufftables(lz_stream,
-         				hufftables_custom, IGZIP_HUFFTABLE_CUSTOM);
-         lz_stream->hufftables = hufftables_custom;
-         DBG("lz_stream = 0x%lx", (long)hufftables_custom);
-       }
 
     // compress and update lz_stream state
     isal_deflate_stateless(lz_stream);
@@ -313,15 +300,17 @@ JNIEXPORT void JNICALL
 Java_com_intel_gkl_compression_IntelDeflater_endNative(JNIEnv *env, jobject obj)
 {
   jint level = env->GetIntField(obj, FID_level);
+   if (level == 1 || level == 2 ) {
+     isal_zstream* lz_stream = (isal_zstream*)env->GetLongField(obj, FID_lz_stream);
+      free(lz_stream->level_buf);
+      free(lz_stream);
 
-  if (level != 1) {
+   }
+  else {
     z_stream* lz_stream = (z_stream*)env->GetLongField(obj, FID_lz_stream);
     deflateEnd(lz_stream);
     free(lz_stream);
   }
-  else {
-   isal_zstream* lz_stream = (isal_zstream*)env->GetLongField(obj, FID_lz_stream);
-   free(lz_stream->hufftables);
-   free(lz_stream);
-  }
+
+
 }

src/main/native/compression/isa-l-master/.travis.yml

@@ -1,11 +1,83 @@
+language: c
 sudo: required
-dist: trusty
+matrix:
+  include:
+    ### OS X
+    - os: osx
+      env: C_COMPILER=clang
+
+    ### linux gcc
+    - dist: trusty
+      env: C_COMPILER=gcc
+
+    ### linux clang
+    - dist: trusty
+      env: C_COMPILER=clang
+
+    ### linux newer clang
+    - dist: trusty
+      addons:
+        apt:
+          sources:
+            - ubuntu-toolchain-r-test
+            - llvm-toolchain-trusty-4.0
+          packages:
+            - clang-4.0
+      env: C_COMPILER=clang-4.0
+
+    ### linux older gcc
+    - dist: trusty
+      addons:
+        apt:
+          sources:
+            - ubuntu-toolchain-r-test
+          packages:
+            - g++-4.7
+      env: C_COMPILER=gcc-4.7
+
+    ### linux newer gcc
+    - dist: trusty
+      addons:
+        apt:
+          sources:
+            - ubuntu-toolchain-r-test
+          packages:
+            - g++-6
+      env: C_COMPILER=gcc-6
+
+    ### linux with new nasm
+    # Removed until travis issue fixed pulling from nasm from debian
+    #- dist: trusty
+    #  addons:
+    #    apt:
+    #      sources:
+    #        - debian-sid
+    #      packages:
+    #        - nasm
+    #  env: C_COMPILER=gcc AS_ASSEMBL=nasm
+
+    ### linux extended tests
+    - dist: trusty
+      addons:
+        apt:
+          sources:
+            - ubuntu-toolchain-r-test
+          packages:
+            - binutils-mingw-w64-x86-64
+            - gcc-mingw-w64-x86-64
+            - wine
+      env: TEST_TYPE=ext
+
+before_install:
+  - if [ -n "${C_COMPILER}" ]; then export CC="${C_COMPILER}"; fi
+  - if [ -n "${AS_ASSEMBL}" ]; then export AS="${AS_ASSEMBL}"; fi
+
 before_script:
-    - sudo apt-get -q update
-    - sudo apt-get install -y yasm nasm
-    - ./autogen.sh
-script: ./configure && make && make check
-language: c
-compiler:
-    - clang
-    - gcc
+  - if [ $TRAVIS_OS_NAME = linux ]; then sudo apt-get -q update; fi
+  - if [ $TRAVIS_OS_NAME = linux ]; then sudo apt-get install yasm indent; fi
+  - if [ $TRAVIS_OS_NAME = osx ]; then brew install yasm; fi
+
+script:
+  - if [ -n "${CC}" ]; then $CC --version; fi
+  - if [ -n "${AS}" ]; then $AS --version; fi
+  - ./tools/test_autorun.sh "${TEST_TYPE}"

src/main/native/compression/isa-l-master/CONTRIBUTING.md

@@ -30,6 +30,10 @@ the included indent script to format C code.
 
     ./tools/iindent your_files.c
 
+And use check format script before submitting.
+
+    ./tools/check_format.sh
+
 [mailing list]:https://lists.01.org/mailman/listinfo/isal
 [license]:LICENSE
 [signed-off-by language]:https://01.org/community/signed-process

src/main/native/compression/isa-l-master/Makefile.am

@@ -31,10 +31,11 @@ include erasure_code/Makefile.am
 include raid/Makefile.am
 include crc/Makefile.am
 include igzip/Makefile.am
+include tests/fuzz/Makefile.am
 
 # LIB version info not necessarily the same as package version
 LIBISAL_CURRENT=2
-LIBISAL_REVISION=19
+LIBISAL_REVISION=21
 LIBISAL_AGE=0
 
 lib_LTLIBRARIES = libisal.la

src/main/native/compression/isa-l-master/Makefile.nmake

@@ -90,6 +90,7 @@ objs =  \
 	bin\xor_gen_sse.obj \
 	bin\crc16_t10dif_01.obj \
 	bin\crc16_t10dif_by4.obj \
+	bin\crc16_t10dif_copy_by4.obj \
 	bin\crc32_gzip.obj \
 	bin\crc32_ieee_01.obj \
 	bin\crc32_ieee_by4.obj \
@@ -120,9 +121,7 @@ objs =  \
 	bin\encode_df.obj \
 	bin\encode_df_04.obj \
 	bin\proc_heap.obj \
-	bin\igzip_icf_body_01.obj \
-	bin\igzip_icf_body_02.obj \
-	bin\igzip_icf_body_04.obj \
+	bin\igzip_icf_body_h1_gr_bt.obj \
 	bin\igzip_icf_finish.obj \
 	bin\igzip_icf_base.obj \
 	bin\igzip_inflate.obj \
@@ -135,7 +134,10 @@ objs =  \
 	bin\crc32_gzip_refl_by8.obj \
 	bin\adler32_sse.obj \
 	bin\adler32_avx2_4.obj \
-	bin\igzip_deflate_hash.obj
+	bin\igzip_deflate_hash.obj \
+	bin\igzip_gen_icf_map_lh1_06.obj \
+	bin\igzip_set_long_icf_fg_06.obj \
+	bin\igzip_icf_body.obj
 
 INCLUDES  = -I./ -Ierasure_code/ -Iraid/ -Icrc/ -Iigzip/ -Iinclude/
 LINKFLAGS = /nologo
@@ -201,6 +203,7 @@ checks = \
 	xor_check_test.exe \
 	pq_check_test.exe \
 	crc16_t10dif_test.exe \
+	crc16_t10dif_copy_test.exe \
 	crc32_ieee_test.exe \
 	crc32_iscsi_test.exe \
 	crc64_funcs_test.exe \

src/main/native/compression/isa-l-master/Makefile.unx

@@ -33,9 +33,13 @@ default: lib
 
 include $(foreach unit,$(units), $(unit)/Makefile.am)
 
+ifneq (,$(findstring igzip,$(units)))
+ include tests/fuzz/Makefile.am
+endif
+
 # Override individual lib names to make one inclusive library.
 lib_name := bin/isa-l.a
 
 include make.inc
 
-VPATH = . $(units) include
+VPATH = . $(units) include tests/fuzz

src/main/native/compression/isa-l-master/README.md

@@ -18,7 +18,8 @@ applications.  ISA-L includes:
 Also see:
 * [ISA-L for updates](https://github.com/01org/isa-l).
 * For crypto functions see [isa-l_crypto on github](https://github.com/01org/isa-l_crypto).
-* The [github wiki](https://github.com/01org/isa-l/wiki).
+* The [github wiki](https://github.com/01org/isa-l/wiki) including a list of
+  [distros/ports](https://github.com/01org/isa-l/wiki/Ports--Repos) offering binary packages.
 * ISA-L [mailing list](https://lists.01.org/mailman/listinfo/isal).
 * [Contributing](CONTRIBUTING.md).
 
@@ -27,10 +28,11 @@ Building ISA-L
 
 ### Prerequisites
 
-* yasm version 1.2.0 or later or nasm v2.11.01 or later.
-* gcc, clang, icc or VC compiler.
-* GNU 'make' or 'nmake' (Windows).
-* Building with autotools requires autoconf/automake packages.
+* Assembler: nasm v2.11.01 or later (nasm v2.13 or better suggested for building in AVX512 support)
+  or yasm version 1.2.0 or later.
+* Compiler: gcc, clang, icc or VC compiler.
+* Make: GNU 'make' or 'nmake' (Windows).
+* Optional: Building with autotools requires autoconf/automake packages.
 
 ### Autotools
 To build and install the library with autotools it is usually sufficient to run:
@@ -57,3 +59,4 @@ Other targets include:
 * `make perfs` : create included performance tests
 * `make ex`    : build examples
 * `make other` : build other utilities such as compression file tests
+* `make doc`   : build API manual

src/main/native/compression/isa-l-master/Release_notes.txt

@@ -1,4 +1,4 @@
-v2.19 Intel Intelligent Storage Acceleration Library Release Notes
+v2.21 Intel Intelligent Storage Acceleration Library Release Notes
 ==================================================================
 
 RELEASE NOTE CONTENTS
@@ -15,6 +15,17 @@ RELEASE NOTE CONTENTS
 
 2. FIXED ISSUES
 ---------------
+v2.20
+
+* Inflate total_out behavior corrected for in-progress decompression.
+  Previously total_out represented the total bytes decompressed into the output
+  buffer or temp internal buffer.  This is changed to be only the bytes put into
+  the output buffer.
+
+* Fixed issue with isal_create_hufftables_subset.  Affects semi-dynamic
+  compression use case when explicitly creating hufftables from histogram.  The
+  _hufftables_subset function could fail to generate length symbols for any
+  length that were never seen.
 
 v2.19
 
@@ -55,6 +66,30 @@ v2.10
 
 3. CHANGE LOG & FEATURES ADDED
 ------------------------------
+v2.21
+
+* Igzip improvements
+  - New compression levels added.  ISA-L fast deflate now has more levels to
+    balance speed vs. target compression level.  Level 0, 1 are as in previous
+    generations.  New levels 2 & 3 target higher compression roughly comparable
+    to zlib levels 2-3.  Level 3 is currently only optimized for processors with
+    AVX512 instructions.
+
+* New T10dif & copy function - crc16_t10dif_copy()
+  - CRC and copy was added to emulate T10dif operations such as DIF insert and
+    strip.  This function stitches together CRC and memcpy operations
+    eliminating an extra data read.
+
+* CRC32 iscsi performance improvements
+  - Fixes issue under some distributions where warm cache performance was
+    reduced.
+
+v2.20
+
+* Igzip improvements
+  - Optimized deflate_hash in compression functions.
+    Improves performance of using preset dictionary.
+  - Removed alignment restrictions on input structure.
 
 v2.19
 

src/main/native/compression/isa-l-master/configure.ac

@@ -3,7 +3,7 @@
 
 AC_PREREQ(2.69)
 AC_INIT([libisal],
-        [2.19.0],
+        [2.21.0],
         [sg.support.isal@intel.com],
         [isa-l],
         [http://01.org/storage-acceleration-library])
@@ -85,8 +85,8 @@ else
     with_modern_nasm=yes
     AC_MSG_RESULT([yes])
     AC_MSG_CHECKING([for optional nasm AVX512 support])
-    AC_LANG_CONFTEST([AC_LANG_SOURCE([[vpshufb zmm0, zmm1, zmm2;]])])
-    sed -i -e '/vpshufb/!d' conftest.c
+    AC_LANG_CONFTEST([AC_LANG_SOURCE([[vinserti32x8 zmm0, ymm1, 1;]])])
+    sed -i -e '/vinsert/!d' conftest.c
     if nasm -f elf64  conftest.c 2> /dev/null; then
       nasm_knows_avx512=yes
       AC_MSG_RESULT([yes])
@@ -109,7 +109,7 @@ if test x"$AS" = x""; then
   elif test x"$with_modern_nasm" = x"yes"; then
     AS=nasm
   else
-    AC_MSG_ERROR([No modern yasm or nasm found as required. Yasm should be 1.2.0 or later, and nasm should be v2.11.01 or later.])
+    AC_MSG_ERROR([No modern yasm or nasm found as required. Yasm should be 1.2.0 or later, and nasm should be v2.11.01 or later (v2.13 for AVX512).])
   fi
 fi
 echo "Using assembler $AS"