feature: runtime detection of CRC32 and ARM64 support.

.gitignore

@@ -12,3 +12,7 @@
 *~
 tags
 *.swo
+test/*.txt
+test/*.o
+test/*.d
+test/ht_test

.travis.yml

@@ -1,49 +1,98 @@
-sudo: required
-dist: xenial
-
 os: linux
+dist: xenial
+sudo: false
 
 language: c
 
+arch:
+  - amd64
+ #- arm64
+
 compiler:
   - gcc
   - clang
 
 addons:
   apt:
     packages:
-    - axel
+    - g++-5
+    - valgrind
     - cpanminus
     - libtest-base-perl
     - libtext-diff-perl
     - libtest-longstring-perl
     - liblist-moreutils-perl
     - libparallel-forkmanager-perl
-    - libgd-dev
     - libmpc-dev
-    - build-essential
     - libgtk2.0-dev
-    - valgrind
 
 env:
   global:
-  - JOBS=3
-  - LUAJIT_PREFIX=/opt/luajit21
-  - LUAJIT_SYSM_PREFIX=/opt/luajit21-sysm
-  - LUAJIT_COMMON_XCFLAGS="-DLUA_USE_APICHECK -DLUA_USE_ASSERT -DLUAJIT_NUMMODE=2 -msse4.2 -O1"
+    - JOBS=6
+    - PREFIX=$HOME/luajit
+    - XCFLAGS="-O1 -DLUA_USE_APICHECK -DLUA_USE_ASSERT -DLUAJIT_NUMMODE=2"
   matrix:
-  - LUAJIT_XCFLAGS="$LUAJIT_COMMON_XCFLAGS"
-  - LUAJIT_XCFLAGS="-DLUAJIT_ENABLE_LUA52COMPAT $LUAJIT_COMMON_XCFLAGS" LUA52=1
-  - LUAJIT_XCFLAGS="-DLUAJIT_USE_VALGRIND -DLUAJIT_USE_SYSMALLOC -DLUAJIT_ENABLE_LUA52COMPAT $LUAJIT_COMMON_XCFLAGS" LUA52=1 FLAGS=-v
-  - LUAJIT_XCFLAGS="-DLUAJIT_DISABLE_GC64 -DLUAJIT_ENABLE_LUA52COMPAT $LUAJIT_COMMON_XCFLAGS" LUA52=1
+    - XCFLAGS="-O3 -DLUAJIT_ENABLE_LUA52COMPAT -DLUAJIT_NUMMODE=2" # OpenResty releases
+    - XCFLAGS="-DLUAJIT_ENABLE_LUA52COMPAT -DLUAJIT_USE_VALGRIND -DLUAJIT_USE_SYSMALLOC $XCFLAGS"
+    - XCFLAGS="-DLUAJIT_DISABLE_JIT $XCFLAGS"
+    - XCFLAGS="-DLUAJIT_DISABLE_GC64 $XCFLAGS"
+    - XCFLAGS="-DLJ_OR_DISABLE_STRHASHCRC32 $XCFLAGS"
+
+matrix:
+  fast_finish: true
+  allow_failures:
+    - arch: arm64
+
+stages:
+  - test
+  - openresty tests
+  - string hashing tests
+
+jobs:
+  allow_failures:
+    - arch: arm64
+  include:
+    - stage: openresty tests
+      script: prove -j$JOBS t
+      env:
+    - stage: openresty tests
+      script: prove -j$JOBS t
+      env:
+      arch: arm64
+    - stage: openresty tests
+      script: prove -j$JOBS t
+      env: TEST_LJ_USE_VALGRIND=1 XCFLAGS="-DLUAJIT_USE_VALGRIND -DLUAJIT_USE_SYSMALLOC $XCFLAGS"
+    - stage: string hashing tests
+      script: make -C test test
+      env:
+    - stage: string hashing tests
+      script: make -C test test
+      env:
+      arch: arm64
+    - stage: string hashing tests
+      script: make -C test test
+      env: WITH_VALGRIND=1 XCFLAGS="-DLUAJIT_USE_VALGRIND -DLUAJIT_USE_SYSMALLOC $XCFLAGS"
+
+before_install:
+  - cpanm --local-lib=$TRAVIS_BUILD_DIR/perl5 local::lib && eval $(perl -I$TRAVIS_BUILD_DIR/perl5/lib/perl5/ -Mlocal::lib)
+  - cpanm --notest IPC::Run3
+  - valgrind --version
+  - export CXX=g++-5
 
 install:
-  - git clone https://github.com/openresty/luajit2-test-suite.git ../luajit2-test-suite
+  - make -j$JOBS CCDEBUG=-g Q= PREFIX=$PREFIX XCFLAGS="$XCFLAGS" >build.log 2>&1 || (cat build.log && exit 1)
+  - make install PREFIX=$PREFIX >build.log 2>&1 || (cat build.log && exit 1)
+  - export PATH="$PREFIX/bin:$PATH"
 
 script:
-  - valgrind --version
-  - cd ../luajit2
-  - make -j$JOBS CCDEBUG=-g Q= PREFIX=$LUAJIT_PREFIX CC=$CC XCFLAGS="$LUAJIT_XCFLAGS" > build.log 2>&1 || (cat build.log && exit 1)
-  - sudo make install PREFIX=$LUAJIT_PREFIX > build.log 2>&1 || (cat build.log && exit 1)
-  - cd ../luajit2-test-suite
-  - ./run-tests -j $JOBS $FLAGS $LUAJIT_PREFIX
+  - if [[ "$XCFLAGS" =~ "LUAJIT_ENABLE_LUA52COMPAT" ]]; then export LUA52=1; fi
+  - if [[ "$XCFLAGS" =~ "LUAJIT_USE_VALGRIND" ]]; then export FLAGS=-v; fi
+  - if [[ "$XCFLAGS" =~ "LJ_OR_DISABLE_STRHASHCRC32" ]]; then export NO_STRHASHCRC32=1; fi
+  - git clone https://github.com/openresty/luajit2-test-suite.git
+  - pushd luajit2-test-suite
+  - ./run-tests -j$JOBS $FLAGS $PREFIX $PREFIX/bin/luajit $CC $CXX
+
+cache:
+  apt: true
+  directories:
+    - perl5

README.md

@@ -13,9 +13,12 @@ openresty/luajit2 - OpenResty's maintained branch of LuaJIT.
         * [table.nkeys](#tablenkeys)
         * [table.clone](#tableclone)
         * [jit.prngstate](#jitprngstate)
+        * [jit.crc32](#jitcrc32)
+        * [jit.strhashcrc32](#jitstrhashcrc32)
         * [thread.exdata](#threadexdata)
     * [New macros](#new-macros)
         * [`OPENRESTY_LUAJIT`](#openresty-luajit)
+        * [`LJ_OR_DISABLE_STRHASHCRC32`](#lj-or-disable-strhashcrc32)
     * [Optimizations](#optimizations)
         * [Updated JIT default parameters](#updated-jit-default-parameters)
         * [String hashing](#string-hashing)
@@ -155,6 +158,45 @@ local newstate = jit.prngstate(123456)
 
 [Back to TOC](#table-of-contents)
 
+### jit.crc32
+
+**syntax:** *ok = jit.crc32()*
+
+Returns a boolean value indicating if the current architecture supports a CRC32
+instruction set. CRC32 support will be checked at runtime on x64 and ARM64
+platforms.
+
+CRC32 support allows for this branch to use an optimized string hashing
+algorithm. See the [String hashing](#string-hashing) section for details on
+how to enable this optimization.
+
+Usage:
+
+```lua
+local ok = jit.crc32()
+```
+
+[Back to TOC](#table-of-contents)
+
+### jit.strhashcrc32
+
+**syntax:** *ok = jit.strhashcrc32()*
+
+Returns a boolean value indicating if the optimized string hashing algorithm
+implemented by this branch is enabled. The `ok` return value will be `true` if
+it is enabled, or `false` otherwise.
+
+See the [String hashing](#string-hashing) section for details on
+how to enable this optimization.
+
+Usage:
+
+```lua
+local ok = jit.strhashcrc32()
+```
+
+[Back to TOC](#table-of-contents)
+
 ### thread.exdata
 
 **syntax:** *exdata = th_exdata(data?)*
@@ -207,6 +249,14 @@ help distinguishing this OpenResty-specific branch of LuaJIT.
 
 [Back to TOC](#table-of-contents)
 
+### `LJ_OR_DISABLE_STRHASHCRC32`
+
+When specified at compilation (`-DLJ_OR_DISABLE_STRHASHCRC32`), this flag will
+disable the string hashing optimization described in the [String
+hashing](#string-hashing) section.
+
+[Back to TOC](#table-of-contents)
+
 ## Optimizations
 
 ### Updated JIT default parameters
@@ -227,13 +277,17 @@ maxmcode=40960  -- in KB
 
 ### String hashing
 
-This optimization only applies to Intel CPUs supporting the SSE 4.2 instruction
-sets. For such CPUs, and when this branch is compiled with `-msse4.2`, the
-string hashing function used for strings interning will be based on an
-optimized crc32 implementation (see `lj_str_new()`).
+This optimizations modifies the string hashing algorithm to use a CRC32-based
+variant. This variant still provides constant-time hashing complexity (`O(n)`)
+but makes hash collision attacks harder for strings up to 127 bytes of size
+(see `lj_str_new()`).
+
+This optimization is only available for x64 and ARM64 architectures, and will
+be enabled if a CRC32 instruction set is detected at runtime (see
+[jit.crc32](#jitcrc32)).
 
-This optimization still provides constant-time hashing complexity (`O(n)`), but
-makes hash collision attacks harder for strings up to 127 bytes of size.
+**Note:** This optimization can be disabled by compiling LuaJIT with
+`-DLJ_OR_DISABLE_STRHASHCRC32`.
 
 [Back to TOC](#table-of-contents)
 

src/Makefile

@@ -490,9 +490,9 @@ LJLIB_O= lib_base.o lib_math.o lib_bit.o lib_string.o lib_table.o \
 LJLIB_C= $(LJLIB_O:.o=.c)
 
 LJCORE_O= lj_gc.o lj_err.o lj_char.o lj_bc.o lj_obj.o lj_buf.o \
-	  lj_str.o lj_tab.o lj_func.o lj_udata.o lj_meta.o lj_debug.o \
-	  lj_state.o lj_dispatch.o lj_vmevent.o lj_vmmath.o lj_strscan.o \
-	  lj_strfmt.o lj_strfmt_num.o lj_api.o lj_profile.o \
+	  lj_str.o lj_str_hash.o lj_tab.o lj_func.o lj_udata.o lj_meta.o \
+	  lj_debug.o lj_state.o lj_dispatch.o lj_vmevent.o lj_vmmath.o \
+	  lj_strscan.o lj_strfmt.o lj_strfmt_num.o lj_api.o lj_profile.o \
 	  lj_lex.o lj_parse.o lj_bcread.o lj_bcwrite.o lj_load.o \
 	  lj_ir.o lj_opt_mem.o lj_opt_fold.o lj_opt_narrow.o \
 	  lj_opt_dce.o lj_opt_loop.o lj_opt_split.o lj_opt_sink.o \
@@ -521,6 +521,15 @@ ALL_GEN= $(LJVM_S) $(ALL_HDRGEN) $(LIB_VMDEFP)
 WIN_RM= *.obj *.lib *.exp *.dll *.exe *.manifest *.pdb *.ilk
 ALL_RM= $(ALL_T) $(ALL_GEN) *.o host/*.o $(WIN_RM)
 
+ifeq (x64,$(TARGET_LJARCH))
+  lj_str_hash-CFLAGS = -msse4.2
+endif
+ifeq (arm64,$(TARGET_LJARCH))
+  lj_str_hash-CFLAGS = -march=armv8-a+crc
+endif
+
+F_CFLAGS = $($(patsubst %.c,%-CFLAGS,$<))
+
 ##############################################################################
 # Build mode handling.
 ##############################################################################
@@ -685,8 +694,8 @@ lj_folddef.h: $(BUILDVM_T) lj_opt_fold.c
 
 %.o: %.c
 	$(E) "CC        $@"
-	$(Q)$(TARGET_DYNCC) $(TARGET_ACFLAGS) -c -o $(@:.o=_dyn.o) $<
-	$(Q)$(TARGET_CC) $(TARGET_ACFLAGS) -c -o $@ $<
+	$(Q)$(TARGET_DYNCC) $(TARGET_ACFLAGS) $(F_CFLAGS) -c -o $(@:.o=_dyn.o) $<
+	$(Q)$(TARGET_CC) $(TARGET_ACFLAGS) $(F_CFLAGS) -c -o $@ $<
 
 %.o: %.S
 	$(E) "ASM       $@"

src/lib_jit.c

@@ -15,6 +15,7 @@
 #include "lj_err.h"
 #include "lj_debug.h"
 #include "lj_str.h"
+#include "lj_str_hash.h"
 #include "lj_tab.h"
 #include "lj_state.h"
 #include "lj_bc.h"
@@ -156,6 +157,23 @@ LJLIB_CF(jit_prngstate)
   return 1;
 }
 
+LJLIB_CF(jit_crc32)
+{
+  setboolV(L->top++, lj_check_crc32_support());
+  return 1;
+}
+
+LJLIB_CF(jit_strhashcrc32)
+{
+#if LJ_OR_STRHASHCRC32
+  global_State *g = G(L);
+  setboolV(L->top++, (g->strhashfn != lj_str_hash_orig) ? 1 : 0);
+#else
+  setboolV(L->top++, 0);
+#endif
+  return 1;
+}
+
 LJLIB_PUSH(top-5) LJLIB_SET(os)
 LJLIB_PUSH(top-4) LJLIB_SET(arch)
 LJLIB_PUSH(top-3) LJLIB_SET(version_num)

src/lj_arch.h

@@ -615,4 +615,13 @@ extern void *LJ_WIN_LOADLIBA(const char *path);
 #define LJ_52			0
 #endif
 
+/* Optimized string hashing, added by OpenResty. */
+#if defined(__GNUC__) && (LUAJIT_TARGET == LUAJIT_ARCH_X64 || LUAJIT_TARGET == LUAJIT_ARCH_ARM64)
+#ifndef LJ_OR_DISABLE_STRHASHCRC32
+#define LJ_OR_STRHASHCRC32 	1
+#endif
+#else
+#define LJ_OR_STRHASHCRC32 	0
+#endif
+
 #endif

src/lj_obj.h

@@ -160,6 +160,11 @@ typedef struct SBuf {
   MRef L;		/* lua_State, used for buffer resizing. */
 } SBuf;
 
+#if LJ_OR_STRHASHCRC32
+/* String hashing functions, added by OpenResty. */
+typedef MSize (*StrHashFunction)(const char *, size_t);
+#endif
+
 /* -- Tags and values ----------------------------------------------------- */
 
 /* Frame link. */
@@ -622,6 +627,9 @@ typedef struct global_State {
   MRef saved_jit_base;  /* saved jit_base for lj_err_throw */
   MRef ctype_state;	/* Pointer to C type state. */
   GCRef gcroot[GCROOT_MAX];  /* GC roots. */
+#if LJ_OR_STRHASHCRC32
+  StrHashFunction strhashfn; /* String hashing function, added by OpenResty */
+#endif
 } global_State;
 
 #define mainthread(g)	(&gcref(g->mainthref)->th)

src/lj_state.c

@@ -204,6 +204,9 @@ LUA_API lua_State *lua_newstate(lua_Alloc f, void *ud)
   setgcref(g->uvhead.prev, obj2gco(&g->uvhead));
   setgcref(g->uvhead.next, obj2gco(&g->uvhead));
   g->strmask = ~(MSize)0;
+#if LJ_OR_STRHASHCRC32
+  lj_init_strhashfn(g);
+#endif
   setnilV(registry(L));
   setnilV(&g->nilnode.val);
   setnilV(&g->nilnode.key);