Merge pull request #13946 from JuliaLang/cb/bit_inline_setindex

Speed up BitArray packing
JuliaLang · Nov 13, 2015 · 8866b4e · 8866b4e
2 parents af36998 + 7fee9ab
commit 8866b4e
Show file tree

Hide file tree

Showing 2 changed files with 29 additions and 22 deletions.
diff --git a/base/bitarray.jl b/base/bitarray.jl
@@ -310,21 +310,17 @@ function convert{T,N}(::Type{BitArray{N}}, A::AbstractArray{T,N})
     ind = 1
     @inbounds begin
         for i = 1:length(Bc)-1
-            u = UInt64(1)
             c = UInt64(0)
             for j = 0:63
-                A[ind]!=0 && (c |= u)
+                c |= (UInt64(A[ind] != 0) << j)
                 ind += 1
-                u <<= 1
             end
             Bc[i] = c
         end
-        u = UInt64(1)
         c = UInt64(0)
         for j = 0:_mod64(l-1)
-            A[ind]!=0 && (c |= u)
+            c |= (UInt64(A[ind] != 0) << j)
             ind += 1
-            u <<= 1
         end
         Bc[end] = c
     end
@@ -359,15 +355,12 @@ end
     i1, i2 = get_chunks_id(i)
     u = UInt64(1) << i2
     @inbounds begin
-        if x
-            Bc[i1] |= u
-        else
-            Bc[i1] &= ~u
-        end
+        c = Bc[i1]
+        Bc[i1] = ifelse(x, c | u, c & ~u)
     end
 end
 
-setindex!(B::BitArray, x, i::Int) = (checkbounds(B, i); unsafe_setindex!(B, x, i))
+@inline setindex!(B::BitArray, x, i::Int) = (checkbounds(B, i); unsafe_setindex!(B, x, i))
 @inline function unsafe_setindex!(B::BitArray, x, i::Int)
     unsafe_bitsetindex!(B.chunks, convert(Bool, x), i)
     return B
@@ -417,11 +410,7 @@ function unsafe_setindex!(B::BitArray, X::AbstractArray, I::BitArray)
             if Imsk & u != 0
                 lx < c && throw_setindex_mismatch(X, c)
                 x = convert(Bool, unsafe_getindex(X, c))
-                if x
-                    C |= u
-                else
-                    C &= ~u
-                end
+                C = ifelse(x, C | u, C & ~u)
                 c += 1
             end
             u <<= 1

diff --git a/base/broadcast.jl b/base/broadcast.jl
@@ -103,16 +103,34 @@ end
 const bitcache_chunks = 64 # this can be changed
 const bitcache_size = 64 * bitcache_chunks # do not change this
 
+function bpack(z::UInt64)
+    z |= z >>> 7
+    z |= z >>> 14
+    z |= z >>> 28
+    z &= 0xFF
+    return z
+end
+
 function dumpbitcache(Bc::Vector{UInt64}, bind::Int, C::Vector{Bool})
     ind = 1
     nc = min(bitcache_chunks, length(Bc)-bind+1)
-    for i = 1:nc
-        u = UInt64(1)
+    C8 = reinterpret(UInt64, C)
+    nc8 = (nc >>> 3) << 3
+    @inbounds for i = 1:nc8
+        c = UInt64(0)
+        for j = 0:8:63
+            c |= (bpack(C8[ind]) << j)
+            ind += 1
+        end
+        Bc[bind] = c
+        bind += 1
+    end
+    ind = (ind-1) << 3 + 1
+    @inbounds for i = (nc8+1):nc
         c = UInt64(0)
-        for j = 1:64
-            C[ind] && (c |= u)
+        for j = 0:63
+            c |= (UInt64(C[ind]) << j)
             ind += 1
-            u <<= 1
         end
         Bc[bind] = c
         bind += 1