[LLM Runtime] Allow CompileBF16 on GCC11 (#655)

* Allow CompileBF16 on GCC11

.github/workflows/cpp-graph-test.yml

@@ -7,6 +7,7 @@ on:
        - '.github/workflows/cpp-graph-test.yml'
        - '.github/workflows/script/models/cpp_graph_inference.sh'
        - 'intel_extension_for_transformers/llm/runtime/graph/**'
+       - 'intel_extension_for_transformers/llm/library/jblas/**'
        - '!intel_extension_for_transformers/llm/runtime/graph/README.md'
   workflow_dispatch:
     inputs:

intel_extension_for_transformers/llm/library/jblas/jblas/jit_blas_utils.h

@@ -37,7 +37,7 @@
 #define CompileAVX512F() (defined(__GNUC__) && (__GNUC__ >= 6))
 #define CompileAVX2() (defined(__GNUC__) && (__GNUC__ >= 5))
 #define CompileAMX() (defined(__GNUC__) && (__GNUC__ >= 11))
-#define CompileBF16() (defined(__GNUC__) && (__GNUC__ >= 13))
+#define CompileBF16() (defined(__GNUC__) && (__GNUC__ >= 11))
 #define CompileFP16() (defined(__GNUC__) && (__GNUC__ >= 13))
 #define CompileAMXBF16() (CompileAMX())
 #define CompileAMXINT8() (CompileAMX())
@@ -77,14 +77,28 @@ struct bf16 {
 
 #if CompileBF16()
 #pragma GCC target("avx512vl", "avx512bf16")
-  explicit bf16(float vf32) : x(bit_cast<uint16_t>(_mm_cvtness_sbh(vf32))) {}
+  static uint16_t f32_to_bf16(float v) {
+    auto mm = _mm_load_ss(&v);
+    auto mm2 = _mm_cvtneps_pbh(mm);
+    uint16_t dst;
+    _mm_storeu_si16(reinterpret_cast<uint16_t*>(&dst), reinterpret_cast<__m128i>(mm2));
+    return dst;
+  }
+
+  explicit bf16(float vf32) : x(bit_cast<uint16_t>(f32_to_bf16(vf32))) {}
 #else
   explicit bf16(float vf32) { fromfloat(vf32); }
 #endif
 
 #if CompileBF16()
 #pragma GCC target("avx512vl", "avx512bf16")
-  float tofloat() const { return static_cast<float>(bit_cast<__bf16>(this->x)); }
+  float tofloat() const {
+    auto mm = _mm_loadu_si16(&(this->x));
+    auto mm2 = _mm_bslli_si128(mm, 2);
+    float dst;
+    _mm_store_ss(&dst, reinterpret_cast<__m128>(mm2));
+    return dst;
+  }
 #else
   float tofloat() const {
     bf16f32 tmp = {0.f};
@@ -103,7 +117,7 @@ struct bf16 {
 
   void fromfloat(float _v) {
 #if CompileBF16()
-    x = bit_cast<uint16_t>(_mm_cvtness_sbh(_v));
+    x = bit_cast<uint16_t>(f32_to_bf16(_v));
 #else
     bf16f32 tmp = {0.f};
     tmp.f32 = _v;

intel_extension_for_transformers/llm/library/jblas/jblas/kernel_avx2.h

@@ -150,6 +150,112 @@ static inline JBLAS_CODE dequant_kblock_s8_f32(int8_t* srcptr, float* dstptr, in
                                            kblock, NPad);
 }
 
+static inline JBLAS_CODE dequant_s32_fp32(const int32_t* srcptr, const int srcstep, float* dstptr, const int dststep,
+                                          const int row, const int col, const float* scaleA, const int ldsa,
+                                          const float* scaleB) {
+  int col8 = utils::padto_le(col, 8);
+  for (int irow = 0; irow < row; irow++) {
+    auto scale = scaleA[irow * ldsa];
+    auto valpha = _mm256_set1_ps(scale);
+    int icol = 0;
+    for (; icol < col8; icol += 8) {
+      auto vwscale = _mm256_loadu_ps(scaleB + icol);
+      auto vscale = _mm256_mul_ps(valpha, vwscale);
+      auto vsrcd = _mm256_loadu_si256((__m256i*)(srcptr + irow * srcstep + icol));
+      auto vsrc = _mm256_cvtepi32_ps(vsrcd);
+      vsrc = _mm256_mul_ps(vsrc, vscale);
+      _mm256_storeu_ps(dstptr + irow * dststep + icol, vsrc);
+    }
+    for (; icol < col; icol += 1) {
+      dstptr[irow * dststep + icol] = scale * scaleB[icol] * srcptr[irow * srcstep + icol];
+    }
+  }
+  return JblasSuccess;
+}
+
+static inline JBLAS_CODE remove_act_zeropoint_bias(float* accptr, int ldacc, int row, int col, uint8_t* zps,
+                                                   float* scales, int lds, const float* reduce) {
+  int constexpr VLen = 8;
+  auto col8 = utils::padto_le(col, VLen);
+  for (int i = 0; i < row; i++) {
+    auto zpf = float(zps[i * lds]) * scales[i * lds];
+    int j = 0;
+    auto vzp = _mm256_set1_ps(-zpf);
+    for (; j < col8; j += VLen) {
+      auto vreduce = _mm256_loadu_ps(reduce + j);
+      auto vacc = _mm256_loadu_ps(&accptr[i * ldacc + j]);
+      vacc = _mm256_fmadd_ps(vzp, vreduce, vacc);
+      _mm256_storeu_ps(&accptr[i * ldacc + j], vacc);
+    }
+    if (j < col) {
+      for (; j < col; j++) {
+        accptr[i * ldacc + j] -= zpf * reduce[j];
+      }
+    }
+  }
+  return JblasSuccess;
+}
+
+static inline JBLAS_CODE remove_wei_zeropoint_bias(float* accptr, int ldacc, int row, int col, int8_t* zps,
+                                                   float* scales, int lds, const float* reduce) {
+  int constexpr VLen = 8;
+  auto col8 = utils::padto_le(col, VLen);
+  const int32_t mask[] = {-1, -1, 0, 0};
+  for (int i = 0; i < row; i++) {
+    auto vreduce = _mm256_set1_ps(-reduce[i * lds]);
+    int j = 0;
+    for (; j < col8; j += VLen) {
+      auto vzp_s32 = _mm256_cvtepi8_epi32(_mm_maskload_epi32((const int*)(zps + j), _mm_loadu_si128((__m128i*)mask)));
+      auto vzp_f32 = _mm256_cvtepi32_ps(vzp_s32);
+      auto vzp = _mm256_mul_ps(vzp_f32, _mm256_loadu_ps(scales + j));
+      auto vacc = _mm256_loadu_ps(&accptr[i * ldacc + j]);
+      vacc = _mm256_fmadd_ps(vzp, vreduce, vacc);
+      _mm256_storeu_ps(&accptr[i * ldacc + j], vacc);
+    }
+    if (j < col) {
+      for (; j < col8; j++) {
+        accptr[i * ldacc + j] -= float(zps[j]) * scales[j] * reduce[i * lds];
+      }
+    }
+  }
+  return JblasSuccess;
+}
+
+static inline JBLAS_CODE remove_zeropoint_bias(float* accptr, int ldacc, int row, int col, uint8_t* zpa, int8_t* zpb,
+                                               float* scalea, float* scaleb, int lds, int k, const float* reducea,
+                                               const float* reduceb) {
+  int constexpr VLen = 8;
+  auto col8 = utils::padto_le(col, VLen);
+  auto vk = _mm256_set1_ps((float)(k));
+  const int32_t mask[] = {-1, -1, 0, 0};
+  for (int i = 0; i < row; i++) {
+    auto vreducea = _mm256_set1_ps(-reducea[i * lds]);
+    auto zpaf = float(zpa[i * lds]) * scalea[i * lds];
+    auto vzpa = _mm256_set1_ps(-zpaf);
+    int j = 0;
+    for (; j < col8; j += VLen) {
+      auto vzp_s32 = _mm256_cvtepi8_epi32(_mm_maskload_epi32((const int*)(zpb + j), _mm_loadu_si128((__m128i*)mask)));
+      auto vzp_f32 = _mm256_cvtepi32_ps(vzp_s32);
+      auto vzpb = _mm256_mul_ps(vzp_f32, _mm256_loadu_ps(scaleb + j));
+      auto vreduceb = _mm256_loadu_ps(reduceb + j);
+      auto vacc = _mm256_loadu_ps(&accptr[i * ldacc + j]);
+      vacc = _mm256_fmadd_ps(vzpa, vreduceb, vacc);
+      vacc = _mm256_fmadd_ps(vzpb, vreducea, vacc);
+      vzpb = _mm256_mul_ps(vzpb, vk);
+      vacc = _mm256_fmadd_ps(vzpa, vzpb, vacc);
+      _mm256_storeu_ps(&accptr[i * ldacc + j], vacc);
+    }
+    if (j < col) {
+      for (; j < col8; j++) {
+        accptr[i * ldacc + j] -= float(zpb[j]) * scaleb[j] * reducea[i * lds];
+        accptr[i * ldacc + j] -= zpaf * reduceb[j];
+        accptr[i * ldacc + j] -= zpaf * float(zpb[j]) * scaleb[j] * k;
+      }
+    }
+  }
+  return JblasSuccess;
+}
+
 template <JBLAS_SIGN_INT_TYPE S4_T>
 static inline JBLAS_CODE decompress_s4_s8(utils::int4x2* srcptr, int8_t* dstptr, int row, int col, int ld_src,
                                           int ld_dst) {

intel_extension_for_transformers/llm/library/jblas/jblas/kernel_avx512_bf16.h

@@ -33,10 +33,13 @@ static inline JBLAS_CODE bf16_cvt_fp32_2D_write_back(const utils::bf16* src_ptr,
     auto dst = dst_ptr + i * dst_step;
     int j = 0;
     for (; j < col_body; j += simd_proc_elt)
-      _mm512_storeu_ps(dst + j, _mm512_cvtpbh_ps((__m256bh)_mm256_loadu_ps(reinterpret_cast<float*>(src + j))));
+      _mm512_storeu_ps(
+          dst + j,  //
+          reinterpret_cast<__m512>(_mm512_bslli_epi128(_mm512_cvtepu16_epi32(_mm256_loadu_epi16(src + j)), 2)));
     if (col_tail > 0)
-      _mm512_mask_storeu_ps(dst + j, tail_mask,
-                            _mm512_cvtpbh_ps((__m256bh)_mm256_loadu_ps(reinterpret_cast<float*>(src + j))));
+      _mm512_mask_storeu_ps(
+          dst + j, tail_mask,
+          reinterpret_cast<__m512>(_mm512_bslli_epi128(_mm512_cvtepu16_epi32(_mm256_loadu_epi16(src + j)), 2)));
     if (zeropadding && npadding) std::memset(dst + col, 0, npadding);
   }
   return JblasSuccess;