[ML-367] Update oneapi to 2024.0.0

mllib-dal/build.sh

@@ -11,9 +11,17 @@ if [[ -z $(which mvn) ]]; then
  exit 1
 fi
 
-if [[ -z $DAALROOT ]]; then
- echo DAALROOT not defined!
- exit 1
+if [[ -n $DAALROOT ]]; then
+  if [[ -e $DAALROOT ]]; then
+      export ONEDAL_VERSION=$(echo "$DAALROOT" | awk -F '/' '{print $(NF)}')
+  fi
+elif [[ -n $DALROOT ]]; then
+  if [[ -e $DALROOT ]]; then
+      export ONEDAL_VERSION=$(echo "$DALROOT" | awk -F '/' '{print $(NF)}')
+  fi
+else
+  echo DAALROOT not defined!
+  exit 1
 fi
 
 if [[ -z $TBBROOT ]]; then
@@ -89,7 +97,11 @@ then
   echo GCC Version: $(gcc -dumpversion)
 fi
 echo JAVA_HOME=$JAVA_HOME
-echo DAALROOT=$DAALROOT
+if [[ -n $DAALROOT ]]; then
+  echo DAALROOT=$DAALROOT
+elif [[ -n $DALROOT ]]; then
+  echo DALROOT=$DALROOT
+fi
 echo TBBROOT=$TBBROOT
 echo CCL_ROOT=$CCL_ROOT
 echo =============================

mllib-dal/src/main/java/com/intel/oap/mllib/LibLoader.java

@@ -59,9 +59,9 @@ public static synchronized void loadLibraries() throws IOException {
    * Load MLlibDAL lib
    */
   private static synchronized void loadLibMLlibDAL() throws IOException {
-    // oneDAL Java API doesn't load correct libtbb version
-    // See https://github.com/oneapi-src/oneDAL/issues/1254
-    // Workaround: Load packaged libtbb & libtbbmalloc & libJavaAPI.so manually
+    // When loading libMLlibDAL.so, it will hang on libtcm.so.
+    // Workaround: loading tbb manually to bypass it.
+    System.loadLibrary("tbb");
     loadFromJar(subDir, "libMLlibDAL.so");
   }
 

mllib-dal/src/main/native/Makefile

@@ -31,50 +31,36 @@ $(info )
 
 CFLAGS_COMMON := -Wall -Wno-deprecated-declarations -fPIC -std=c++17 \
                  -I $(I_MPI_ROOT)/include \
-                 -I $(DAALROOT)/include \
-                 -I $(CCL_ROOT)/include/cpu/oneapi/ \
-                 -I $(CMPLR_ROOT)/linux/include \
-                 -I $(CMPLR_ROOT)/linux/include/sycl
+                 -I $(DALROOT)/include \
+                 -I $(CCL_ROOT)/include/oneapi/ \
+                 -I $(CMPLR_ROOT)/include \
+                 -I $(CMPLR_ROOT)/include/sycl
 
 ifeq ($(PLATFORM_PROFILE),CPU_ONLY_PROFILE)
   CFLAGS := $(CFLAGS_COMMON)
 else ifeq ($(PLATFORM_PROFILE),CPU_GPU_PROFILE)
   CFLAGS := $(CFLAGS_COMMON) -fsycl \
             -fsycl-device-code-split=per_kernel \
-            -I $(CCL_ROOT)/include/cpu_gpu_dpcpp/oneapi/
+            -fno-sycl-id-queries-fit-in-int
 else
   $(error Unknow building profile, should be CPU_ONLY_PROFILE or CPU_GPU_PROFILE)
   exit 1
 endif
 
-INCS := -I $(CCL_ROOT)/include/cpu \
+INCS := -I $(CCL_ROOT)/include \
         -I $(JAVA_HOME)/include \
         -I $(JAVA_HOME)/include/linux \
-        -I $(DAALROOT)/include \
+        -I $(DALROOT)/include \
         -I ./javah \
         -I ./
 
-ifeq ($(PLATFORM_PROFILE),CPU_ONLY_PROFILE)
-  INCS := $(INCS)
-else ifeq ($(PLATFORM_PROFILE),CPU_GPU_PROFILE)
-  INCS := $(INCS) -I $(CCL_ROOT)/include/cpu_gpu_dpcpp
-else
-  $(error Unknow building profile, should be CPU_ONLY_PROFILE or CPU_GPU_PROFILE)
-  exit 1
-endif
-
 # Use static link if possible, TBB is only available as dynamic libs
-LIBS_COMMON := -L$(CCL_ROOT)/lib/cpu -lccl \
-        -L$(CMPLR_ROOT)/linux/compiler/lib/intel64_lin -l:libirc.a \
-        -L$(DAALROOT)/lib/intel64 -lonedal_core -lonedal_thread -lonedal_dpc \
+LIBS_COMMON := -L$(CCL_ROOT)/lib -lccl \
+        -L$(CMPLR_ROOT)/lib -l:libirc.a \
+        -L$(DALROOT)/lib/intel64 -lonedal_core -lonedal_thread -lonedal_dpc -lonedal_parameters_dpc \
         -L$(TBBROOT)/lib/intel64/gcc4.8 -ltbb -ltbbmalloc \
         -L$(I_MPI_ROOT)
 
-ifeq ($(PLATFORM_PROFILE),CPU_GPU_PROFILE)
-      LIBS_COMMON := $(LIBS_COMMON) \
-                     -L$(CCL_ROOT)/lib/cpu_gpu_dpcpp -lccl
-endif
-
 ifeq ($(PLATFORM_PROFILE),CPU_ONLY_PROFILE)
   LIBS := $(LIBS_COMMON) $(ONEDAL_LIBS)
 else ifeq ($(PLATFORM_PROFILE),CPU_GPU_PROFILE)

mllib-dal/src/main/native/Makefile_2023.2.0

@@ -0,0 +1,172 @@
+# Copyright 2020 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Use gcc for CPU and dpcpp for GPU
+ifeq ($(PLATFORM_PROFILE),CPU_ONLY_PROFILE)
+  CC := gcc
+  CXX := g++
+else ifeq ($(PLATFORM_PROFILE),CPU_GPU_PROFILE)
+  CC := icpx
+  CXX := icpx
+endif
+
+RM := rm -rf
+
+PLATFORM_PROFILE ?= CPU_ONLY_PROFILE
+
+$(info )
+$(info === Profile is $(PLATFORM_PROFILE) ===)
+$(info )
+
+CFLAGS_COMMON := -Wall -Wno-deprecated-declarations -fPIC -std=c++17 \
+                 -I $(I_MPI_ROOT)/include \
+                 -I $(DAALROOT)/include \
+                 -I $(CCL_ROOT)/include/cpu/oneapi/ \
+                 -I $(CMPLR_ROOT)/linux/include \
+                 -I $(CMPLR_ROOT)/linux/include/sycl
+
+ifeq ($(PLATFORM_PROFILE),CPU_ONLY_PROFILE)
+  CFLAGS := $(CFLAGS_COMMON)
+else ifeq ($(PLATFORM_PROFILE),CPU_GPU_PROFILE)
+  CFLAGS := $(CFLAGS_COMMON) -fsycl \
+            -fsycl-device-code-split=per_kernel \
+            -I $(CCL_ROOT)/include/cpu_gpu_dpcpp/oneapi/
+else
+  $(error Unknow building profile, should be CPU_ONLY_PROFILE or CPU_GPU_PROFILE)
+  exit 1
+endif
+
+INCS := -I $(CCL_ROOT)/include/cpu \
+        -I $(JAVA_HOME)/include \
+        -I $(JAVA_HOME)/include/linux \
+        -I $(DAALROOT)/include \
+        -I ./javah \
+        -I ./
+
+ifeq ($(PLATFORM_PROFILE),CPU_ONLY_PROFILE)
+  INCS := $(INCS)
+else ifeq ($(PLATFORM_PROFILE),CPU_GPU_PROFILE)
+  INCS := $(INCS) -I $(CCL_ROOT)/include/cpu_gpu_dpcpp
+else
+  $(error Unknow building profile, should be CPU_ONLY_PROFILE or CPU_GPU_PROFILE)
+  exit 1
+endif
+
+# Use static link if possible, TBB is only available as dynamic libs
+LIBS_COMMON := -L$(CCL_ROOT)/lib/cpu -lccl \
+        -L$(CMPLR_ROOT)/linux/compiler/lib/intel64_lin -l:libirc.a \
+        -L$(DAALROOT)/lib/intel64 -lonedal_core -lonedal_thread -lonedal_dpc \
+        -L$(TBBROOT)/lib/intel64/gcc4.8 -ltbb -ltbbmalloc \
+        -L$(I_MPI_ROOT)
+
+ifeq ($(PLATFORM_PROFILE),CPU_GPU_PROFILE)
+      LIBS_COMMON := $(LIBS_COMMON) \
+                     -L$(CCL_ROOT)/lib/cpu_gpu_dpcpp -lccl
+endif
+
+ifeq ($(PLATFORM_PROFILE),CPU_ONLY_PROFILE)
+  LIBS := $(LIBS_COMMON) $(ONEDAL_LIBS)
+else ifeq ($(PLATFORM_PROFILE),CPU_GPU_PROFILE)
+  LIBS := $(LIBS_COMMON) $(ONEDAL_LIBS) -l:libonedal_sycl.a
+endif
+
+CPP_SRCS += \
+  ./service.cpp ./error_handling.cpp \
+  ./daal/csr_numeric_table_impl.cpp \
+  ./daal/homogen_numeric_table_byte_buffer_impl.cpp \
+  ./daal/merged_numeric_table_impl.cpp \
+  ./daal/numeric_table_impl.cpp \
+  ./daal/row_merged_numeric_table_impl.cpp \
+  ./daal/data_dictionary.cpp \
+  ./daal/data_feature.cpp \
+  ./oneapi/dal/HomogenTableImpl.cpp \
+  ./oneapi/dal/SimpleMetadataImpl.cpp \
+  ./oneapi/dal/ColumnAccessorImpl.cpp \
+  ./oneapi/dal/RowAccessorImpl.cpp \
+  ./OneCCL.cpp ./OneDAL.cpp \
+  ./Logger.cpp \
+  ./KMeansImpl.cpp \
+  ./PCAImpl.cpp \
+  ./ALSDALImpl.cpp ./ALSShuffle.cpp \
+  ./NaiveBayesDALImpl.cpp \
+  ./LinearRegressionImpl.cpp \
+  ./CorrelationImpl.cpp \
+  ./SummarizerImpl.cpp \
+  ./DecisionForestClassifierImpl.cpp \
+  ./DecisionForestRegressorImpl.cpp
+
+
+
+OBJS += \
+  ./service.o ./error_handling.o \
+  ./daal/csr_numeric_table_impl.o \
+  ./daal/homogen_numeric_table_byte_buffer_impl.o \
+  ./daal/merged_numeric_table_impl.o \
+  ./daal/numeric_table_impl.o \
+  ./daal/row_merged_numeric_table_impl.o \
+  ./daal/data_dictionary.o \
+  ./daal/data_feature.o \
+  ./oneapi/dal/HomogenTableImpl.o \
+  ./oneapi/dal/SimpleMetadataImpl.o \
+  ./oneapi/dal/ColumnAccessorImpl.o \
+  ./oneapi/dal/RowAccessorImpl.o \
+  ./OneCCL.o ./OneDAL.o \
+  ./Logger.o\
+  ./KMeansImpl.o \
+  ./PCAImpl.o \
+  ./ALSDALImpl.o ./ALSShuffle.o \
+  ./NaiveBayesDALImpl.o \
+  ./LinearRegressionImpl.o \
+  ./CorrelationImpl.o \
+  ./SummarizerImpl.o \
+  ./DecisionForestClassifierImpl.o \
+  ./DecisionForestRegressorImpl.o
+
+# Compile cpp with a compiler version before onedal 2023.2.0
+DEFINES=-D$(PLATFORM_PROFILE)
+ifneq ($(ONEDAL_VERSION),2023.2.0)
+     DEFINES+=-DONEDAL_VERSION_LESS_THAN_2023_2_0
+endif
+
+ifeq ($(PLATFORM_PROFILE),CPU_GPU_PROFILE)
+  CPP_SRCS += ./GPU.cpp
+  OBJS += ./GPU.o
+endif
+
+# Output Binary
+OUTPUT = ../../../src/main/resources/lib/libMLlibDAL.so
+
+all: $(OUTPUT)
+
+# Compile
+%.o: %.cpp
+	@echo 'Building file: $<'
+	$(CXX) $(CFLAGS) $(INCS) $(DEFINES) -c -o "$@" "$<"
+	@echo 'Finished building: $<'
+	@echo ' '
+
+# Link
+$(OUTPUT): $(OBJS)
+	@echo 'Building target: $@'
+	@echo 'Invoking: Linker'
+	$(CXX) $(CFLAGS) -shared -o $(OUTPUT) $(OBJS) $(LIBS)
+	@echo 'Finished building target: $@'
+	@echo ' '
+
+clean:
+	@echo 'Cleaning up'
+	-$(RM) $(OBJS) $(OUTPUT)
+	-@echo ' '
+
+.PHONY: all clean

mllib-dal/src/main/native/build-cpu-gpu.sh

@@ -16,5 +16,4 @@
 
 export PLATFORM_PROFILE=CPU_GPU_PROFILE
 
-make clean
-make -j
+./build.sh

mllib-dal/src/main/native/build.sh

@@ -18,5 +18,37 @@ if [[ $OAP_MLLIB_TESTING == "true" ]]; then
   exit 0
 fi
 
-make clean
-make -j
+if [[ -z $ONEDAL_VERSION ]]; then
+  if [[ -n $DAALROOT ]]; then
+    if [[ -e $DAALROOT ]]; then
+        export ONEDAL_VERSION=$(echo "$DAALROOT" | awk -F '/' '{print $(NF)}')
+    fi
+  elif [[ -n $DALROOT ]]; then
+    if [[ -e $DALROOT ]]; then
+        export ONEDAL_VERSION=$(echo "$DALROOT" | awk -F '/' '{print $(NF)}')
+    fi
+  else
+    echo DAALROOT not defined!
+    exit 1
+  fi
+fi
+
+verlte() {
+    expr $(printf '%s\n%s' "$1" "$2" | sort -t '.' -k 1,1 -k 2,2 -k 3,3 -g | sed -n 2p) != "$2"
+}
+# Reference version
+reference_version="2023.2.0"
+
+# Compare versions
+result=$(verlte "$ONEDAL_VERSION" "$reference_version")
+
+#Check the result of the comparison
+if [ "$result" -eq 1 ]; then
+    echo "$ONEDAL_VERSION is greater than $reference_version"
+    make clean
+    make -f Makefile -j
+elif [ "$result" -eq 0 ]; then
+    echo "$ONEDAL_VERSION is less than or equal $reference_version"
+    make clean
+    make -f Makefile_2023.2.0 -j
+fi

mllib-dal/src/main/native/service.h

@@ -43,7 +43,11 @@ using namespace daal::data_management;
 
 #include "Logger.h"
 #include "error_handling.h"
+#ifdef ONEDAL_VERSION_LESS_THAN_2023_2_0
 #include "oneapi/dal/table/detail/csr.hpp"
+#else
+#include "oneapi/dal/table/csr.hpp"
+#endif
 #include "oneapi/dal/table/homogen.hpp"
 
 using namespace oneapi::dal;

mllib-dal/test.sh

@@ -12,11 +12,17 @@ SCRIPT_DIR=$( cd $(dirname ${BASH_SOURCE[0]}) && pwd )
 OAP_MLLIB_ROOT=$(cd $SCRIPT_DIR/.. && pwd)
 source $OAP_MLLIB_ROOT/RELEASE
 
-if [[ -z $DAALROOT ]]; then
+if [[ -z $DAALROOT && -z $DALROOT ]]; then
  echo DAALROOT not defined!
  exit 1
 fi
 
+if [[ -e $DAALROOT ]]; then
+  export ONEDAL_VERSION=$(echo "$DAALROOT" | awk -F '/' '{print $(NF)}')
+elif [[ -e $DALROOT ]]; then
+  export ONEDAL_VERSION=$(echo "$DALROOT" | awk -F '/' '{print $(NF)}')
+fi
+
 if [[ -z $TBBROOT ]]; then
  echo TBBROOT not defined!
  exit 1