diff --git a/mllib-dal/build.sh b/mllib-dal/build.sh index 47b699728..57e5e9e4b 100755 --- a/mllib-dal/build.sh +++ b/mllib-dal/build.sh @@ -11,9 +11,17 @@ if [[ -z $(which mvn) ]]; then exit 1 fi -if [[ -z $DAALROOT ]]; then - echo DAALROOT not defined! - exit 1 +if [[ -n $DAALROOT ]]; then + if [[ -e $DAALROOT ]]; then + export ONEDAL_VERSION=$(echo "$DAALROOT" | awk -F '/' '{print $(NF)}') + fi +elif [[ -n $DALROOT ]]; then + if [[ -e $DALROOT ]]; then + export ONEDAL_VERSION=$(echo "$DALROOT" | awk -F '/' '{print $(NF)}') + fi +else + echo DAALROOT not defined! + exit 1 fi if [[ -z $TBBROOT ]]; then @@ -89,7 +97,11 @@ then echo GCC Version: $(gcc -dumpversion) fi echo JAVA_HOME=$JAVA_HOME -echo DAALROOT=$DAALROOT +if [[ -n $DAALROOT ]]; then + echo DAALROOT=$DAALROOT +elif [[ -n $DALROOT ]]; then + echo DALROOT=$DALROOT +fi echo TBBROOT=$TBBROOT echo CCL_ROOT=$CCL_ROOT echo ============================= diff --git a/mllib-dal/src/main/java/com/intel/oap/mllib/LibLoader.java b/mllib-dal/src/main/java/com/intel/oap/mllib/LibLoader.java index d8cdf4fc8..ac71d19db 100644 --- a/mllib-dal/src/main/java/com/intel/oap/mllib/LibLoader.java +++ b/mllib-dal/src/main/java/com/intel/oap/mllib/LibLoader.java @@ -59,9 +59,9 @@ public static synchronized void loadLibraries() throws IOException { * Load MLlibDAL lib */ private static synchronized void loadLibMLlibDAL() throws IOException { - // oneDAL Java API doesn't load correct libtbb version - // See https://github.com/oneapi-src/oneDAL/issues/1254 - // Workaround: Load packaged libtbb & libtbbmalloc & libJavaAPI.so manually + // When loading libMLlibDAL.so, it will hang on libtcm.so. + // Workaround: loading tbb manually to bypass it. + System.loadLibrary("tbb"); loadFromJar(subDir, "libMLlibDAL.so"); } diff --git a/mllib-dal/src/main/native/Makefile b/mllib-dal/src/main/native/Makefile index 953b9cb6a..7085a95d9 100644 --- a/mllib-dal/src/main/native/Makefile +++ b/mllib-dal/src/main/native/Makefile @@ -31,50 +31,36 @@ $(info ) CFLAGS_COMMON := -Wall -Wno-deprecated-declarations -fPIC -std=c++17 \ -I $(I_MPI_ROOT)/include \ - -I $(DAALROOT)/include \ - -I $(CCL_ROOT)/include/cpu/oneapi/ \ - -I $(CMPLR_ROOT)/linux/include \ - -I $(CMPLR_ROOT)/linux/include/sycl + -I $(DALROOT)/include \ + -I $(CCL_ROOT)/include/oneapi/ \ + -I $(CMPLR_ROOT)/include \ + -I $(CMPLR_ROOT)/include/sycl ifeq ($(PLATFORM_PROFILE),CPU_ONLY_PROFILE) CFLAGS := $(CFLAGS_COMMON) else ifeq ($(PLATFORM_PROFILE),CPU_GPU_PROFILE) CFLAGS := $(CFLAGS_COMMON) -fsycl \ -fsycl-device-code-split=per_kernel \ - -I $(CCL_ROOT)/include/cpu_gpu_dpcpp/oneapi/ + -fno-sycl-id-queries-fit-in-int else $(error Unknow building profile, should be CPU_ONLY_PROFILE or CPU_GPU_PROFILE) exit 1 endif -INCS := -I $(CCL_ROOT)/include/cpu \ +INCS := -I $(CCL_ROOT)/include \ -I $(JAVA_HOME)/include \ -I $(JAVA_HOME)/include/linux \ - -I $(DAALROOT)/include \ + -I $(DALROOT)/include \ -I ./javah \ -I ./ -ifeq ($(PLATFORM_PROFILE),CPU_ONLY_PROFILE) - INCS := $(INCS) -else ifeq ($(PLATFORM_PROFILE),CPU_GPU_PROFILE) - INCS := $(INCS) -I $(CCL_ROOT)/include/cpu_gpu_dpcpp -else - $(error Unknow building profile, should be CPU_ONLY_PROFILE or CPU_GPU_PROFILE) - exit 1 -endif - # Use static link if possible, TBB is only available as dynamic libs -LIBS_COMMON := -L$(CCL_ROOT)/lib/cpu -lccl \ - -L$(CMPLR_ROOT)/linux/compiler/lib/intel64_lin -l:libirc.a \ - -L$(DAALROOT)/lib/intel64 -lonedal_core -lonedal_thread -lonedal_dpc \ +LIBS_COMMON := -L$(CCL_ROOT)/lib -lccl \ + -L$(CMPLR_ROOT)/lib -l:libirc.a \ + -L$(DALROOT)/lib/intel64 -lonedal_core -lonedal_thread -lonedal_dpc -lonedal_parameters_dpc \ -L$(TBBROOT)/lib/intel64/gcc4.8 -ltbb -ltbbmalloc \ -L$(I_MPI_ROOT) -ifeq ($(PLATFORM_PROFILE),CPU_GPU_PROFILE) - LIBS_COMMON := $(LIBS_COMMON) \ - -L$(CCL_ROOT)/lib/cpu_gpu_dpcpp -lccl -endif - ifeq ($(PLATFORM_PROFILE),CPU_ONLY_PROFILE) LIBS := $(LIBS_COMMON) $(ONEDAL_LIBS) else ifeq ($(PLATFORM_PROFILE),CPU_GPU_PROFILE) diff --git a/mllib-dal/src/main/native/Makefile_2023.2.0 b/mllib-dal/src/main/native/Makefile_2023.2.0 new file mode 100644 index 000000000..a3d54998d --- /dev/null +++ b/mllib-dal/src/main/native/Makefile_2023.2.0 @@ -0,0 +1,172 @@ +# Copyright 2020 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Use gcc for CPU and dpcpp for GPU +ifeq ($(PLATFORM_PROFILE),CPU_ONLY_PROFILE) + CC := gcc + CXX := g++ +else ifeq ($(PLATFORM_PROFILE),CPU_GPU_PROFILE) + CC := icpx + CXX := icpx +endif + +RM := rm -rf + +PLATFORM_PROFILE ?= CPU_ONLY_PROFILE + +$(info ) +$(info === Profile is $(PLATFORM_PROFILE) ===) +$(info ) + +CFLAGS_COMMON := -Wall -Wno-deprecated-declarations -fPIC -std=c++17 \ + -I $(I_MPI_ROOT)/include \ + -I $(DAALROOT)/include \ + -I $(CCL_ROOT)/include/cpu/oneapi/ \ + -I $(CMPLR_ROOT)/linux/include \ + -I $(CMPLR_ROOT)/linux/include/sycl + +ifeq ($(PLATFORM_PROFILE),CPU_ONLY_PROFILE) + CFLAGS := $(CFLAGS_COMMON) +else ifeq ($(PLATFORM_PROFILE),CPU_GPU_PROFILE) + CFLAGS := $(CFLAGS_COMMON) -fsycl \ + -fsycl-device-code-split=per_kernel \ + -I $(CCL_ROOT)/include/cpu_gpu_dpcpp/oneapi/ +else + $(error Unknow building profile, should be CPU_ONLY_PROFILE or CPU_GPU_PROFILE) + exit 1 +endif + +INCS := -I $(CCL_ROOT)/include/cpu \ + -I $(JAVA_HOME)/include \ + -I $(JAVA_HOME)/include/linux \ + -I $(DAALROOT)/include \ + -I ./javah \ + -I ./ + +ifeq ($(PLATFORM_PROFILE),CPU_ONLY_PROFILE) + INCS := $(INCS) +else ifeq ($(PLATFORM_PROFILE),CPU_GPU_PROFILE) + INCS := $(INCS) -I $(CCL_ROOT)/include/cpu_gpu_dpcpp +else + $(error Unknow building profile, should be CPU_ONLY_PROFILE or CPU_GPU_PROFILE) + exit 1 +endif + +# Use static link if possible, TBB is only available as dynamic libs +LIBS_COMMON := -L$(CCL_ROOT)/lib/cpu -lccl \ + -L$(CMPLR_ROOT)/linux/compiler/lib/intel64_lin -l:libirc.a \ + -L$(DAALROOT)/lib/intel64 -lonedal_core -lonedal_thread -lonedal_dpc \ + -L$(TBBROOT)/lib/intel64/gcc4.8 -ltbb -ltbbmalloc \ + -L$(I_MPI_ROOT) + +ifeq ($(PLATFORM_PROFILE),CPU_GPU_PROFILE) + LIBS_COMMON := $(LIBS_COMMON) \ + -L$(CCL_ROOT)/lib/cpu_gpu_dpcpp -lccl +endif + +ifeq ($(PLATFORM_PROFILE),CPU_ONLY_PROFILE) + LIBS := $(LIBS_COMMON) $(ONEDAL_LIBS) +else ifeq ($(PLATFORM_PROFILE),CPU_GPU_PROFILE) + LIBS := $(LIBS_COMMON) $(ONEDAL_LIBS) -l:libonedal_sycl.a +endif + +CPP_SRCS += \ + ./service.cpp ./error_handling.cpp \ + ./daal/csr_numeric_table_impl.cpp \ + ./daal/homogen_numeric_table_byte_buffer_impl.cpp \ + ./daal/merged_numeric_table_impl.cpp \ + ./daal/numeric_table_impl.cpp \ + ./daal/row_merged_numeric_table_impl.cpp \ + ./daal/data_dictionary.cpp \ + ./daal/data_feature.cpp \ + ./oneapi/dal/HomogenTableImpl.cpp \ + ./oneapi/dal/SimpleMetadataImpl.cpp \ + ./oneapi/dal/ColumnAccessorImpl.cpp \ + ./oneapi/dal/RowAccessorImpl.cpp \ + ./OneCCL.cpp ./OneDAL.cpp \ + ./Logger.cpp \ + ./KMeansImpl.cpp \ + ./PCAImpl.cpp \ + ./ALSDALImpl.cpp ./ALSShuffle.cpp \ + ./NaiveBayesDALImpl.cpp \ + ./LinearRegressionImpl.cpp \ + ./CorrelationImpl.cpp \ + ./SummarizerImpl.cpp \ + ./DecisionForestClassifierImpl.cpp \ + ./DecisionForestRegressorImpl.cpp + + + +OBJS += \ + ./service.o ./error_handling.o \ + ./daal/csr_numeric_table_impl.o \ + ./daal/homogen_numeric_table_byte_buffer_impl.o \ + ./daal/merged_numeric_table_impl.o \ + ./daal/numeric_table_impl.o \ + ./daal/row_merged_numeric_table_impl.o \ + ./daal/data_dictionary.o \ + ./daal/data_feature.o \ + ./oneapi/dal/HomogenTableImpl.o \ + ./oneapi/dal/SimpleMetadataImpl.o \ + ./oneapi/dal/ColumnAccessorImpl.o \ + ./oneapi/dal/RowAccessorImpl.o \ + ./OneCCL.o ./OneDAL.o \ + ./Logger.o\ + ./KMeansImpl.o \ + ./PCAImpl.o \ + ./ALSDALImpl.o ./ALSShuffle.o \ + ./NaiveBayesDALImpl.o \ + ./LinearRegressionImpl.o \ + ./CorrelationImpl.o \ + ./SummarizerImpl.o \ + ./DecisionForestClassifierImpl.o \ + ./DecisionForestRegressorImpl.o + +# Compile cpp with a compiler version before onedal 2023.2.0 +DEFINES=-D$(PLATFORM_PROFILE) +ifneq ($(ONEDAL_VERSION),2023.2.0) + DEFINES+=-DONEDAL_VERSION_LESS_THAN_2023_2_0 +endif + +ifeq ($(PLATFORM_PROFILE),CPU_GPU_PROFILE) + CPP_SRCS += ./GPU.cpp + OBJS += ./GPU.o +endif + +# Output Binary +OUTPUT = ../../../src/main/resources/lib/libMLlibDAL.so + +all: $(OUTPUT) + +# Compile +%.o: %.cpp + @echo 'Building file: $<' + $(CXX) $(CFLAGS) $(INCS) $(DEFINES) -c -o "$@" "$<" + @echo 'Finished building: $<' + @echo ' ' + +# Link +$(OUTPUT): $(OBJS) + @echo 'Building target: $@' + @echo 'Invoking: Linker' + $(CXX) $(CFLAGS) -shared -o $(OUTPUT) $(OBJS) $(LIBS) + @echo 'Finished building target: $@' + @echo ' ' + +clean: + @echo 'Cleaning up' + -$(RM) $(OBJS) $(OUTPUT) + -@echo ' ' + +.PHONY: all clean diff --git a/mllib-dal/src/main/native/build-cpu-gpu.sh b/mllib-dal/src/main/native/build-cpu-gpu.sh index 81d591516..12630e4a4 100755 --- a/mllib-dal/src/main/native/build-cpu-gpu.sh +++ b/mllib-dal/src/main/native/build-cpu-gpu.sh @@ -16,5 +16,4 @@ export PLATFORM_PROFILE=CPU_GPU_PROFILE -make clean -make -j +./build.sh diff --git a/mllib-dal/src/main/native/build.sh b/mllib-dal/src/main/native/build.sh index cfa1ef844..79d934a3b 100755 --- a/mllib-dal/src/main/native/build.sh +++ b/mllib-dal/src/main/native/build.sh @@ -18,5 +18,37 @@ if [[ $OAP_MLLIB_TESTING == "true" ]]; then exit 0 fi -make clean -make -j +if [[ -z $ONEDAL_VERSION ]]; then + if [[ -n $DAALROOT ]]; then + if [[ -e $DAALROOT ]]; then + export ONEDAL_VERSION=$(echo "$DAALROOT" | awk -F '/' '{print $(NF)}') + fi + elif [[ -n $DALROOT ]]; then + if [[ -e $DALROOT ]]; then + export ONEDAL_VERSION=$(echo "$DALROOT" | awk -F '/' '{print $(NF)}') + fi + else + echo DAALROOT not defined! + exit 1 + fi +fi + +verlte() { + expr $(printf '%s\n%s' "$1" "$2" | sort -t '.' -k 1,1 -k 2,2 -k 3,3 -g | sed -n 2p) != "$2" +} +# Reference version +reference_version="2023.2.0" + +# Compare versions +result=$(verlte "$ONEDAL_VERSION" "$reference_version") + +#Check the result of the comparison +if [ "$result" -eq 1 ]; then + echo "$ONEDAL_VERSION is greater than $reference_version" + make clean + make -f Makefile -j +elif [ "$result" -eq 0 ]; then + echo "$ONEDAL_VERSION is less than or equal $reference_version" + make clean + make -f Makefile_2023.2.0 -j +fi diff --git a/mllib-dal/src/main/native/service.h b/mllib-dal/src/main/native/service.h index b9579b84a..bdc7acc6a 100644 --- a/mllib-dal/src/main/native/service.h +++ b/mllib-dal/src/main/native/service.h @@ -43,7 +43,11 @@ using namespace daal::data_management; #include "Logger.h" #include "error_handling.h" +#ifdef ONEDAL_VERSION_LESS_THAN_2023_2_0 #include "oneapi/dal/table/detail/csr.hpp" +#else +#include "oneapi/dal/table/csr.hpp" +#endif #include "oneapi/dal/table/homogen.hpp" using namespace oneapi::dal; diff --git a/mllib-dal/test.sh b/mllib-dal/test.sh index 0d509481b..355d71b3c 100755 --- a/mllib-dal/test.sh +++ b/mllib-dal/test.sh @@ -12,11 +12,17 @@ SCRIPT_DIR=$( cd $(dirname ${BASH_SOURCE[0]}) && pwd ) OAP_MLLIB_ROOT=$(cd $SCRIPT_DIR/.. && pwd) source $OAP_MLLIB_ROOT/RELEASE -if [[ -z $DAALROOT ]]; then +if [[ -z $DAALROOT && -z $DALROOT ]]; then echo DAALROOT not defined! exit 1 fi +if [[ -e $DAALROOT ]]; then + export ONEDAL_VERSION=$(echo "$DAALROOT" | awk -F '/' '{print $(NF)}') +elif [[ -e $DALROOT ]]; then + export ONEDAL_VERSION=$(echo "$DALROOT" | awk -F '/' '{print $(NF)}') +fi + if [[ -z $TBBROOT ]]; then echo TBBROOT not defined! exit 1