From be55d63e6dc6ac0e631ae59c5eb5b254db63607f Mon Sep 17 00:00:00 2001 From: minmingzhu Date: Mon, 18 Sep 2023 07:20:38 +0000 Subject: [PATCH 01/19] update spark to 3.3.3 Signed-off-by: minmingzhu --- README.md | 1 + examples/scala/pom-parent.xml | 2 +- mllib-dal/pom.xml | 2 +- .../com/intel/oap/mllib/classification/NaiveBayesShim.scala | 5 +++-- .../mllib/classification/RandomForestClassifierShim.scala | 6 +++--- .../scala/com/intel/oap/mllib/clustering/KMeansShim.scala | 5 +++-- .../main/scala/com/intel/oap/mllib/feature/PCAShim.scala | 5 +++-- .../scala/com/intel/oap/mllib/recommendation/ALSShim.scala | 4 ++-- .../intel/oap/mllib/regression/LinearRegressionShim.scala | 4 ++-- .../oap/mllib/regression/RandomForestRegressorShim.scala | 5 +++-- .../scala/com/intel/oap/mllib/stat/CorrelationShim.scala | 5 +++-- .../scala/com/intel/oap/mllib/stat/SummarizerShim.scala | 5 +++-- .../classification/{spark322 => spark333}/NaiveBayes.scala | 2 +- .../{spark322 => spark333}/RandomForestClassifier.scala | 2 +- .../spark/ml/clustering/{spark322 => spark333}/KMeans.scala | 2 +- .../spark/ml/feature/{spark322 => spark333}/PCA.scala | 2 +- .../ml/recommendation/{spark322 => spark333}/ALS.scala | 2 +- .../{spark322 => spark333}/LinearRegression.scala | 2 +- .../{spark322 => spark333}/RandomForestRegressor.scala | 2 +- .../spark/ml/stat/{spark322 => spark333}/Correlation.scala | 2 +- .../mllib/stat/{spark322 => spark333}/Statistics.scala | 2 +- 21 files changed, 37 insertions(+), 30 deletions(-) rename mllib-dal/src/main/scala/org/apache/spark/ml/classification/{spark322 => spark333}/NaiveBayes.scala (99%) rename mllib-dal/src/main/scala/org/apache/spark/ml/classification/{spark322 => spark333}/RandomForestClassifier.scala (99%) rename mllib-dal/src/main/scala/org/apache/spark/ml/clustering/{spark322 => spark333}/KMeans.scala (99%) rename mllib-dal/src/main/scala/org/apache/spark/ml/feature/{spark322 => spark333}/PCA.scala (98%) rename mllib-dal/src/main/scala/org/apache/spark/ml/recommendation/{spark322 => spark333}/ALS.scala (99%) rename mllib-dal/src/main/scala/org/apache/spark/ml/regression/{spark322 => spark333}/LinearRegression.scala (99%) rename mllib-dal/src/main/scala/org/apache/spark/ml/regression/{spark322 => spark333}/RandomForestRegressor.scala (99%) rename mllib-dal/src/main/scala/org/apache/spark/ml/stat/{spark322 => spark333}/Correlation.scala (99%) rename mllib-dal/src/main/scala/org/apache/spark/mllib/stat/{spark322 => spark333}/Statistics.scala (98%) diff --git a/README.md b/README.md index 5e2e0724d..55afbf855 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,7 @@ The following runtime packages with all their dependencies should be installed i * Apache Spark 3.2.0 * Apache Spark 3.2.1 * Apache Spark 3.2.2 +* Apache Spark 3.3.3 ### Supported Intel® oneAPI Toolkits diff --git a/examples/scala/pom-parent.xml b/examples/scala/pom-parent.xml index d1fec21cc..7b20fa05b 100644 --- a/examples/scala/pom-parent.xml +++ b/examples/scala/pom-parent.xml @@ -29,7 +29,7 @@ 1.6.0 2.12.15 2.12 - 3.2.2 + 3.3.3 diff --git a/mllib-dal/pom.xml b/mllib-dal/pom.xml index caedfa050..f88829542 100644 --- a/mllib-dal/pom.xml +++ b/mllib-dal/pom.xml @@ -31,7 +31,7 @@ 2.12.15 2.12 3.2.9 - 3.2.2 + 3.3.3 2023.1.0.31217 src/assembly/assembly.xml diff --git a/mllib-dal/src/main/scala/com/intel/oap/mllib/classification/NaiveBayesShim.scala b/mllib-dal/src/main/scala/com/intel/oap/mllib/classification/NaiveBayesShim.scala index dff679027..522a7f6af 100644 --- a/mllib-dal/src/main/scala/com/intel/oap/mllib/classification/NaiveBayesShim.scala +++ b/mllib-dal/src/main/scala/com/intel/oap/mllib/classification/NaiveBayesShim.scala @@ -20,7 +20,7 @@ import com.intel.oap.mllib.Utils import org.apache.spark.internal.Logging import org.apache.spark.ml.classification.NaiveBayesModel -import org.apache.spark.ml.classification.spark322.{NaiveBayes => NaiveBayesSpark322} +import org.apache.spark.ml.classification.spark333.{NaiveBayes => NaiveBayesSpark333} import org.apache.spark.ml.param.ParamMap import org.apache.spark.sql.Dataset import org.apache.spark.{SPARK_VERSION, SparkException} @@ -35,7 +35,8 @@ object NaiveBayesShim extends Logging { logInfo(s"Loading NaiveBayes for Spark $SPARK_VERSION") val shim = Utils.getSparkVersion() match { - case "3.1.1" | "3.1.2" | "3.1.3" | "3.2.0" | "3.2.1" | "3.2.2" => new NaiveBayesSpark322(uid) + case "3.1.1" | "3.1.2" | "3.1.3" | "3.2.0" | "3.2.1" | "3.2.2" | "3.3.3" => + new NaiveBayesSpark333(uid) case _ => throw new SparkException(s"Unsupported Spark version $SPARK_VERSION") } shim diff --git a/mllib-dal/src/main/scala/com/intel/oap/mllib/classification/RandomForestClassifierShim.scala b/mllib-dal/src/main/scala/com/intel/oap/mllib/classification/RandomForestClassifierShim.scala index 94831c8d7..66c39f9d3 100644 --- a/mllib-dal/src/main/scala/com/intel/oap/mllib/classification/RandomForestClassifierShim.scala +++ b/mllib-dal/src/main/scala/com/intel/oap/mllib/classification/RandomForestClassifierShim.scala @@ -19,7 +19,7 @@ import com.intel.oap.mllib.Utils import org.apache.spark.internal.Logging import org.apache.spark.ml.classification.RandomForestClassificationModel import org.apache.spark.{SPARK_VERSION, SparkException} -import org.apache.spark.ml.classification.spark322.{RandomForestClassifier => RandomForestClassifier322} +import org.apache.spark.ml.classification.spark333.{RandomForestClassifier => RandomForestClassifier333} import org.apache.spark.ml.param.ParamMap import org.apache.spark.sql.Dataset @@ -33,8 +33,8 @@ object RandomForestClassifierShim extends Logging { logInfo(s"Loading RandomForestClassifier for Spark $SPARK_VERSION") val shim = Utils.getSparkVersion() match { - case "3.1.1" | "3.1.2" | "3.1.3" | "3.2.0" | "3.2.1" | "3.2.2" => - new RandomForestClassifier322(uid) + case "3.1.1" | "3.1.2" | "3.1.3" | "3.2.0" | "3.2.1" | "3.2.2" | "3.3.3" => + new RandomForestClassifier333(uid) case _ => throw new SparkException(s"Unsupported Spark version $SPARK_VERSION") } shim diff --git a/mllib-dal/src/main/scala/com/intel/oap/mllib/clustering/KMeansShim.scala b/mllib-dal/src/main/scala/com/intel/oap/mllib/clustering/KMeansShim.scala index 66e16c754..8c969142b 100644 --- a/mllib-dal/src/main/scala/com/intel/oap/mllib/clustering/KMeansShim.scala +++ b/mllib-dal/src/main/scala/com/intel/oap/mllib/clustering/KMeansShim.scala @@ -20,7 +20,7 @@ import com.intel.oap.mllib.Utils import org.apache.spark.internal.Logging import org.apache.spark.ml.clustering.{KMeans, KMeansModel} -import org.apache.spark.ml.clustering.spark322.{KMeans => KMeansSpark322} +import org.apache.spark.ml.clustering.spark333.{KMeans => KMeansSpark333} import org.apache.spark.ml.param.ParamMap import org.apache.spark.sql.Dataset import org.apache.spark.{SPARK_VERSION, SparkException} @@ -34,7 +34,8 @@ object KMeansShim extends Logging { def create(uid: String): KMeansShim = { logInfo(s"Loading KMeans for Spark $SPARK_VERSION") val kmeans = Utils.getSparkVersion() match { - case "3.1.1" | "3.1.2" | "3.1.3" | "3.2.0" | "3.2.1" | "3.2.2" => new KMeansSpark322(uid) + case "3.1.1" | "3.1.2" | "3.1.3" | "3.2.0" | "3.2.1" | "3.2.2" | "3.3.3" => + new KMeansSpark333(uid) case _ => throw new SparkException(s"Unsupported Spark version $SPARK_VERSION") } kmeans diff --git a/mllib-dal/src/main/scala/com/intel/oap/mllib/feature/PCAShim.scala b/mllib-dal/src/main/scala/com/intel/oap/mllib/feature/PCAShim.scala index 0f2df5e10..4b656bf6e 100644 --- a/mllib-dal/src/main/scala/com/intel/oap/mllib/feature/PCAShim.scala +++ b/mllib-dal/src/main/scala/com/intel/oap/mllib/feature/PCAShim.scala @@ -20,7 +20,7 @@ import com.intel.oap.mllib.Utils import org.apache.spark.internal.Logging import org.apache.spark.ml.feature.PCAModel -import org.apache.spark.ml.feature.spark322.{PCA => PCASpark322} +import org.apache.spark.ml.feature.spark333.{PCA => PCASpark333} import org.apache.spark.ml.param.ParamMap import org.apache.spark.sql.Dataset import org.apache.spark.{SPARK_VERSION, SparkException} @@ -34,7 +34,8 @@ object PCAShim extends Logging { def create(uid: String): PCAShim = { logInfo(s"Loading PCA for Spark $SPARK_VERSION") val pca = Utils.getSparkVersion() match { - case "3.1.1" | "3.1.2" | "3.1.3" | "3.2.0" | "3.2.1" | "3.2.2" => new PCASpark322(uid) + case "3.1.1" | "3.1.2" | "3.1.3" | "3.2.0" | "3.2.1" | "3.2.2" | "3.3.3" => + new PCASpark333(uid) case _ => throw new SparkException(s"Unsupported Spark version $SPARK_VERSION") } pca diff --git a/mllib-dal/src/main/scala/com/intel/oap/mllib/recommendation/ALSShim.scala b/mllib-dal/src/main/scala/com/intel/oap/mllib/recommendation/ALSShim.scala index f754df8b9..e2c50f41e 100644 --- a/mllib-dal/src/main/scala/com/intel/oap/mllib/recommendation/ALSShim.scala +++ b/mllib-dal/src/main/scala/com/intel/oap/mllib/recommendation/ALSShim.scala @@ -21,7 +21,7 @@ import com.intel.oap.mllib.Utils import org.apache.spark.internal.Logging import org.apache.spark.ml.recommendation.ALS.Rating import org.apache.spark.ml.recommendation.spark313.{ALS => ALSSpark313} -import org.apache.spark.ml.recommendation.spark322.{ALS => ALSSpark322} +import org.apache.spark.ml.recommendation.spark333.{ALS => ALSSpark333} import org.apache.spark.rdd.RDD import org.apache.spark.storage.StorageLevel import org.apache.spark.{SPARK_VERSION, SparkException} @@ -50,7 +50,7 @@ object ALSShim extends Logging { logInfo(s"Loading ALS for Spark $SPARK_VERSION") val als = Utils.getSparkVersion() match { case "3.1.1" | "3.1.2" | "3.1.3" => new ALSSpark313() - case "3.2.0" | "3.2.1" | "3.2.2" => new ALSSpark322() + case "3.2.0" | "3.2.1" | "3.2.2" | "3.3.3" => new ALSSpark333() case _ => throw new SparkException(s"Unsupported Spark version $SPARK_VERSION") } als diff --git a/mllib-dal/src/main/scala/com/intel/oap/mllib/regression/LinearRegressionShim.scala b/mllib-dal/src/main/scala/com/intel/oap/mllib/regression/LinearRegressionShim.scala index 8782d7095..fc9aea021 100644 --- a/mllib-dal/src/main/scala/com/intel/oap/mllib/regression/LinearRegressionShim.scala +++ b/mllib-dal/src/main/scala/com/intel/oap/mllib/regression/LinearRegressionShim.scala @@ -22,7 +22,7 @@ import org.apache.spark.internal.Logging import org.apache.spark.ml.param.ParamMap import org.apache.spark.ml.regression.LinearRegressionModel import org.apache.spark.ml.regression.spark313.{LinearRegression => LinearRegressionSpark313} -import org.apache.spark.ml.regression.spark322.{LinearRegression => LinearRegressionSpark322} +import org.apache.spark.ml.regression.spark333.{LinearRegression => LinearRegressionSpark333} import org.apache.spark.sql.Dataset import org.apache.spark.{SPARK_VERSION, SparkException} @@ -36,7 +36,7 @@ object LinearRegressionShim extends Logging { logInfo(s"Loading ALS for Spark $SPARK_VERSION") val linearRegression = Utils.getSparkVersion() match { case "3.1.1" | "3.1.2" | "3.1.3" => new LinearRegressionSpark313(uid) - case "3.2.0" | "3.2.1" | "3.2.2" => new LinearRegressionSpark322(uid) + case "3.2.0" | "3.2.1" | "3.2.2" | "3.3.3" => new LinearRegressionSpark333(uid) case _ => throw new SparkException(s"Unsupported Spark version $SPARK_VERSION") } linearRegression diff --git a/mllib-dal/src/main/scala/com/intel/oap/mllib/regression/RandomForestRegressorShim.scala b/mllib-dal/src/main/scala/com/intel/oap/mllib/regression/RandomForestRegressorShim.scala index 6cffe1e11..a056fd08e 100644 --- a/mllib-dal/src/main/scala/com/intel/oap/mllib/regression/RandomForestRegressorShim.scala +++ b/mllib-dal/src/main/scala/com/intel/oap/mllib/regression/RandomForestRegressorShim.scala @@ -20,7 +20,7 @@ import org.apache.spark.internal.Logging import org.apache.spark.{SPARK_VERSION, SparkException} import org.apache.spark.ml.param.ParamMap import org.apache.spark.ml.regression.RandomForestRegressionModel -import org.apache.spark.ml.regression.spark322.{RandomForestRegressor => RandomForestRegressor322} +import org.apache.spark.ml.regression.spark333.{RandomForestRegressor => RandomForestRegressor333} import org.apache.spark.sql.Dataset trait RandomForestRegressorShim extends Logging { @@ -33,7 +33,8 @@ object RandomForestRegressorShim extends Logging { logInfo(s"Loading RandomForestClassifier for Spark $SPARK_VERSION") val shim = Utils.getSparkVersion() match { - case "3.1.1" | "3.1.2" | "3.1.3" | "3.2.0" | "3.2.1" | "3.2.2" => new RandomForestRegressor322(uid) + case "3.1.1" | "3.1.2" | "3.1.3" | "3.2.0" | "3.2.1" | "3.2.2" | "3.3.3" => + new RandomForestRegressor333(uid) case _ => throw new SparkException(s"Unsupported Spark version $SPARK_VERSION") } shim diff --git a/mllib-dal/src/main/scala/com/intel/oap/mllib/stat/CorrelationShim.scala b/mllib-dal/src/main/scala/com/intel/oap/mllib/stat/CorrelationShim.scala index 90b9f6d29..036955223 100644 --- a/mllib-dal/src/main/scala/com/intel/oap/mllib/stat/CorrelationShim.scala +++ b/mllib-dal/src/main/scala/com/intel/oap/mllib/stat/CorrelationShim.scala @@ -25,7 +25,7 @@ import org.apache.spark.sql.{DataFrame, Dataset} import org.apache.spark.storage.StorageLevel import scala.reflect.ClassTag -import org.apache.spark.ml.stat.spark322.{Correlation => CorrelationSpark322} +import org.apache.spark.ml.stat.spark333.{Correlation => CorrelationSpark333} trait CorrelationShim extends Serializable with Logging { def corr(dataset: Dataset[_], column: String, method: String): DataFrame @@ -35,7 +35,8 @@ object CorrelationShim extends Logging { def create(): CorrelationShim = { logInfo(s"Loading Correlation for Spark $SPARK_VERSION") val als = Utils.getSparkVersion() match { - case "3.1.1" | "3.1.2" | "3.1.3" | "3.2.0" | "3.2.1" | "3.2.2" => new CorrelationSpark322() + case "3.1.1" | "3.1.2" | "3.1.3" | "3.2.0" | "3.2.1" | "3.2.2" | "3.3.3" => + new CorrelationSpark333() case _ => throw new SparkException(s"Unsupported Spark version $SPARK_VERSION") } als diff --git a/mllib-dal/src/main/scala/com/intel/oap/mllib/stat/SummarizerShim.scala b/mllib-dal/src/main/scala/com/intel/oap/mllib/stat/SummarizerShim.scala index 5f3ff92df..38efa04a0 100644 --- a/mllib-dal/src/main/scala/com/intel/oap/mllib/stat/SummarizerShim.scala +++ b/mllib-dal/src/main/scala/com/intel/oap/mllib/stat/SummarizerShim.scala @@ -24,7 +24,7 @@ import org.apache.spark.mllib.linalg.Vector import org.apache.spark.mllib.stat.MultivariateStatisticalSummary import org.apache.spark.rdd.RDD import org.apache.spark.sql.{DataFrame, Dataset} -import org.apache.spark.mllib.stat.spark322.{Statistics => SummarizerSpark322} +import org.apache.spark.mllib.stat.spark333.{Statistics => SummarizerSpark333} trait SummarizerShim extends Serializable with Logging { def colStats(X: RDD[Vector]): MultivariateStatisticalSummary @@ -35,7 +35,8 @@ object SummarizerShim extends Logging { def create(): SummarizerShim = { logInfo(s"Loading Summarizer for Spark $SPARK_VERSION") val summarizer = Utils.getSparkVersion() match { - case "3.1.1" | "3.1.2" | "3.1.3" | "3.2.0" | "3.2.1" | "3.2.2" => new SummarizerSpark322() + case "3.1.1" | "3.1.2" | "3.1.3" | "3.2.0" | "3.2.1" | "3.2.2" | "3.3.3" => + new SummarizerSpark333() case _ => throw new SparkException(s"Unsupported Spark version $SPARK_VERSION") } summarizer diff --git a/mllib-dal/src/main/scala/org/apache/spark/ml/classification/spark322/NaiveBayes.scala b/mllib-dal/src/main/scala/org/apache/spark/ml/classification/spark333/NaiveBayes.scala similarity index 99% rename from mllib-dal/src/main/scala/org/apache/spark/ml/classification/spark322/NaiveBayes.scala rename to mllib-dal/src/main/scala/org/apache/spark/ml/classification/spark333/NaiveBayes.scala index d59654305..12936022e 100644 --- a/mllib-dal/src/main/scala/org/apache/spark/ml/classification/spark322/NaiveBayes.scala +++ b/mllib-dal/src/main/scala/org/apache/spark/ml/classification/spark333/NaiveBayes.scala @@ -17,7 +17,7 @@ */ // scalastyle:on -package org.apache.spark.ml.classification.spark322 +package org.apache.spark.ml.classification.spark333 import com.intel.oap.mllib.Utils import com.intel.oap.mllib.classification.{NaiveBayesDALImpl, NaiveBayesShim} diff --git a/mllib-dal/src/main/scala/org/apache/spark/ml/classification/spark322/RandomForestClassifier.scala b/mllib-dal/src/main/scala/org/apache/spark/ml/classification/spark333/RandomForestClassifier.scala similarity index 99% rename from mllib-dal/src/main/scala/org/apache/spark/ml/classification/spark322/RandomForestClassifier.scala rename to mllib-dal/src/main/scala/org/apache/spark/ml/classification/spark333/RandomForestClassifier.scala index 6ce3e8261..9777b1413 100644 --- a/mllib-dal/src/main/scala/org/apache/spark/ml/classification/spark322/RandomForestClassifier.scala +++ b/mllib-dal/src/main/scala/org/apache/spark/ml/classification/spark333/RandomForestClassifier.scala @@ -17,7 +17,7 @@ */ // scalastyle:on -package org.apache.spark.ml.classification.spark322 +package org.apache.spark.ml.classification.spark333 import com.intel.oap.mllib.Utils import com.intel.oap.mllib.classification.{LearningNode => LearningNodeDAL, RandomForestClassifierDALImpl, RandomForestClassifierShim} diff --git a/mllib-dal/src/main/scala/org/apache/spark/ml/clustering/spark322/KMeans.scala b/mllib-dal/src/main/scala/org/apache/spark/ml/clustering/spark333/KMeans.scala similarity index 99% rename from mllib-dal/src/main/scala/org/apache/spark/ml/clustering/spark322/KMeans.scala rename to mllib-dal/src/main/scala/org/apache/spark/ml/clustering/spark333/KMeans.scala index 8d47377ac..07b56a837 100644 --- a/mllib-dal/src/main/scala/org/apache/spark/ml/clustering/spark322/KMeans.scala +++ b/mllib-dal/src/main/scala/org/apache/spark/ml/clustering/spark333/KMeans.scala @@ -17,7 +17,7 @@ */ // scalastyle:on -package org.apache.spark.ml.clustering.spark322 +package org.apache.spark.ml.clustering.spark333 import com.intel.oap.mllib.Utils import com.intel.oap.mllib.clustering.{KMeansDALImpl, KMeansShim} diff --git a/mllib-dal/src/main/scala/org/apache/spark/ml/feature/spark322/PCA.scala b/mllib-dal/src/main/scala/org/apache/spark/ml/feature/spark333/PCA.scala similarity index 98% rename from mllib-dal/src/main/scala/org/apache/spark/ml/feature/spark322/PCA.scala rename to mllib-dal/src/main/scala/org/apache/spark/ml/feature/spark333/PCA.scala index cc99b1779..e0b32fc22 100644 --- a/mllib-dal/src/main/scala/org/apache/spark/ml/feature/spark322/PCA.scala +++ b/mllib-dal/src/main/scala/org/apache/spark/ml/feature/spark333/PCA.scala @@ -17,7 +17,7 @@ */ // scalastyle:on -package org.apache.spark.ml.feature.spark322 +package org.apache.spark.ml.feature.spark333 import com.intel.oap.mllib.Utils import com.intel.oap.mllib.feature.{PCADALImpl, PCAShim} diff --git a/mllib-dal/src/main/scala/org/apache/spark/ml/recommendation/spark322/ALS.scala b/mllib-dal/src/main/scala/org/apache/spark/ml/recommendation/spark333/ALS.scala similarity index 99% rename from mllib-dal/src/main/scala/org/apache/spark/ml/recommendation/spark322/ALS.scala rename to mllib-dal/src/main/scala/org/apache/spark/ml/recommendation/spark333/ALS.scala index 81e2d8300..2fe5cd29f 100644 --- a/mllib-dal/src/main/scala/org/apache/spark/ml/recommendation/spark322/ALS.scala +++ b/mllib-dal/src/main/scala/org/apache/spark/ml/recommendation/spark333/ALS.scala @@ -17,7 +17,7 @@ */ // scalastyle:on -package org.apache.spark.ml.recommendation.spark322 +package org.apache.spark.ml.recommendation.spark333 import com.github.fommil.netlib.BLAS.{getInstance => blas} import com.intel.oap.mllib.{Utils => DALUtils} diff --git a/mllib-dal/src/main/scala/org/apache/spark/ml/regression/spark322/LinearRegression.scala b/mllib-dal/src/main/scala/org/apache/spark/ml/regression/spark333/LinearRegression.scala similarity index 99% rename from mllib-dal/src/main/scala/org/apache/spark/ml/regression/spark322/LinearRegression.scala rename to mllib-dal/src/main/scala/org/apache/spark/ml/regression/spark333/LinearRegression.scala index ac861b2a6..a921dfbfc 100644 --- a/mllib-dal/src/main/scala/org/apache/spark/ml/regression/spark322/LinearRegression.scala +++ b/mllib-dal/src/main/scala/org/apache/spark/ml/regression/spark333/LinearRegression.scala @@ -17,7 +17,7 @@ */ // scalastyle:on -package org.apache.spark.ml.regression.spark322 +package org.apache.spark.ml.regression.spark333 import breeze.linalg.{DenseVector => BDV} import breeze.optimize.{ diff --git a/mllib-dal/src/main/scala/org/apache/spark/ml/regression/spark322/RandomForestRegressor.scala b/mllib-dal/src/main/scala/org/apache/spark/ml/regression/spark333/RandomForestRegressor.scala similarity index 99% rename from mllib-dal/src/main/scala/org/apache/spark/ml/regression/spark322/RandomForestRegressor.scala rename to mllib-dal/src/main/scala/org/apache/spark/ml/regression/spark333/RandomForestRegressor.scala index 2b1557fc2..479e7fdd1 100644 --- a/mllib-dal/src/main/scala/org/apache/spark/ml/regression/spark322/RandomForestRegressor.scala +++ b/mllib-dal/src/main/scala/org/apache/spark/ml/regression/spark333/RandomForestRegressor.scala @@ -17,7 +17,7 @@ */ // scalastyle:on -package org.apache.spark.ml.regression.spark322 +package org.apache.spark.ml.regression.spark333 import com.intel.oap.mllib.Utils import com.intel.oap.mllib.classification.{LearningNode => LearningNodeDAL} diff --git a/mllib-dal/src/main/scala/org/apache/spark/ml/stat/spark322/Correlation.scala b/mllib-dal/src/main/scala/org/apache/spark/ml/stat/spark333/Correlation.scala similarity index 99% rename from mllib-dal/src/main/scala/org/apache/spark/ml/stat/spark322/Correlation.scala rename to mllib-dal/src/main/scala/org/apache/spark/ml/stat/spark333/Correlation.scala index 1d456ea62..2fe66fe85 100644 --- a/mllib-dal/src/main/scala/org/apache/spark/ml/stat/spark322/Correlation.scala +++ b/mllib-dal/src/main/scala/org/apache/spark/ml/stat/spark333/Correlation.scala @@ -17,7 +17,7 @@ */ // scalastyle:on -package org.apache.spark.ml.stat.spark322 +package org.apache.spark.ml.stat.spark333 import com.intel.oap.mllib.Utils import com.intel.oap.mllib.stat.{CorrelationDALImpl, CorrelationShim} diff --git a/mllib-dal/src/main/scala/org/apache/spark/mllib/stat/spark322/Statistics.scala b/mllib-dal/src/main/scala/org/apache/spark/mllib/stat/spark333/Statistics.scala similarity index 98% rename from mllib-dal/src/main/scala/org/apache/spark/mllib/stat/spark322/Statistics.scala rename to mllib-dal/src/main/scala/org/apache/spark/mllib/stat/spark333/Statistics.scala index c36fd18bd..222c1139e 100644 --- a/mllib-dal/src/main/scala/org/apache/spark/mllib/stat/spark322/Statistics.scala +++ b/mllib-dal/src/main/scala/org/apache/spark/mllib/stat/spark333/Statistics.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.apache.spark.mllib.stat.spark322 +package org.apache.spark.mllib.stat.spark333 import com.intel.oap.mllib.Utils import com.intel.oap.mllib.stat.{SummarizerDALImpl, SummarizerShim} From 3bc7b4510c2f896bd1cc5d2b79ce3b8f88c4c40c Mon Sep 17 00:00:00 2001 From: minmingzhu Date: Mon, 22 Jan 2024 03:31:31 +0000 Subject: [PATCH 02/19] support oneapi 2024.0.0 Signed-off-by: minmingzhu --- .../java/com/intel/oap/mllib/LibLoader.java | 1 + mllib-dal/src/main/native/Makefile | 35 ++-- mllib-dal/src/main/native/Makefile_2023.2.0 | 168 ++++++++++++++++++ mllib-dal/src/main/native/service.h | 4 + 4 files changed, 184 insertions(+), 24 deletions(-) create mode 100644 mllib-dal/src/main/native/Makefile_2023.2.0 diff --git a/mllib-dal/src/main/java/com/intel/oap/mllib/LibLoader.java b/mllib-dal/src/main/java/com/intel/oap/mllib/LibLoader.java index d8cdf4fc8..ce902ea66 100644 --- a/mllib-dal/src/main/java/com/intel/oap/mllib/LibLoader.java +++ b/mllib-dal/src/main/java/com/intel/oap/mllib/LibLoader.java @@ -62,6 +62,7 @@ private static synchronized void loadLibMLlibDAL() throws IOException { // oneDAL Java API doesn't load correct libtbb version // See https://github.com/oneapi-src/oneDAL/issues/1254 // Workaround: Load packaged libtbb & libtbbmalloc & libJavaAPI.so manually + System.loadLibrary("onedal_thread"); loadFromJar(subDir, "libMLlibDAL.so"); } diff --git a/mllib-dal/src/main/native/Makefile b/mllib-dal/src/main/native/Makefile index 953b9cb6a..d851bcd09 100644 --- a/mllib-dal/src/main/native/Makefile +++ b/mllib-dal/src/main/native/Makefile @@ -24,6 +24,7 @@ endif RM := rm -rf PLATFORM_PROFILE ?= CPU_ONLY_PROFILE +ONEDAL_VERSION=2023.2.0 $(info ) $(info === Profile is $(PLATFORM_PROFILE) ===) @@ -31,8 +32,8 @@ $(info ) CFLAGS_COMMON := -Wall -Wno-deprecated-declarations -fPIC -std=c++17 \ -I $(I_MPI_ROOT)/include \ - -I $(DAALROOT)/include \ - -I $(CCL_ROOT)/include/cpu/oneapi/ \ + -I $(DALROOT)/include \ + -I $(CCL_ROOT)/include/oneapi/ \ -I $(CMPLR_ROOT)/linux/include \ -I $(CMPLR_ROOT)/linux/include/sycl @@ -41,40 +42,26 @@ ifeq ($(PLATFORM_PROFILE),CPU_ONLY_PROFILE) else ifeq ($(PLATFORM_PROFILE),CPU_GPU_PROFILE) CFLAGS := $(CFLAGS_COMMON) -fsycl \ -fsycl-device-code-split=per_kernel \ - -I $(CCL_ROOT)/include/cpu_gpu_dpcpp/oneapi/ + -fno-sycl-id-queries-fit-in-int else $(error Unknow building profile, should be CPU_ONLY_PROFILE or CPU_GPU_PROFILE) exit 1 endif -INCS := -I $(CCL_ROOT)/include/cpu \ +INCS := -I $(CCL_ROOT)/include \ -I $(JAVA_HOME)/include \ -I $(JAVA_HOME)/include/linux \ - -I $(DAALROOT)/include \ + -I $(DALROOT)/include \ -I ./javah \ -I ./ -ifeq ($(PLATFORM_PROFILE),CPU_ONLY_PROFILE) - INCS := $(INCS) -else ifeq ($(PLATFORM_PROFILE),CPU_GPU_PROFILE) - INCS := $(INCS) -I $(CCL_ROOT)/include/cpu_gpu_dpcpp -else - $(error Unknow building profile, should be CPU_ONLY_PROFILE or CPU_GPU_PROFILE) - exit 1 -endif - # Use static link if possible, TBB is only available as dynamic libs -LIBS_COMMON := -L$(CCL_ROOT)/lib/cpu -lccl \ - -L$(CMPLR_ROOT)/linux/compiler/lib/intel64_lin -l:libirc.a \ - -L$(DAALROOT)/lib/intel64 -lonedal_core -lonedal_thread -lonedal_dpc \ +LIBS_COMMON := -L$(CCL_ROOT)/lib -lccl \ + -L$(CMPLR_ROOT)/linux/compiler/lib -l:libirc.a \ + -L$(DALROOT)/lib/intel64 -lonedal_core -lonedal_thread -lonedal_dpc -lonedal_parameters_dpc \ -L$(TBBROOT)/lib/intel64/gcc4.8 -ltbb -ltbbmalloc \ -L$(I_MPI_ROOT) -ifeq ($(PLATFORM_PROFILE),CPU_GPU_PROFILE) - LIBS_COMMON := $(LIBS_COMMON) \ - -L$(CCL_ROOT)/lib/cpu_gpu_dpcpp -lccl -endif - ifeq ($(PLATFORM_PROFILE),CPU_ONLY_PROFILE) LIBS := $(LIBS_COMMON) $(ONEDAL_LIBS) else ifeq ($(PLATFORM_PROFILE),CPU_GPU_PROFILE) @@ -133,7 +120,7 @@ OBJS += \ ./DecisionForestClassifierImpl.o \ ./DecisionForestRegressorImpl.o -DEFINES=-D$(PLATFORM_PROFILE) +DEFINES=-D$(PLATFORM_PROFILE) -D$(ONEDAL_VERSION) ifeq ($(PLATFORM_PROFILE),CPU_GPU_PROFILE) CPP_SRCS += ./GPU.cpp @@ -165,4 +152,4 @@ clean: -$(RM) $(OBJS) $(OUTPUT) -@echo ' ' -.PHONY: all clean +.PHONY: all clean \ No newline at end of file diff --git a/mllib-dal/src/main/native/Makefile_2023.2.0 b/mllib-dal/src/main/native/Makefile_2023.2.0 new file mode 100644 index 000000000..953b9cb6a --- /dev/null +++ b/mllib-dal/src/main/native/Makefile_2023.2.0 @@ -0,0 +1,168 @@ +# Copyright 2020 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Use gcc for CPU and dpcpp for GPU +ifeq ($(PLATFORM_PROFILE),CPU_ONLY_PROFILE) + CC := gcc + CXX := g++ +else ifeq ($(PLATFORM_PROFILE),CPU_GPU_PROFILE) + CC := icpx + CXX := icpx +endif + +RM := rm -rf + +PLATFORM_PROFILE ?= CPU_ONLY_PROFILE + +$(info ) +$(info === Profile is $(PLATFORM_PROFILE) ===) +$(info ) + +CFLAGS_COMMON := -Wall -Wno-deprecated-declarations -fPIC -std=c++17 \ + -I $(I_MPI_ROOT)/include \ + -I $(DAALROOT)/include \ + -I $(CCL_ROOT)/include/cpu/oneapi/ \ + -I $(CMPLR_ROOT)/linux/include \ + -I $(CMPLR_ROOT)/linux/include/sycl + +ifeq ($(PLATFORM_PROFILE),CPU_ONLY_PROFILE) + CFLAGS := $(CFLAGS_COMMON) +else ifeq ($(PLATFORM_PROFILE),CPU_GPU_PROFILE) + CFLAGS := $(CFLAGS_COMMON) -fsycl \ + -fsycl-device-code-split=per_kernel \ + -I $(CCL_ROOT)/include/cpu_gpu_dpcpp/oneapi/ +else + $(error Unknow building profile, should be CPU_ONLY_PROFILE or CPU_GPU_PROFILE) + exit 1 +endif + +INCS := -I $(CCL_ROOT)/include/cpu \ + -I $(JAVA_HOME)/include \ + -I $(JAVA_HOME)/include/linux \ + -I $(DAALROOT)/include \ + -I ./javah \ + -I ./ + +ifeq ($(PLATFORM_PROFILE),CPU_ONLY_PROFILE) + INCS := $(INCS) +else ifeq ($(PLATFORM_PROFILE),CPU_GPU_PROFILE) + INCS := $(INCS) -I $(CCL_ROOT)/include/cpu_gpu_dpcpp +else + $(error Unknow building profile, should be CPU_ONLY_PROFILE or CPU_GPU_PROFILE) + exit 1 +endif + +# Use static link if possible, TBB is only available as dynamic libs +LIBS_COMMON := -L$(CCL_ROOT)/lib/cpu -lccl \ + -L$(CMPLR_ROOT)/linux/compiler/lib/intel64_lin -l:libirc.a \ + -L$(DAALROOT)/lib/intel64 -lonedal_core -lonedal_thread -lonedal_dpc \ + -L$(TBBROOT)/lib/intel64/gcc4.8 -ltbb -ltbbmalloc \ + -L$(I_MPI_ROOT) + +ifeq ($(PLATFORM_PROFILE),CPU_GPU_PROFILE) + LIBS_COMMON := $(LIBS_COMMON) \ + -L$(CCL_ROOT)/lib/cpu_gpu_dpcpp -lccl +endif + +ifeq ($(PLATFORM_PROFILE),CPU_ONLY_PROFILE) + LIBS := $(LIBS_COMMON) $(ONEDAL_LIBS) +else ifeq ($(PLATFORM_PROFILE),CPU_GPU_PROFILE) + LIBS := $(LIBS_COMMON) $(ONEDAL_LIBS) -l:libonedal_sycl.a +endif + +CPP_SRCS += \ + ./service.cpp ./error_handling.cpp \ + ./daal/csr_numeric_table_impl.cpp \ + ./daal/homogen_numeric_table_byte_buffer_impl.cpp \ + ./daal/merged_numeric_table_impl.cpp \ + ./daal/numeric_table_impl.cpp \ + ./daal/row_merged_numeric_table_impl.cpp \ + ./daal/data_dictionary.cpp \ + ./daal/data_feature.cpp \ + ./oneapi/dal/HomogenTableImpl.cpp \ + ./oneapi/dal/SimpleMetadataImpl.cpp \ + ./oneapi/dal/ColumnAccessorImpl.cpp \ + ./oneapi/dal/RowAccessorImpl.cpp \ + ./OneCCL.cpp ./OneDAL.cpp \ + ./Logger.cpp \ + ./KMeansImpl.cpp \ + ./PCAImpl.cpp \ + ./ALSDALImpl.cpp ./ALSShuffle.cpp \ + ./NaiveBayesDALImpl.cpp \ + ./LinearRegressionImpl.cpp \ + ./CorrelationImpl.cpp \ + ./SummarizerImpl.cpp \ + ./DecisionForestClassifierImpl.cpp \ + ./DecisionForestRegressorImpl.cpp + + + +OBJS += \ + ./service.o ./error_handling.o \ + ./daal/csr_numeric_table_impl.o \ + ./daal/homogen_numeric_table_byte_buffer_impl.o \ + ./daal/merged_numeric_table_impl.o \ + ./daal/numeric_table_impl.o \ + ./daal/row_merged_numeric_table_impl.o \ + ./daal/data_dictionary.o \ + ./daal/data_feature.o \ + ./oneapi/dal/HomogenTableImpl.o \ + ./oneapi/dal/SimpleMetadataImpl.o \ + ./oneapi/dal/ColumnAccessorImpl.o \ + ./oneapi/dal/RowAccessorImpl.o \ + ./OneCCL.o ./OneDAL.o \ + ./Logger.o\ + ./KMeansImpl.o \ + ./PCAImpl.o \ + ./ALSDALImpl.o ./ALSShuffle.o \ + ./NaiveBayesDALImpl.o \ + ./LinearRegressionImpl.o \ + ./CorrelationImpl.o \ + ./SummarizerImpl.o \ + ./DecisionForestClassifierImpl.o \ + ./DecisionForestRegressorImpl.o + +DEFINES=-D$(PLATFORM_PROFILE) + +ifeq ($(PLATFORM_PROFILE),CPU_GPU_PROFILE) + CPP_SRCS += ./GPU.cpp + OBJS += ./GPU.o +endif + +# Output Binary +OUTPUT = ../../../src/main/resources/lib/libMLlibDAL.so + +all: $(OUTPUT) + +# Compile +%.o: %.cpp + @echo 'Building file: $<' + $(CXX) $(CFLAGS) $(INCS) $(DEFINES) -c -o "$@" "$<" + @echo 'Finished building: $<' + @echo ' ' + +# Link +$(OUTPUT): $(OBJS) + @echo 'Building target: $@' + @echo 'Invoking: Linker' + $(CXX) $(CFLAGS) -shared -o $(OUTPUT) $(OBJS) $(LIBS) + @echo 'Finished building target: $@' + @echo ' ' + +clean: + @echo 'Cleaning up' + -$(RM) $(OBJS) $(OUTPUT) + -@echo ' ' + +.PHONY: all clean diff --git a/mllib-dal/src/main/native/service.h b/mllib-dal/src/main/native/service.h index b9579b84a..c04f7b359 100644 --- a/mllib-dal/src/main/native/service.h +++ b/mllib-dal/src/main/native/service.h @@ -43,7 +43,11 @@ using namespace daal::data_management; #include "Logger.h" #include "error_handling.h" +#ifdef ONEDAL_VERSION #include "oneapi/dal/table/detail/csr.hpp" +#else +#include "oneapi/dal/table/csr.hpp" +#endif #include "oneapi/dal/table/homogen.hpp" using namespace oneapi::dal; From d52f7219a789ca4a36113437f0baeef365aa1e0b Mon Sep 17 00:00:00 2001 From: minmingzhu Date: Mon, 22 Jan 2024 03:38:41 +0000 Subject: [PATCH 03/19] update Signed-off-by: minmingzhu --- mllib-dal/src/main/native/Makefile | 3 +-- mllib-dal/src/main/native/Makefile_2023.2.0 | 3 ++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mllib-dal/src/main/native/Makefile b/mllib-dal/src/main/native/Makefile index d851bcd09..16dfe8413 100644 --- a/mllib-dal/src/main/native/Makefile +++ b/mllib-dal/src/main/native/Makefile @@ -24,7 +24,6 @@ endif RM := rm -rf PLATFORM_PROFILE ?= CPU_ONLY_PROFILE -ONEDAL_VERSION=2023.2.0 $(info ) $(info === Profile is $(PLATFORM_PROFILE) ===) @@ -120,7 +119,7 @@ OBJS += \ ./DecisionForestClassifierImpl.o \ ./DecisionForestRegressorImpl.o -DEFINES=-D$(PLATFORM_PROFILE) -D$(ONEDAL_VERSION) +DEFINES=-D$(PLATFORM_PROFILE) ifeq ($(PLATFORM_PROFILE),CPU_GPU_PROFILE) CPP_SRCS += ./GPU.cpp diff --git a/mllib-dal/src/main/native/Makefile_2023.2.0 b/mllib-dal/src/main/native/Makefile_2023.2.0 index 953b9cb6a..514930a8c 100644 --- a/mllib-dal/src/main/native/Makefile_2023.2.0 +++ b/mllib-dal/src/main/native/Makefile_2023.2.0 @@ -24,6 +24,7 @@ endif RM := rm -rf PLATFORM_PROFILE ?= CPU_ONLY_PROFILE +ONEDAL_VERSION=2023.2.0 $(info ) $(info === Profile is $(PLATFORM_PROFILE) ===) @@ -133,7 +134,7 @@ OBJS += \ ./DecisionForestClassifierImpl.o \ ./DecisionForestRegressorImpl.o -DEFINES=-D$(PLATFORM_PROFILE) +DEFINES=-D$(PLATFORM_PROFILE) -D$(ONEDAL_VERSION) ifeq ($(PLATFORM_PROFILE),CPU_GPU_PROFILE) CPP_SRCS += ./GPU.cpp From d899a673c7cfb4e938594be50e95badd0f91669f Mon Sep 17 00:00:00 2001 From: minmingzhu <45281494+minmingzhu@users.noreply.github.com> Date: Mon, 22 Jan 2024 11:39:55 +0800 Subject: [PATCH 04/19] Update Makefile --- mllib-dal/src/main/native/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mllib-dal/src/main/native/Makefile b/mllib-dal/src/main/native/Makefile index 16dfe8413..ea2c73e07 100644 --- a/mllib-dal/src/main/native/Makefile +++ b/mllib-dal/src/main/native/Makefile @@ -151,4 +151,4 @@ clean: -$(RM) $(OBJS) $(OUTPUT) -@echo ' ' -.PHONY: all clean \ No newline at end of file +.PHONY: all clean From f84876cbc477b50d65c7cc7c4a6ff62a69b5ab40 Mon Sep 17 00:00:00 2001 From: minmingzhu Date: Mon, 22 Jan 2024 07:40:32 +0000 Subject: [PATCH 05/19] update script Signed-off-by: minmingzhu --- mllib-dal/build.sh | 16 ++++++- mllib-dal/src/main/native/build-cpu-gpu.sh | 3 +- mllib-dal/src/main/native/build.sh | 53 +++++++++++++++++++++- 3 files changed, 66 insertions(+), 6 deletions(-) diff --git a/mllib-dal/build.sh b/mllib-dal/build.sh index 47b699728..eee3467b0 100755 --- a/mllib-dal/build.sh +++ b/mllib-dal/build.sh @@ -11,11 +11,18 @@ if [[ -z $(which mvn) ]]; then exit 1 fi -if [[ -z $DAALROOT ]]; then +if [[ -z $DAALROOT && -z $DALROOT ]]; then echo DAALROOT not defined! exit 1 fi +if [[ -e $DAALROOT ]]; then + export ONEDAL_VERSION=$(echo "$DAALROOT" | awk -F '/' '{print $(NF)}') +elif [[ -e $DALROOT ]]; then + export ONEDAL_VERSION=$(echo "$DALROOT" | awk -F '/' '{print $(NF)}') +fi +echo $ONEDAL_VERSION + if [[ -z $TBBROOT ]]; then echo TBBROOT not defined! exit 1 @@ -74,6 +81,7 @@ OAP_MLLIB_ROOT=$(cd $SCRIPT_DIR/.. && pwd) source $OAP_MLLIB_ROOT/RELEASE # Set default PLATFORM_PROFILE from RELEASE envs export PLATFORM_PROFILE=${PLATFORM_OPT:-$PLATFORM_PROFILE} +echo $PLATFORM_PROFILE echo @@ -89,7 +97,11 @@ then echo GCC Version: $(gcc -dumpversion) fi echo JAVA_HOME=$JAVA_HOME -echo DAALROOT=$DAALROOT +if [[ -e $DAALROOT ]]; then + echo DAALROOT=$DAALROOT +elif [[ -e $DALROOT ]]; then + echo DAALROOT=$DALROOT +fi echo TBBROOT=$TBBROOT echo CCL_ROOT=$CCL_ROOT echo ============================= diff --git a/mllib-dal/src/main/native/build-cpu-gpu.sh b/mllib-dal/src/main/native/build-cpu-gpu.sh index 81d591516..09ae89768 100755 --- a/mllib-dal/src/main/native/build-cpu-gpu.sh +++ b/mllib-dal/src/main/native/build-cpu-gpu.sh @@ -16,5 +16,4 @@ export PLATFORM_PROFILE=CPU_GPU_PROFILE -make clean -make -j +bash build.sh \ No newline at end of file diff --git a/mllib-dal/src/main/native/build.sh b/mllib-dal/src/main/native/build.sh index cfa1ef844..5529bd135 100755 --- a/mllib-dal/src/main/native/build.sh +++ b/mllib-dal/src/main/native/build.sh @@ -18,5 +18,54 @@ if [[ $OAP_MLLIB_TESTING == "true" ]]; then exit 0 fi -make clean -make -j +if [[ -z $ONEDAL_VERSION ]]; then + if [[ -e $DAALROOT ]]; then + export ONEDAL_VERSION=$(echo "$DAALROOT" | awk -F '/' '{print $(NF)}') + elif [[ -e $DALROOT ]]; then + export ONEDAL_VERSION=$(echo "$DALROOT" | awk -F '/' '{print $(NF)}') + else + echo DAALROOT not defined! + exit 1 + fi + echo $ONEDAL_VERSION +fi + +# Function to compare version strings +compare_versions() { + local v1=$1 + local v2=$2 + + # Convert versions to arrays + IFS='.' read -ra v1_array <<< "$v1" + IFS='.' read -ra v2_array <<< "$v2" + + # Iterate through each segment and compare numerically + for i in {0..2}; do + if ((v1_array[i] > v2_array[i])); then + return 0 # v1 > v2 + elif ((v1_array[i] < v2_array[i])); then + return 1 # v1 < v2 + fi + done + + return 1 # v1 == v2 +} + +# Reference version +reference_version="2023.2.0" + +# Compare versions +compare_versions "$ONEDAL_VERSION" "$reference_version" +result=$? + +# Check the result of the comparison +if [ "$result" -eq 0 ]; then + echo "$ONEDAL_VERSION is greater than $reference_version" + make clean + make -f Makefile -j +elif [ "$result" -eq 1 ]; then + echo "$ONEDAL_VERSION is less than or equal $reference_version" + make clean + make -f Makefile_2023.2.0 -j + +fi From 8d85475e22cdcbc9712726f71b050344712edb44 Mon Sep 17 00:00:00 2001 From: minmingzhu <45281494+minmingzhu@users.noreply.github.com> Date: Mon, 22 Jan 2024 15:50:15 +0800 Subject: [PATCH 06/19] Update build-cpu-gpu.sh --- mllib-dal/src/main/native/build-cpu-gpu.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mllib-dal/src/main/native/build-cpu-gpu.sh b/mllib-dal/src/main/native/build-cpu-gpu.sh index 09ae89768..b970f00d8 100755 --- a/mllib-dal/src/main/native/build-cpu-gpu.sh +++ b/mllib-dal/src/main/native/build-cpu-gpu.sh @@ -16,4 +16,4 @@ export PLATFORM_PROFILE=CPU_GPU_PROFILE -bash build.sh \ No newline at end of file +bash build.sh From cca6198281c07035970b42d68909e7a7a8877eab Mon Sep 17 00:00:00 2001 From: minmingzhu Date: Mon, 22 Jan 2024 07:54:24 +0000 Subject: [PATCH 07/19] update Signed-off-by: minmingzhu --- mllib-dal/src/main/native/Makefile_2023.2.0 | 2 +- mllib-dal/src/main/native/service.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mllib-dal/src/main/native/Makefile_2023.2.0 b/mllib-dal/src/main/native/Makefile_2023.2.0 index 514930a8c..43fc8c089 100644 --- a/mllib-dal/src/main/native/Makefile_2023.2.0 +++ b/mllib-dal/src/main/native/Makefile_2023.2.0 @@ -134,7 +134,7 @@ OBJS += \ ./DecisionForestClassifierImpl.o \ ./DecisionForestRegressorImpl.o -DEFINES=-D$(PLATFORM_PROFILE) -D$(ONEDAL_VERSION) +DEFINES=-D$(PLATFORM_PROFILE) -DONEDAL_VERSION=$(ONEDAL_VERSION) ifeq ($(PLATFORM_PROFILE),CPU_GPU_PROFILE) CPP_SRCS += ./GPU.cpp diff --git a/mllib-dal/src/main/native/service.h b/mllib-dal/src/main/native/service.h index c04f7b359..49db54c82 100644 --- a/mllib-dal/src/main/native/service.h +++ b/mllib-dal/src/main/native/service.h @@ -43,7 +43,7 @@ using namespace daal::data_management; #include "Logger.h" #include "error_handling.h" -#ifdef ONEDAL_VERSION +#ifdef ONEDAL_VERSION == "2023.2.0" #include "oneapi/dal/table/detail/csr.hpp" #else #include "oneapi/dal/table/csr.hpp" From 6dac863308acaf48074ca961ef0b1f634842e8e9 Mon Sep 17 00:00:00 2001 From: minmingzhu Date: Mon, 22 Jan 2024 08:26:29 +0000 Subject: [PATCH 08/19] 1. remove debug print 2. support multi onedal version in test.sh Signed-off-by: minmingzhu --- mllib-dal/build.sh | 2 -- mllib-dal/test.sh | 6 ++++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/mllib-dal/build.sh b/mllib-dal/build.sh index eee3467b0..1744c0856 100755 --- a/mllib-dal/build.sh +++ b/mllib-dal/build.sh @@ -21,7 +21,6 @@ if [[ -e $DAALROOT ]]; then elif [[ -e $DALROOT ]]; then export ONEDAL_VERSION=$(echo "$DALROOT" | awk -F '/' '{print $(NF)}') fi -echo $ONEDAL_VERSION if [[ -z $TBBROOT ]]; then echo TBBROOT not defined! @@ -81,7 +80,6 @@ OAP_MLLIB_ROOT=$(cd $SCRIPT_DIR/.. && pwd) source $OAP_MLLIB_ROOT/RELEASE # Set default PLATFORM_PROFILE from RELEASE envs export PLATFORM_PROFILE=${PLATFORM_OPT:-$PLATFORM_PROFILE} -echo $PLATFORM_PROFILE echo diff --git a/mllib-dal/test.sh b/mllib-dal/test.sh index 0d509481b..6ea9de1a4 100755 --- a/mllib-dal/test.sh +++ b/mllib-dal/test.sh @@ -17,6 +17,12 @@ if [[ -z $DAALROOT ]]; then exit 1 fi +if [[ -e $DAALROOT ]]; then + export ONEDAL_VERSION=$(echo "$DAALROOT" | awk -F '/' '{print $(NF)}') +elif [[ -e $DALROOT ]]; then + export ONEDAL_VERSION=$(echo "$DALROOT" | awk -F '/' '{print $(NF)}') +fi + if [[ -z $TBBROOT ]]; then echo TBBROOT not defined! exit 1 From 595bba68897cfbe8878825af1b7d4ac3709c41d3 Mon Sep 17 00:00:00 2001 From: minmingzhu <45281494+minmingzhu@users.noreply.github.com> Date: Mon, 22 Jan 2024 16:28:26 +0800 Subject: [PATCH 09/19] Update test.sh --- mllib-dal/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mllib-dal/test.sh b/mllib-dal/test.sh index 6ea9de1a4..355d71b3c 100755 --- a/mllib-dal/test.sh +++ b/mllib-dal/test.sh @@ -12,7 +12,7 @@ SCRIPT_DIR=$( cd $(dirname ${BASH_SOURCE[0]}) && pwd ) OAP_MLLIB_ROOT=$(cd $SCRIPT_DIR/.. && pwd) source $OAP_MLLIB_ROOT/RELEASE -if [[ -z $DAALROOT ]]; then +if [[ -z $DAALROOT && -z $DALROOT ]]; then echo DAALROOT not defined! exit 1 fi From 0afb6f17d63a63ec55cdd4bc15f9d30968eb8018 Mon Sep 17 00:00:00 2001 From: minmingzhu Date: Mon, 29 Jan 2024 05:48:18 +0000 Subject: [PATCH 10/19] fix comments Signed-off-by: minmingzhu --- mllib-dal/build.sh | 23 ++++++------- .../java/com/intel/oap/mllib/LibLoader.java | 4 --- mllib-dal/src/main/native/Makefile_2023.2.0 | 3 +- mllib-dal/src/main/native/build-cpu-gpu.sh | 2 +- mllib-dal/src/main/native/build.sh | 32 ++++--------------- mllib-dal/src/main/native/service.h | 2 +- 6 files changed, 19 insertions(+), 47 deletions(-) diff --git a/mllib-dal/build.sh b/mllib-dal/build.sh index 1744c0856..6dfb4d591 100755 --- a/mllib-dal/build.sh +++ b/mllib-dal/build.sh @@ -11,15 +11,14 @@ if [[ -z $(which mvn) ]]; then exit 1 fi -if [[ -z $DAALROOT && -z $DALROOT ]]; then - echo DAALROOT not defined! - exit 1 -fi - -if [[ -e $DAALROOT ]]; then - export ONEDAL_VERSION=$(echo "$DAALROOT" | awk -F '/' '{print $(NF)}') -elif [[ -e $DALROOT ]]; then - export ONEDAL_VERSION=$(echo "$DALROOT" | awk -F '/' '{print $(NF)}') +if [[ -z $DAALROOT ]]; then + export ONEDAL_VERSION=$(echo "$DAALROOT" | awk -F '/' '{print $(NF)}') +elif [[ -z $DALROOT ]]; then + export ONEDAL_VERSION=$(echo "$DALROOT" | awk -F '/' '{print $(NF)}') + DAALROOT=$DALROOT +else + echo DAALROOT not defined! + exit 1 fi if [[ -z $TBBROOT ]]; then @@ -95,11 +94,7 @@ then echo GCC Version: $(gcc -dumpversion) fi echo JAVA_HOME=$JAVA_HOME -if [[ -e $DAALROOT ]]; then - echo DAALROOT=$DAALROOT -elif [[ -e $DALROOT ]]; then - echo DAALROOT=$DALROOT -fi +echo DAALROOT=$DAALROOT echo TBBROOT=$TBBROOT echo CCL_ROOT=$CCL_ROOT echo ============================= diff --git a/mllib-dal/src/main/java/com/intel/oap/mllib/LibLoader.java b/mllib-dal/src/main/java/com/intel/oap/mllib/LibLoader.java index ce902ea66..c85394490 100644 --- a/mllib-dal/src/main/java/com/intel/oap/mllib/LibLoader.java +++ b/mllib-dal/src/main/java/com/intel/oap/mllib/LibLoader.java @@ -59,10 +59,6 @@ public static synchronized void loadLibraries() throws IOException { * Load MLlibDAL lib */ private static synchronized void loadLibMLlibDAL() throws IOException { - // oneDAL Java API doesn't load correct libtbb version - // See https://github.com/oneapi-src/oneDAL/issues/1254 - // Workaround: Load packaged libtbb & libtbbmalloc & libJavaAPI.so manually - System.loadLibrary("onedal_thread"); loadFromJar(subDir, "libMLlibDAL.so"); } diff --git a/mllib-dal/src/main/native/Makefile_2023.2.0 b/mllib-dal/src/main/native/Makefile_2023.2.0 index 43fc8c089..ccece0186 100644 --- a/mllib-dal/src/main/native/Makefile_2023.2.0 +++ b/mllib-dal/src/main/native/Makefile_2023.2.0 @@ -134,7 +134,8 @@ OBJS += \ ./DecisionForestClassifierImpl.o \ ./DecisionForestRegressorImpl.o -DEFINES=-D$(PLATFORM_PROFILE) -DONEDAL_VERSION=$(ONEDAL_VERSION) +# Compile cpp with a compiler version before onedal 2023.2.0 +DEFINES=-D$(PLATFORM_PROFILE) -DONEDAL_VERSION_BEFORE_2023_2_0 ifeq ($(PLATFORM_PROFILE),CPU_GPU_PROFILE) CPP_SRCS += ./GPU.cpp diff --git a/mllib-dal/src/main/native/build-cpu-gpu.sh b/mllib-dal/src/main/native/build-cpu-gpu.sh index b970f00d8..12630e4a4 100755 --- a/mllib-dal/src/main/native/build-cpu-gpu.sh +++ b/mllib-dal/src/main/native/build-cpu-gpu.sh @@ -16,4 +16,4 @@ export PLATFORM_PROFILE=CPU_GPU_PROFILE -bash build.sh +./build.sh diff --git a/mllib-dal/src/main/native/build.sh b/mllib-dal/src/main/native/build.sh index 5529bd135..24649c94b 100755 --- a/mllib-dal/src/main/native/build.sh +++ b/mllib-dal/src/main/native/build.sh @@ -27,45 +27,25 @@ if [[ -z $ONEDAL_VERSION ]]; then echo DAALROOT not defined! exit 1 fi - echo $ONEDAL_VERSION fi -# Function to compare version strings -compare_versions() { - local v1=$1 - local v2=$2 - - # Convert versions to arrays - IFS='.' read -ra v1_array <<< "$v1" - IFS='.' read -ra v2_array <<< "$v2" - - # Iterate through each segment and compare numerically - for i in {0..2}; do - if ((v1_array[i] > v2_array[i])); then - return 0 # v1 > v2 - elif ((v1_array[i] < v2_array[i])); then - return 1 # v1 < v2 - fi - done - - return 1 # v1 == v2 +verlte() { + printf '%s\n%s' "$1" "$2" | sort -C -V } - # Reference version reference_version="2023.2.0" # Compare versions -compare_versions "$ONEDAL_VERSION" "$reference_version" +verlte "$ONEDAL_VERSION" "$reference_version" && echo "1" || echo "0" result=$? -# Check the result of the comparison +#Check the result of the comparison if [ "$result" -eq 0 ]; then echo "$ONEDAL_VERSION is greater than $reference_version" make clean - make -f Makefile -j + make -f Makefile -j elif [ "$result" -eq 1 ]; then echo "$ONEDAL_VERSION is less than or equal $reference_version" make clean - make -f Makefile_2023.2.0 -j - + make -f Makefile_2023.2.0 -j fi diff --git a/mllib-dal/src/main/native/service.h b/mllib-dal/src/main/native/service.h index 49db54c82..a4e344f04 100644 --- a/mllib-dal/src/main/native/service.h +++ b/mllib-dal/src/main/native/service.h @@ -43,7 +43,7 @@ using namespace daal::data_management; #include "Logger.h" #include "error_handling.h" -#ifdef ONEDAL_VERSION == "2023.2.0" +#ifdef ONEDAL_VERSION_BEFORE_2023_2_0 #include "oneapi/dal/table/detail/csr.hpp" #else #include "oneapi/dal/table/csr.hpp" From 029ad7637a81b095c3933549cfb1f64fef7cf885 Mon Sep 17 00:00:00 2001 From: minmingzhu Date: Mon, 29 Jan 2024 05:58:17 +0000 Subject: [PATCH 11/19] CI testing Signed-off-by: minmingzhu --- mllib-dal/build.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mllib-dal/build.sh b/mllib-dal/build.sh index 6dfb4d591..77b5902b3 100755 --- a/mllib-dal/build.sh +++ b/mllib-dal/build.sh @@ -12,11 +12,15 @@ if [[ -z $(which mvn) ]]; then fi if [[ -z $DAALROOT ]]; then + echo $DAALROOT export ONEDAL_VERSION=$(echo "$DAALROOT" | awk -F '/' '{print $(NF)}') elif [[ -z $DALROOT ]]; then + echo $DALROOT export ONEDAL_VERSION=$(echo "$DALROOT" | awk -F '/' '{print $(NF)}') DAALROOT=$DALROOT else + echo $DAALROOT + echo $DALROOT echo DAALROOT not defined! exit 1 fi From 00496754afc4b44fc798462516fd979adbf1581b Mon Sep 17 00:00:00 2001 From: minmingzhu Date: Mon, 29 Jan 2024 06:07:56 +0000 Subject: [PATCH 12/19] CI testing Signed-off-by: minmingzhu --- mllib-dal/build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mllib-dal/build.sh b/mllib-dal/build.sh index 77b5902b3..3bd314380 100755 --- a/mllib-dal/build.sh +++ b/mllib-dal/build.sh @@ -11,10 +11,10 @@ if [[ -z $(which mvn) ]]; then exit 1 fi -if [[ -z $DAALROOT ]]; then +if [[ -e $DAALROOT ]]; then echo $DAALROOT export ONEDAL_VERSION=$(echo "$DAALROOT" | awk -F '/' '{print $(NF)}') -elif [[ -z $DALROOT ]]; then +elif [[ -e $DALROOT ]]; then echo $DALROOT export ONEDAL_VERSION=$(echo "$DALROOT" | awk -F '/' '{print $(NF)}') DAALROOT=$DALROOT From 2a55b951ab823fe980ff3ceca71217def6e9e149 Mon Sep 17 00:00:00 2001 From: minmingzhu Date: Mon, 29 Jan 2024 06:28:28 +0000 Subject: [PATCH 13/19] CI testing Signed-off-by: minmingzhu --- mllib-dal/src/main/native/build.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mllib-dal/src/main/native/build.sh b/mllib-dal/src/main/native/build.sh index 24649c94b..8331aa483 100755 --- a/mllib-dal/src/main/native/build.sh +++ b/mllib-dal/src/main/native/build.sh @@ -30,7 +30,7 @@ if [[ -z $ONEDAL_VERSION ]]; then fi verlte() { - printf '%s\n%s' "$1" "$2" | sort -C -V + expr $(printf '%s\n%s' "$1" "$2" | sort -t '.' -k 1,1 -k 2,2 -k 3,3 -g | sed -n 2p) != "2023.2.0" } # Reference version reference_version="2023.2.0" @@ -40,11 +40,11 @@ verlte "$ONEDAL_VERSION" "$reference_version" && echo "1" || echo "0" result=$? #Check the result of the comparison -if [ "$result" -eq 0 ]; then +if [ "$result" -eq 1 ]; then echo "$ONEDAL_VERSION is greater than $reference_version" make clean make -f Makefile -j -elif [ "$result" -eq 1 ]; then +elif [ "$result" -eq 0 ]; then echo "$ONEDAL_VERSION is less than or equal $reference_version" make clean make -f Makefile_2023.2.0 -j From 53b0091492fae24c83708b100b0bf5346b7fbdd8 Mon Sep 17 00:00:00 2001 From: minmingzhu Date: Mon, 29 Jan 2024 07:03:39 +0000 Subject: [PATCH 14/19] Remove debug log Signed-off-by: minmingzhu --- mllib-dal/build.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/mllib-dal/build.sh b/mllib-dal/build.sh index 3bd314380..2baab6603 100755 --- a/mllib-dal/build.sh +++ b/mllib-dal/build.sh @@ -12,15 +12,11 @@ if [[ -z $(which mvn) ]]; then fi if [[ -e $DAALROOT ]]; then - echo $DAALROOT export ONEDAL_VERSION=$(echo "$DAALROOT" | awk -F '/' '{print $(NF)}') elif [[ -e $DALROOT ]]; then - echo $DALROOT export ONEDAL_VERSION=$(echo "$DALROOT" | awk -F '/' '{print $(NF)}') DAALROOT=$DALROOT else - echo $DAALROOT - echo $DALROOT echo DAALROOT not defined! exit 1 fi From 6f5f90307ffe30905937e5e05238d999b7da3b3b Mon Sep 17 00:00:00 2001 From: minmingzhu Date: Tue, 30 Jan 2024 06:37:33 +0000 Subject: [PATCH 15/19] fix comments Signed-off-by: minmingzhu --- mllib-dal/build.sh | 23 ++++++++++++++------- mllib-dal/src/main/native/Makefile | 6 +++--- mllib-dal/src/main/native/Makefile_2023.2.0 | 2 +- mllib-dal/src/main/native/build.sh | 17 ++++++++------- mllib-dal/src/main/native/service.h | 2 +- 5 files changed, 30 insertions(+), 20 deletions(-) diff --git a/mllib-dal/build.sh b/mllib-dal/build.sh index 2baab6603..57e5e9e4b 100755 --- a/mllib-dal/build.sh +++ b/mllib-dal/build.sh @@ -11,14 +11,17 @@ if [[ -z $(which mvn) ]]; then exit 1 fi -if [[ -e $DAALROOT ]]; then - export ONEDAL_VERSION=$(echo "$DAALROOT" | awk -F '/' '{print $(NF)}') -elif [[ -e $DALROOT ]]; then - export ONEDAL_VERSION=$(echo "$DALROOT" | awk -F '/' '{print $(NF)}') - DAALROOT=$DALROOT +if [[ -n $DAALROOT ]]; then + if [[ -e $DAALROOT ]]; then + export ONEDAL_VERSION=$(echo "$DAALROOT" | awk -F '/' '{print $(NF)}') + fi +elif [[ -n $DALROOT ]]; then + if [[ -e $DALROOT ]]; then + export ONEDAL_VERSION=$(echo "$DALROOT" | awk -F '/' '{print $(NF)}') + fi else - echo DAALROOT not defined! - exit 1 + echo DAALROOT not defined! + exit 1 fi if [[ -z $TBBROOT ]]; then @@ -94,7 +97,11 @@ then echo GCC Version: $(gcc -dumpversion) fi echo JAVA_HOME=$JAVA_HOME -echo DAALROOT=$DAALROOT +if [[ -n $DAALROOT ]]; then + echo DAALROOT=$DAALROOT +elif [[ -n $DALROOT ]]; then + echo DALROOT=$DALROOT +fi echo TBBROOT=$TBBROOT echo CCL_ROOT=$CCL_ROOT echo ============================= diff --git a/mllib-dal/src/main/native/Makefile b/mllib-dal/src/main/native/Makefile index ea2c73e07..7085a95d9 100644 --- a/mllib-dal/src/main/native/Makefile +++ b/mllib-dal/src/main/native/Makefile @@ -33,8 +33,8 @@ CFLAGS_COMMON := -Wall -Wno-deprecated-declarations -fPIC -std=c++17 \ -I $(I_MPI_ROOT)/include \ -I $(DALROOT)/include \ -I $(CCL_ROOT)/include/oneapi/ \ - -I $(CMPLR_ROOT)/linux/include \ - -I $(CMPLR_ROOT)/linux/include/sycl + -I $(CMPLR_ROOT)/include \ + -I $(CMPLR_ROOT)/include/sycl ifeq ($(PLATFORM_PROFILE),CPU_ONLY_PROFILE) CFLAGS := $(CFLAGS_COMMON) @@ -56,7 +56,7 @@ INCS := -I $(CCL_ROOT)/include \ # Use static link if possible, TBB is only available as dynamic libs LIBS_COMMON := -L$(CCL_ROOT)/lib -lccl \ - -L$(CMPLR_ROOT)/linux/compiler/lib -l:libirc.a \ + -L$(CMPLR_ROOT)/lib -l:libirc.a \ -L$(DALROOT)/lib/intel64 -lonedal_core -lonedal_thread -lonedal_dpc -lonedal_parameters_dpc \ -L$(TBBROOT)/lib/intel64/gcc4.8 -ltbb -ltbbmalloc \ -L$(I_MPI_ROOT) diff --git a/mllib-dal/src/main/native/Makefile_2023.2.0 b/mllib-dal/src/main/native/Makefile_2023.2.0 index ccece0186..2e4e51b69 100644 --- a/mllib-dal/src/main/native/Makefile_2023.2.0 +++ b/mllib-dal/src/main/native/Makefile_2023.2.0 @@ -135,7 +135,7 @@ OBJS += \ ./DecisionForestRegressorImpl.o # Compile cpp with a compiler version before onedal 2023.2.0 -DEFINES=-D$(PLATFORM_PROFILE) -DONEDAL_VERSION_BEFORE_2023_2_0 +DEFINES=-D$(PLATFORM_PROFILE) -DONEDAL_VERSION_LESS_THAN_OR_EQUAL_TO_2023.2.0 ifeq ($(PLATFORM_PROFILE),CPU_GPU_PROFILE) CPP_SRCS += ./GPU.cpp diff --git a/mllib-dal/src/main/native/build.sh b/mllib-dal/src/main/native/build.sh index 8331aa483..79d934a3b 100755 --- a/mllib-dal/src/main/native/build.sh +++ b/mllib-dal/src/main/native/build.sh @@ -19,10 +19,14 @@ if [[ $OAP_MLLIB_TESTING == "true" ]]; then fi if [[ -z $ONEDAL_VERSION ]]; then - if [[ -e $DAALROOT ]]; then - export ONEDAL_VERSION=$(echo "$DAALROOT" | awk -F '/' '{print $(NF)}') - elif [[ -e $DALROOT ]]; then - export ONEDAL_VERSION=$(echo "$DALROOT" | awk -F '/' '{print $(NF)}') + if [[ -n $DAALROOT ]]; then + if [[ -e $DAALROOT ]]; then + export ONEDAL_VERSION=$(echo "$DAALROOT" | awk -F '/' '{print $(NF)}') + fi + elif [[ -n $DALROOT ]]; then + if [[ -e $DALROOT ]]; then + export ONEDAL_VERSION=$(echo "$DALROOT" | awk -F '/' '{print $(NF)}') + fi else echo DAALROOT not defined! exit 1 @@ -30,14 +34,13 @@ if [[ -z $ONEDAL_VERSION ]]; then fi verlte() { - expr $(printf '%s\n%s' "$1" "$2" | sort -t '.' -k 1,1 -k 2,2 -k 3,3 -g | sed -n 2p) != "2023.2.0" + expr $(printf '%s\n%s' "$1" "$2" | sort -t '.' -k 1,1 -k 2,2 -k 3,3 -g | sed -n 2p) != "$2" } # Reference version reference_version="2023.2.0" # Compare versions -verlte "$ONEDAL_VERSION" "$reference_version" && echo "1" || echo "0" -result=$? +result=$(verlte "$ONEDAL_VERSION" "$reference_version") #Check the result of the comparison if [ "$result" -eq 1 ]; then diff --git a/mllib-dal/src/main/native/service.h b/mllib-dal/src/main/native/service.h index a4e344f04..a8e8882b8 100644 --- a/mllib-dal/src/main/native/service.h +++ b/mllib-dal/src/main/native/service.h @@ -43,7 +43,7 @@ using namespace daal::data_management; #include "Logger.h" #include "error_handling.h" -#ifdef ONEDAL_VERSION_BEFORE_2023_2_0 +#ifdef ONEDAL_VERSION_LESS_THAN_OR_EQUAL_TO_2023.2.0 #include "oneapi/dal/table/detail/csr.hpp" #else #include "oneapi/dal/table/csr.hpp" From 186647405c9865d99b9b42a30e0e5e1e667dfb31 Mon Sep 17 00:00:00 2001 From: minmingzhu Date: Tue, 30 Jan 2024 06:57:52 +0000 Subject: [PATCH 16/19] update Signed-off-by: minmingzhu --- mllib-dal/src/main/native/Makefile_2023.2.0 | 2 +- mllib-dal/src/main/native/service.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mllib-dal/src/main/native/Makefile_2023.2.0 b/mllib-dal/src/main/native/Makefile_2023.2.0 index 2e4e51b69..59069c47e 100644 --- a/mllib-dal/src/main/native/Makefile_2023.2.0 +++ b/mllib-dal/src/main/native/Makefile_2023.2.0 @@ -135,7 +135,7 @@ OBJS += \ ./DecisionForestRegressorImpl.o # Compile cpp with a compiler version before onedal 2023.2.0 -DEFINES=-D$(PLATFORM_PROFILE) -DONEDAL_VERSION_LESS_THAN_OR_EQUAL_TO_2023.2.0 +DEFINES=-D$(PLATFORM_PROFILE) -DONEDAL_VERSION_LESS_THAN_OR_EQUAL_TO_2023_2_0 ifeq ($(PLATFORM_PROFILE),CPU_GPU_PROFILE) CPP_SRCS += ./GPU.cpp diff --git a/mllib-dal/src/main/native/service.h b/mllib-dal/src/main/native/service.h index a8e8882b8..2064d709d 100644 --- a/mllib-dal/src/main/native/service.h +++ b/mllib-dal/src/main/native/service.h @@ -43,7 +43,7 @@ using namespace daal::data_management; #include "Logger.h" #include "error_handling.h" -#ifdef ONEDAL_VERSION_LESS_THAN_OR_EQUAL_TO_2023.2.0 +#ifdef ONEDAL_VERSION_LESS_THAN_OR_EQUAL_TO_2023_2_0 #include "oneapi/dal/table/detail/csr.hpp" #else #include "oneapi/dal/table/csr.hpp" From 3e3270905e78c153e8bea48f3ddd1b3b4a96b85f Mon Sep 17 00:00:00 2001 From: minmingzhu Date: Wed, 31 Jan 2024 05:43:01 +0000 Subject: [PATCH 17/19] preload tbb Signed-off-by: minmingzhu --- mllib-dal/src/main/java/com/intel/oap/mllib/LibLoader.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mllib-dal/src/main/java/com/intel/oap/mllib/LibLoader.java b/mllib-dal/src/main/java/com/intel/oap/mllib/LibLoader.java index c85394490..ac71d19db 100644 --- a/mllib-dal/src/main/java/com/intel/oap/mllib/LibLoader.java +++ b/mllib-dal/src/main/java/com/intel/oap/mllib/LibLoader.java @@ -59,6 +59,9 @@ public static synchronized void loadLibraries() throws IOException { * Load MLlibDAL lib */ private static synchronized void loadLibMLlibDAL() throws IOException { + // When loading libMLlibDAL.so, it will hang on libtcm.so. + // Workaround: loading tbb manually to bypass it. + System.loadLibrary("tbb"); loadFromJar(subDir, "libMLlibDAL.so"); } From f78ba680c2f03a4221d903dde8a9b94180b73ff5 Mon Sep 17 00:00:00 2001 From: minmingzhu Date: Thu, 1 Feb 2024 05:30:38 +0000 Subject: [PATCH 18/19] update Makefile_2023.2.0 and service.h Signed-off-by: minmingzhu --- mllib-dal/src/main/native/Makefile_2023.2.0 | 6 ++++-- mllib-dal/src/main/native/service.h | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/mllib-dal/src/main/native/Makefile_2023.2.0 b/mllib-dal/src/main/native/Makefile_2023.2.0 index 59069c47e..c077f5458 100644 --- a/mllib-dal/src/main/native/Makefile_2023.2.0 +++ b/mllib-dal/src/main/native/Makefile_2023.2.0 @@ -24,7 +24,6 @@ endif RM := rm -rf PLATFORM_PROFILE ?= CPU_ONLY_PROFILE -ONEDAL_VERSION=2023.2.0 $(info ) $(info === Profile is $(PLATFORM_PROFILE) ===) @@ -135,7 +134,10 @@ OBJS += \ ./DecisionForestRegressorImpl.o # Compile cpp with a compiler version before onedal 2023.2.0 -DEFINES=-D$(PLATFORM_PROFILE) -DONEDAL_VERSION_LESS_THAN_OR_EQUAL_TO_2023_2_0 +DEFINES=-D$(PLATFORM_PROFILE) +ifneq ($(ONEDAL_VERSION),"2023.2.0") + DEFINES+=-DONEDAL_VERSION_LESS_THAN_2023_2_0 +endif ifeq ($(PLATFORM_PROFILE),CPU_GPU_PROFILE) CPP_SRCS += ./GPU.cpp diff --git a/mllib-dal/src/main/native/service.h b/mllib-dal/src/main/native/service.h index 2064d709d..bdc7acc6a 100644 --- a/mllib-dal/src/main/native/service.h +++ b/mllib-dal/src/main/native/service.h @@ -43,7 +43,7 @@ using namespace daal::data_management; #include "Logger.h" #include "error_handling.h" -#ifdef ONEDAL_VERSION_LESS_THAN_OR_EQUAL_TO_2023_2_0 +#ifdef ONEDAL_VERSION_LESS_THAN_2023_2_0 #include "oneapi/dal/table/detail/csr.hpp" #else #include "oneapi/dal/table/csr.hpp" From 7b062c030b2d037505d5352c0967908c290bb252 Mon Sep 17 00:00:00 2001 From: minmingzhu Date: Thu, 1 Feb 2024 06:10:30 +0000 Subject: [PATCH 19/19] update Makefile_2023.2.0 Signed-off-by: minmingzhu --- mllib-dal/src/main/native/Makefile_2023.2.0 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mllib-dal/src/main/native/Makefile_2023.2.0 b/mllib-dal/src/main/native/Makefile_2023.2.0 index c077f5458..a3d54998d 100644 --- a/mllib-dal/src/main/native/Makefile_2023.2.0 +++ b/mllib-dal/src/main/native/Makefile_2023.2.0 @@ -135,7 +135,7 @@ OBJS += \ # Compile cpp with a compiler version before onedal 2023.2.0 DEFINES=-D$(PLATFORM_PROFILE) -ifneq ($(ONEDAL_VERSION),"2023.2.0") +ifneq ($(ONEDAL_VERSION),2023.2.0) DEFINES+=-DONEDAL_VERSION_LESS_THAN_2023_2_0 endif