Fix Bloop project generation in buildall [skip ci] (NVIDIA#6156)

This PR fixes NVIDIA#5706

- Generates a Bloop project for each Spark dependency specified via `--profile`
- Explains how to activate a particular project
- No longer tries to have multiple Spark dependencies in the same .bloop
directory
- Fixes incorrect BUILD_ALL_DEBUG tests
- Fixes match for .bloop in gitignore 

Signed-off-by: Gera Shegalov <gera@apache.org>

.gitignore

@@ -1,7 +1,7 @@
 # keep the lines below sorted
 
 # standard .bloop and version-specific .bloop3xy generated by buildall -gb
-.bloop*/
+.bloop*
 .cache
 .classpath
 .DS_Store

CONTRIBUTING.md

@@ -218,9 +218,21 @@ not clobbered by repeated `bloopInstall` Maven plugin invocations, and it uses
 [jq](https://stedolan.github.io/jq/) to post-process JSON-formatted project files such that they
 compile project classes into non-overlapping set of output directories.
 
+To activate the Spark dependency version 3XY you currently are working with update
+the symlink `.bloop` to point to the corresponding directory `.bloop-spark3XY`
+
+Example usage:
+```Bash
+./build/buildall --generate-bloop --profile=311,330
+rm -vf .bloop
+ln -s .bloop-spark330 .bloop
+```
+
 You can now open the spark-rapids as a
 [BSP project in IDEA](https://www.jetbrains.com/help/idea/bsp-support.html)
 
+Read on for VS Code Scala Metals instructions.
+
 # Bloop, Scala Metals, and Visual Studio Code
 
 _Last tested with 1.63.0-insider (Universal) Commit: bedf867b5b02c1c800fbaf4d6ce09cefba_

build/buildall

@@ -22,6 +22,7 @@ shopt -s extglob
 BLOOP_VERSION=${BLOOP_VERSION:-"1.4.13"}
 BLOOP_SCALA_VERSION=${BLOOP_SCALA_VERSION:-"2.13"}
 SKIP_CLEAN=1
+BUILD_ALL_DEBUG=0
 
 function print_usage() {
   echo "Usage: buildall [OPTION]"
@@ -52,18 +53,20 @@ function print_usage() {
 }
 
 function bloopInstall() {
-  BLOOP_DIR="${BLOOP_DIR:-$PWD/.bloop}"
-  mkdir -p $BLOOP_DIR
-  rm -f $BLOOP_DIR/*
 
-  time (
-    for bv in $SPARK_SHIM_VERSIONS; do
-      bloop_config_dir="$PWD/.bloop$bv"
-      mkdir -p "$bloop_config_dir"
-      rm -f "$bloop_config_dir"/*
+  [[ "$BUILD_ALL_DEBUG" == "1" ]] && set -x
+
+  local bloopTmpDir=$(mktemp -d /tmp/tmp.bloop.XXXXXX)
 
-      $MVN install ch.epfl.scala:maven-bloop_${BLOOP_SCALA_VERSION}:${BLOOP_VERSION}:bloopInstall -pl dist -am \
-        -Dbloop.configDirectory="$bloop_config_dir" \
+  time (
+    bloopDirsGenerated=()
+    for bv in "${SPARK_SHIM_VERSIONS[@]}"; do
+      bloopTmpConfigDir="$bloopTmpDir/.bloop$bv"
+      mkdir -p $bloopTmpConfigDir
+      $MVN -B clean install \
+        ch.epfl.scala:maven-bloop_${BLOOP_SCALA_VERSION}:${BLOOP_VERSION}:bloopInstall \
+        -pl aggregator -am \
+        -Dbloop.configDirectory="$bloopTmpConfigDir" \
         -DdownloadSources=true \
         -Dbuildver="$bv" \
         -DskipTests \
@@ -73,24 +76,21 @@ function bloopInstall() {
         -Dmaven.updateconfig.skip=true
 
       specifier="spark$bv"
-      for bloop_json in $(echo $bloop_config_dir/*.json); do
-        IFS="/" <<< "$bloop_json" read -ra bloop_json_parts
-        last_idx=$((${#bloop_json_parts[@]} - 1))
-        file="${bloop_json_parts[$last_idx]}"
-        project="${file%.json}-$specifier"
-        < $bloop_json jq \
-          --arg specifier "$specifier" \
-          '.project.out=.project.out + "/" + $specifier | .project.name=.project.name + "-" + $specifier' \
-           > "$BLOOP_DIR/$project.json"
-      done
+      bloopDir=$PWD/.bloop-$specifier
+      rm -rf $bloopDir
+      mv $bloopTmpConfigDir $bloopDir
+      echo "generated bloop files under $bloopDir"
+      bloopDirsGenerated+=($bloopDir)
     done
-
-    echo "Generated Bloop files under $BLOOP_DIR"
+    echo "#### Created bloop projects ${bloopDirsGenerated[@]}"
+    echo "Execute"
+    echo "  ln -s .bloop-spark3XY .bloop"
+    echo "to make it an active Bloop project in VS Code Scala Metals"
   )
 }
 
 function versionsFromDistProfile() {
-  [[ BUILD_ALL_DEBUG == "1" ]] && set -x
+  [[ "$BUILD_ALL_DEBUG" == "1" ]] && set -x
   versionRawStr=$(mvn -B help:evaluate -q -pl dist -P"$1" -Dexpression=included_buildvers -DforceStdout)
   versionStr=${versionRawStr//[$'\n',]/}
   echo -n $versionStr
@@ -124,8 +124,8 @@ case "$1" in
   ;;
 
 --debug)
+  BUILD_ALL_DEBUG=1
   set -x
-  BUILD_ALL_DEBUG="1"
   ;;
 
 --clean)
@@ -182,6 +182,7 @@ case $DIST_PROFILE in
 esac
 
 echo "Spark versions involved: ${SPARK_SHIM_VERSIONS[@]} ..."
+export MVN_BASE_DIR=$($MVN help:evaluate -Dexpression=project.basedir -q -DforceStdout)
 
 if [[ "$GEN_BLOOP" == "true" ]]; then
   bloopInstall
@@ -202,10 +203,8 @@ fi
 
 echo "Building a combined dist jar with Shims for ${SPARK_SHIM_VERSIONS[@]} ..."
 
-export MVN_BASE_DIR=$($MVN help:evaluate -Dexpression=project.basedir -q -DforceStdout)
-
 function build_single_shim() {
-  [[ BUILD_ALL_DEBUG == "1" ]] && set -x
+  [[ "$BUILD_ALL_DEBUG" == "1" ]] && set -x
   BUILD_VER=$1
   mkdir -p "$MVN_BASE_DIR/target"
   (( BUILD_PARALLEL == 1 || NUM_SHIMS == 1 )) && LOG_FILE="/dev/tty" || \