try to run in parallel

docs/_scripts/execute_notebooks.sh

@@ -3,33 +3,30 @@
 # Read the list of notebooks to skip from the JSON file
 SKIP_NOTEBOOKS=$(python -c "import json; print(' '.join(json.load(open('docs/notebooks_no_execution.json'))))")
 
-errors=()  # Initialize an array to collect errors
-
-for file in $(find docs/docs/tutorials docs/docs/how-tos -name "*.ipynb" | grep -v ".ipynb_checkpoints")
-do
-  # Check if the file is in the list of notebooks to skip
-  if [[ $SKIP_NOTEBOOKS == *"$file"* ]]; then
-    echo "Skipping $file (in no-execution list)"
-    continue
-  fi
-
-  echo "Executing $file"
-  start_time=$(date +%s)
-  if ! output=$(time poetry run jupyter execute --allow-errors "$file" 2>&1); then
+# Function to execute a single notebook
+execute_notebook() {
+    file="$1"
+    echo "Starting execution of $file"
+    start_time=$(date +%s)
+    if ! output=$(time poetry run jupyter execute --allow-errors "$file" 2>&1); then
+        end_time=$(date +%s)
+        execution_time=$((end_time - start_time))
+        echo "Error in $file. Execution time: $execution_time seconds"
+        echo "Error details: $output"
+        exit 1
+    fi
     end_time=$(date +%s)
     execution_time=$((end_time - start_time))
-    echo "Execution time: $execution_time seconds"
-    errors+=("$file: $output")  # Add a tuple of the file and error message to the errors list
-    printf '%s\n' "${errors[@]}"
+    echo "Finished $file. Execution time: $execution_time seconds"
+}
+
+export -f execute_notebook
+
+# Find all notebooks and filter out those in the skip list
+notebooks=$(find docs/docs/tutorials docs/docs/how-tos -name "*.ipynb" | grep -v ".ipynb_checkpoints" | grep -vFf <(echo "$SKIP_NOTEBOOKS"))
+
+# Run notebooks in parallel
+if ! parallel execute_notebook ::: $notebooks; then
+    echo "Errors occurred during notebook execution"
     exit 1
-  fi
-  end_time=$(date +%s)
-  execution_time=$((end_time - start_time))
-  echo "Execution time: $execution_time seconds"
-done
-# Optionally, print the errors
-if [ ${#errors[@]} -ne 0 ]; then
-  echo "Errors occurred in the following files:"
-  printf '%s\n' "${errors[@]}"
-  exit 1  # Exit with an error code if there are errors
 fi