Add support for SimSYCL and AddressSanitizer (+CI)

.github/workflows/build_matrix.json

@@ -8,7 +8,9 @@
         { "sycl": "hipsycl", "sycl-version": "d2bd9fc7", "ubuntu-version": "22.04", "platform": "nvidia", "build-type": "Debug" },
         { "sycl": "hipsycl", "sycl-version": "d2bd9fc7", "ubuntu-version": "22.04", "platform": "nvidia", "build-type": "Release" },
         { "sycl": "hipsycl", "sycl-version": "latest", "ubuntu-version": "23.04", "platform": "nvidia", "build-type": "Debug" },
-        { "sycl": "hipsycl", "sycl-version": "latest", "ubuntu-version": "23.04", "platform": "nvidia", "build-type": "Release" }
+        { "sycl": "hipsycl", "sycl-version": "latest", "ubuntu-version": "23.04", "platform": "nvidia", "build-type": "Release" },
+        { "sycl": "simsycl", "sycl-version": "HEAD", "ubuntu-version": "22.04", "platform": "intel", "build-type": "Release" },
+        { "sycl": "simsycl", "sycl-version": "HEAD", "ubuntu-version": "22.04", "platform": "intel", "build-type": "Debug" }
     ],
     "nightly": [
         { "sycl": "dpcpp", "sycl-version": "HEAD", "ubuntu-version": "22.04", "platform": "intel", "build-type": "Debug" },

.github/workflows/celerity_ci.yml

@@ -149,8 +149,14 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
     steps:
-      - name: Set container workspace environment variable
-        run: echo "container-workspace=$GITHUB_WORKSPACE" > $GITHUB_ENV
+      # We limit DPC++ to Level-Zero devices (instead of e.g. also showing OpenCL or CUDA devices).
+      # Most importantly, despite its naming, this currently also avoids the "Unified Runtime over Level-Zero", which
+      # doesn't seem to be quite ready for production use yet (= it crashes).
+      - name: Set environment variables
+        run: |
+          echo "container-workspace=$GITHUB_WORKSPACE" > $GITHUB_ENV
+          echo "ONEAPI_DEVICE_SELECTOR=level_zero:*" >> $GITHUB_ENV
+          echo "SIMSYCL_SYSTEM=$GITHUB_WORKSPACE/ci/simsycl-system.json" >> $GITHUB_ENV
       - name: Print exact SYCL revision used for this CI run
         run: cat /VERSION
       - uses: actions/checkout@v3
@@ -167,12 +173,20 @@ jobs:
           path: ${{ env.build-name }}.log
       - name: Build examples against installed Celerity
         run: bash /root/build-examples.sh ${{ env.container-workspace }}/examples --build-type ${{ matrix.build-type }}
-      # Limit DPC++ to Level-Zero devices (instead of e.g. also showing OpenCL or CUDA devices).
-      # Most importantly, despite its naming, this currently also avoids the "Unified Runtime over Level-Zero", which
-      # doesn't seem to be quite ready for production use yet (= it crashes).
-      - name: Limit visible devices
+      # For a run with ASan, we need to fake dlclose() so all shared libraries are still mapped when leaks are evaluated
+      # - otherwise, we will see "<unknown module>" instead of a proper trace if it originated in an unloaded library
+      # (see https://github.com/google/sanitizers/issues/89#issuecomment-406316683). LD_PRELOAD will be set to CI_LD_PRELOAD
+      # by /root/capture-backtrace.sh (otherwise we would also preload for other processes like bash, which we don't want to do).
+      # Also, hwloc (used by OpenMPI) will attempt to dlopen(RTLD_DEEPBIND) for GPU backend discovery which is incompatible
+      # with sanitizers but also irrelevant for a SimSYCL build, so we set HWLOC_COMPONENTS accordingly.
+      - name: Prepare Sanitizer run
+        if: matrix.sycl == 'simsycl' && matrix.build-type == 'Debug'
         run: |
-          echo "ONEAPI_DEVICE_SELECTOR=level_zero:*" >> $GITHUB_ENV
+          echo "ASAN_OPTIONS=detect_leaks=1" >> $GITHUB_ENV
+          echo "LSAN_OPTIONS=suppressions=$GITHUB_WORKSPACE/ci/lsan.suppressions" >> $GITHUB_ENV
+          echo "int dlclose(void *p) { return 0; }" | cc -x c -shared -o libfake_dlclose.so -
+          echo "CI_LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libasan.so.6:$(pwd)/libfake_dlclose.so" >> $GITHUB_ENV
+          echo "HWLOC_COMPONENTS=-cuda,-opencl,-rsmi" >> $GITHUB_ENV
       - name: Run unit tests
         timeout-minutes: 5
         working-directory: ${{ env.build-dir }}
@@ -183,7 +197,7 @@ jobs:
         working-directory: ${{ env.examples-build-dir }}
         run: ${{ env.container-workspace }}/ci/run-examples.sh /data/Lenna.png 1 2 4
       - name: Run debugging tests
-        if: matrix.build-type == 'Debug' && matrix.sycl == 'hipsycl' # newer DPC++ generates DWARF5 which is incompatible with Ubuntu 20.04's GDB
+        if: matrix.build-type == 'Debug' && matrix.sycl != 'dpcpp' # newer DPC++ generates DWARF5 which is incompatible with Ubuntu 20.04's GDB
         run: ${{ env.container-workspace }}/test/debug/pretty-print-test.py ${{ env.build-dir }}/test/debug/pretty_printables
       - name: Run system tests
         working-directory: ${{ env.build-dir }}

CHANGELOG.md

@@ -6,6 +6,21 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic
 Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [Unreleased]
+
+We recommend using the following SYCL versions with this release:
+
+- DPC++: ???
+- hipSYCL++: ???
+- SimSYCL++: ???
+
+See our [platform support guide](docs/platform-support.md) for a complete list of all officially supported configurations.
+
+### Added
+
+- Add support for SimSYCL as a SYCL implementation (#238)
+- Extend compiler support to GCC (optionally with sanitizers) and C++20 code bases (#238)
+
 ## [0.5.0] - 2023-12-21
 
 We recommend using the following SYCL versions with this release:

CMakeLists.txt

@@ -33,7 +33,7 @@ set(CMAKE_MODULE_PATH "${CMAKE_MODULE_PATH}" "${CELERITY_CMAKE_DIR}")
 find_package(MPI 2.0 REQUIRED)
 find_package(Threads REQUIRED)
 
-set(CELERITY_SYCL_IMPL "OFF" CACHE STRING "hipSYCL|DPC++")
+set(CELERITY_SYCL_IMPL "OFF" CACHE STRING "hipSYCL|DPC++|SimSYCL")
 
 include("${CELERITY_CMAKE_DIR}/TestDPCXX.cmake")
 test_cxx_compiler_is_dpcpp(CXX_COMPILER_IS_DPCPP)
@@ -48,6 +48,8 @@ elseif(CELERITY_SYCL_IMPL STREQUAL "DPC++")
   if(NOT CXX_COMPILER_IS_DPCPP)
     message(FATAL_ERROR "CELERITY_SYCL_IMPL set to DPC++, but CXX compiler at ${CMAKE_CXX_COMPILER} is not DPC++")
   endif()
+elseif(CELERITY_SYCL_IMPL STREQUAL "SimSYCL")
+  find_package(SimSYCL CONFIG REQUIRED)
 elseif(NOT "${CELERITY_SYCL_IMPL}x" STREQUAL "OFFx")
   message(FATAL_ERROR "Invalid SYCL implementation ${CELERITY_SYCL_IMPL} specified")
 else()
@@ -61,13 +63,20 @@ else()
     list(APPEND AVAILABLE_SYCL_IMPLS hipSYCL)
   endif()
 
+  find_package(SimSYCL QUIET CONFIG)
+  if(SimSYCL_FOUND)
+    message(STATUS "Found SimSYCL: ${SimSYCL_DIR}")
+    list(APPEND AVAILABLE_SYCL_IMPLS SimSYCL)
+  endif()
+
   list(LENGTH AVAILABLE_SYCL_IMPLS NUM_AVAILABLE_SYCL_IMPLS)
   if(NUM_AVAILABLE_SYCL_IMPLS EQUAL 0)
-    message(FATAL_ERROR "No SYCL implementation found. You might need to add an install path for hipSYCL to CMAKE_PREFIX_PATH.")
+    message(FATAL_ERROR "No SYCL implementation found. You might need to add an install path for hipSYCL "
+    "or SimSYCL to CMAKE_PREFIX_PATH.")
   elseif(NUM_AVAILABLE_SYCL_IMPLS GREATER 1)
     list(JOIN AVAILABLE_SYCL_IMPLS " and " AVAILABLE_SYCL_IMPLS_STR)
     message(FATAL_ERROR "More than one SYCL implementation available: Found ${AVAILABLE_SYCL_IMPLS_STR}. "
-      "Please choose one implementation using -DCELERITY_SYCL_IMPL=hipSYCL|DPC++.")
+      "Please choose one implementation using -DCELERITY_SYCL_IMPL=hipSYCL|DPC++|SimSYCL.")
   else()
     set(CELERITY_SYCL_IMPL "${AVAILABLE_SYCL_IMPLS}")
     message(STATUS "Automatically choosing ${CELERITY_SYCL_IMPL} because it is the only SYCL implementation available")
@@ -103,12 +112,18 @@ endif()
 
 if(CELERITY_SYCL_IMPL STREQUAL hipSYCL AND HIPSYCL_SUPPORTS_SYCL_2020_REDUCTIONS)
   set(CELERITY_FEATURE_SCALAR_REDUCTIONS ON)
-elseif(CELERITY_SYCL_IMPL STREQUAL "DPC++")
+elseif(CELERITY_SYCL_IMPL STREQUAL "DPC++" OR CELERITY_SYCL_IMPL STREQUAL "SimSYCL")
   set(CELERITY_FEATURE_SCALAR_REDUCTIONS ON)
 else()
   set(CELERITY_FEATURE_SCALAR_REDUCTIONS OFF)
 endif()
 
+if(CELERITY_SYCL_IMPL STREQUAL SimSYCL)
+  set(CELERITY_CXX_STANDARD 20)
+else()
+  set(CELERITY_CXX_STANDARD 17)
+endif()
+
 set(CELERITY_RUNTIME_LIBRARY celerity_runtime)
 include("${CMAKE_CURRENT_LIST_DIR}/cmake/AddToTarget.cmake")
 
@@ -120,6 +135,11 @@ if(CELERITY_SYCL_IMPL STREQUAL "DPC++")
   # See https://github.com/oneapi-src/unified-runtime/issues/803
   message(STATUS "Not enabling mimalloc by default because it breaks with oneAPI plugin loading")
   set(CELERITY_USE_MIMALLOC_DEFAULT OFF)
+elseif(CELERITY_SYCL_IMPL STREQUAL "SimSYCL" AND SIMSYCL_ENABLE_ASAN)
+  message(STATUS "Not enabling mimalloc by default because SimSYCL is built with AddressSanitizer")
+  set(CELERITY_USE_MIMALLOC_DEFAULT OFF)
+  list(APPEND CELERITY_CXX_FLAGS -fsanitize=address)
+  list(APPEND CELERITY_LINK_FLAGS -fsanitize=address)
 else()
   set(CELERITY_USE_MIMALLOC_DEFAULT ON)
 endif()
@@ -129,13 +149,24 @@ option(CELERITY_USE_MIMALLOC "Use the mimalloc memory allocator" ${CELERITY_USE_
 include(FetchContent)
 
 macro(fetch_content_from_submodule DEPNAME RELPATH)
-  if(NOT EXISTS "${PROJECT_SOURCE_DIR}/${RELPATH}/CMakeLists.txt")
-    message(FATAL_ERROR "The git submodule ${RELPATH} is missing.\nTry running `git submodule update --init`.")
+  # Dependency might already have been satisfied previously by
+  #   - the top-level CMake project including celerity-runtime as a dependency, but not its first one
+  #   - another depencency of celerity-runtime that it transitively shares a depencency with (SimSYCL -> libenvpp)
+  # The top-level CMake project must ensure that there are no conflicts between dependency versions in this case.
+  if(${DEPNAME}_FOUND)
+    message(STATUS "Using system ${DEPNAME} in ${${DEPNAME}_DIR}")
+    set(CELERITY_DETAIL_USE_SYSTEM_${DEPNAME} ON)
+  else()
+    message(STATUS "Fetching ${DEPNAME} from submodule")
+    set(CELERITY_DETAIL_USE_SYSTEM_${DEPNAME} OFF)
+    if(NOT EXISTS "${PROJECT_SOURCE_DIR}/${RELPATH}/CMakeLists.txt")
+      message(FATAL_ERROR "The git submodule ${RELPATH} is missing.\nTry running `git submodule update --init`.")
+    endif()
+    FetchContent_Declare(${DEPNAME}
+      URL "${PROJECT_SOURCE_DIR}/${RELPATH}"
+    )
+    FetchContent_MakeAvailable(${DEPNAME})
   endif()
-  FetchContent_Declare(${DEPNAME}
-    URL "${PROJECT_SOURCE_DIR}/${RELPATH}"
-  )
-  FetchContent_MakeAvailable(${DEPNAME})
 endmacro()
 
 set(FMT_INSTALL ON CACHE BOOL "" FORCE)
@@ -224,7 +255,7 @@ add_library(
   ${INCLUDES}
 )
 
-set_property(TARGET celerity_runtime PROPERTY CXX_STANDARD 17)
+set_property(TARGET celerity_runtime PROPERTY CXX_STANDARD "${CELERITY_CXX_STANDARD}")
 
 target_include_directories(celerity_runtime PUBLIC
   $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/include>
@@ -241,6 +272,8 @@ if(WIN32 AND CELERITY_SYCL_IMPL STREQUAL "DPC++")
     $<$<CONFIG:Debug>:sycl6d>
     $<$<CONFIG:Release>:sycl6>
   )
+elseif(CELERITY_SYCL_IMPL STREQUAL SimSYCL)
+  set(SYCL_LIB SimSYCL::simsycl)
 endif()
 
 if(CELERITY_USE_MIMALLOC)
@@ -320,6 +353,7 @@ elseif(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang|AppleClang")
 endif()
 
 target_compile_options(celerity_runtime PUBLIC "${CELERITY_CXX_FLAGS}")
+target_link_options(celerity_runtime PUBLIC "${CELERITY_LINK_FLAGS}")
 
 # Examples
 

README.md

@@ -77,10 +77,11 @@ installed first.
 
 - A supported SYCL implementation, either
     - [hipSYCL](https://github.com/illuhad/hipsycl),
-    - [DPC++](https://github.com/intel/llvm)
+    - [DPC++](https://github.com/intel/llvm), or
+    - [SimSYCL](https://github.com/celerity/SimSYCL)
 - A MPI 2 implementation (tested with OpenMPI 4.0, MPICH 3.3 should work as well)
 - [CMake](https://www.cmake.org) (3.13 or newer)
-- A C++17 compiler
+- A C++17 compiler (C++20 when working with SimSYCL)
 
 See the [platform support guide](docs/platform-support.md) on which library and OS versions are supported and
 automatically tested.

ci/lsan.suppressions

@@ -0,0 +1,10 @@
+# Leaks in third-party libraries we do not control
+leak:libmpi.so
+leak:libpmix.so
+leak:libopen-pal.so
+leak:lib/openmpi
+leak:libfontconfig.so
+leak:libhwloc.so
+leak:libopen-rte.so
+# There is one leak for which the stack trace ends at __vasprintf_internal
+leak:__vasprintf_internal

ci/run-system-tests.sh

@@ -21,6 +21,6 @@ for e in "${!SYSTEM_TESTS[@]}"; do
 
     for n in "${NUM_NODES[@]}"; do
         echo -e "\n\n ---- Running \"$EXE\" on $n node(s) ----\n\n" >&2
-        mpirun --bind-to none -n ${n} bash /root/capture-backtrace.sh ${CMD} || exit 1
+        mpirun --bind-to none -n ${n} bash /root/capture-backtrace.sh "$CMD" || exit 1
     done
 done