CMakeLists.txt

# Preamble ####################################################################
#
cmake_minimum_required(VERSION 3.20.0)
project(WarpX VERSION 23.06)

include(${WarpX_SOURCE_DIR}/cmake/WarpXFunctions.cmake)

# In-source tree builds are messy and can screw up the build system.
# Avoid building at least in the same dir as the root dir:
if(CMAKE_BINARY_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
    message(FATAL_ERROR "Building in-source is not supported! "
            "Create a build directory and remove "
            "${CMAKE_SOURCE_DIR}/CMakeCache.txt ${CMAKE_SOURCE_DIR}/CMakeFiles/")
endif()


# CMake policies ##############################################################
#
# AMReX 21.06+ supports CUDA_ARCHITECTURES with CMake 3.20+
# CMake 3.18+: CMAKE_CUDA_ARCHITECTURES
# https://cmake.org/cmake/help/latest/policy/CMP0104.html
if(POLICY CMP0104)
    cmake_policy(SET CMP0104 OLD)
endif()

# We use simple syntax in cmake_dependent_option, so we are compatible with the
# extended syntax in CMake 3.22+
# https://cmake.org/cmake/help/v3.22/policy/CMP0127.html
if(POLICY CMP0127)
    cmake_policy(SET CMP0127 NEW)
endif()


# C++ Standard in Superbuilds #################################################
#
# This is the easiest way to push up a C++17 requirement for AMReX, PICSAR and
# openPMD-api until they increase their requirement.
set_cxx17_superbuild()


# CCache Support ##############################################################
#
# this is an optional tool that stores compiled object files; allows fast
# re-builds even with "make clean" in between. Mainly used to store AMReX
# objects
set_ccache()


# Output Directories ##########################################################
#
# temporary build directories
set_default_build_dirs()

# install directories
set_default_install_dirs()


# Options and Variants ########################################################
#
include(CMakeDependentOption)
option(WarpX_APP           "Build the WarpX executable application"     ON)
option(WarpX_ASCENT        "Ascent in situ diagnostics"                 OFF)
option(WarpX_EB            "Embedded boundary support"                  OFF)
cmake_dependent_option(WarpX_GPUCLOCK
                           "Add GPU kernel timers (cost function)"      ON
                           "WarpX_COMPUTE STREQUAL CUDA OR WarpX_COMPUTE STREQUAL HIP" OFF)
option(WarpX_LIB           "Build WarpX as a shared library"            OFF)
option(WarpX_MPI           "Multi-node support (message-passing)"       ON)
option(WarpX_OPENPMD       "openPMD I/O (HDF5, ADIOS)"                  ON)
option(WarpX_PSATD         "spectral solver support"                    OFF)
option(WarpX_SENSEI        "SENSEI in situ diagnostics"                 OFF)
option(WarpX_QED           "QED support (requires PICSAR)"                    ON)
option(WarpX_QED_TABLE_GEN "QED table generation (requires PICSAR and Boost)" OFF)

set(WarpX_DIMS_VALUES 1 2 3 RZ)
set(WarpX_DIMS 3 CACHE STRING "Simulation dimensionality <1;2;3;RZ>")
list(REMOVE_DUPLICATES WarpX_DIMS)
foreach(D IN LISTS WarpX_DIMS)
    if(NOT D IN_LIST WarpX_DIMS_VALUES)
       message(FATAL_ERROR "WarpX_DIMS=${D} is not allowed."
          " Must be one of ${WarpX_DIMS_VALUES}")
    endif()
endforeach()

# for some targets need to be triggered once, so any dim dependency will do
list(LENGTH WarpX_DIMS list_len)
math(EXPR list_last "${list_len} - 1")
list(GET WarpX_DIMS ${list_last} WarpX_DIMS_LAST)
warpx_set_suffix_dims(WarpX_DIMS_LAST ${WarpX_DIMS_LAST})

set(WarpX_PRECISION_VALUES SINGLE DOUBLE)
set(WarpX_PRECISION DOUBLE CACHE STRING "Floating point precision (SINGLE/DOUBLE)")
set_property(CACHE WarpX_PRECISION PROPERTY STRINGS ${WarpX_PRECISION_VALUES})
if(NOT WarpX_PRECISION IN_LIST WarpX_PRECISION_VALUES)
    message(FATAL_ERROR "WarpX_PRECISION (${WarpX_PRECISION}) must be one of ${WarpX_PRECISION_VALUES}")
endif()

set(WarpX_PARTICLE_PRECISION_VALUES SINGLE DOUBLE)
set(WarpX_PARTICLE_PRECISION ${WarpX_PRECISION} CACHE STRING "Particle floating point precision (SINGLE/DOUBLE)")
set_property(CACHE WarpX_PARTICLE_PRECISION PROPERTY STRINGS ${WarpX_PARTICLE_PRECISION_VALUES})
if(NOT WarpX_PARTICLE_PRECISION IN_LIST WarpX_PARTICLE_PRECISION_VALUES)
    message(FATAL_ERROR "WarpX_PARTICLE_PRECISION (${WarpX_PARTICLE_PRECISION}) must be one of ${WarpX_PARTICLE_PRECISION_VALUES}")
endif()

set(WarpX_COMPUTE_VALUES NOACC OMP CUDA SYCL HIP)
set(WarpX_COMPUTE OMP CACHE STRING "On-node, accelerated computing backend (NOACC/OMP/CUDA/SYCL/HIP)")
set_property(CACHE WarpX_COMPUTE PROPERTY STRINGS ${WarpX_COMPUTE_VALUES})
if(NOT WarpX_COMPUTE IN_LIST WarpX_COMPUTE_VALUES)
    message(FATAL_ERROR "WarpX_COMPUTE (${WarpX_COMPUTE}) must be one of ${WarpX_COMPUTE_VALUES}")
endif()

option(WarpX_MPI_THREAD_MULTIPLE "MPI thread-multiple support, i.e. for async_io" ON)
mark_as_advanced(WarpX_MPI_THREAD_MULTIPLE)

option(WarpX_amrex_internal                    "Download & build AMReX" ON)

# change the default build type to Release (or RelWithDebInfo) instead of Debug
set_default_build_type("Release")

# Option to enable interprocedural optimization
# (also know as "link-time optimization" or "whole program optimization")
option(WarpX_IPO                                "Compile WarpX with interprocedural optimization (will take more time)" OFF)

# note: we could skip this if we solely build WarpX_APP, but if we build a
# shared WarpX library or a third party, like ImpactX, uses ablastr in a
# shared library (e.g., for Python bindings), then we need relocatable code.
option(ABLASTR_POSITION_INDEPENDENT_CODE
    "Build ABLASTR with position independent code" ${WarpX_LIB})
mark_as_advanced(ABLASTR_POSITION_INDEPENDENT_CODE)

# this defined the variable BUILD_TESTING which is ON by default
#include(CTest)


# Dependencies ################################################################
#

# AMReX
#   builds AMReX from source (default) or finds an existing install
set(WarpX_amrex_dim ${WarpX_DIMS})  # RZ is AMReX 2D
list(TRANSFORM WarpX_amrex_dim REPLACE RZ 2)
list(REMOVE_DUPLICATES WarpX_amrex_dim)

include(${WarpX_SOURCE_DIR}/cmake/dependencies/AMReX.cmake)
foreach(D IN LISTS WarpX_amrex_dim)
    #   suppress warnings in AMReX headers (use -isystem instead of -I)
    warpx_make_third_party_includes_system(AMReX::amrex_${D}d amrex_${D}d)
endforeach()
# For <AMReX_buildInfo.H>
include(AMReXBuildInfo)

# PICSAR
#   builds PICSAR from source
include(${WarpX_SOURCE_DIR}/cmake/dependencies/PICSAR.cmake)

# openPMD
#   builds openPMD-api from source (default) or finds an existing install
include(${WarpX_SOURCE_DIR}/cmake/dependencies/openPMD.cmake)

# PSATD
include(${WarpX_SOURCE_DIR}/cmake/dependencies/FFT.cmake)
if(WarpX_PSATD)
    # BLASPP and LAPACKPP
    if(RZ IN_LIST WarpX_DIMS)
        find_package(blaspp CONFIG REQUIRED)
        find_package(lapackpp CONFIG REQUIRED)
        find_package(OpenMP REQUIRED)  # pulled by the two above
    endif()
endif()


# Targets #####################################################################
#
set(_ALL_TARGETS)
foreach(D IN LISTS WarpX_DIMS)
    warpx_set_suffix_dims(SD ${D})
    # collect all objects for compilation
    add_library(WarpX_${SD} OBJECT)
    add_library(ablastr_${SD})

    # ABLASTR library
    set(_BUILDINFO_SRC ablastr_${SD})
    list(APPEND _ALL_TARGETS WarpX_${SD} ablastr_${SD})
    add_library(WarpX::ablastr_${SD} ALIAS ablastr_${SD})

    # executable application
    #   note: we currently avoid a dependency on a core library
    #         for simpler usage, but could make this an option
    if(WarpX_APP)
        add_executable(app_${SD})
        add_executable(WarpX::app_${SD} ALIAS app_${SD})
        target_link_libraries(app_${SD} PRIVATE WarpX_${SD} ablastr_${SD})
        set(_BUILDINFO_SRC app_${SD})
        list(APPEND _ALL_TARGETS app_${SD})
    endif()

    # link into a shared library
    if(WarpX_LIB)
        add_library(shared_${SD} MODULE)
        add_library(WarpX::shared_${SD} ALIAS shared_${SD})
        target_link_libraries(shared_${SD} PUBLIC WarpX_${SD} ablastr_${SD})
        set(_BUILDINFO_SRC shared_${SD})
        list(APPEND _ALL_TARGETS shared_${SD})

        set_target_properties(WarpX_${SD} shared_${SD} PROPERTIES
            POSITION_INDEPENDENT_CODE ON
            WINDOWS_EXPORT_ALL_SYMBOLS ON
        )
        set(ABLASTR_POSITION_INDEPENDENT_CODE ON CACHE BOOL
            "Build ABLASTR with position independent code" FORCE)
    endif()

    # ABLASTR library (static or shared)
    set_target_properties(ablastr_${SD} PROPERTIES
        WINDOWS_EXPORT_ALL_SYMBOLS ON
    )
    if(ABLASTR_POSITION_INDEPENDENT_CODE)
        set_target_properties(ablastr_${SD} PROPERTIES
            POSITION_INDEPENDENT_CODE ON
        )
    endif()

    # own headers
    target_include_directories(WarpX_${SD} PUBLIC
        $<BUILD_INTERFACE:${WarpX_SOURCE_DIR}/Source>
        $<BUILD_INTERFACE:${WarpX_BINARY_DIR}/Source>
    )
    target_include_directories(ablastr_${SD} PUBLIC
        # future: own directory root
        $<BUILD_INTERFACE:${WarpX_SOURCE_DIR}/Source>
    )

    # if we include <AMReX_buildInfo.H> we will need to call:
    generate_buildinfo(${_BUILDINFO_SRC} "${WarpX_SOURCE_DIR}")
    target_link_libraries(WarpX_${SD} PRIVATE buildInfo::${_BUILDINFO_SRC})
    unset(_BUILDINFO_SRC)

    # add sources
    target_sources(WarpX_${SD} PRIVATE Source/WarpX.cpp)
    if(WarpX_APP)
        target_sources(app_${SD} PRIVATE Source/main.cpp)
    endif()
endforeach()

add_subdirectory(Source/ablastr)
add_subdirectory(Source/AcceleratorLattice)
add_subdirectory(Source/BoundaryConditions)
add_subdirectory(Source/Diagnostics)
add_subdirectory(Source/EmbeddedBoundary)
add_subdirectory(Source/Evolve)
add_subdirectory(Source/FieldSolver)
add_subdirectory(Source/Filter)
add_subdirectory(Source/Initialization)
add_subdirectory(Source/Laser)
add_subdirectory(Source/Parallelization)
add_subdirectory(Source/Particles)
add_subdirectory(Source/Python)
add_subdirectory(Source/Utils)

# Interprocedural optimization
if(WarpX_IPO)
    enable_IPO("${_ALL_TARGETS}")
endif()

# link dependencies
foreach(D IN LISTS WarpX_DIMS)
    warpx_set_suffix_dims(SD ${D})
    if(D STREQUAL "RZ")
        target_link_libraries(ablastr_${SD} PUBLIC WarpX::thirdparty::amrex_2d)
    else()
        target_link_libraries(ablastr_${SD} PUBLIC WarpX::thirdparty::amrex_${D}d)
    endif()
    target_link_libraries(WarpX_${SD} PUBLIC ablastr_${SD})

    if(WarpX_PSATD)
        target_link_libraries(ablastr_${SD} PUBLIC WarpX::thirdparty::FFT)
        if(D STREQUAL "RZ")
            target_link_libraries(ablastr_${SD} PUBLIC blaspp)
            target_link_libraries(ablastr_${SD} PUBLIC lapackpp)

            # BLAS++ forgets to declare cuBLAS and cudaRT dependencies
            if(WarpX_COMPUTE STREQUAL CUDA)
                find_package(CUDAToolkit REQUIRED)
                target_link_libraries(ablastr_${SD} PUBLIC CUDA::cudart CUDA::cublas)
            endif()
        endif()
    endif()

    if(WarpX_OPENPMD)
        target_link_libraries(ablastr_${SD} PUBLIC openPMD::openPMD)
    endif()

    if(WarpX_QED)
        target_compile_definitions(ablastr_${SD} PUBLIC WARPX_QED)
        if(WarpX_QED_TABLE_GEN)
            target_compile_definitions(ablastr_${SD} PUBLIC WARPX_QED_TABLE_GEN)
        endif()
        target_link_libraries(ablastr_${SD} PUBLIC PXRMP_QED::PXRMP_QED)
    endif()
endforeach()

# C++ properties: at least a C++17 capable compiler is needed
if(WarpX_COMPUTE STREQUAL CUDA)
    # AMReX helper function: propagate CUDA specific target & source properties
    foreach(warpx_tgt IN LISTS _ALL_TARGETS)
        setup_target_for_cuda_compilation(${warpx_tgt})
    endforeach()
    foreach(warpx_tgt IN LISTS _ALL_TARGETS)
        target_compile_features(${warpx_tgt} PUBLIC cuda_std_17)
    endforeach()
    set_target_properties(${_ALL_TARGETS} PROPERTIES
        CUDA_EXTENSIONS OFF
        CUDA_STANDARD_REQUIRED ON
    )
else()
    foreach(warpx_tgt IN LISTS _ALL_TARGETS)
        target_compile_features(${warpx_tgt} PUBLIC cxx_std_17)
    endforeach()
    set_target_properties(${_ALL_TARGETS} PROPERTIES
        CXX_EXTENSIONS OFF
        CXX_STANDARD_REQUIRED ON
    )
endif()

foreach(D IN LISTS WarpX_DIMS)
    warpx_set_suffix_dims(SD ${D})

    # avoid building all object files if we are only used as ABLASTR library
    if(NOT WarpX_APP AND NOT WarpX_LIB)
        set_target_properties(WarpX_${SD} PROPERTIES
            EXCLUDE_FROM_ALL 1
            EXCLUDE_FROM_DEFAULT_BUILD 1
        )
    endif()

    # fancy binary name for build variants
    set_warpx_binary_name(${D})
endforeach()


# Defines #####################################################################
#
foreach(D IN LISTS WarpX_DIMS)
    warpx_set_suffix_dims(SD ${D})
    if(D STREQUAL 3)
        target_compile_definitions(ablastr_${SD} PUBLIC WARPX_DIM_3D WARPX_ZINDEX=2)
    elseif(D STREQUAL 2)
        target_compile_definitions(ablastr_${SD} PUBLIC WARPX_DIM_XZ WARPX_ZINDEX=1)
    elseif(D STREQUAL 1)
        target_compile_definitions(ablastr_${SD} PUBLIC WARPX_DIM_1D_Z WARPX_ZINDEX=0)
    elseif(D STREQUAL "RZ")
        target_compile_definitions(ablastr_${SD} PUBLIC WARPX_DIM_RZ WARPX_ZINDEX=1)
    endif()

    if(WarpX_GPUCLOCK)
        target_compile_definitions(ablastr_${SD} PUBLIC WARPX_USE_GPUCLOCK)
    endif()

    if(WarpX_OPENPMD)
        target_compile_definitions(ablastr_${SD} PUBLIC WARPX_USE_OPENPMD)
    endif()

    if(WarpX_QED)
        target_compile_definitions(ablastr_${SD} PUBLIC WARPX_QED)
        if(WarpX_QED_TABLE_GEN)
            target_compile_definitions(ablastr_${SD} PUBLIC WarpX_QED_TABLE_GEN)
        endif()
    endif()

    if(WarpX_PSATD)
        target_compile_definitions(ablastr_${SD} PUBLIC WARPX_USE_PSATD)
    endif()

    # <cmath>: M_PI
    if(WIN32)
        target_compile_definitions(ablastr_${SD} PUBLIC _USE_MATH_DEFINES)
    endif()
endforeach()


# Warnings ####################################################################
#
set_cxx_warnings()


# Generate Configuration and .pc Files ########################################
#
get_source_version(WarpX_${WarpX_DIMS_LAST} ${WarpX_SOURCE_DIR})
configure_file(
    ${WarpX_SOURCE_DIR}/Source/Utils/WarpXVersion.H.in
    ${WarpX_BINARY_DIR}/Source/Utils/WarpXVersion.H
    @ONLY
)

# these files are used if WarpX is installed and picked up by a downstream
# project (not needed yet)

#include(CMakePackageConfigHelpers)
#write_basic_package_version_file("WarpXConfigVersion.cmake"
#    VERSION ${WarpX_VERSION}
#    COMPATIBILITY SameMajorVersion
#)


# Installs ####################################################################
#
# headers, libraries and executables
set(WarpX_INSTALL_TARGET_NAMES)
foreach(D IN LISTS WarpX_DIMS)
    warpx_set_suffix_dims(SD ${D})
    list(APPEND WarpX_INSTALL_TARGET_NAMES ablastr_${SD})
    if(WarpX_APP)
        list(APPEND WarpX_INSTALL_TARGET_NAMES app_${SD})
    endif()
    if(WarpX_LIB)
        list(APPEND WarpX_INSTALL_TARGET_NAMES shared_${SD})
    endif()

    install(TARGETS ${WarpX_INSTALL_TARGET_NAMES}
        EXPORT WarpXTargets
        LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
        ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
        RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
        INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
    )

    # simplified library alias
    # this is currently expected by Python bindings
    if(WarpX_LIB)
        if(D STREQUAL "RZ")
            set(lib_dim "rz")
        else()
            set(lib_dim "${WarpX_DIMS}d")
        endif()
        if(IS_ABSOLUTE ${CMAKE_INSTALL_LIBDIR})
            set(ABS_INSTALL_LIB_DIR ${CMAKE_INSTALL_LIBDIR})
        else()
            set(ABS_INSTALL_LIB_DIR ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR})
        endif()
        # escape spaces for generated cmake_install.cmake file
        file(TO_CMAKE_PATH "${ABS_INSTALL_LIB_DIR}" ABS_INSTALL_LIB_DIR)

        install(CODE "file(CREATE_LINK
            $<TARGET_FILE_NAME:shared_${SD}>
            \"${ABS_INSTALL_LIB_DIR}/libwarpx.${lib_dim}$<TARGET_FILE_SUFFIX:shared_${SD}>\"
            COPY_ON_ERROR SYMBOLIC)")
    endif()

    # CMake package file for find_package(WarpX::WarpX) in depending projects
    #install(EXPORT WarpXTargets
    #    FILE WarpXTargets.cmake
    #    NAMESPACE WarpX::
    #    DESTINATION ${WarpX_INSTALL_CMAKEDIR}
    #)
    #install(
    #    FILES
    #        ${WarpX_BINARY_DIR}/WarpXConfig.cmake
    #        ${WarpX_BINARY_DIR}/WarpXConfigVersion.cmake
    #    DESTINATION ${WarpX_INSTALL_CMAKEDIR}
    #)
endforeach()


# pip helpers for the pywarpx package #########################################
#
if(WarpX_LIB)
    set(PYINSTALLOPTIONS "" CACHE STRING
        "Additional parameters to pass to `pip install`")

    # add a prefix to custom targets so we do not collide if used as a subproject
    if(CMAKE_SOURCE_DIR STREQUAL PROJECT_SOURCE_DIR)
        set(_WarpX_CUSTOM_TARGET_PREFIX_DEFAULT "")
    else()
        set(_WarpX_CUSTOM_TARGET_PREFIX_DEFAULT "warpx_")
    endif()
    set(WarpX_CUSTOM_TARGET_PREFIX "${_WarpX_CUSTOM_TARGET_PREFIX_DEFAULT}"
            CACHE STRING "Prefix for custom targets")

    # build the wheel by re-using the shared library we build
    add_custom_target(${WarpX_CUSTOM_TARGET_PREFIX}pip_wheel
        ${CMAKE_COMMAND} -E rm -f -r warpx-whl
        COMMAND
            ${CMAKE_COMMAND} -E env PYWARPX_LIB_DIR=$<TARGET_FILE_DIR:shared_${WarpX_DIMS_LAST}>
                python3 -m pip wheel -v --no-build-isolation --no-deps --wheel-dir=warpx-whl ${WarpX_SOURCE_DIR}
        WORKING_DIRECTORY
            ${WarpX_BINARY_DIR}
        DEPENDS
            shared_${WarpX_DIMS_LAST}
    )

    # this will also upgrade/downgrade dependencies, e.g., when the version of picmistandard changes
    if(WarpX_MPI)
        set(pyWarpX_REQUIREMENT_FILE "requirements_mpi.txt")
    else()
        set(pyWarpX_REQUIREMENT_FILE "requirements.txt")
    endif()
    add_custom_target(${WarpX_CUSTOM_TARGET_PREFIX}pip_install_requirements
        python3 -m pip install ${PYINSTALLOPTIONS} -r "${WarpX_SOURCE_DIR}/${pyWarpX_REQUIREMENT_FILE}"
        WORKING_DIRECTORY
            ${WarpX_BINARY_DIR}
    )

    # We force-install because in development, it is likely that the version of
    # the package does not change, but the code did change. We need --no-deps,
    # because otherwise pip would also force reinstall all dependencies.
    add_custom_target(${WarpX_CUSTOM_TARGET_PREFIX}pip_install
        ${CMAKE_COMMAND} -E env WARPX_MPI=${WarpX_MPI}
            python3 -m pip install --force-reinstall --no-index --no-deps ${PYINSTALLOPTIONS} --find-links=warpx-whl pywarpx
        WORKING_DIRECTORY
            ${WarpX_BINARY_DIR}
        DEPENDS
            shared_${WarpX_DIMS_LAST}
            ${WarpX_CUSTOM_TARGET_PREFIX}pip_wheel
            ${WarpX_CUSTOM_TARGET_PREFIX}pip_install_requirements
    )
endif()


# Tests #######################################################################
#

#if(BUILD_TESTING)
#    enable_testing()
#
#    add_test(...)
#endif()


# Status Summary for Build Options ############################################
#
warpx_print_summary()