Merge branch 'development' of github.com:ECP-WarpX/WarpX into tps_exa…

…mple

.github/workflows/cuda.yml

@@ -131,7 +131,7 @@ jobs:
         which nvcc || echo "nvcc not in PATH!"
 
         git clone https://github.com/AMReX-Codes/amrex.git ../amrex
-        cd ../amrex && git checkout --detach 028638564f7be0694b9898f8d4088cdbf9a6f9f5 && cd -
+        cd ../amrex && git checkout --detach 103d0e9f7c7e7fbedbeb61512d3f37f3e2b03770 && cd -
         make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_FFT=TRUE USE_CCACHE=TRUE -j 4
 
         ccache -s

.github/workflows/windows.yml

@@ -10,7 +10,9 @@ jobs:
   build_win_msvc:
     name: MSVC C++17 w/o MPI
     runs-on: windows-latest
-    if: github.event.pull_request.draft == false
+    # disabled due to issues in #5230
+    if: 0
+    #if: github.event.pull_request.draft == false
     steps:
     - uses: actions/checkout@v4
     - uses: actions/setup-python@v5

.pre-commit-config.yaml

@@ -69,7 +69,7 @@ repos:
 # Python: Ruff linter & formatter
 #   https://docs.astral.sh/ruff/
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.6.5
+  rev: v0.6.8
   hooks:
     # Run the linter
     - id: ruff

.readthedocs.yml

@@ -9,14 +9,17 @@ version: 2
 build:
   os: ubuntu-22.04
   tools:
-    python: "3.11"
+    python: "mambaforge-latest"
+    # python: "3.11"
 
 sphinx:
-   configuration: Docs/source/conf.py
+  configuration: Docs/source/conf.py
 
-python:
-   install:
-   - requirements: Docs/requirements.txt
+conda:
+  environment: Docs/conda.yml
+# python:
+#   install:
+#   - requirements: Docs/requirements.txt
 
 formats:
   - htmlzip

Docs/Doxyfile

@@ -2245,15 +2245,15 @@ ENABLE_PREPROCESSING   = YES
 # The default value is: NO.
 # This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
 
-MACRO_EXPANSION        = NO
+MACRO_EXPANSION        = YES
 
 # If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES then
 # the macro expansion is limited to the macros specified with the PREDEFINED and
 # EXPAND_AS_DEFINED tags.
 # The default value is: NO.
 # This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
 
-EXPAND_ONLY_PREDEF     = NO
+EXPAND_ONLY_PREDEF     = YES
 
 # If the SEARCH_INCLUDES tag is set to YES, the include files in the
 # INCLUDE_PATH will be searched if a #include is found.
@@ -2305,14 +2305,16 @@ PREDEFINED             = AMREX_Linux=1 \
                          WARPX_QED=1 \
                          WARPX_QED_TABLE_GEN=1
 
+PREDEFINED           += "AMREX_ENUM(CLASS,...)=\"enum class CLASS : int { __VA_ARGS__ };\""
+
 # If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then this
 # tag can be used to specify a list of macro names that should be expanded. The
 # macro definition that is found in the sources will be used. Use the PREDEFINED
 # tag if you want to use a different macro definition that overrules the
 # definition found in the source code.
 # This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
 
-EXPAND_AS_DEFINED      =
+EXPAND_AS_DEFINED      = AMREX_ENUM
 
 # If the SKIP_FUNCTION_MACROS tag is set to YES then doxygen's preprocessor will
 # remove all references to function-like macros that are alone on a line, have

Docs/README.md

@@ -9,12 +9,13 @@ More information can be found in Docs/source/developers/documentation.rst.
 
 Install the Python requirements for compiling the documentation:
 ```
-python3 -m pip install -r Docs/requirements.txt
+cd Docs/
+python3 -m pip install -r requirements.txt
 ```
 
 ### Compiling the documentation
 
-`cd` into the `Docs/` directory and type
+Still in the `Docs/` directory, type
 ```
 make html
 ```

Docs/conda.yml

@@ -0,0 +1,12 @@
+name: readthedocs
+
+channels:
+  - conda-forge
+  - nodefaults
+
+dependencies:
+  - python
+  - doxygen
+  - pip
+  - pip:
+    - -r requirements.txt

Docs/requirements.txt

@@ -5,7 +5,7 @@
 # License: BSD-3-Clause-LBNL
 
 # WarpX PICMI bindings w/o C++ component (used for autoclass docs)
--e Python
+-e ../Python
 breathe
 docutils>=0.17.1
 

Docs/source/developers/documentation.rst

@@ -56,16 +56,15 @@ First, make sure you are in the root directory of WarpX's source and install the
 
 .. code-block:: sh
 
-    python3 -m pip install -r Docs/requirements.txt
+    cd Docs/
+    python3 -m pip install -r requirements.txt
 
 You will also need Doxygen (macOS: ``brew install doxygen``; Ubuntu: ``sudo apt install doxygen``).
 
-Then, to compile the documentation, use
+Still in the ``Docs/`` directory, compile the documentation via
 
 .. code-block:: sh
 
-    cd Docs/
-
     make html
     # This will first compile the Doxygen documentation (execute doxygen)
     # and then build html pages from rst files using sphinx and breathe.

Docs/source/developers/fields.rst

@@ -37,6 +37,13 @@ The ``MultiFab`` constructor (for, e.g., ``Ex`` on level ``lev``) is called in `
 
 By default, the ``MultiFab`` are set to ``0`` at initialization. They can be assigned a different value in ``WarpX::InitLevelData``.
 
+Field Names
+-----------
+
+The commonly used WarpX field names are defined in:
+
+.. doxygenenum:: warpx::fields::FieldType
+
 Field solver
 ------------
 

Docs/source/developers/testing.rst

@@ -175,6 +175,8 @@ If you need a new Python package dependency for testing, please add it in `Regre
 
 Sometimes two or more tests share a large number of input parameters. The shared input parameters can be collected in a "base" input file that can be passed as a runtime parameter in the actual test input files through the parameter ``FILE``.
 
+If the new test is added in a new directory that did not exist before, please add the name of that directory with the command ``add_subdirectory`` in `Physics_applications/CMakeLists.txt <https://github.com/ECP-WarpX/WarpX/tree/development/Examples/Physics_applications/CMakeLists.txt>`__ or `Tests/CMakeLists.txt <https://github.com/ECP-WarpX/WarpX/tree/development/Examples/Tests/CMakeLists.txt>`__, depending on where the new test directory is located.
+
 Naming conventions for automated tests
 --------------------------------------
 

Docs/source/highlights.rst

@@ -90,6 +90,11 @@ Scientific works in laser-ion acceleration and laser-matter interaction.
    Physical Review Research **6**, 033148, 2024.
    `DOI:10.1103/PhysRevResearch.6.033148 <https://doi.org/10.1103/PhysRevResearch.6.033148>`__
 
+#. Zaïm N, Sainte-Marie A, Fedeli L, Bartoli P, Huebl A, Leblanc A, Vay J-L, Vincenti H.
+   **Light-matter interaction near the Schwinger limit using tightly focused doppler-boosted lasers**.
+   Physical Review Letters **132**, 175002, 2024.
+   `DOI:10.1103/PhysRevLett.132.175002 <https://doi.org/10.1103/PhysRevLett.132.175002>`__
+
 #. Knight B, Gautam C, Stoner C, Egner B, Smith J, Orban C, Manfredi J, Frische K, Dexter M, Chowdhury E, Patnaik A (2023).
    **Detailed Characterization of a kHz-rate Laser-Driven Fusion at a Thin Liquid Sheet with a Neutron Detection Suite**.
    High Power Laser Science and Engineering, 1-13, 2023.
@@ -110,6 +115,11 @@ Scientific works in laser-ion acceleration and laser-matter interaction.
    Phys. Rev. Accel. Beams **25**, 093402, 2022.
    `DOI:10.1103/PhysRevAccelBeams.25.093402 <https://doi.org/10.1103/PhysRevAccelBeams.25.093402>`__
 
+#. Fedeli L, Sainte-Marie A, Zaïm N, Thévenet M, Vay J-L, Myers A, Quéré F, Vincenti H.
+   **Probing strong-field QED with Doppler-boosted PetaWatt-class lasers**.
+   Physical Review Letters **127**, 114801, 2021.
+   `DOI:10.1103/PhysRevLett.127.114801 <https://doi.org/10.1103/PhysRevLett.127.114801>`__
+
 
 Particle Accelerator & Beam Physics
 ***********************************

Docs/source/install/dependencies.rst

@@ -117,7 +117,7 @@ For Nvidia CUDA GPU support, you will need to have `a recent CUDA driver install
 
 .. code-block:: bash
 
-   conda install -c nvidia -c conda-forge cuda cupy
+   conda install -c nvidia -c conda-forge cuda cuda-nvtx-dev cupy
 
 More info for `CUDA-enabled ML packages <https://twitter.com/jeremyphoward/status/1697435241152127369>`__.
 

Docs/source/install/hpc.rst

@@ -43,6 +43,7 @@ This section documents quick-start guides for a selection of supercomputers that
    hpc/lassen
    hpc/lawrencium
    hpc/leonardo
+   hpc/lonestar6
    hpc/lumi
    hpc/lxplus
    hpc/ookami

Docs/source/install/hpc/lonestar6.rst

@@ -0,0 +1,139 @@
+.. _building-lonestar6:
+
+Lonestar6 (TACC)
+================
+
+The `Lonestar6 cluster <https://portal.tacc.utexas.edu/user-guides/lonestar6>`_ is located at `TACC <https://www.tacc.utexas.edu>`__.
+
+
+Introduction
+------------
+
+If you are new to this system, **please see the following resources**:
+
+* `TACC user guide <https://portal.tacc.utexas.edu/user-guides/>`__
+* Batch system: `Slurm <https://portal.tacc.utexas.edu/user-guides/lonestar6#job-management>`__
+* `Jupyter service <https://tacc.github.io/ctls2017/docs/intro_to_python/intro_to_python_011_jupyter.html>`__
+* `Filesystem directories <https://portal.tacc.utexas.edu/user-guides/lonestar6#managing-files-on-lonestar6>`__:
+
+  * ``$HOME``: per-user home directory, backed up (10 GB)
+  * ``$WORK``: per-user production directory, not backed up, not purged, Lustre (1 TB)
+  * ``$SCRATCH``: per-user production directory, not backed up, purged every 10 days, Lustre (no limits, 8PByte total)
+
+
+Installation
+------------
+
+Use the following commands to download the WarpX source code and switch to the correct branch:
+
+.. code-block:: bash
+
+   git clone https://github.com/ECP-WarpX/WarpX.git $WORK/src/warpx
+
+We use system software modules, add environment hints and further dependencies via the file ``$HOME/lonestar6_warpx_a100.profile``.
+Create it now:
+
+.. code-block:: bash
+
+   cp $HOME/src/warpx/Tools/machines/lonestar6-tacc/lonestar6_warpx_a100.profile.example $HOME/lonestar6_warpx_a100.profile
+
+.. dropdown:: Script Details
+   :color: light
+   :icon: info
+   :animate: fade-in-slide-down
+
+   .. literalinclude:: ../../../../Tools/machines/lonestar6-tacc/lonestar6_warpx_a100.profile.example
+      :language: bash
+
+Edit the 2nd line of this script, which sets the ``export proj=""`` variable.
+For example, if you are member of the project ``abcde``, then run ``nano $HOME/lonestar6_warpx_a100.profile`` and edit line 2 to read:
+
+.. code-block:: bash
+
+   export proj="abcde"
+
+Exit the ``nano`` editor with ``Ctrl`` + ``O`` (save) and then ``Ctrl`` + ``X`` (exit).
+
+.. important::
+
+   Now, and as the first step on future logins to Lonestar6, activate these environment settings:
+
+   .. code-block:: bash
+
+      source $HOME/lonestar6_warpx_a100.profile
+
+Finally, since Lonestar6 does not yet provide software modules for some of our dependencies, install them once:
+
+.. code-block:: bash
+
+   bash $HOME/src/warpx/Tools/machines/lonestar6-tacc/install_a100_dependencies.sh
+   source ${SW_DIR}/venvs/warpx-a100/bin/activate
+
+.. dropdown:: Script Details
+   :color: light
+   :icon: info
+   :animate: fade-in-slide-down
+
+   .. literalinclude:: ../../../../Tools/machines/lonestar6-tacc/install_a100_dependencies.sh
+      :language: bash
+
+
+.. _building-lonestar6-compilation:
+
+Compilation
+-----------
+
+Use the following :ref:`cmake commands <building-cmake>` to compile the application executable:
+
+.. code-block:: bash
+
+   cd $HOME/src/warpx
+   rm -rf build_pm_gpu
+
+   cmake -S . -B build_gpu -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3"
+   cmake --build build_gpu -j 16
+
+The WarpX application executables are now in ``$HOME/src/warpx/build_gpu/bin/``.
+Additionally, the following commands will install WarpX as a Python module:
+
+.. code-block:: bash
+
+   cd $HOME/src/warpx
+   rm -rf build_pm_gpu_py
+
+   cmake -S . -B build_gpu_py -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3"
+   cmake --build build_gpu_py -j 16 --target pip_install
+
+Now, you can :ref:`submit Lonestar6 compute jobs <running-cpp-lonestar6>` for WarpX :ref:`Python (PICMI) scripts <usage-picmi>` (:ref:`example scripts <usage-examples>`).
+Or, you can use the WarpX executables to submit Lonestar6 jobs (:ref:`example inputs <usage-examples>`).
+For executables, you can reference their location in your :ref:`job script <running-cpp-lonestar6>` or copy them to a location in ``$WORK`` or ``$SCRATCH``.
+
+
+.. _running-cpp-lonestar6:
+
+Running
+-------
+
+.. _running-cpp-lonestar6-A100-GPUs:
+
+A100 GPUs (40 GB)
+^^^^^^^^^^^^^^^^^
+
+`84 GPU nodes, each with 2 A100 GPUs (40 GB) <https://portal.tacc.utexas.edu/user-guides/lonestar6#system-gpu>`__.
+
+The batch script below can be used to run a WarpX simulation on multiple nodes (change ``-N`` accordingly) on the supercomputer lonestar6 at tacc.
+Replace descriptions between chevrons ``<>`` by relevant values, for instance ``<input file>`` could be ``plasma_mirror_inputs``.
+Note that we run one MPI rank per GPU.
+
+
+.. literalinclude:: ../../../../Tools/machines/lonestar6-tacc/lonestar6_a100.sbatch
+   :language: bash
+   :caption: You can copy this file from ``Tools/machines/lonestar6-tacc/lonestar6_a100.sbatch``.
+
+To run a simulation, copy the lines above to a file ``lonestar6.sbatch`` and run
+
+.. code-block:: bash
+
+   sbatch lonestar6_a100.sbatch
+
+to submit the job.

Docs/source/install/hpc/perlmutter.rst

@@ -153,7 +153,7 @@ Use the following :ref:`cmake commands <building-cmake>` to compile the applicat
          cd $HOME/src/warpx
          rm -rf build_pm_gpu
 
-         cmake -S . -B build_pm_gpu -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_HEFFTE=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3"
+         cmake -S . -B build_pm_gpu -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3"
          cmake --build build_pm_gpu -j 16
 
       The WarpX application executables are now in ``$HOME/src/warpx/build_pm_gpu/bin/``.
@@ -164,7 +164,7 @@ Use the following :ref:`cmake commands <building-cmake>` to compile the applicat
          cd $HOME/src/warpx
          rm -rf build_pm_gpu_py
 
-         cmake -S . -B build_pm_gpu_py -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_HEFFTE=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3"
+         cmake -S . -B build_pm_gpu_py -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3"
          cmake --build build_pm_gpu_py -j 16 --target pip_install
 
    .. tab-item:: CPU Nodes
@@ -174,7 +174,7 @@ Use the following :ref:`cmake commands <building-cmake>` to compile the applicat
          cd $HOME/src/warpx
          rm -rf build_pm_cpu
 
-         cmake -S . -B build_pm_cpu -DWarpX_COMPUTE=OMP -DWarpX_FFT=ON -DWarpX_HEFFTE=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3"
+         cmake -S . -B build_pm_cpu -DWarpX_COMPUTE=OMP -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3"
          cmake --build build_pm_cpu -j 16
 
       The WarpX application executables are now in ``$HOME/src/warpx/build_pm_cpu/bin/``.
@@ -184,7 +184,7 @@ Use the following :ref:`cmake commands <building-cmake>` to compile the applicat
 
          rm -rf build_pm_cpu_py
 
-         cmake -S . -B build_pm_cpu_py -DWarpX_COMPUTE=OMP -DWarpX_FFT=ON -DWarpX_HEFFTE=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3"
+         cmake -S . -B build_pm_cpu_py -DWarpX_COMPUTE=OMP -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3"
          cmake --build build_pm_cpu_py -j 16 --target pip_install
 
 Now, you can :ref:`submit Perlmutter compute jobs <running-cpp-perlmutter>` for WarpX :ref:`Python (PICMI) scripts <usage-picmi>` (:ref:`example scripts <usage-examples>`).