diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index e262083a..71d8cef1 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -82,7 +82,7 @@ option(pyAMReX_amrex_internal "Download & build AMReX" ON) set(pyAMReX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(pyAMReX_amrex_internal)") -set(pyAMReX_amrex_branch "6b294a5cd3e5c55b36c7d0afe2993501775eb42f" +set(pyAMReX_amrex_branch "ee492f47704ae8b094ad9bb4b6f758d82606a61e" CACHE STRING "Repository branch for pyAMReX_amrex_repo if(pyAMReX_amrex_internal)") diff --git a/src/Particle/ParticleContainer.H b/src/Particle/ParticleContainer.H index fdd80055..02f5c34f 100644 --- a/src/Particle/ParticleContainer.H +++ b/src/Particle/ParticleContainer.H @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -24,26 +25,38 @@ #include namespace py = pybind11; -using namespace amrex; +template +std::string particle_type_suffix () +{ + std::string suffix; + if constexpr (T_ParticleType::is_soa_particle) + suffix = "pureSoA_" + + std::to_string(T_NArrayReal) + "_" + + std::to_string(T_NArrayInt); + else + suffix = std::to_string(T_ParticleType::NReal) + "_" + + std::to_string(T_ParticleType::NInt) + "_" + + std::to_string(T_NArrayReal) + "_" + + std::to_string(T_NArrayInt); + + return suffix; +} template void make_Base_Iterators (py::module &m, std::string allocstr) { using iterator_base = T_ParIterBase; using container = typename iterator_base::ContainerType; - constexpr int NStructReal = container::NStructReal; - constexpr int NStructInt = container::NStructInt; + using ParticleType = typename container::ParticleType; constexpr int NArrayReal = container::NArrayReal; constexpr int NArrayInt = container::NArrayInt; + std::string const suffix = particle_type_suffix(); std::string particle_it_base_name = std::string("Par"); if (is_const) particle_it_base_name += "Const"; - particle_it_base_name += "IterBase_" + - std::to_string(NStructReal) + "_" + std::to_string(NStructInt) + "_" + - std::to_string(NArrayReal) + "_" + std::to_string(NArrayInt) + "_" + - allocstr; - py::class_(m, particle_it_base_name.c_str()) + particle_it_base_name += "IterBase_" + suffix + "_" + allocstr; + auto py_it_base = py::class_(m, particle_it_base_name.c_str()) .def(py::init(), py::arg("particle_container"), py::arg("level")) .def(py::init(), @@ -51,11 +64,10 @@ void make_Base_Iterators (py::module &m, std::string allocstr) .def("particle_tile", &iterator_base::GetParticleTile, py::return_value_policy::reference_internal) - .def("aos", &iterator_base::GetArrayOfStructs, - py::return_value_policy::reference_internal) .def("soa", &iterator_base::GetStructOfArrays, py::return_value_policy::reference_internal) + .def_property_readonly_static("is_soa_particle", [](const py::object&){ return ParticleType::is_soa_particle;}) .def_property_readonly("num_particles", &iterator_base::numParticles) .def_property_readonly("num_real_particles", &iterator_base::numRealParticles) .def_property_readonly("num_neighbor_particles", &iterator_base::numNeighborParticles) @@ -75,6 +87,12 @@ void make_Base_Iterators (py::module &m, std::string allocstr) py::return_value_policy::reference_internal ) ; + + // only legacy particle has an AoS data structure for positions and id+cpu + if constexpr (!ParticleType::is_soa_particle) + py_it_base.def("aos", + &iterator_base::GetArrayOfStructs, + py::return_value_policy::reference_internal); } template class Allocator=DefaultAllocator> @@ -82,19 +100,17 @@ void make_Iterators (py::module &m, std::string allocstr) { using iterator = T_ParIter; using container = typename iterator::ContainerType; - constexpr int NStructReal = container::NStructReal; - constexpr int NStructInt = container::NStructInt; + using ParticleType = typename container::ParticleType; constexpr int NArrayReal = container::NArrayReal; constexpr int NArrayInt = container::NArrayInt; - using iterator_base = amrex::ParIterBase; + using iterator_base = amrex::ParIterBase_impl; make_Base_Iterators< is_const, iterator_base >(m, allocstr); + std::string const suffix = particle_type_suffix(); auto particle_it_name = std::string("Par"); if (is_const) particle_it_name += "Const"; - particle_it_name += std::string("Iter_") + std::to_string(NStructReal) + "_" + - std::to_string(NStructInt) + "_" + std::to_string(NArrayReal) + "_" + - std::to_string(NArrayInt) + "_" + allocstr; + particle_it_name += std::string("Iter_") + suffix + "_" + allocstr; py::class_(m, particle_it_name.c_str()) .def("__repr__", [particle_it_name](iterator const & pti) { @@ -108,53 +124,61 @@ void make_Iterators (py::module &m, std::string allocstr) py::arg("particle_container"), py::arg("level")) .def(py::init(), py::arg("particle_container"), py::arg("level"), py::arg("info")) + .def_property_readonly_static("is_soa_particle", [](const py::object&){ return ParticleType::is_soa_particle;}) ; } -template +template void make_ParticleInitData (py::module &m) { - using ParticleInitData = ParticleInitType; + using ParticleType = T_ParticleType; + using ParticleInitData = ParticleInitType; + // depends on https://github.com/AMReX-Codes/amrex/pull/3280 + // using ParticleInitData = ParticleInitType; + std::string const suffix = particle_type_suffix(); auto const particle_init_data_type = - std::string("ParticleInitType_") + std::to_string(T_NStructReal) + "_" + - std::to_string(T_NStructInt) + "_" + std::to_string(T_NArrayReal) + "_" + - std::to_string(T_NArrayInt); - py::class_(m, particle_init_data_type.c_str()) + std::string("ParticleInitType_") + suffix; + auto py_particle_init_data = py::class_(m, particle_init_data_type.c_str()) .def(py::init<>()) - .def_readwrite("real_struct_data", &ParticleInitData::real_struct_data) - .def_readwrite("int_struct_data", &ParticleInitData::int_struct_data) + .def_property_readonly_static("is_soa_particle", [](const py::object&){return ParticleType::is_soa_particle;}) .def_readwrite("real_array_data", &ParticleInitData::real_array_data) .def_readwrite("int_array_data", &ParticleInitData::int_array_data) ; + + // only legacy particle has an AoS data structure for positions and id+cpu + if constexpr (!ParticleType::is_soa_particle) + py_particle_init_data + .def_readwrite("real_struct_data", &ParticleInitData::real_struct_data) + .def_readwrite("int_struct_data", &ParticleInitData::int_struct_data); } -template class Allocator=DefaultAllocator> void make_ParticleContainer_and_Iterators (py::module &m, std::string allocstr) { - using ParticleContainerType = ParticleContainer< - T_NStructReal, T_NStructInt, T_NArrayReal, T_NArrayInt, + using ParticleType = T_ParticleType; + using ParticleContainerType = ParticleContainer_impl< + ParticleType, T_NArrayReal, T_NArrayInt, Allocator >; using ParticleInitData = typename ParticleContainerType::ParticleInitData; using ParticleTileType = typename ParticleContainerType::ParticleTileType; - //using AoS = typename ParticleContainerType::AoS; - auto const particle_container_type = std::string("ParticleContainer_") + std::to_string(T_NStructReal) + "_" + - std::to_string(T_NStructInt) + "_" + std::to_string(T_NArrayReal) + "_" + - std::to_string(T_NArrayInt) + "_" + allocstr; - py::class_(m, particle_container_type.c_str()) + std::string const suffix = particle_type_suffix(); + auto const particle_container_type = std::string("ParticleContainer_") + suffix + "_" + allocstr; + auto py_pc = py::class_(m, particle_container_type.c_str()) .def(py::init()) .def(py::init()) .def(py::init&, - const Vector&, - const Vector&, - const Vector&>()) + const Vector&, + const Vector&, + const Vector&>()) .def(py::init&, - const Vector&, - const Vector&, - const Vector&>()) + const Vector&, + const Vector&, + const Vector&>()) + .def_property_readonly_static("is_soa_particle", [](const py::object&){return ParticleType::is_soa_particle;}) .def_property_readonly_static("NStructReal", [](const py::object&){return ParticleContainerType::NStructReal; }) .def_property_readonly_static("NStructInt", [](const py::object&){return ParticleContainerType::NStructInt; }) .def_property_readonly_static("NArrayReal", [](const py::object&){return ParticleContainerType::NArrayReal; }) @@ -203,13 +227,8 @@ void make_ParticleContainer_and_Iterators (py::module &m, std::string allocstr) // const ParticleInitData& mass, // bool serialize = false, RealBox bx = RealBox()); - .def("InitRandom", py::overload_cast(&ParticleContainerType::InitRandom)) - - .def("InitRandomPerBox", py::overload_cast(&ParticleContainerType::InitRandomPerBox)) - .def("InitOnePerCell", &ParticleContainerType::InitOnePerCell) - - .def("Increment", &ParticleContainerType::Increment) - .def("IncrementWithTotal", &ParticleContainerType::IncrementWithTotal, py::arg("mf"), py::arg("level"), py::arg("local")=false) + .def("Increment", &ParticleContainerType::Increment) // TODO pure SoA + //.def("IncrementWithTotal", &ParticleContainerType::IncrementWithTotal, py::arg("mf"), py::arg("level"), py::arg("local")=false) // TODO pure SoA .def("Redistribute", &ParticleContainerType::Redistribute, py::arg("lev_min")=0, py::arg("lev_max")=-1, py::arg("nGrow")=0, py::arg("local")=0, py::arg("remove_negative")=true) .def("SortParticlesByCell", &ParticleContainerType::SortParticlesByCell) @@ -235,12 +254,12 @@ void make_ParticleContainer_and_Iterators (py::module &m, std::string allocstr) // void CreateVirtualParticles (int level, AoS& virts) const; //.def("CreateVirtualParticles", py::overload_cast(&ParticleContainerType::CreateVirtualParticles, py::const_), // py::arg("level"), py::arg("virts")) - .def("CreateVirtualParticles", py::overload_cast(&ParticleContainerType::CreateVirtualParticles, py::const_), - py::arg("level"), py::arg("virts")) + //.def("CreateVirtualParticles", py::overload_cast(&ParticleContainerType::CreateVirtualParticles, py::const_), + // py::arg("level"), py::arg("virts")) // TODO pure SoA //.def("CreateGhostParticles", py::overload_cast(&ParticleContainerType::CreateGhostParticles, py::const_), // py::arg("level"), py::arg("ngrow"), py::arg("ghosts")) - .def("CreateGhostParticles", py::overload_cast(&ParticleContainerType::CreateGhostParticles, py::const_), - py::arg("level"), py::arg("ngrow"), py::arg("ghosts")) + //.def("CreateGhostParticles", py::overload_cast(&ParticleContainerType::CreateGhostParticles, py::const_), + // py::arg("level"), py::arg("ngrow"), py::arg("ghosts")) // TODO pure SoA //.def("AddParticlesAtLevel", py::overload_cast(&ParticleContainerType::AddParticlesAtLevel), // py::arg("particles"), py::arg("level"), py::arg("ngrow")=0) .def("AddParticlesAtLevel", py::overload_cast(&ParticleContainerType::AddParticlesAtLevel), @@ -342,46 +361,60 @@ void make_ParticleContainer_and_Iterators (py::module &m, std::string allocstr) // } ; - using iterator = amrex::ParIter; + // TODO for pure SoA + // depends on https://github.com/AMReX-Codes/amrex/pull/3280 + if constexpr (!T_ParticleType::is_soa_particle) { + py_pc + .def("InitRandom", py::overload_cast(&ParticleContainerType::InitRandom)) // TODO pure SoA + .def("InitRandomPerBox", py::overload_cast(&ParticleContainerType::InitRandomPerBox)) // TODO pure SoA + .def("InitOnePerCell", &ParticleContainerType::InitOnePerCell); + } + + using iterator = amrex::ParIter_impl; make_Iterators< false, iterator, Allocator >(m, allocstr); - using const_iterator = amrex::ParConstIter; + using const_iterator = amrex::ParConstIter_impl; make_Iterators< true, const_iterator, Allocator >(m, allocstr); } -template +/** Create ParticleContainers and Iterators + */ +template void make_ParticleContainer_and_Iterators (py::module &m) { - make_ParticleInitData(m); + // TODO for pure SoA + // depends on https://github.com/AMReX-Codes/amrex/pull/3280 + if constexpr (!T_ParticleType::is_soa_particle) + make_ParticleInitData(m); // see Src/Base/AMReX_GpuContainers.H // !AMREX_USE_GPU: DefaultAllocator = std::allocator // AMREX_USE_GPU: DefaultAllocator = amrex::ArenaAllocator // work-around for https://github.com/pybind/pybind11/pull/4581 - //make_ParticleContainer_and_Iterators(m, "std"); // CPU DefaultAllocator - //make_ParticleContainer_and_Iterators(m, "arena"); // GPU DefaultAllocator #ifdef AMREX_USE_GPU - make_ParticleContainer_and_Iterators(m, "std"); - make_ParticleContainer_and_Iterators(m, "default"); // amrex::ArenaAllocator #else - make_ParticleContainer_and_Iterators(m, "default"); // std::allocator - make_ParticleContainer_and_Iterators(m, "arena"); + make_ParticleContainer_and_Iterators(m, "default"); // std::allocator + make_ParticleContainer_and_Iterators(m, "arena"); #endif // end work-around - make_ParticleContainer_and_Iterators(m, "pinned"); + make_ParticleContainer_and_Iterators(m, "pinned"); #ifdef AMREX_USE_GPU - make_ParticleContainer_and_Iterators(m, "device"); - make_ParticleContainer_and_Iterators(m, "managed"); - make_ParticleContainer_and_Iterators(m, "async"); #endif } diff --git a/src/Particle/ParticleContainer.cpp b/src/Particle/ParticleContainer.cpp index bdfcc1b0..b72b7e9d 100644 --- a/src/Particle/ParticleContainer.cpp +++ b/src/Particle/ParticleContainer.cpp @@ -5,14 +5,19 @@ */ #include "ParticleContainer.H" +#include + + void init_ParticleContainer_HiPACE(py::module& m); void init_ParticleContainer_ImpactX(py::module& m); void init_ParticleContainer_WarpX(py::module& m); void init_ParticleContainer(py::module& m) { + using namespace amrex; + // TODO: we might need to move all or most of the defines in here into a // test/example submodule, so they do not collide with downstream projects - make_ParticleContainer_and_Iterators< 1, 1, 2, 1> (m); // tests + make_ParticleContainer_and_Iterators, 2, 1>(m); // tests init_ParticleContainer_HiPACE(m); init_ParticleContainer_ImpactX(m); diff --git a/src/Particle/ParticleContainer_HiPACE.cpp b/src/Particle/ParticleContainer_HiPACE.cpp index 2b8bea9b..4342879f 100644 --- a/src/Particle/ParticleContainer_HiPACE.cpp +++ b/src/Particle/ParticleContainer_HiPACE.cpp @@ -5,10 +5,14 @@ */ #include "ParticleContainer.H" +#include + void init_ParticleContainer_HiPACE(py::module& /* m */) { + using namespace amrex; + // TODO: we might need to move all or most of the defines in here into a // test/example submodule, so they do not collide with downstream projects - //make_ParticleContainer_and_Iterators< 0, 0, 4, 0> (m); // HiPACE++ 22.07 - //make_ParticleContainer_and_Iterators< 0, 0, 37, 1> (m); // HiPACE++ 22.07 + //make_ParticleContainer_and_Iterators, 4, 0> (m); // HiPACE++ 22.07 + //make_ParticleContainer_and_Iterators, 37, 1> (m); // HiPACE++ 22.07 } diff --git a/src/Particle/ParticleContainer_ImpactX.cpp b/src/Particle/ParticleContainer_ImpactX.cpp index ef3d3191..f75729e6 100644 --- a/src/Particle/ParticleContainer_ImpactX.cpp +++ b/src/Particle/ParticleContainer_ImpactX.cpp @@ -5,9 +5,15 @@ */ #include "ParticleContainer.H" +#include +#include + void init_ParticleContainer_ImpactX(py::module& m) { + using namespace amrex; + // TODO: we might need to move all or most of the defines in here into a // test/example submodule, so they do not collide with downstream projects - make_ParticleContainer_and_Iterators< 0, 0, 5, 0> (m); // ImpactX 22.07 + make_ParticleContainer_and_Iterators, 5, 0>(m); // ImpactX 22.07 - 23.04 + make_ParticleContainer_and_Iterators, 8, 2>(m); // ImpactX 23.05+ } diff --git a/src/Particle/ParticleContainer_WarpX.cpp b/src/Particle/ParticleContainer_WarpX.cpp index ba36cb09..da8f328f 100644 --- a/src/Particle/ParticleContainer_WarpX.cpp +++ b/src/Particle/ParticleContainer_WarpX.cpp @@ -5,10 +5,14 @@ */ #include "ParticleContainer.H" +#include + void init_ParticleContainer_WarpX(py::module& m) { + using namespace amrex; + // TODO: we might need to move all or most of the defines in here into a // test/example submodule, so they do not collide with downstream projects - make_ParticleContainer_and_Iterators< 0, 0, 4, 0> (m); // WarpX 22.07 1D-3D - //make_ParticleContainer_and_Iterators< 0, 0, 5, 0> (m); // WarpX 22.07 RZ + make_ParticleContainer_and_Iterators, 4, 0>(m); // WarpX 22.07 - 23.04 1D-3D + //make_ParticleContainer_and_Iterators, 5, 0> (m); // WarpX 22.07 - 23.04 RZ } diff --git a/src/Particle/StructOfArrays.cpp b/src/Particle/StructOfArrays.cpp index 895e6ce4..613d302f 100644 --- a/src/Particle/StructOfArrays.cpp +++ b/src/Particle/StructOfArrays.cpp @@ -72,6 +72,7 @@ void make_StructOfArrays(py::module &m) void init_StructOfArrays(py::module& m) { make_StructOfArrays< 2, 1>(m); make_StructOfArrays< 4, 0>(m); // HiPACE++ 22.07 - make_StructOfArrays< 5, 0>(m); // ImpactX 22.07 + make_StructOfArrays< 5, 0>(m); // ImpactX 22.07 - 23.04 + make_StructOfArrays< 8, 2>(m); // ImpactX 23.04+ make_StructOfArrays<37, 1>(m); // HiPACE++ 22.07 }