diff --git a/bindings/python/ADIOSPy.cpp b/bindings/python/ADIOSPy.cpp deleted file mode 100644 index 7f196f40a1..0000000000 --- a/bindings/python/ADIOSPy.cpp +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * ADIOSPy.cpp - * - * Created on: Mar 13, 2017 - * Author: William F Godoy godoywf@ornl.gov - */ - -#include "ADIOSPy.h" - -#include "adiosPyTypes.h" - -#include "adios2/ADIOSMPI.h" - -namespace adios2 -{ - -ADIOSPy::ADIOSPy(const std::string configFile, MPI_Comm mpiComm, - const bool debugMode) -: m_DebugMode(debugMode), - m_ADIOS(std::make_shared(configFile, mpiComm, debugMode)) -{ -} - -ADIOSPy::ADIOSPy(MPI_Comm mpiComm, const bool debugMode) -: ADIOSPy("", mpiComm, debugMode) -{ -} - -ADIOSPy::ADIOSPy(const std::string configFile, const bool debugMode) -: ADIOSPy(configFile, MPI_COMM_SELF, debugMode) -{ -} - -ADIOSPy::ADIOSPy(const bool debugMode) : ADIOSPy("", MPI_COMM_SELF, debugMode) -{ -} - -IOPy ADIOSPy::DeclareIO(const std::string name) -{ - return IOPy(m_ADIOS->DeclareIO(name), m_DebugMode); -} - -} // end namespace adios diff --git a/bindings/python/ADIOSPy.h b/bindings/python/ADIOSPy.h deleted file mode 100644 index d3c9f5e85d..0000000000 --- a/bindings/python/ADIOSPy.h +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * ADIOSPy.h python binding to ADIOS class - * - * Created on: Mar 13, 2017 - * Author: William F Godoy godoywf@ornl.gov - */ - -#ifndef ADIOS2_BINDINGS_PYTHON_SOURCE_ADIOSPY_H_ -#define ADIOS2_BINDINGS_PYTHON_SOURCE_ADIOSPY_H_ - -/// \cond EXCLUDE_FROM_DOXYGEN -#include //std::shared_ptr -#include -/// \endcond - -#include - -#include "IOPy.h" -#include "adios2/ADIOSMPICommOnly.h" - -namespace adios2 -{ - -class ADIOSPy -{ - -public: - ADIOSPy(const std::string configFile, MPI_Comm mpiComm, - const bool debugMode); - ADIOSPy(MPI_Comm mpiComm, const bool debugMode); - ADIOSPy(const std::string configFile, const bool debugMode); - ADIOSPy(const bool debugMode); - - ~ADIOSPy() = default; - - IOPy DeclareIO(const std::string name); - -private: - const bool m_DebugMode; - std::shared_ptr m_ADIOS; -}; - -} // end namespace adios - -#endif /* BINDINGS_PYTHON_SOURCE_ADIOSPY_H_ */ diff --git a/bindings/python/CMakeLists.txt b/bindings/python/CMakeLists.txt index a6ced9fc20..bb4245170f 100644 --- a/bindings/python/CMakeLists.txt +++ b/bindings/python/CMakeLists.txt @@ -1,34 +1,31 @@ pybind11_add_module(adios2py MODULE - ADIOSPy.h - ADIOSPy.cpp - adiosPyFunctions.h - adiosPyFunctions.cpp - adiosPyTypes.h - EnginePy.h - EnginePy.inl - EnginePy.cpp - gluePyBind11.cpp - IOPy.h - IOPy.cpp - VariablePy.h - VariablePy.cpp + adios2py.cpp + PyAttribute.cpp + PyADIOS.cpp + PyEnums.cpp + PyEngine.cpp + PyIO.cpp + PyVariable.cpp ) +target_include_directories(adios2py PRIVATE ${ADIOS2_SOURCE_DIR}/source/adios2/engine/pyengine) target_link_libraries(adios2py PRIVATE adios2) if(ADIOS2_HAVE_MPI) target_link_libraries(adios2py PRIVATE PythonModule::mpi4py) endif() -string(REGEX REPLACE - "^${PYTHON_PREFIX}/[^/]*/python" "${CMAKE_INSTALL_LIBDIR}/python" - CMAKE_INSTALL_PYTHONDIR "${PYTHON_SITE_PACKAGES}" +set(PY_VERSION_STRING "${PYTHON_VERSION_MAJOR}.${PYTHON_VERSION_MINOR}.${PYTHON_VERSION_PATCH}") +set(CMAKE_INSTALL_PYTHONDIR + "${CMAKE_INSTALL_LIBDIR}/python${PY_VERSION_STRING}/site-packages" ) set(CMAKE_INSTALL_PYTHONDIR "${CMAKE_INSTALL_PYTHONDIR}" CACHE INTERNAL "" FORCE ) +message(" ******** adios2py output directory: ${ADIOS2_BINARY_DIR}/${CMAKE_INSTALL_PYTHONDIR}") set_target_properties(adios2py PROPERTIES OUTPUT_NAME adios2 LIBRARY_OUTPUT_DIRECTORY ${ADIOS2_BINARY_DIR}/${CMAKE_INSTALL_PYTHONDIR} RUNTIME_OUTPUT_DIRECTORY ${ADIOS2_BINARY_DIR}/${CMAKE_INSTALL_PYTHONDIR} + ARCHIVE_OUTPUT_DIRECTORY ${ADIOS2_BINARY_DIR}/${CMAKE_INSTALL_PYTHONDIR} ) install(TARGETS adios2py DESTINATION ${CMAKE_INSTALL_PYTHONDIR} diff --git a/bindings/python/EnginePy.cpp b/bindings/python/EnginePy.cpp deleted file mode 100644 index 3a203a5d65..0000000000 --- a/bindings/python/EnginePy.cpp +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * EnginePy.cpp - * - * Created on: Mar 15, 2017 - * Author: wgodoy - */ - -#include "EnginePy.h" - -#include "adiosPyFunctions.h" - -namespace adios2 -{ - -EnginePy::EnginePy(IO &io, const std::string &name, const OpenMode openMode, - MPI_Comm mpiComm) -: m_IO(io), m_Engine(m_IO.Open(name, openMode, mpiComm)), - m_DebugMode(m_IO.m_DebugMode) -{ -} - -void EnginePy::Write(VariablePy &variable, const pyArray &array) -{ - if (variable.m_IsDefined) - { - // do nothing, not supporting compound types in Python, yet - } -#define declare_type(T) \ - else if (pybind11::isinstance>(array)) \ - { \ - DefineVariableInIO(variable); \ - } - ADIOS2_FOREACH_TYPE_1ARG(declare_type) -#undef declare_type - - if (!variable.m_IsDefined) - { - if (m_DebugMode) - { - throw std::runtime_error("ERROR: variable " + variable.m_Name + - " couldn't not be created in IO " + - m_IO.m_Name + " , in call to Write\n"); - } - } -#define declare_type(T) \ - else if (pybind11::isinstance>(array)) \ - { \ - WriteInIO(variable, array); \ - } - ADIOS2_FOREACH_TYPE_1ARG(declare_type) -#undef declare_type -} - -void EnginePy::Advance(const float timeoutSeconds) -{ - m_Engine->Advance(timeoutSeconds); -} - -void EnginePy::Close(const int transportIndex) -{ - m_Engine->Close(transportIndex); -} - -} // end namespace adios diff --git a/bindings/python/EnginePy.h b/bindings/python/EnginePy.h deleted file mode 100644 index 10046b3a65..0000000000 --- a/bindings/python/EnginePy.h +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * EnginePy.h - * - * Created on: Mar 15, 2017 - * Author: William F Godoy godoywf@ornl.gov - */ - -#ifndef ADIOS2_BINDINGS_PYTHON_SOURCE_ENGINEPY_H_ -#define ADIOS2_BINDINGS_PYTHON_SOURCE_ENGINEPY_H_ - -/// \cond EXCLUDE_FROM_DOXYGEN -#include //std::shared_ptr -#include -/// \endcond - -#include - -#include "VariablePy.h" -#include "adiosPyTypes.h" //pyArray - -namespace adios2 -{ - -class EnginePy -{ - -public: - EnginePy(IO &io, const std::string &name, const OpenMode openMode, - MPI_Comm mpiComm); - - ~EnginePy() = default; - - void Write(VariablePy &variable, const pyArray &array); - - void Advance(const float timeoutSeconds = 0.); - - void Close(const int transportIndex = -1); - -private: - IO &m_IO; - std::shared_ptr m_Engine; - const bool m_DebugMode; - - template - void DefineVariableInIO(VariablePy &variable); - - template - void WriteInIO(VariablePy &variable, const pyArray &array); -}; - -} // end namespace adios - -#include "EnginePy.inl" - -#endif /* BINDINGS_PYTHON_SOURCE_ENGINEPY_H_ */ diff --git a/bindings/python/EnginePy.inl b/bindings/python/EnginePy.inl deleted file mode 100644 index eea3e71aa3..0000000000 --- a/bindings/python/EnginePy.inl +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * EnginePy.tcc - * - * Created on: Jun 8, 2017 - * Author: William F Godoy godoywf@ornl.gov - */ - -#ifndef ADIOS2_BINDINGS_PYTHON_SOURCE_ENGINEPY_INL_ -#define ADIOS2_BINDINGS_PYTHON_SOURCE_ENGINEPY_INL_ -#ifndef ADIOS2_BINDINGS_PYTHON_SOURCE_ENGINEPY_H_ -#error "Inline file should only be included from it's header, never on it's own" -#endif - -#include "adiosPyFunctions.h" - -namespace adios2 -{ - -template -void EnginePy::DefineVariableInIO(VariablePy &variable) -{ - auto &var = m_IO.DefineVariable( - variable.m_Name, PyListToDims(variable.m_Shape), - PyListToDims(variable.m_Start), PyListToDims(variable.m_Count), - variable.m_IsConstantDims); - - variable.m_VariableBase = &var; - variable.m_IsDefined = true; -} - -template -void EnginePy::WriteInIO(VariablePy &variable, const pyArray &array) -{ - m_Engine->Write(*dynamic_cast *>(variable.m_VariableBase), - reinterpret_cast(array.data())); -} - -} // end namespace adios - -#endif /* BINDINGS_PYTHON_SOURCE_ENGINEPY_TCC_ */ diff --git a/bindings/python/IOPy.cpp b/bindings/python/IOPy.cpp deleted file mode 100644 index 1ae9475b1f..0000000000 --- a/bindings/python/IOPy.cpp +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * IOPy.cpp - * - * Created on: Mar 14, 2017 - * Author: William F Godoy godoywf@ornl.gov - */ - -#include "IOPy.h" - -namespace adios2 -{ - -IOPy::IOPy(IO &io, const bool debugMode) : m_IO(io), m_DebugMode(debugMode) -{ - m_IO.m_HostLanguage = "Python"; -} - -void IOPy::SetEngine(const std::string engine) { m_IO.SetEngine(engine); } - -void IOPy::SetParameters(const pyKwargs &kwargs) noexcept -{ - m_IO.SetParameters(KwargsToParams(kwargs)); -} - -unsigned int IOPy::AddTransport(const std::string type, - const pyKwargs &kwargs) noexcept -{ - return m_IO.AddTransport(type, KwargsToParams(kwargs)); -} - -VariablePy &IOPy::DefineVariable(const std::string &name, const pyList shape, - const pyList start, const pyList count, - const bool isConstantDims) -{ - if (m_DebugMode) - { - if (m_Variables.count(name) == 1) - { - throw std::invalid_argument("ERROR: variable " + name + - " already exists, use GetVariable, in " - "call to DefineVariable\n"); - } - } - - auto itVariableEmplace = - m_Variables.emplace(name, VariablePy(name, shape, start, count, - isConstantDims, m_DebugMode)); - return itVariableEmplace.first->second; -} - -VariablePy &IOPy::GetVariable(const std::string &name) -{ - auto itVariable = m_Variables.find(name); - - if (m_DebugMode) - { - if (itVariable == m_Variables.end()) - { - throw std::invalid_argument("ERROR: variable " + name + - " doesn't exist, in " - "call to GetVariable\n"); - } - } - return itVariable->second; -} - -EnginePy IOPy::Open(const std::string &name, const int openMode) -{ - return EnginePy(m_IO, name, static_cast(openMode), - m_IO.m_MPIComm); -} - -} // end namespace adios diff --git a/bindings/python/IOPy.h b/bindings/python/IOPy.h deleted file mode 100644 index c56255c51b..0000000000 --- a/bindings/python/IOPy.h +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * IOPy.h - * - * Created on: Mar 14, 2017 - * Author: William F Godoy godoywf@ornl.gov - */ - -#ifndef ADIOS2_BINDINGS_PYTHON_SOURCE_IOPY_H_ -#define ADIOS2_BINDINGS_PYTHON_SOURCE_IOPY_H_ - -/// \cond EXCLUDE_FROM_DOXYGEN -#include -/// \endcond - -#include "EnginePy.h" -#include "adiosPyTypes.h" - -namespace adios2 -{ - -class IOPy -{ - -public: - IO &m_IO; - const bool m_DebugMode; - - IOPy(IO &io, const bool debugMode); - - ~IOPy() = default; - - void SetEngine(const std::string engineType); - - void SetParameters(const pyKwargs &kwargs) noexcept; - unsigned int AddTransport(const std::string type, - const pyKwargs &kwargs) noexcept; - - VariablePy &DefineVariable(const std::string &name, const pyList shape, - const pyList start, const pyList count, - const bool isConstantDims); - - VariablePy &GetVariable(const std::string &name); - - EnginePy Open(const std::string &name, const int openMode); - -private: - /** - * Extra map needed as Variables are not created in ADIOS at - * DefineVariable, but until Write when type is known from numpy - */ - std::map m_Variables; -}; - -} // end namespace adios - -#endif /* BINDINGS_PYTHON_SOURCE_IOPY_H_ */ diff --git a/bindings/python/PyADIOS.cpp b/bindings/python/PyADIOS.cpp new file mode 100644 index 0000000000..c4ab8b5e52 --- /dev/null +++ b/bindings/python/PyADIOS.cpp @@ -0,0 +1,53 @@ +#include + +#include + +#ifdef ADIOS2_HAVE_MPI +#include +#endif + +namespace adios2 +{ +template +void GeneratePythonBindings(pybind11::module &m); + +template <> +void GeneratePythonBindings(pybind11::module &m) +{ + pybind11::class_(m, "ADIOS") + .def(pybind11::init([](const bool debug) { + auto a = new ADIOS(debug); + a->m_HostLanguage = "Python"; + return a; + }), + pybind11::arg("debug") = true) + .def(pybind11::init([](const std::string config, const bool debug) { + auto a = new ADIOS(config, debug); + a->m_HostLanguage = "Python"; + return a; + }), + pybind11::arg("config").none(false), pybind11::arg("debug") = true) +#ifdef ADIOS2_HAVE_MPI + .def(pybind11::init([](pybind11::object comm, const bool debug) { + import_mpi4py__MPI(); + auto a = new ADIOS(*PyMPIComm_Get(comm.ptr()), debug); + a->m_HostLanguage = "Python"; + return a; + }), + pybind11::arg("comm").none(false), pybind11::arg("debug") = true) + .def(pybind11::init([](pybind11::object comm, const std::string config, + const bool debug) { + import_mpi4py__MPI(); + auto a = new ADIOS(config, *PyMPIComm_Get(comm.ptr()), debug); + a->m_HostLanguage = "Python"; + return a; + }), + pybind11::arg("comm").none(false), + pybind11::arg("config").none(false), pybind11::arg("debug") = true) +#endif + .def_readonly("HostLanguage", &ADIOS::m_HostLanguage) + .def("DeclareIO", &ADIOS::DeclareIO) + .def("GetIO", &ADIOS::GetIO); +} + +} // end namespace adios2 diff --git a/bindings/python/PyAttribute.cpp b/bindings/python/PyAttribute.cpp new file mode 100644 index 0000000000..a6dc79e942 --- /dev/null +++ b/bindings/python/PyAttribute.cpp @@ -0,0 +1,29 @@ +#include +#include + +#include + +namespace adios2 +{ +template +void GeneratePythonBindings(pybind11::module &m); + +template <> +void GeneratePythonBindings(pybind11::module &m) +{ + pybind11::class_(m, "Attribute") + .def_readonly("Name", &AttributeBase::m_Name) + .def_readonly("Type", &AttributeBase::m_Type) + .def_readonly("Elements", &AttributeBase::m_Elements) + .def_readonly("Name", &AttributeBase::m_IsSingleValue); + +#define pyattr(T, L) \ + pybind11::class_, AttributeBase>( \ + m, "Attribute" ADIOS2_STRINGIFY(L)) \ + .def_readwrite("DataArray", &Attribute::m_DataArray) \ + .def_readwrite("DataSingleValue", &Attribute::m_DataSingleValue); + ADIOS2_FOREACH_TYPE_2ARGS(pyattr) +#undef pyvar +} + +} // end namespace adios2 diff --git a/bindings/python/PyEngine.cpp b/bindings/python/PyEngine.cpp new file mode 100644 index 0000000000..30f750837b --- /dev/null +++ b/bindings/python/PyEngine.cpp @@ -0,0 +1,73 @@ +#include +#include +#include + +#include + +#include "PyEngineBase.h" +#include + +namespace adios2 +{ +template +void GeneratePythonBindings(pybind11::module &m); + +template <> +void GeneratePythonBindings(pybind11::module &m) +{ + // Wrapping for internal Engine classes + pybind11::class_> engine(m, "EngineBase"); + engine.def("Write", + [](Engine &e, VariableBase &variable, pybind11::array values) { + e.Write(variable, values.data()); + }); + engine.def("Write", + [](Engine &e, const std::string name, pybind11::array values) { + e.Write(name, values.data()); + }); + engine.def("InquireVariable", + [](Engine &e, const std::string name) -> VariableBase * { + VariableBase *var; +#define inquire(T) \ + if (var = e.InquireVariable(name, false)) \ + { \ + return var; \ + } + ADIOS2_FOREACH_TYPE_1ARG(inquire) +#undef inquire + return nullptr; + }); + engine.def("Close", &Engine::Close, pybind11::arg("transportIndex") = -1); + + // + // Wrappings to extend Engine in Python + // + + // This is the trampoline class used by pybind11 to implement the + // inheritance model and virtual springboard + class PyEngine : public PyEngineBase + { + public: + using PyEngineBase::PyEngineBase; + void Init() override { PYBIND11_OVERLOAD(void, PyEngineBase, Init, ); } + void DoWrite(VariableBase *var, pybind11::array values) override + { + PYBIND11_OVERLOAD_PURE(void, PyEngineBase, DoWrite, var, values); + } + void Close(const int transportIndex) + { + PYBIND11_OVERLOAD_PURE(void, PyEngineBase, Close, transportIndex); + } + }; + + pybind11::class_>( + m, "Engine", engine) + .def(pybind11::init()) + .def("Init", &PyEngineBase::Init) + .def("DoWrite", + (void (PyEngineBase::*)(VariableBase *, pybind11::array)) & + PyEngineBase::DoWrite) + .def("Close", &PyEngineBase::Close); +} +} // end namespace adios2 diff --git a/bindings/python/PyEnums.cpp b/bindings/python/PyEnums.cpp new file mode 100644 index 0000000000..35322e1733 --- /dev/null +++ b/bindings/python/PyEnums.cpp @@ -0,0 +1,68 @@ +#include + +#include + +namespace adios2 +{ +template +void GeneratePythonBindings(pybind11::module &m); + +template <> +void GeneratePythonBindings(pybind11::module &m) +{ + pybind11::enum_(m, "ShapeID") + .value("GlobalValue", ShapeID::GlobalValue) + .value("GlobalArray", ShapeID::GlobalArray) + .value("JoinedArray", ShapeID::JoinedArray) + .value("LocalValue", ShapeID::LocalValue) + .value("LocalArray", ShapeID::LocalArray) + .export_values(); + + pybind11::enum_(m, "IOMode") + .value("Independent", IOMode::Independent) + .value("Collective", IOMode::Collective) + .export_values(); + + pybind11::enum_(m, "OpenMode") + .value("Undefined", OpenMode::Undefined) + .value("Write", OpenMode::Write) + .value("Read", OpenMode::Read) + .value("Append", OpenMode::Append) + .value("ReadWrite", OpenMode::ReadWrite) + .export_values(); + + pybind11::enum_(m, "ReadMultiplexPattern") + .value("GlobalReaders", ReadMultiplexPattern::GlobalReaders) + .value("RoundRobin", ReadMultiplexPattern::RoundRobin) + .value("FirstInFirstOut", ReadMultiplexPattern::FirstInFirstOut) + .value("OpenAllSteps", ReadMultiplexPattern::OpenAllSteps) + .export_values(); + + pybind11::enum_(m, "ReadMode") + .value("NonBlocking", ReadMode::Blocking) + .value("Blocking", ReadMode::Blocking) + .export_values(); + + pybind11::enum_(m, "AdvanceMode") + .value("Append", AdvanceMode::Append) + .value("Update", AdvanceMode::Update) + .value("NextAvailable", AdvanceMode::NextAvailable) + .value("LatestAvailable", AdvanceMode::LatestAvailable) + .export_values(); + + pybind11::enum_(m, "AdvanceStatus") + .value("OK", AdvanceStatus::OK) + .value("StepNotReady", AdvanceStatus::StepNotReady) + .value("EndOfStream", AdvanceStatus::EndOfStream) + .value("OtherError", AdvanceStatus::OtherError) + .export_values(); + + pybind11::enum_(m, "SelectionType") + .value("BoundingBox", SelectionType::BoundingBox) + .value("Points", SelectionType::Points) + .value("WriteBlock", SelectionType::WriteBlock) + .value("Auto", SelectionType::Auto) + .export_values(); +} + +} // end namespace adios2 diff --git a/bindings/python/PyIO.cpp b/bindings/python/PyIO.cpp new file mode 100644 index 0000000000..d798200f93 --- /dev/null +++ b/bindings/python/PyIO.cpp @@ -0,0 +1,60 @@ +#include +#include +#include + +#include + +namespace adios2 +{ +template +void GeneratePythonBindings(pybind11::module &m); + +template <> +void GeneratePythonBindings(pybind11::module &m) +{ + pybind11::class_(m, "IO") + .def_property("EngineType", &IO::GetEngine, &IO::SetEngine) + .def_property("Parameters", (Params & (IO::*)()) & IO::GetParameters, + &IO::SetParameters) + .def("AddTransport", &IO::AddTransport) + .def_property_readonly("TransportParameters", + &IO::GetTransportParameters) + .def("DefineVariable", + [](IO &io, std::string &name, Dims &shape, Dims &start, + Dims &count, bool constantDims, pybind11::object type) { + pybind11::dtype dt = pybind11::dtype::from_args(type); + if (type.is_none()) + { + // Delay variable creation + } +#define define_variable(T) \ + else if (dt.type() == pybind11::dtype::of().type()) \ + { \ + return pybind11::cast( \ + io.DefineVariable(name, shape, start, count, constantDims)); \ + } + ADIOS2_FOREACH_TYPE_1ARG(define_variable) +#undef define_variable + else + { + throw std::invalid_argument( + "Error: IO::DefineVariable called " + "with unsupported datatype"); + } + }, + pybind11::arg("name").none(false), + pybind11::arg("shape").none(false) = Dims{}, + pybind11::arg("start").none(false) = Dims{}, + pybind11::arg("count").none(false) = Dims{}, + pybind11::arg("constantDims") = false, + pybind11::arg("type").none(false)) + .def("RemoveVariable", &IO::RemoveVariable, + pybind11::arg("name").none(false)) + .def("GetVariable", &IO::GetVariableBase, + pybind11::arg("name").none(false)) + .def("Open", [](IO &io, const std::string &name, + const OpenMode mode) { return io.Open(name, mode); }, + pybind11::return_value_policy::reference); +} + +} // end namespace adios2 diff --git a/bindings/python/PySelection.cpp b/bindings/python/PySelection.cpp new file mode 100644 index 0000000000..ec57d03505 --- /dev/null +++ b/bindings/python/PySelection.cpp @@ -0,0 +1,25 @@ +#include + +#include +#include + +namespace adios2 +{ +template +void GeneratePythonBindings(pybind11::module &m); + +template <> +void GeneratePythonBindings(pybind11::module &m) +{ + pybind11::class_(m, "Selection") + .def_readonly("Type", &Selection::m_Type); + + pybind11::class_("SelectionBoundingBox") + .def(pybind11::init(), + pybind11::arg("start"), pybind11::arg("count"), + pybind11::arg("debug") = false) + .def_readwrite("Start", &SelectionBoundingBox.m_Start) + .def_readwrite("Count", &SelectionBoundingBox.m_Count); +} + +} // end namespace adios2 diff --git a/bindings/python/PyVariable.cpp b/bindings/python/PyVariable.cpp new file mode 100644 index 0000000000..19d65d993f --- /dev/null +++ b/bindings/python/PyVariable.cpp @@ -0,0 +1,42 @@ +#include +#include +#include + +#include + +namespace adios2 +{ +template +void GeneratePythonBindings(pybind11::module &m); + +template <> +void GeneratePythonBindings(pybind11::module &m) +{ + class PyVariableBase : public VariableBase + { + public: + using VariableBase::VariableBase; + pybind11::dtype m_DType; + }; + + pybind11::class_ varBase(m, "Variable"); + varBase.def_readonly("Name", &VariableBase::m_Name); + varBase.def_readonly("ElementSize", &VariableBase::m_ElementSize); + varBase.def_readonly("Shape", &VariableBase::m_Shape); + varBase.def_readonly("Start", &VariableBase::m_Start); + varBase.def_readonly("Count", &VariableBase::m_Count); + varBase.def_readonly("IsScalar", &VariableBase::m_SingleValue); + varBase.def_readonly("HasConstantDims", &VariableBase::m_ConstantDims); + varBase.def_property_readonly("PayLoadSize", &VariableBase::PayLoadSize); + varBase.def_property_readonly("TotalSize", &VariableBase::TotalSize); + varBase.def_property_readonly( + "Type", [](PyVariableBase &self) { return self.m_DType; }); + +#define pyvar(T, L) \ + pybind11::class_>(m, "Variable" ADIOS2_STRINGIFY(L), varBase) \ + .def_readwrite("Data", &Variable::m_Data); + ADIOS2_FOREACH_TYPE_2ARGS(pyvar) +#undef pyvar +} + +} // end namespace adios2 diff --git a/bindings/python/VariablePy.cpp b/bindings/python/VariablePy.cpp deleted file mode 100644 index 4946db480d..0000000000 --- a/bindings/python/VariablePy.cpp +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * VariablePy.cpp - * - * Created on: Mar 17, 2017 - * Author: William F Godoy godoywf@ornl.gov - */ - -#include "VariablePy.h" - -namespace adios2 -{ - -VariablePy::VariablePy(const std::string &name, const pyList shape, - const pyList start, const pyList count, - const bool isConstantDims, const bool debugMode) -: m_Name(name), m_Shape(shape), m_Start(start), m_Count(count), - m_IsConstantDims(isConstantDims), m_DebugMode(debugMode) -{ -} - -void VariablePy::SetDimensions(const pyList shape, const pyList start, - const pyList count) -{ - if (m_DebugMode) - { - if (m_IsConstantDims) - { - throw std::invalid_argument( - "ERROR: variable " + m_Name + - " dimensions are constant, in call from SetDimensions\n"); - } - } - - m_Shape = shape; - m_Start = start; - m_Count = count; -} - -std::string VariablePy::GetType() const noexcept -{ - return m_VariableBase->m_Type; -} - -} // end namespace adios diff --git a/bindings/python/VariablePy.h b/bindings/python/VariablePy.h deleted file mode 100644 index 0c6bd3635c..0000000000 --- a/bindings/python/VariablePy.h +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * VariablePy.h - * - * Created on: Mar 13, 2017 - * Author: William F Godoy godoywf@ornl.gov - */ - -#ifndef ADIOS2_BINDINGS_PYTHON_SOURCE_VARIABLEPY_H_ -#define ADIOS2_BINDINGS_PYTHON_SOURCE_VARIABLEPY_H_ - -#include - -#include "adiosPyFunctions.h" - -namespace adios2 -{ - -class VariablePy -{ - -public: - const std::string m_Name; - pyList m_Shape; - pyList m_Start; - pyList m_Count; - const bool m_IsConstantDims; - - VariableBase *m_VariableBase = nullptr; - bool m_IsDefined = false; - - VariablePy(const std::string &name, const pyList shape, const pyList start, - const pyList count, const bool isConstantDims, - const bool debugMode); - - ~VariablePy() = default; - - void SetDimensions(const pyList shape, const pyList start, - const pyList count); - - std::string GetType() const noexcept; - -private: - const bool m_DebugMode; -}; - -} // end namespace adios - -#endif /* BINDINGS_PYTHON_SOURCE_VARIABLEPY_H_ */ diff --git a/bindings/python/adios2py.cpp b/bindings/python/adios2py.cpp new file mode 100644 index 0000000000..8b3d275e14 --- /dev/null +++ b/bindings/python/adios2py.cpp @@ -0,0 +1,19 @@ +#include + +#include + +namespace adios2 +{ +template +void GeneratePythonBindings(pybind11::module &m); +} + +PYBIND11_MODULE(adios2, m) +{ + adios2::GeneratePythonBindings(m); + adios2::GeneratePythonBindings(m); + adios2::GeneratePythonBindings(m); + adios2::GeneratePythonBindings(m); + adios2::GeneratePythonBindings(m); + adios2::GeneratePythonBindings(m); +} diff --git a/bindings/python/adiosPyFunctions.cpp b/bindings/python/adiosPyFunctions.cpp deleted file mode 100644 index 4b4a2f61a5..0000000000 --- a/bindings/python/adiosPyFunctions.cpp +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * adiosPyFunctions.cpp - * - * Created on: Mar 13, 2017 - * Author: William F Godoy godoywf@ornl.gov - */ - -#include "adiosPyFunctions.h" - -namespace adios2 -{ - -Dims PyListToDims(const pyList list) noexcept -{ - const unsigned int length = pybind11::len(list); - Dims dimensions; - dimensions.reserve(length); - - for (unsigned int i = 0; i < length; ++i) - { - dimensions.push_back(pybind11::cast(list[i])); - } - - return dimensions; -} - -Params KwargsToParams(const pyKwargs &kwargs) noexcept -{ - Params parameters; - - for (const auto &pair : kwargs) - { - parameters.emplace(pybind11::cast(pair.first), - pybind11::cast(pair.second)); - } - return parameters; -} - -} // end namespace adios diff --git a/bindings/python/adiosPyFunctions.h b/bindings/python/adiosPyFunctions.h deleted file mode 100644 index 9138deb75e..0000000000 --- a/bindings/python/adiosPyFunctions.h +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * adiosPyFunctions.h - * - * Created on: Mar 13, 2017 - * Author: William F Godoy godoywf@ornl.gov - */ - -#ifndef ADIOS2_BINDINGS_PYTHON_SOURCE_ADIOSPYFUNCTIONS_H_ -#define ADIOS2_BINDINGS_PYTHON_SOURCE_ADIOSPYFUNCTIONS_H_ - -#include -#include -#include - -#include "adios2/ADIOSTypes.h" -#include "adiosPyTypes.h" - -namespace adios2 -{ - -/** - * Python list to vector of dimensions (Dims) - * @param list python list of numbers - * @return adios::Dims - */ -Dims PyListToDims(const pyList list) noexcept; - -/** - * Python dictionary kwargs to adios::Params (std::map) - * @param kwargs dictionary - * @return adios::Params - */ -Params KwargsToParams(const pyKwargs &kwargs) noexcept; - -} // end namespace adios - -#endif /* BINDINGS_PYTHON_SOURCE_ADIOSPYFUNCTIONS_H_ */ diff --git a/bindings/python/adiosPyTypes.h b/bindings/python/adiosPyTypes.h deleted file mode 100644 index e4ba524784..0000000000 --- a/bindings/python/adiosPyTypes.h +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * adiosPyTypes.h - * - * Created on: Jun 7, 2017 - * Author: William F Godoy godoywf@ornl.gov - */ - -#ifndef ADIOS2_BINDINGS_PYTHON_SOURCE_ADIOSPYTYPES_H_ -#define ADIOS2_BINDINGS_PYTHON_SOURCE_ADIOSPYTYPES_H_ - -#include -#include -#include - -namespace adios2 -{ -// pytypes -using pyObject = pybind11::object; -using pyTuple = pybind11::tuple; -using pyDict = pybind11::dict; -using pyKwargs = pybind11::kwargs; -using pyList = pybind11::list; - -// numpy -using pyArray = pybind11::array; -using pyDType = pybind11::dtype; -} - -#endif /* BINDINGS_PYTHON_SOURCE_ADIOSPYTYPES_H_ */ diff --git a/bindings/python/gluePyBind11.cpp b/bindings/python/gluePyBind11.cpp deleted file mode 100644 index 85f5ec2194..0000000000 --- a/bindings/python/gluePyBind11.cpp +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * gluePyBind11.cpp - * - * Created on: Mar 16, 2017 - * Author: William F Godoy godoywf@ornl.gov - */ - -#include - -#include -#include - -#ifdef ADIOS2_HAVE_MPI -#include -#endif - -#include "ADIOSPy.h" -#include "EnginePy.h" -#include "IOPy.h" -#include "VariablePy.h" -#include "adiosPyFunctions.h" -#include "adiosPyTypes.h" - -#ifdef ADIOS2_HAVE_MPI -adios2::ADIOSPy ADIOSPyInitConfig(const std::string configFile, - adios2::pyObject &object, - const bool debugMode) -{ - MPI_Comm *mpiCommPtr = PyMPIComm_Get(object.ptr()); - - if (import_mpi4py() < 0) - { - throw std::runtime_error("ERROR: could not import mpi4py " - "communicator, in call to ADIOS " - "constructor\n"); - } - - if (mpiCommPtr == nullptr) - { - throw std::runtime_error("ERROR: mpi4py communicator is null, in call " - "to ADIOS constructor\n"); - } - return adios2::ADIOSPy(configFile, *mpiCommPtr, debugMode); -} - -adios2::ADIOSPy ADIOSPyInit(adios2::pyObject &object, const bool debugMode) -{ - MPI_Comm *mpiCommPtr = PyMPIComm_Get(object.ptr()); - - if (import_mpi4py() < 0) - { - throw std::runtime_error("ERROR: could not import mpi4py " - "communicator, in call to ADIOS " - "constructor\n"); - } - - if (mpiCommPtr == nullptr) - { - throw std::runtime_error("ERROR: mpi4py communicator is null, in call " - "to ADIOS constructor\n"); - } - return adios2::ADIOSPy(*mpiCommPtr, debugMode); -} -#else -adios2::ADIOSPy ADIOSPyInitConfig(const std::string configFile, - const bool debugMode) -{ - return adios2::ADIOSPy(configFile, debugMode); -} - -adios2::ADIOSPy ADIOSPyInit(const bool debugMode) -{ - return adios2::ADIOSPy(debugMode); -} -#endif - -PYBIND11_PLUGIN(adios2) -{ -#ifdef ADIOS2_HAVE_MPI - if (import_mpi4py() < 0) - { - throw std::runtime_error( - "ERROR: mpi4py not loaded correctly\n"); /* Python 2.X */ - } -#endif - - pybind11::module m("adios2", "ADIOS2 Python bindings using pybind11"); - m.attr("DebugON") = true; - m.attr("DebugOFF") = false; - m.attr("ConstantDims") = true; - m.attr("OpenModeWrite") = static_cast(adios2::OpenMode::Write); - m.attr("OpenModeRead") = static_cast(adios2::OpenMode::Read); - m.attr("OpenModeAppend") = static_cast(adios2::OpenMode::Append); - m.attr("OpenModeReadWrite") = static_cast(adios2::OpenMode::ReadWrite); - m.def("ADIOS", &ADIOSPyInit, "Function that creates an ADIOS class object"); - m.def("ADIOS", &ADIOSPyInitConfig, - "Function that creates an ADIOS class object using a config file"); - - pybind11::class_(m, "ADIOSPy") - .def("DeclareIO", &adios2::ADIOSPy::DeclareIO); - - pybind11::class_(m, "IOPy") - .def("SetEngine", &adios2::IOPy::SetEngine) - .def("SetParameters", &adios2::IOPy::SetParameters) - .def("AddTransport", &adios2::IOPy::AddTransport) - .def("DefineVariable", &adios2::IOPy::DefineVariable, - pybind11::return_value_policy::reference_internal, - pybind11::arg("name"), pybind11::arg("shape") = adios2::pyList(), - pybind11::arg("start") = adios2::pyList(), - pybind11::arg("count") = adios2::pyList(), - pybind11::arg("isConstantDims") = false) - .def("GetVariable", &adios2::IOPy::GetVariable, - pybind11::return_value_policy::reference_internal) - .def("Open", (adios2::EnginePy (adios2::IOPy::*)(const std::string &, - const int)) & - adios2::IOPy::Open); - - pybind11::class_(m, "VariablePy") - .def("SetDimensions", &adios2::VariablePy::SetDimensions); - - pybind11::class_(m, "EnginePy") - .def("Write", &adios2::EnginePy::Write) - .def("Advance", &adios2::EnginePy::Advance, - pybind11::arg("timeoutSeconds") = 0.) - .def("Close", &adios2::EnginePy::Close, - pybind11::arg("transportIndex") = -1); - - return m.ptr(); -} diff --git a/cmake/ADIOSFunctions.cmake b/cmake/ADIOSFunctions.cmake index 8864a202fd..ce9c701acc 100644 --- a/cmake/ADIOSFunctions.cmake +++ b/cmake/ADIOSFunctions.cmake @@ -24,8 +24,13 @@ endfunction() function(python_add_test) set(options) + # NAME: test name + # ADDTOPYPATH: Provide this if python modules need to be found in specific directories + # USE_MPI_FOR_PYTHON_TESTS: Set to anything for mpi mode set(oneValueArgs NAME + ADDTOPYPATH + USE_MPI_FOR_PYTHON_TESTS ) # EXEC_WRAPPER: Any extra arguments to pass on the command line before test case # SCRIPT: Script name and corresponding comand line inputs @@ -34,8 +39,27 @@ function(python_add_test) add_test(NAME ${ARGS_NAME} COMMAND ${ARGS_EXEC_WRAPPER} ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/${ARGS_SCRIPT} ) - set_property(TEST ${ARGS_NAME} PROPERTY - ENVIRONMENT "PYTHONPATH=${ADIOS2_BINARY_DIR}/${CMAKE_INSTALL_PYTHONDIR}:$ENV{PYTHONPATH}" + + set(testsPythonPath "${ADIOS2_BINARY_DIR}/${CMAKE_INSTALL_PYTHONDIR};$ENV{PYTHONPATH}") + + if(DEFINED ARGS_ADDTOPYPATH) + set(testsPythonPath "${ARGS_ADDTOPYPATH};${testsPythonPath}") + endif() + + if(ARGS_USE_MPI_FOR_PYTHON_TESTS) + set(mpi_flag "ADIOS2_PYTHON_TESTS_USE_MPI=${ARGS_USE_MPI_FOR_PYTHON_TESTS}") + endif() + + if(WIN32) + set(path_var "PATH=${ADIOS2_BINARY_DIR}/bin;$ENV{PATH}") + else() + string(REPLACE ";" ":" testsPythonPath ${testsPythonPath}) + endif() + + set_property(TEST ${ARGS_NAME} PROPERTY ENVIRONMENT + "PYTHONPATH=${testsPythonPath}" + ${mpi_flag} + ${path_var} ) endfunction() @@ -54,6 +78,13 @@ function(GenerateADIOSHeaderConfig) endif() endforeach() + if(ADIOS2_HAVE_Python) + string(APPEND ADIOS2_CONFIG_DEFINES " +/* Python site-packages directory discovered at configure time */ +#define ADIOS2_PYTHON_SITE_PACKAGES_DIRECTORY \"@PYTHON_SITE_PACKAGES@\" +") + endif() + configure_file( ${ADIOS2_SOURCE_DIR}/source/adios2/ADIOSConfig.h.in ${ADIOS2_BINARY_DIR}/source/adios2/ADIOSConfig.h.in diff --git a/cmake/FindPythonFull.cmake b/cmake/FindPythonFull.cmake index 6b70423eaa..2db706b648 100644 --- a/cmake/FindPythonFull.cmake +++ b/cmake/FindPythonFull.cmake @@ -29,6 +29,9 @@ include(CMakeFindDependencyMacro) set(_req_vars) foreach(comp IN LISTS PythonFull_FIND_COMPONENTS) if(comp STREQUAL Interp) + if((NOT PYTHON_EXECUTABLE) AND (NOT "$ENV{PYTHON}" STREQUAL "")) + set(PYTHON_EXECUTABLE "$ENV{PYTHON}" CACHE FILEPATH "") + endif() find_package(PythonInterp) set(PythonFull_${comp}_FOUND ${PYTHONINTERP_FOUND}) list(APPEND _req_vars PYTHON_EXECUTABLE) diff --git a/examples/hello/bpWriter/CMakeLists.txt b/examples/hello/bpWriter/CMakeLists.txt index 3f3f19f4ad..d4b9ee1728 100644 --- a/examples/hello/bpWriter/CMakeLists.txt +++ b/examples/hello/bpWriter/CMakeLists.txt @@ -6,15 +6,15 @@ if(ADIOS2_HAVE_MPI) add_executable(hello_bpWriter helloBPWriter.cpp) target_link_libraries(hello_bpWriter MPI::MPI_C) - + add_executable(hello_bpWriter_c helloBPWriter.c) target_link_libraries(hello_bpWriter_c MPI::MPI_C) - + if(ADIOS2_HAVE_Fortran) add_executable(hello_bpWriter_f helloBPWriter.f90) target_link_libraries(hello_bpWriter_f MPI::MPI_Fortran adios2_f) endif() - + else() add_executable(hello_bpWriter helloBPWriter_nompi.cpp) add_executable(hello_bpWriter_c helloBPWriter_nompi.c) @@ -27,3 +27,8 @@ endif() target_link_libraries(hello_bpWriter adios2) target_link_libraries(hello_bpWriter_c adios2) + +if(ADIOS2_HAVE_Python) + add_executable(examplePythonPlugin examplePythonPlugin.cpp) + target_link_libraries(examplePythonPlugin PUBLIC adios2) +endif() diff --git a/examples/hello/bpWriter/examplePythonPlugin.cpp b/examples/hello/bpWriter/examplePythonPlugin.cpp new file mode 100644 index 0000000000..cf4c0a1993 --- /dev/null +++ b/examples/hello/bpWriter/examplePythonPlugin.cpp @@ -0,0 +1,73 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * examplePythonPlugin.cpp Example of using an Engine defined in python + * + * Created on: Sept 21, 2017 + * Author: Scott Wittenburg //std::ios_base::failure +#include //std::cout +#include //std::invalid_argument std::exception +#include + +#include + +int main(int argc, char *argv[]) +{ + /** Application variable */ + std::vector myFloats = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; + const std::size_t Nx = myFloats.size(); + + try + { + /** ADIOS class factory of IO class objects, DebugON is recommended */ + adios2::ADIOS adios(adios2::DebugON); + + /*** IO class object: settings and factory of Settings: Variables, + * Parameters, Transports, and Execution: Engines */ + adios2::IO &io = adios.DeclareIO("PythonPluginIO"); + + /** global array: name, { shape (total dimensions) }, { start (local) }, + * { count (local) }, all are constant dimensions */ + adios2::Variable &var = io.DefineVariable( + "data", {}, {}, {Nx}, adios2::ConstantDims); + + /** Engine derived class, spawned to start IO operations */ + io.SetEngine("PythonEngine"); + io.SetParameters({{"PluginName", "FirstPythonPlugin"}, + {"PluginModule", "TestPythonEngine"}, + {"PluginClass", "TestPythonEngine"}}); + auto writer = io.Open("TestPythonPlugin", adios2::OpenMode::Write); + + if (!writer) + { + throw std::ios_base::failure("ERROR: writer not created at Open\n"); + } + + /** Write variable for buffering */ + writer->Write(var, myFloats.data()); + + /** Create bp file, engine becomes unreachable after this*/ + writer->Close(); + } + catch (std::invalid_argument &e) + { + std::cout << "Invalid argument exception, STOPPING PROGRAM\n"; + std::cout << e.what() << "\n"; + } + catch (std::ios_base::failure &e) + { + std::cout << "IO System base failure exception, STOPPING PROGRAM\n"; + std::cout << e.what() << "\n"; + } + catch (std::exception &e) + { + std::cout << "Exception, STOPPING PROGRAM from rank\n"; + std::cout << e.what() << "\n"; + } + + return 0; +} \ No newline at end of file diff --git a/flake8.cfg b/flake8.cfg index 48317219e0..700efc45d5 100644 --- a/flake8.cfg +++ b/flake8.cfg @@ -2,5 +2,5 @@ max-line-length = 80 max-complexity = 14 format = pylint -ignore = F403,F405,F999 +ignore = F401,F403,F405,F999 exclude = thirdparty/ diff --git a/source/adios2/ADIOSMacros.h b/source/adios2/ADIOSMacros.h index 1f01f9ea60..63eced280b 100644 --- a/source/adios2/ADIOSMacros.h +++ b/source/adios2/ADIOSMacros.h @@ -11,6 +11,10 @@ #include #include "adios2/ADIOSTypes.h" + +#define ADIOS2_STRINGIFY_HELPER(X) #X +#define ADIOS2_STRINGIFY(X) ADIOS2_STRINGIFY_HELPER(X) + // The ADIOS_FOREACH_TYPE_1ARG macro assumes the given argument is a macro which // takes a single argument that is a type and then inserts the given MACRO for // each of the known primitive types @@ -108,6 +112,7 @@ // #define ADIOS2_FOREACH_TYPE_2ARGS(MACRO) \ MACRO(char, Char) \ + MACRO(signed char, SChar) \ MACRO(unsigned char, UChar) \ MACRO(short, Short) \ MACRO(unsigned short, UShort) \ @@ -126,6 +131,7 @@ #define ADIOS2_FOREACH_PRIMITIVE_TYPE_2ARGS(MACRO) \ MACRO(char, Char) \ + MACRO(signed char, SChar) \ MACRO(unsigned char, UChar) \ MACRO(short, Short) \ MACRO(unsigned short, UShort) \ diff --git a/source/adios2/CMakeLists.txt b/source/adios2/CMakeLists.txt index ab0ddc0279..7f206828e4 100644 --- a/source/adios2/CMakeLists.txt +++ b/source/adios2/CMakeLists.txt @@ -135,6 +135,19 @@ if(ADIOS2_HAVE_HDF5) target_link_libraries(adios2 PRIVATE ${HDF5_C_LIBRARIES}) endif() +if(ADIOS2_HAVE_Python) + target_sources(adios2 PRIVATE + helper/PythonInterpreter.h + helper/PythonInterpreter.cpp + helper/PythonModuleHelper.h + engine/pyengine/PythonEngine.h + engine/pyengine/PythonEngine.cpp + engine/pyengine/PyEngineBase.h + ) + target_include_directories(adios2 PRIVATE ${PYBIND11_INCLUDE_DIR} ${PYTHON_INCLUDE_DIRS}) + target_link_libraries(adios2 PRIVATE pybind11::embed) +endif() + # Set library version information set_target_properties(adios2 PROPERTIES VERSION ${ADIOS2_VERSION} diff --git a/source/adios2/core/Engine.cpp b/source/adios2/core/Engine.cpp index 9d19e892b7..9131c98a17 100644 --- a/source/adios2/core/Engine.cpp +++ b/source/adios2/core/Engine.cpp @@ -18,6 +18,8 @@ #include "adios2/helper/adiosFunctions.h" //GetType +#include + namespace adios2 { @@ -62,7 +64,7 @@ void Engine::Release() {} void Engine::PerformReads(ReadMode /*mode*/){}; // PROTECTED -void Engine::Init() {} +void Engine::Init() { std::cout << "Inside Engine::Init()" << std::endl; } void Engine::InitParameters() {} @@ -130,6 +132,12 @@ VariableBase *Engine::InquireVariableUnknown(const std::string &name, } #define define(T, L) \ + template <> \ + Variable *Engine::InquireVariable(const std::string &variableName, \ + const bool readIn) \ + { \ + return InquireVariable##L(variableName, readIn); \ + } \ Variable *Engine::InquireVariable##L(const std::string &name, \ const bool readIn) \ { \ diff --git a/source/adios2/core/Engine.h b/source/adios2/core/Engine.h index 3a405452f0..94c4823d77 100644 --- a/source/adios2/core/Engine.h +++ b/source/adios2/core/Engine.h @@ -294,7 +294,7 @@ class Engine * reading * @return a vector of strings */ - std::vector VariableNames(); + std::vector VariableNames() const; /** * Closes a particular transport, or all if -1 diff --git a/source/adios2/core/Engine.tcc b/source/adios2/core/Engine.tcc index 74666c7324..bdaaba3b14 100644 --- a/source/adios2/core/Engine.tcc +++ b/source/adios2/core/Engine.tcc @@ -47,130 +47,6 @@ void Engine::Write(const std::string &variableName, const T values) Write(m_IO.GetVariable(variableName), &values); } -template <> -Variable *Engine::InquireVariable(const std::string &variableName, - const bool readIn) -{ - return InquireVariableChar(variableName, readIn); -} - -template <> -Variable * -Engine::InquireVariable(const std::string &variableName, - const bool readIn) -{ - return InquireVariableUChar(variableName, readIn); -} - -template <> -Variable *Engine::InquireVariable(const std::string &variableName, - const bool readIn) -{ - return InquireVariableShort(variableName, readIn); -} - -template <> -Variable * -Engine::InquireVariable(const std::string &variableName, - const bool readIn) -{ - return InquireVariableUShort(variableName, readIn); -} - -template <> -Variable *Engine::InquireVariable(const std::string &variableName, - const bool readIn) -{ - return InquireVariableInt(variableName, readIn); -} - -template <> -Variable * -Engine::InquireVariable(const std::string &variableName, - const bool readIn) -{ - return InquireVariableUInt(variableName, readIn); -} - -template <> -Variable * -Engine::InquireVariable(const std::string &variableName, - const bool readIn) -{ - return InquireVariableLInt(variableName, readIn); -} - -template <> -Variable * -Engine::InquireVariable(const std::string &variableName, - const bool readIn) -{ - return InquireVariableLLInt(variableName, readIn); -} - -template <> -Variable * -Engine::InquireVariable(const std::string &variableName, - const bool readIn) -{ - return InquireVariableULInt(variableName, readIn); -} - -template <> -Variable * -Engine::InquireVariable(const std::string &variableName, - const bool readIn) -{ - return InquireVariableULLInt(variableName, readIn); -} - -template <> -Variable *Engine::InquireVariable(const std::string &variableName, - const bool readIn) -{ - return InquireVariableFloat(variableName, readIn); -} - -template <> -Variable * -Engine::InquireVariable(const std::string &variableName, - const bool readIn) -{ - return InquireVariableDouble(variableName, readIn); -} - -template <> -Variable * -Engine::InquireVariable(const std::string &variableName, - const bool readIn) -{ - return InquireVariableLDouble(variableName, readIn); -} - -template <> -Variable * -Engine::InquireVariable(const std::string &variableName, - const bool readIn) -{ - return InquireVariableCFloat(variableName, readIn); -} - -template <> -Variable * -Engine::InquireVariable(const std::string &variableName, - const bool readIn) -{ - return InquireVariableCDouble(variableName, readIn); -} - -template <> -Variable * -Engine::InquireVariable(const std::string &variableName, - const bool readIn) -{ - return InquireVariableCLDouble(variableName, readIn); -} - } // end namespace adios #endif /** ADIOS2_CORE_ENGINE_TCC_ */ diff --git a/source/adios2/core/IO.cpp b/source/adios2/core/IO.cpp index 1de67f0987..34148b2d34 100644 --- a/source/adios2/core/IO.cpp +++ b/source/adios2/core/IO.cpp @@ -34,6 +34,10 @@ #endif #endif +#ifdef ADIOS2_HAVE_PYTHON +#include "adios2/engine/pyengine/PythonEngine.h" +#endif + namespace adios2 { @@ -45,7 +49,10 @@ IO::IO(const std::string name, MPI_Comm mpiComm, const bool inConfigFile, } void IO::SetEngine(const std::string engineType) { m_EngineType = engineType; } +const std::string IO::GetEngine() const { return m_EngineType; } + void IO::SetIOMode(const IOMode ioMode) { m_IOMode = ioMode; }; +const IOMode IO::GetIOMode() const { return m_IOMode; } void IO::SetParameters(const Params ¶meters) { m_Parameters = parameters; } @@ -55,6 +62,8 @@ void IO::SetSingleParameter(const std::string key, m_Parameters[key] = value; } +Params &IO::GetParameters() { return m_Parameters; } + const Params &IO::GetParameters() const { return m_Parameters; } unsigned int IO::AddTransport(const std::string type, const Params ¶meters) @@ -70,6 +79,11 @@ unsigned int IO::AddTransport(const std::string type, const Params ¶meters) return static_cast(m_TransportsParameters.size() - 1); } +const std::vector &IO::GetTransportParameters() const +{ + return m_TransportsParameters; +} + void IO::SetTransportSingleParameter(const unsigned int transportIndex, const std::string key, const std::string value) @@ -310,6 +324,15 @@ std::shared_ptr IO::Open(const std::string &name, { engine = std::make_shared(*this, name, openMode, mpiComm); } + else if (m_EngineType == "PythonEngine") + { +#ifdef ADIOS2_HAVE_PYTHON + engine = std::make_shared(*this, name, openMode, mpiComm); +#else + throw std::invalid_argument("ERROR: this version didn't compile with " + "Python enabled, can't use PythonEngine\n"); +#endif + } else { if (m_DebugMode) diff --git a/source/adios2/core/IO.h b/source/adios2/core/IO.h index f29191dac3..9e81a97374 100644 --- a/source/adios2/core/IO.h +++ b/source/adios2/core/IO.h @@ -80,10 +80,21 @@ class IO */ void SetEngine(const std::string engine); + /** + * Gets the engine type for this IO class object + * @param engine + */ + const std::string GetEngine() const; + /** Set the IO mode (collective or independent) * @param IO mode */ void SetIOMode(const IOMode mode); + /** + * Get the IO mode (collective or independent) + */ + const IOMode GetIOMode() const; + /** * Version that passes a map to fill out parameters * initializer list = { "param1", "value1" }, {"param2", "value2"}, @@ -102,6 +113,7 @@ class IO /** * Retrieve existing parameter set */ + Params &GetParameters(); const Params &GetParameters() const; /** @@ -113,6 +125,8 @@ class IO unsigned int AddTransport(const std::string type, const Params ¶ms = Params()); + const std::vector &GetTransportParameters() const; + /** * Set a single parameter to an existing transport identified with a * transportIndex handler from AddTransport. This function overwrites diff --git a/source/adios2/engine/pyengine/PyEngineBase.h b/source/adios2/engine/pyengine/PyEngineBase.h new file mode 100644 index 0000000000..83ea04415d --- /dev/null +++ b/source/adios2/engine/pyengine/PyEngineBase.h @@ -0,0 +1,57 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * PyEngineBase.h: This is the base class used to translate the protected + * template functions into public members that can be overridden in python. + * + * Created on: Oct 9, 2017 + * Author: Chuck Atkins + */ + +#ifndef ADIOS2_ENGINE_PYENGINE_PYENGINEBASE_H_ +#define ADIOS2_ENGINE_PYENGINE_PYENGINEBASE_H_ + +#include +#include + +#include "adios2/core/Engine.h" + +namespace adios2 +{ + +class PyEngineBase : public Engine +{ + // Make sure PythonEngine can forward all its calls here + friend class PythonEngine; + +public: + PyEngineBase(const std::string engineType, IO &io, const std::string &name, + const OpenMode openMode) + : Engine(engineType, io, name, openMode, io.m_MPIComm) + { + } + + virtual ~PyEngineBase() = default; + + using Engine::Init; + virtual void DoWrite(VariableBase *var, pybind11::array values) = 0; + using Engine::Close; + +protected: +// The C++ code calls this implementation which will redirect to the +// python implementation. +#define define_dowrite(T) \ + void DoWrite(Variable &var, const T *values) override \ + { \ + DoWrite(&var, \ + pybind11::array_t(std::accumulate(var.m_Count.begin(), \ + var.m_Count.end(), 0), \ + values)); \ + } + ADIOS2_FOREACH_TYPE_1ARG(define_dowrite) +#undef define_dowrite +}; +} + +#endif /* ADIOS2_ENGINE_PYENGINE_PYENGINEBASE_H_ */ diff --git a/source/adios2/engine/pyengine/PythonEngine.cpp b/source/adios2/engine/pyengine/PythonEngine.cpp new file mode 100644 index 0000000000..b15a6be1d4 --- /dev/null +++ b/source/adios2/engine/pyengine/PythonEngine.cpp @@ -0,0 +1,160 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * PythonEngine.cpp + * + * Created on: Sept 21, 2017 + * Author: Scott Wittenburg + */ + +#include "PythonEngine.h" +#include "PyEngineBase.h" + +#include +#include +#include +#include +#include +#include + +#include "adios2/helper/PythonInterpreter.h" +#include "adios2/helper/PythonModuleHelper.h" + +#include +#include + +namespace adios2 +{ + +/******************************************************************************/ + +struct PythonEngine::Impl +{ + std::string m_PluginName; + pybind11::object enginePyClass; + pybind11::object enginePyObject; + std::shared_ptr eng; +}; + +/******************************************************************************/ + +PythonEngine::PythonEngine(IO &io, const std::string &name, + const OpenMode openMode, MPI_Comm mpiComm) +: Engine("PythonEngine", io, name, openMode, mpiComm), m_Impl(new Impl) +{ + Init(); + m_Impl->enginePyObject = + m_Impl->enginePyClass("PythonEngine", io, name, openMode); + m_Impl->eng = m_Impl->enginePyObject.cast>(); +} + +PythonEngine::~PythonEngine() {} + +void PythonEngine::PerformReads(ReadMode mode) +{ + m_Impl->eng->PerformReads(mode); +} + +void PythonEngine::Release() { m_Impl->eng->Release(); } + +void PythonEngine::Advance(const float timeoutSeconds) +{ + m_Impl->eng->Advance(timeoutSeconds); +} + +void PythonEngine::Advance(const AdvanceMode mode, const float timeoutSeconds) +{ + m_Impl->eng->Advance(mode, timeoutSeconds); +} + +void PythonEngine::AdvanceAsync(const AdvanceMode mode, + AdvanceAsyncCallback callback) +{ + m_Impl->eng->AdvanceAsync(mode, callback); +} + +void PythonEngine::SetCallBack( + std::function)> + callback) +{ + m_Impl->eng->SetCallBack(callback); +} + +void PythonEngine::Close(const int transportIndex) { m_Impl->eng->Close(); } + +void PythonEngine::Init() +{ + auto paramPluginNameIt = m_IO.m_Parameters.find("PluginName"); + if (paramPluginNameIt == m_IO.m_Parameters.end()) + { + throw std::invalid_argument("PythonEngine: PluginName must be " + "specified in engine parameters"); + } + m_Impl->m_PluginName = paramPluginNameIt->second; + + // Get the python engine plugin module name, if provided + std::string *pluginModuleName = nullptr; + auto paramPluginModuleIt = m_IO.m_Parameters.find("PluginModule"); + if (paramPluginModuleIt != m_IO.m_Parameters.end()) + { + pluginModuleName = &(paramPluginModuleIt->second); + } + + // Get the python engine plugin class name + auto paramPluginClassIt = m_IO.m_Parameters.find("PluginClass"); + if (paramPluginClassIt == m_IO.m_Parameters.end()) + { + throw std::invalid_argument("PythonEngine: PluginClass must be " + "specified in engine parameters"); + } + std::string pluginClassName = paramPluginClassIt->second; + + // Initialize python interpreter if it's not already running + adios2::PythonInterpreter::instance().initialize(); + + m_Impl->enginePyClass = adios2::PythonModuleHelper::FindPythonClass( + pluginClassName, pluginModuleName); +} + +#define define(T) \ + void PythonEngine::DoWrite(Variable &variable, const T *values) \ + { \ + m_Impl->eng->DoWrite(variable, values); \ + } \ + void PythonEngine::DoScheduleRead(Variable &variable, const T *values) \ + { \ + m_Impl->eng->DoScheduleRead(variable, values); \ + } \ + void PythonEngine::DoScheduleRead(const std::string &variableName, \ + const T *values) \ + { \ + m_Impl->eng->DoScheduleRead(variableName, values); \ + } +ADIOS2_FOREACH_TYPE_1ARG(define) +#undef define + +void PythonEngine::DoWrite(VariableCompound &variable, const void *values) +{ + std::cout << "PythonEngine::DoWrite(VariableCompound &variable, " + << "const void *values) is not yet implemented" << std::endl; + // m_Impl->eng->DoWrite(variable, values); +} + +#define define(T, L) \ + Variable *PythonEngine::InquireVariable##L(const std::string &name, \ + const bool readIn) \ + { \ + return m_Impl->eng->InquireVariable##L(name, readIn); \ + } +ADIOS2_FOREACH_TYPE_2ARGS(define) +#undef define + +VariableBase *PythonEngine::InquireVariableUnknown(const std::string &name, + const bool readIn) +{ + return m_Impl->eng->InquireVariableUnknown(name, readIn); +} + +} // end namespace adios2 diff --git a/source/adios2/engine/pyengine/PythonEngine.h b/source/adios2/engine/pyengine/PythonEngine.h new file mode 100644 index 0000000000..6350b711fd --- /dev/null +++ b/source/adios2/engine/pyengine/PythonEngine.h @@ -0,0 +1,81 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * PythonEngine.h Support for an engine implemented in python + * + * Created on: Sept 21, 2017 + * Author: Scott Wittenburg + */ + +#ifndef ADIOS2_ENGINE_PYENGINE_PYTHONENGINE_H_ +#define ADIOS2_ENGINE_PYENGINE_PYTHONENGINE_H_ + +#include // for function +#include // for unique_ptr +#include // for string +#include // for add_pointer +#include // for vector + +#include "adios2/ADIOSMPICommOnly.h" +#include "adios2/ADIOSMacros.h" +#include "adios2/ADIOSTypes.h" +#include "adios2/core/Engine.h" +#include "adios2/core/IO.h" +#include "adios2/core/Variable.h" +#include "adios2/core/VariableCompound.h" + +namespace adios2 +{ + +/** A front-end wrapper for an engine implemented in python */ +class PythonEngine : public Engine +{ + +public: + PythonEngine(IO &io, const std::string &name, const OpenMode openMode, + MPI_Comm mpiComm); + virtual ~PythonEngine(); + + void PerformReads(ReadMode mode) override; + void Release() override; + void Advance(const float timeoutSeconds = 0.0) override; + void Advance(const AdvanceMode mode, + const float timeoutSeconds = 0.0) override; + void AdvanceAsync(const AdvanceMode mode, + AdvanceAsyncCallback callback) override; + + void SetCallBack(std::function)> + callback) override; + + void Close(const int transportIndex = -1) override; + +protected: + void Init() override; + +#define declare(T) \ + void DoWrite(Variable &variable, const T *values) override; \ + void DoScheduleRead(Variable &variable, const T *values) override; \ + void DoScheduleRead(const std::string &variableName, const T *values) \ + override; + ADIOS2_FOREACH_TYPE_1ARG(declare) +#undef declare + void DoWrite(VariableCompound &variable, const void *values) override; + +#define declare(T, L) \ + Variable *InquireVariable##L(const std::string &name, \ + const bool readIn) override; + ADIOS2_FOREACH_TYPE_2ARGS(declare) +#undef declare + VariableBase *InquireVariableUnknown(const std::string &name, + const bool readIn) override; + +private: + struct Impl; + std::unique_ptr m_Impl; +}; + +} // end namespace adios2 + +#endif /* ADIOS2_ENGINE_PYENGINE_PYTHONENGINE_H_ */ diff --git a/source/adios2/helper/PythonInterpreter.cpp b/source/adios2/helper/PythonInterpreter.cpp new file mode 100644 index 0000000000..2dcc9e5cc4 --- /dev/null +++ b/source/adios2/helper/PythonInterpreter.cpp @@ -0,0 +1,117 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * PythonInterpreter.cpp + * + * Created on: Sept 27, 2017 + * Author: Scott Wittenburg + */ + +#include "PythonInterpreter.h" + +#include +#include +#include +#include + +#include "adios2sys/SystemTools.hxx" + +#include "adios2/ADIOSConfig.h" + +#if WIN32 +#include +extern __declspec(dllimport) int Py_NoSiteFlag; +#endif + +#if defined(__linux) +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif +#include +#elif defined(__unix) +#include +#endif + +namespace adios2 +{ + +PythonInterpreter PythonInterpreter::m_instance; + +PythonInterpreter &PythonInterpreter::instance() +{ + return PythonInterpreter::m_instance; +} + +PythonInterpreter::PythonInterpreter() : m_embedded(false) +{ + this->initialize(); +} + +PythonInterpreter::~PythonInterpreter() { this->finalize(); } + +bool PythonInterpreter::isInitialized() const +{ + return Py_IsInitialized() != 0; +} + +void PythonInterpreter::initialize() +{ + if (this->isInitialized()) + { + return; + } + + if (adios2sys::SystemTools::HasEnv("PYTHONHOME")) + { + char *pyHomeVar = + const_cast(adios2sys::SystemTools::GetEnv("PYTHONHOME")); + std::cout << "Will use PYTHONHOME (" << pyHomeVar << ") as argument to " + << "Py_SetProgramName" << std::endl; + Py_SetProgramName(pyHomeVar); + } +#if defined(__linux) || defined(__unix) + else + { + // Find the actual library file where the symbol below is defined, + // then use that path to help find Python run-time libraries. + void *handle = dlsym(RTLD_NEXT, "Py_SetProgramName"); + if (handle) + { + Dl_info di; + int ret = dladdr(handle, &di); + if (ret != 0 && di.dli_saddr && di.dli_fname) + { + char *pyHome = const_cast(di.dli_fname); + std::cout << "Will use location of 'Py_SetProgramName' (" + << pyHome << ") as argument to Py_SetProgramName" + << std::endl; + Py_SetProgramName(pyHome); + } + } + } +#endif + + m_embedded = true; + Py_NoSiteFlag = 1; + pybind11::initialize_interpreter(); + +// If, at configure time, we found PYTHON_SITE_PACKAGES, then we +// use it now. This allows us to find modules in your virtual +// environment site-packages directory, or extra modules you may +// have installed with your system python. +#ifdef ADIOS2_PYTHON_SITE_PACKAGES_DIRECTORY + pybind11::module sys = pybind11::module::import("sys"); + sys.attr("path").attr("append")(ADIOS2_PYTHON_SITE_PACKAGES_DIRECTORY); +#endif +} + +void PythonInterpreter::finalize() +{ + if (this->isInitialized()) + { + pybind11::finalize_interpreter(); + m_embedded = false; + } +} +} diff --git a/source/adios2/helper/PythonInterpreter.h b/source/adios2/helper/PythonInterpreter.h new file mode 100644 index 0000000000..f3f155eb54 --- /dev/null +++ b/source/adios2/helper/PythonInterpreter.h @@ -0,0 +1,49 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * PythonInterpreter.h Singleton python interpreter functionality + * + * Created on: Sept 27, 2017 + * Author: Scott Wittenburg + */ + +#ifndef ADIOS2_HELPER_PYTHONINTERPRETER_H_ +#define ADIOS2_HELPER_PYTHONINTERPRETER_H_ + +namespace adios2 +{ + +/** @brief Singleton class for encapsulating the embedded interpreter. + Ensures that the interpreter is initialized/finalized at static + initialization/deinitialization. It is left to the user to ensure + that their python modules can be found on the python path. + */ +class PythonInterpreter +{ +public: + static PythonInterpreter &instance(); + + // Check if python is initialized. + bool isInitialized() const; + + // Initialize the embedded python + void initialize(); + + // Finalize the embedded python. + void finalize(); + + // Returns true if the embedded python session has been initialized. + bool isEmbedded() const { return m_embedded; } + +private: + PythonInterpreter(); + virtual ~PythonInterpreter(); + + static PythonInterpreter m_instance; + + bool m_embedded; +}; +} + +#endif /* ADIOS2_HELPER_PYTHONINTERPRETER_H_ */ diff --git a/source/adios2/helper/PythonModuleHelper.h b/source/adios2/helper/PythonModuleHelper.h new file mode 100644 index 0000000000..8d86d30bb4 --- /dev/null +++ b/source/adios2/helper/PythonModuleHelper.h @@ -0,0 +1,59 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * PythonModuleHelper.h Helper method to find a python class, importing + * it if necessary. + * + * Created on: Oct 02, 2017 + * Author: Scott Wittenburg + */ + +#ifndef ADIOS2_HELPER_PYTHONMODULEHELPER_H_ +#define ADIOS2_HELPER_PYTHONMODULEHELPER_H_ + +#include +#include + +namespace adios2 +{ +namespace PythonModuleHelper +{ +pybind11::object FindPythonClass(const std::string &className, + const std::string *moduleName = nullptr) +{ + pybind11::dict globals = pybind11::globals(); + + if (moduleName != nullptr) + { + pybind11::object moduleObject; + + if (globals.contains((*moduleName).c_str())) + { + moduleObject = globals[(*moduleName).c_str()]; + } + else + { + moduleObject = pybind11::module::import((*moduleName).c_str()); + } + + pybind11::object constructor = moduleObject.attr(className.c_str()); + return constructor; + } + else if (globals.contains(className.c_str())) + { + pybind11::object constructor = globals[className.c_str()]; + return constructor; + } + + // Unable to instantiate the object in this case + throw std::runtime_error("PythonModuleHelper::FindPythonClass " + "Specified class was not present in main " + "module, nor was a module name provided in " + "the parameters. Unable to instantiate " + "python class."); +} +} +} + +#endif /* ADIOS2_HELPER_PYTHONMODULEHELPER_H_ */ diff --git a/source/adios2/helper/adiosType.h b/source/adios2/helper/adiosType.h index 529ab97eed..6dc051d439 100644 --- a/source/adios2/helper/adiosType.h +++ b/source/adios2/helper/adiosType.h @@ -30,6 +30,14 @@ namespace adios2 template inline std::string GetType() noexcept; +/** + * Gets type description from template parameter T + * @return string with type description, sutable for use in variable and class + * names + */ +template +inline std::string GetTypeDescription() noexcept; + /** * Check in types set if "type" is one of the aliases for a certain type, * (e.g. if type = integer is an accepted alias for "int", returning true) diff --git a/source/adios2/helper/adiosType.inl b/source/adios2/helper/adiosType.inl index b81bb59818..0a6b1a6e0a 100644 --- a/source/adios2/helper/adiosType.inl +++ b/source/adios2/helper/adiosType.inl @@ -14,6 +14,8 @@ #error "Inline file should only be included from it's header, never on it's own" #endif +#include "adios2/ADIOSMacros.h" + namespace adios2 { @@ -120,6 +122,15 @@ inline std::string GetType>() noexcept return "long double complex"; } +#define define_impl(T, L) \ + template <> \ + inline std::string GetTypeDescription() noexcept \ + { \ + return ADIOS2_STRINGIFY(L); \ + } +ADIOS2_FOREACH_TYPE_2ARGS(define_impl) +#undef define_impl + template bool IsTypeAlias( const std::string type, diff --git a/testing/CMakeLists.txt b/testing/CMakeLists.txt index 17740cd079..f9af5252df 100644 --- a/testing/CMakeLists.txt +++ b/testing/CMakeLists.txt @@ -7,7 +7,7 @@ include(GoogleTest) if(ADIOS2_HAVE_MPI) set(MPIEXEC_COMMAND - ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} + ${MPIEXEC_EXECUTABLE} -oversubscribe ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ) endif() add_subdirectory(adios2) diff --git a/testing/adios2/bindings/C/CMakeLists.txt b/testing/adios2/bindings/C/CMakeLists.txt index 3e8f220fea..ab680174a5 100644 --- a/testing/adios2/bindings/C/CMakeLists.txt +++ b/testing/adios2/bindings/C/CMakeLists.txt @@ -8,10 +8,7 @@ target_link_libraries(TestBPWriteTypes_c adios2 gtest) if(ADIOS2_HAVE_MPI) target_link_libraries(TestBPWriteTypes_c MPI::MPI_C) - set(extra_test_args - EXEC_WRAPPER - ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} - ) + set(extra_test_args EXEC_WRAPPER ${MPIEXEC_COMMAND}) endif() gtest_add_tests(TARGET TestBPWriteTypes_c ${extra_test_args}) diff --git a/testing/adios2/bindings/python/Adios2PythonTestBase.py b/testing/adios2/bindings/python/Adios2PythonTestBase.py new file mode 100644 index 0000000000..8776048dde --- /dev/null +++ b/testing/adios2/bindings/python/Adios2PythonTestBase.py @@ -0,0 +1,16 @@ +import unittest +import os + + +class Adios2PythonTestBase(unittest.TestCase): + # Convenience method to let python tests know if MPI is enabled + @staticmethod + def isUsingMpi(): + if 'ADIOS2_PYTHON_TESTS_USE_MPI' in os.environ: + return True + return False + + # Expose main method so python tests don't need to import unittest + @staticmethod + def main(): + unittest.main() diff --git a/testing/adios2/bindings/python/CMakeLists.txt b/testing/adios2/bindings/python/CMakeLists.txt index 6286000b81..1a67599786 100644 --- a/testing/adios2/bindings/python/CMakeLists.txt +++ b/testing/adios2/bindings/python/CMakeLists.txt @@ -3,13 +3,27 @@ # accompanying file Copyright.txt for details. #------------------------------------------------------------------------------# -if(NOT ADIOS2_HAVE_MPI) - python_add_test(NAME PythonBPWrite SCRIPT TestBPWriteTypes_nompi.py) -endif() - if(ADIOS2_HAVE_MPI) - set(test_parameters - EXEC_WRAPPER - ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS}) - python_add_test(NAME PythonBPWrite ${test_parameters} SCRIPT TestBPWriteTypes.py) + set(extra_test_args EXEC_WRAPPER ${MPIEXEC_COMMAND}) + set(use_mpi USE_MPI_FOR_PYTHON_TESTS "TRUE") endif() + +python_add_test(NAME PythonBPWrite SCRIPT TestBPWriteTypes.py + ${extra_test_args} ${use_mpi}) +python_add_test(NAME TestOpenPythonEngineFromPython SCRIPT TestOpenPythonEngineFromPython.py + ${extra_test_args} ${use_mpi} + ADDTOPYPATH ${CMAKE_CURRENT_SOURCE_DIR}) +python_add_test(NAME TestImportFromCppOpenFromPython SCRIPT TestImportFromCppOpenFromPython.py + ${extra_test_args} ${use_mpi} + ADDTOPYPATH ${CMAKE_CURRENT_SOURCE_DIR}) +python_add_test(NAME TestOpenInlinePythonEngineFromPython SCRIPT TestOpenInlinePythonEngineFromPython.py + ${extra_test_args} ${use_mpi}) + +add_executable(TestPythonEngineFromCpp TestPythonEngineFromCpp.cpp) +target_link_libraries(TestPythonEngineFromCpp adios2 gtest gtest_main) + +gtest_add_tests(TARGET TestPythonEngineFromCpp ${extra_test_args} TEST_LIST pyEngineTests) + +set_property(TEST ${pyEngineTests} PROPERTY + ENVIRONMENT "PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR}:${ADIOS2_BINARY_DIR}/${CMAKE_INSTALL_PYTHONDIR}:$ENV{PYTHONPATH}" +) diff --git a/testing/adios2/bindings/python/TestBPWriteTypes.py b/testing/adios2/bindings/python/TestBPWriteTypes.py index 36cfe7243a..d1fa56f2ee 100644 --- a/testing/adios2/bindings/python/TestBPWriteTypes.py +++ b/testing/adios2/bindings/python/TestBPWriteTypes.py @@ -8,61 +8,101 @@ # Created on: Feb 2, 2017 # Author: William F Godoy godoywf@ornl.gov - +from Adios2PythonTestBase import Adios2PythonTestBase from adios2NPTypes import SmallTestData -from mpi4py import MPI import adios2 +if Adios2PythonTestBase.isUsingMpi(): + from mpi4py import MPI + + +# +# Create a testcase class with some tests +# +class TestBPWriteTypes(Adios2PythonTestBase): + def testWriteTypes(self): + # Test data + data = SmallTestData() + rank = 0 + size = 1 + + if self.isUsingMpi(): + comm = MPI.COMM_WORLD + rank = comm.Get_rank() + size = comm.Get_size() + adios = adios2.ADIOS(comm) + else: + adios = adios2.ADIOS() + + bpIO = adios.DeclareIO("NPTypes") + + # ADIOS Variable name, shape, start, offset, constant dims + # All local variables + varI8 = bpIO.DefineVariable( + "varI8", [size * data.I8.size], [rank * data.I8.size], + [data.I8.size], True, data.I8.dtype) + varI16 = bpIO.DefineVariable( + "varI16", [size * data.I16.size], [rank * data.I16.size], + [data.I16.size], True, data.I16.dtype) -# Test data -data = SmallTestData() + # varI32 = bpIO.DefineVariable( + # "varI32", [size * data.I32.size], [rank * data.I32.size], + # [data.I32.size], True, data.I32.dtype) -comm = MPI.COMM_WORLD -adios = adios2.ADIOS(comm, adios2.DebugON) + # FIXME: Uncomment below when 64 bit ints work properly + # varI64 = bpIO.DefineVariable( + # "varI64", [], [], [data.I64.size], True, data.I64.dtype) -bpIO = adios.DeclareIO("NPTypes") + varU8 = bpIO.DefineVariable( + "varUI8", [size * data.U8.size], [rank * data.U8.size], + [data.U8.size], True, data.U8.dtype) + varU16 = bpIO.DefineVariable( + "varUI16", [size * data.U16.size], [rank * data.U16.size], + [data.U16.size], True, data.U16.dtype) -# ADIOS Variable name, shape, start, offset, constant dims -# All local variables -varI8 = bpIO.DefineVariable( - "varI8", [], [], [data.I8.size], adios2.ConstantDims) -varI16 = bpIO.DefineVariable( - "varI16", [], [], [data.I16.size], adios2.ConstantDims) -varI32 = bpIO.DefineVariable( - "varI32", [], [], [data.I32.size], adios2.ConstantDims) -varI64 = bpIO.DefineVariable( - "varI64", [], [], [data.I64.size], adios2.ConstantDims) + # varU32 = bpIO.DefineVariable( + # "varUI32", [size * data.U32.size], [rank * data.U32.size], + # [data.U32.size], True, data.U32.dtype) -varU8 = bpIO.DefineVariable( - "varUI8", [], [], [data.U8.size], adios2.ConstantDims) -varU16 = bpIO.DefineVariable( - "varUI16", [], [], [data.U16.size], adios2.ConstantDims) -varU32 = bpIO.DefineVariable( - "varUI32", [], [], [data.U32.size], adios2.ConstantDims) -varU64 = bpIO.DefineVariable( - "varUI64", [], [], [data.U64.size], adios2.ConstantDims) + # FIXME: Uncomment below when 64 bit ints work properly + # varU64 = bpIO.DefineVariable( + # "varUI64", [], [], [data.U64.size], True, data.U64.dtype) -varR32 = bpIO.DefineVariable( - "varR32", [], [], [data.R32.size], adios2.ConstantDims) + varR32 = bpIO.DefineVariable( + "varR32", [size * data.R32.size], [rank * data.R32.size], + [data.R32.size], True, data.R32.dtype) -varR64 = bpIO.DefineVariable( - "varR64", [], [], [data.R64.size], adios2.ConstantDims) + varR64 = bpIO.DefineVariable( + "varR64", [size * data.R64.size], [rank * data.R64.size], + [data.R64.size], True, data.R64.dtype) + # ADIOS Engine + bpFileWriter = bpIO.Open("npTypes.bp", adios2.OpenMode.Write) -# ADIOS Engine -bpFileWriter = bpIO.Open("npTypes.bp", adios2.OpenModeWrite) + bpFileWriter.Write(varI8, data.I8) + bpFileWriter.Write(varI16, data.I16) -bpFileWriter.Write(varI8, data.I8) -bpFileWriter.Write(varI16, data.I16) -bpFileWriter.Write(varI32, data.I32) -bpFileWriter.Write(varI64, data.I64) + # bpFileWriter.Write(varI32, data.I32) -bpFileWriter.Write(varU8, data.U8) -bpFileWriter.Write(varU16, data.U16) -bpFileWriter.Write(varU32, data.U32) -bpFileWriter.Write(varU64, data.U64) + # FIXME: Uncomment below when 64 bit ints work properly + # bpFileWriter.Write(varI64, data.I64) -bpFileWriter.Write(varR32, data.R32) -bpFileWriter.Write(varR64, data.R64) + bpFileWriter.Write(varU8, data.U8) + bpFileWriter.Write(varU16, data.U16) -bpFileWriter.Close() + # bpFileWriter.Write(varU32, data.U32) + + # FIXME: Uncomment below when 64 bit ints work properly + # bpFileWriter.Write(varU64, data.U64) + + bpFileWriter.Write(varR32, data.R32) + bpFileWriter.Write(varR64, data.R64) + + bpFileWriter.Close() + + +# +# Trigger the tests +# +if __name__ == '__main__': + Adios2PythonTestBase.main() diff --git a/testing/adios2/bindings/python/TestBPWriteTypes_nompi.py b/testing/adios2/bindings/python/TestBPWriteTypes_nompi.py deleted file mode 100644 index 842adacd8c..0000000000 --- a/testing/adios2/bindings/python/TestBPWriteTypes_nompi.py +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env python - -# -# Distributed under the OSI-approved Apache License, Version 2.0. See -# accompanying file Copyright.txt for details. -# -# TestBPWriteTypes.py: test Python numpy types in ADIOS2 File Write -# Created on: Feb 2, 2017 -# Author: William F Godoy godoywf@ornl.gov - - -from adios2NPTypes import SmallTestData -import adios2 - - -# Test data -data = SmallTestData() - -adios = adios2.ADIOS(adios2.DebugON) - -bpIO = adios.DeclareIO("NPTypes") - -# ADIOS Variable name, shape, start, offset, constant dims -# All local variables -varI8 = bpIO.DefineVariable( - "varI8", [], [], [data.I8.size], adios2.ConstantDims) -varI16 = bpIO.DefineVariable( - "varI16", [], [], [data.I16.size], adios2.ConstantDims) -varI32 = bpIO.DefineVariable( - "varI32", [], [], [data.I32.size], adios2.ConstantDims) -varI64 = bpIO.DefineVariable( - "varI64", [], [], [data.I64.size], adios2.ConstantDims) - -varU8 = bpIO.DefineVariable( - "varUI8", [], [], [data.U8.size], adios2.ConstantDims) -varU16 = bpIO.DefineVariable( - "varUI16", [], [], [data.U16.size], adios2.ConstantDims) -varU32 = bpIO.DefineVariable( - "varUI32", [], [], [data.U32.size], adios2.ConstantDims) -varU64 = bpIO.DefineVariable( - "varUI64", [], [], [data.U64.size], adios2.ConstantDims) - -varR32 = bpIO.DefineVariable( - "varR32", [], [], [data.R32.size], adios2.ConstantDims) - -varR64 = bpIO.DefineVariable( - "varR64", [], [], [data.R64.size], adios2.ConstantDims) - - -# ADIOS Engine -bpFileWriter = bpIO.Open("npTypes.bp", adios2.OpenModeWrite) - -bpFileWriter.Write(varI8, data.I8) -bpFileWriter.Write(varI16, data.I16) -bpFileWriter.Write(varI32, data.I32) -bpFileWriter.Write(varI64, data.I64) - -bpFileWriter.Write(varU8, data.U8) -bpFileWriter.Write(varU16, data.U16) -bpFileWriter.Write(varU32, data.U32) -bpFileWriter.Write(varU64, data.U64) - -bpFileWriter.Write(varR32, data.R32) -bpFileWriter.Write(varR64, data.R64) - -bpFileWriter.Close() diff --git a/testing/adios2/bindings/python/TestImportFromCppOpenFromPython.py b/testing/adios2/bindings/python/TestImportFromCppOpenFromPython.py new file mode 100644 index 0000000000..af38d6a326 --- /dev/null +++ b/testing/adios2/bindings/python/TestImportFromCppOpenFromPython.py @@ -0,0 +1,61 @@ +from Adios2PythonTestBase import Adios2PythonTestBase + +import numpy as np +import adios2 + +if Adios2PythonTestBase.isUsingMpi(): + from mpi4py import MPI + + +# +# Create a testcase class with some tests +# +class TestOpenPythonEngineFromPython(Adios2PythonTestBase): + def testCreateEngine(self): + rank = 0 + size = 1 + + if self.isUsingMpi(): + # MPI + comm = MPI.COMM_WORLD + rank = comm.Get_rank() + size = comm.Get_size() + adios = adios2.ADIOS(comm) + else: + adios = adios2.ADIOS() + + # User data + myArray = np.array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9], dtype=np.uint8) + Nx = myArray.size + + # ADIOS IO + bpIO = adios.DeclareIO("BPFile_N2N") + + # ADIOS Variable name, shape, start, offset, constant dims + ioArray = bpIO.DefineVariable( + "bpArray", [size * Nx], [rank * Nx], [Nx], True, myArray.dtype) + + # Engine derived class, spawned to start IO operations + bpIO.EngineType = "PythonEngine" + + # Since we did not import the TestPythonEngine class directly, we must + # provide the "PluginModule" parameter indicating the module containing + # the engine class we want. + bpIO.Parameters = { + "PluginName": "DoNotReallyCare", + "PluginModule": "TestPythonEngine", + "PluginClass": "TestPythonEngine" + } + + # ADIOS Engine + bpFileWriter = bpIO.Open("npArray.bp", adios2.OpenMode.Write) + bpFileWriter.Write(ioArray, myArray) + + bpFileWriter.Close() + + +# +# Trigger the tests +# +if __name__ == '__main__': + Adios2PythonTestBase.main() diff --git a/testing/adios2/bindings/python/TestOpenInlinePythonEngineFromPython.py b/testing/adios2/bindings/python/TestOpenInlinePythonEngineFromPython.py new file mode 100644 index 0000000000..aca1c0af31 --- /dev/null +++ b/testing/adios2/bindings/python/TestOpenInlinePythonEngineFromPython.py @@ -0,0 +1,117 @@ +from Adios2PythonTestBase import Adios2PythonTestBase + +import numpy +import adios2 + +if Adios2PythonTestBase.isUsingMpi(): + from mpi4py import MPI + +MODULE_LEVEL_VARIABLE = 0 + + +# +# Define a new engine type in python +# +class TestInlinePythonEngine(adios2.Engine): + def __init__(self, engineType, io, name, openMode): + adios2.Engine.__init__(self, engineType, io, name, openMode) + + # Calls Engine::Init() if no Init() method is defined here + self.Init() + + # def Init(self): + # print('Inside TestPythonEngine Init()') + + def DoWrite(self, variable, values): + global MODULE_LEVEL_VARIABLE + MODULE_LEVEL_VARIABLE += 1 + print('Inside TestInlinePythonEngine.DoWrite') + print(variable) + print(values) + + def Advance(self, timeoutSeconds): + print('Inside TestInlinePythonEngine.Advance') + + def Close(self, transportIndex): + print('Inside TestInlinePythonEngine.Close') + + +# +# Create a testcase class with some tests +# +class TestOpenInlinePythonEngineFromPython(Adios2PythonTestBase): + def testCreateEngine(self): + global MODULE_LEVEL_VARIABLE + self.assertEqual(MODULE_LEVEL_VARIABLE, 0) + MODULE_LEVEL_VARIABLE += 1 + self.assertEqual(MODULE_LEVEL_VARIABLE, 1) + + rank = 0 + size = 1 + + if self.isUsingMpi(): + # MPI + comm = MPI.COMM_WORLD + rank = comm.Get_rank() + size = comm.Get_size() + adios = adios2.ADIOS(comm) + else: + adios = adios2.ADIOS() + + # User data + myArray = numpy.array([0., 1., 2., 3., 4., 5., 6., 7., 8., 9.]) + Nx = myArray.size + + # ADIOS IO + bpIO = adios.DeclareIO("BPFile_N2N") + + # ADIOS Variable name, shape, start, offset, constant dims + ioArray = bpIO.DefineVariable( + "bpArray", [size * Nx], [rank * Nx], [Nx], True, myArray.dtype) + + # Engine derived class, spawned to start IO operations + bpIO.EngineType = "PythonEngine" + bpIO.Parameters = { + "PluginName": "FirstPythonPlugin", + "PluginClass": "TestInlinePythonEngine" + } + + # ADIOS Engine + bpFileWriter = bpIO.Open("npArray.bp", adios2.OpenMode.Write) + bpFileWriter.Write(ioArray, myArray) + + self.assertEqual(MODULE_LEVEL_VARIABLE, 2) + + bpFileWriter.Close() + + def testCreateEngineWithWrongName(self): + if self.isUsingMpi(): + # MPI + comm = MPI.COMM_WORLD + adios = adios2.ADIOS(comm) + else: + adios = adios2.ADIOS() + + bpIO = adios.DeclareIO("BPFile_N2N") + + bpIO.EngineType = "PythonEngine" + bpIO.Parameters = { + "PluginName": "FirstPythonPlugin", + "PluginClass": "MissingEngine" + } + + bpFileWriter = None + + # Make sure exception is raised due to wrong engine name + with self.assertRaises(RuntimeError): + bpFileWriter = bpIO.Open("npArray.bp", adios2.OpenMode.Write) + + if bpFileWriter: + bpFileWriter.Close() + + +# +# Trigger the tests +# +if __name__ == '__main__': + Adios2PythonTestBase.main() diff --git a/testing/adios2/bindings/python/TestOpenPythonEngineFromPython.py b/testing/adios2/bindings/python/TestOpenPythonEngineFromPython.py new file mode 100644 index 0000000000..d389599422 --- /dev/null +++ b/testing/adios2/bindings/python/TestOpenPythonEngineFromPython.py @@ -0,0 +1,62 @@ +from Adios2PythonTestBase import Adios2PythonTestBase + +import numpy +import adios2 + +# Import this module so the IO factory can find the engine class it needs +from TestPythonEngine import TestPythonEngine + +if Adios2PythonTestBase.isUsingMpi(): + from mpi4py import MPI + + +# +# Create a testcase class with some tests +# +class TestOpenPythonEngineFromPython(Adios2PythonTestBase): + def testCreateEngine(self): + rank = 0 + size = 1 + + if self.isUsingMpi(): + # MPI + comm = MPI.COMM_WORLD + rank = comm.Get_rank() + size = comm.Get_size() + adios = adios2.ADIOS(comm) + else: + adios = adios2.ADIOS() + + # User data + myArray = numpy.array([0., 1., 2., 3., 4., 5., 6., 7., 8., 9.]) + Nx = myArray.size + + # ADIOS IO + bpIO = adios.DeclareIO("BPFile_N2N") + + # ADIOS Variable name, shape, start, offset, constant dims + ioArray = bpIO.DefineVariable( + "bpArray", [size * Nx], [rank * Nx], [Nx], True, myArray.dtype) + + # Engine derived class, spawned to start IO operations + bpIO.EngineType = "PythonEngine" + + # Since we import the TestPythonEngine class directly, we should not + # need to provide a "PluginModule" parameter + bpIO.Parameters = { + "PluginName": "DoNotReallyCare", + "PluginClass": "TestPythonEngine" + } + + # ADIOS Engine + bpFileWriter = bpIO.Open("npArray.bp", adios2.OpenMode.Write) + bpFileWriter.Write(ioArray, myArray) + + bpFileWriter.Close() + + +# +# Trigger the tests +# +if __name__ == '__main__': + Adios2PythonTestBase.main() diff --git a/testing/adios2/bindings/python/TestPythonEngine.py b/testing/adios2/bindings/python/TestPythonEngine.py new file mode 100644 index 0000000000..7c25e29805 --- /dev/null +++ b/testing/adios2/bindings/python/TestPythonEngine.py @@ -0,0 +1,28 @@ + +import adios2 +import numpy as np + +print('Inside TestPythonEngine module') + + +class TestPythonEngine(adios2.Engine): + def __init__(self, engineType, io, name, openMode): + adios2.Engine.__init__(self, engineType, io, name, openMode) + print('Inside TestPythonEngine python ctor, just called parent ctor') + + # Calls Engine::Init() if no Init() method is defined here + self.Init() + + # def Init(self): + # print('Inside TestPythonEngine Init()') + + def DoWrite(self, variable, values): + print('Inside TestPythonEngine.DoWrite') + print(variable) + print(values) + + def Advance(self, timeoutSeconds): + print('Inside TestPythonEngine.Advance') + + def Close(self, transportIndex): + print('Inside TestPythonEngine.Close') diff --git a/testing/adios2/bindings/python/TestPythonEngineFromCpp.cpp b/testing/adios2/bindings/python/TestPythonEngineFromCpp.cpp new file mode 100644 index 0000000000..3c5cd8bef6 --- /dev/null +++ b/testing/adios2/bindings/python/TestPythonEngineFromCpp.cpp @@ -0,0 +1,101 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + */ + +#include +#include + +/** + * Test that with the proper plugin module and class specified, we + * can successfully create the python engine from C++. + */ +TEST(PythonEngineTest, CreatePythonEngineFromCPlusPlus) +{ + int mpiRank = 0, mpiSize = 1; + + // Application variable + std::vector myDoubles = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; + const std::size_t Nx = myDoubles.size(); + +#ifdef ADIOS2_HAVE_MPI + MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); + MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + adios2::ADIOS adios(MPI_COMM_WORLD, adios2::DebugON); +#else + adios2::ADIOS adios(adios2::DebugON); +#endif + + adios2::IO &io = adios.DeclareIO("PythonPluginIO"); + + adios2::Variable &var = io.DefineVariable( + "data", {mpiSize * Nx}, {mpiRank * Nx}, {Nx}, adios2::ConstantDims); + + io.SetEngine("PythonEngine"); + io.SetParameters({{"PluginName", "TestPythonPlugin"}, + {"PluginModule", "TestPythonEngine"}, + {"PluginClass", "TestPythonEngine"}}); + + std::shared_ptr writer; + EXPECT_NO_THROW( + { writer = io.Open("TestPythonPlugin", adios2::OpenMode::Write); }); + + ASSERT_NE(writer.get(), nullptr); + + /** Write variable for buffering */ + writer->Write(var, myDoubles.data()); + + writer->Close(); +} + +/** + * Test that without the proper plugin module specified, IO fails to + * instantiate the engine and throws the correct exception type. + */ +TEST(PythonEngineTest, CreatePythonEngineFromCPlusPlusExpectImportError) +{ + int mpiRank = 0, mpiSize = 1; + +#ifdef ADIOS2_HAVE_MPI + MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); + MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + adios2::ADIOS adios(MPI_COMM_WORLD, adios2::DebugON); +#else + adios2::ADIOS adios(adios2::DebugON); +#endif + + adios2::IO &io = adios.DeclareIO("PythonPluginIO"); + + io.SetEngine("PythonEngine"); + + // By not specifying "PluginModule" parameter, we should fail during + // the "Open" call... + io.SetParameters({{"PluginName", "TestPythonPlugin"}, + {"PluginClass", "TestPythonEngine"}}); + + std::shared_ptr writer; + + ASSERT_THROW( + { writer = io.Open("TestPythonPlugin", adios2::OpenMode::Write); }, + std::runtime_error); +} + +//****************************************************************************** +// main +//****************************************************************************** + +int main(int argc, char **argv) +{ +#ifdef ADIOS2_HAVE_MPI + MPI_Init(nullptr, nullptr); +#endif + + ::testing::InitGoogleTest(&argc, argv); + int result = RUN_ALL_TESTS(); + +#ifdef ADIOS2_HAVE_MPI + MPI_Finalize(); +#endif + + return result; +} diff --git a/thirdparty/pybind11/pybind11/include/pybind11/numpy.h b/thirdparty/pybind11/pybind11/include/pybind11/numpy.h index 55bb816984..abf389163b 100644 --- a/thirdparty/pybind11/pybind11/include/pybind11/numpy.h +++ b/thirdparty/pybind11/pybind11/include/pybind11/numpy.h @@ -471,6 +471,11 @@ class dtype : public object { return detail::array_descriptor_proxy(m_ptr)->kind; } + /// + char type() const { + return detail::array_descriptor_proxy(m_ptr)->type; + } + private: static object _dtype_from_pep3118() { static PyObject *obj = module::import("numpy.core._internal")