From 85488ca758832b866cae1d7563c50337f049dc9e Mon Sep 17 00:00:00 2001 From: Chuck Atkins Date: Tue, 26 Sep 2017 17:04:22 -0400 Subject: [PATCH 01/24] Initial re-work of ADIOS, IO, and Variable python bindings. --- bindings/python/ADIOSPy.cpp | 46 ---------- bindings/python/ADIOSPy.h | 48 ---------- bindings/python/CMakeLists.txt | 28 +++--- bindings/python/EnginePy.cpp | 67 -------------- bindings/python/EnginePy.h | 58 ------------ bindings/python/EnginePy.inl | 43 --------- bindings/python/IOPy.cpp | 76 --------------- bindings/python/IOPy.h | 59 ------------ bindings/python/PyADIOS.cpp | 55 +++++++++++ bindings/python/PyIO.cpp | 34 +++++++ bindings/python/PyVariable.cpp | 34 +++++++ bindings/python/VariablePy.cpp | 47 ---------- bindings/python/VariablePy.h | 51 ----------- bindings/python/adios2py.cpp | 16 ++++ bindings/python/adiosPyFunctions.cpp | 42 --------- bindings/python/adiosPyFunctions.h | 41 --------- bindings/python/adiosPyTypes.h | 32 ------- bindings/python/gluePyBind11.cpp | 132 --------------------------- 18 files changed, 154 insertions(+), 755 deletions(-) delete mode 100644 bindings/python/ADIOSPy.cpp delete mode 100644 bindings/python/ADIOSPy.h delete mode 100644 bindings/python/EnginePy.cpp delete mode 100644 bindings/python/EnginePy.h delete mode 100644 bindings/python/EnginePy.inl delete mode 100644 bindings/python/IOPy.cpp delete mode 100644 bindings/python/IOPy.h create mode 100644 bindings/python/PyADIOS.cpp create mode 100644 bindings/python/PyIO.cpp create mode 100644 bindings/python/PyVariable.cpp delete mode 100644 bindings/python/VariablePy.cpp delete mode 100644 bindings/python/VariablePy.h create mode 100644 bindings/python/adios2py.cpp delete mode 100644 bindings/python/adiosPyFunctions.cpp delete mode 100644 bindings/python/adiosPyFunctions.h delete mode 100644 bindings/python/adiosPyTypes.h delete mode 100644 bindings/python/gluePyBind11.cpp diff --git a/bindings/python/ADIOSPy.cpp b/bindings/python/ADIOSPy.cpp deleted file mode 100644 index 7f196f40a1..0000000000 --- a/bindings/python/ADIOSPy.cpp +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * ADIOSPy.cpp - * - * Created on: Mar 13, 2017 - * Author: William F Godoy godoywf@ornl.gov - */ - -#include "ADIOSPy.h" - -#include "adiosPyTypes.h" - -#include "adios2/ADIOSMPI.h" - -namespace adios2 -{ - -ADIOSPy::ADIOSPy(const std::string configFile, MPI_Comm mpiComm, - const bool debugMode) -: m_DebugMode(debugMode), - m_ADIOS(std::make_shared(configFile, mpiComm, debugMode)) -{ -} - -ADIOSPy::ADIOSPy(MPI_Comm mpiComm, const bool debugMode) -: ADIOSPy("", mpiComm, debugMode) -{ -} - -ADIOSPy::ADIOSPy(const std::string configFile, const bool debugMode) -: ADIOSPy(configFile, MPI_COMM_SELF, debugMode) -{ -} - -ADIOSPy::ADIOSPy(const bool debugMode) : ADIOSPy("", MPI_COMM_SELF, debugMode) -{ -} - -IOPy ADIOSPy::DeclareIO(const std::string name) -{ - return IOPy(m_ADIOS->DeclareIO(name), m_DebugMode); -} - -} // end namespace adios diff --git a/bindings/python/ADIOSPy.h b/bindings/python/ADIOSPy.h deleted file mode 100644 index d3c9f5e85d..0000000000 --- a/bindings/python/ADIOSPy.h +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * ADIOSPy.h python binding to ADIOS class - * - * Created on: Mar 13, 2017 - * Author: William F Godoy godoywf@ornl.gov - */ - -#ifndef ADIOS2_BINDINGS_PYTHON_SOURCE_ADIOSPY_H_ -#define ADIOS2_BINDINGS_PYTHON_SOURCE_ADIOSPY_H_ - -/// \cond EXCLUDE_FROM_DOXYGEN -#include //std::shared_ptr -#include -/// \endcond - -#include - -#include "IOPy.h" -#include "adios2/ADIOSMPICommOnly.h" - -namespace adios2 -{ - -class ADIOSPy -{ - -public: - ADIOSPy(const std::string configFile, MPI_Comm mpiComm, - const bool debugMode); - ADIOSPy(MPI_Comm mpiComm, const bool debugMode); - ADIOSPy(const std::string configFile, const bool debugMode); - ADIOSPy(const bool debugMode); - - ~ADIOSPy() = default; - - IOPy DeclareIO(const std::string name); - -private: - const bool m_DebugMode; - std::shared_ptr m_ADIOS; -}; - -} // end namespace adios - -#endif /* BINDINGS_PYTHON_SOURCE_ADIOSPY_H_ */ diff --git a/bindings/python/CMakeLists.txt b/bindings/python/CMakeLists.txt index a6ced9fc20..c1f9f384c8 100644 --- a/bindings/python/CMakeLists.txt +++ b/bindings/python/CMakeLists.txt @@ -1,17 +1,19 @@ pybind11_add_module(adios2py MODULE - ADIOSPy.h - ADIOSPy.cpp - adiosPyFunctions.h - adiosPyFunctions.cpp - adiosPyTypes.h - EnginePy.h - EnginePy.inl - EnginePy.cpp - gluePyBind11.cpp - IOPy.h - IOPy.cpp - VariablePy.h - VariablePy.cpp + adios2py.cpp + PyADIOS.cpp + PyIO.cpp + PyVariable.cpp +# adiosPyFunctions.h +# adiosPyFunctions.cpp +# adiosPyTypes.h +# EnginePy.h +# EnginePy.inl +# EnginePy.cpp +# gluePyBind11.cpp +# IOPy.h +# IOPy.cpp +# VariablePy.h +# VariablePy.cpp ) target_link_libraries(adios2py PRIVATE adios2) if(ADIOS2_HAVE_MPI) diff --git a/bindings/python/EnginePy.cpp b/bindings/python/EnginePy.cpp deleted file mode 100644 index 3a203a5d65..0000000000 --- a/bindings/python/EnginePy.cpp +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * EnginePy.cpp - * - * Created on: Mar 15, 2017 - * Author: wgodoy - */ - -#include "EnginePy.h" - -#include "adiosPyFunctions.h" - -namespace adios2 -{ - -EnginePy::EnginePy(IO &io, const std::string &name, const OpenMode openMode, - MPI_Comm mpiComm) -: m_IO(io), m_Engine(m_IO.Open(name, openMode, mpiComm)), - m_DebugMode(m_IO.m_DebugMode) -{ -} - -void EnginePy::Write(VariablePy &variable, const pyArray &array) -{ - if (variable.m_IsDefined) - { - // do nothing, not supporting compound types in Python, yet - } -#define declare_type(T) \ - else if (pybind11::isinstance>(array)) \ - { \ - DefineVariableInIO(variable); \ - } - ADIOS2_FOREACH_TYPE_1ARG(declare_type) -#undef declare_type - - if (!variable.m_IsDefined) - { - if (m_DebugMode) - { - throw std::runtime_error("ERROR: variable " + variable.m_Name + - " couldn't not be created in IO " + - m_IO.m_Name + " , in call to Write\n"); - } - } -#define declare_type(T) \ - else if (pybind11::isinstance>(array)) \ - { \ - WriteInIO(variable, array); \ - } - ADIOS2_FOREACH_TYPE_1ARG(declare_type) -#undef declare_type -} - -void EnginePy::Advance(const float timeoutSeconds) -{ - m_Engine->Advance(timeoutSeconds); -} - -void EnginePy::Close(const int transportIndex) -{ - m_Engine->Close(transportIndex); -} - -} // end namespace adios diff --git a/bindings/python/EnginePy.h b/bindings/python/EnginePy.h deleted file mode 100644 index 10046b3a65..0000000000 --- a/bindings/python/EnginePy.h +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * EnginePy.h - * - * Created on: Mar 15, 2017 - * Author: William F Godoy godoywf@ornl.gov - */ - -#ifndef ADIOS2_BINDINGS_PYTHON_SOURCE_ENGINEPY_H_ -#define ADIOS2_BINDINGS_PYTHON_SOURCE_ENGINEPY_H_ - -/// \cond EXCLUDE_FROM_DOXYGEN -#include //std::shared_ptr -#include -/// \endcond - -#include - -#include "VariablePy.h" -#include "adiosPyTypes.h" //pyArray - -namespace adios2 -{ - -class EnginePy -{ - -public: - EnginePy(IO &io, const std::string &name, const OpenMode openMode, - MPI_Comm mpiComm); - - ~EnginePy() = default; - - void Write(VariablePy &variable, const pyArray &array); - - void Advance(const float timeoutSeconds = 0.); - - void Close(const int transportIndex = -1); - -private: - IO &m_IO; - std::shared_ptr m_Engine; - const bool m_DebugMode; - - template - void DefineVariableInIO(VariablePy &variable); - - template - void WriteInIO(VariablePy &variable, const pyArray &array); -}; - -} // end namespace adios - -#include "EnginePy.inl" - -#endif /* BINDINGS_PYTHON_SOURCE_ENGINEPY_H_ */ diff --git a/bindings/python/EnginePy.inl b/bindings/python/EnginePy.inl deleted file mode 100644 index eea3e71aa3..0000000000 --- a/bindings/python/EnginePy.inl +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * EnginePy.tcc - * - * Created on: Jun 8, 2017 - * Author: William F Godoy godoywf@ornl.gov - */ - -#ifndef ADIOS2_BINDINGS_PYTHON_SOURCE_ENGINEPY_INL_ -#define ADIOS2_BINDINGS_PYTHON_SOURCE_ENGINEPY_INL_ -#ifndef ADIOS2_BINDINGS_PYTHON_SOURCE_ENGINEPY_H_ -#error "Inline file should only be included from it's header, never on it's own" -#endif - -#include "adiosPyFunctions.h" - -namespace adios2 -{ - -template -void EnginePy::DefineVariableInIO(VariablePy &variable) -{ - auto &var = m_IO.DefineVariable( - variable.m_Name, PyListToDims(variable.m_Shape), - PyListToDims(variable.m_Start), PyListToDims(variable.m_Count), - variable.m_IsConstantDims); - - variable.m_VariableBase = &var; - variable.m_IsDefined = true; -} - -template -void EnginePy::WriteInIO(VariablePy &variable, const pyArray &array) -{ - m_Engine->Write(*dynamic_cast *>(variable.m_VariableBase), - reinterpret_cast(array.data())); -} - -} // end namespace adios - -#endif /* BINDINGS_PYTHON_SOURCE_ENGINEPY_TCC_ */ diff --git a/bindings/python/IOPy.cpp b/bindings/python/IOPy.cpp deleted file mode 100644 index 1ae9475b1f..0000000000 --- a/bindings/python/IOPy.cpp +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * IOPy.cpp - * - * Created on: Mar 14, 2017 - * Author: William F Godoy godoywf@ornl.gov - */ - -#include "IOPy.h" - -namespace adios2 -{ - -IOPy::IOPy(IO &io, const bool debugMode) : m_IO(io), m_DebugMode(debugMode) -{ - m_IO.m_HostLanguage = "Python"; -} - -void IOPy::SetEngine(const std::string engine) { m_IO.SetEngine(engine); } - -void IOPy::SetParameters(const pyKwargs &kwargs) noexcept -{ - m_IO.SetParameters(KwargsToParams(kwargs)); -} - -unsigned int IOPy::AddTransport(const std::string type, - const pyKwargs &kwargs) noexcept -{ - return m_IO.AddTransport(type, KwargsToParams(kwargs)); -} - -VariablePy &IOPy::DefineVariable(const std::string &name, const pyList shape, - const pyList start, const pyList count, - const bool isConstantDims) -{ - if (m_DebugMode) - { - if (m_Variables.count(name) == 1) - { - throw std::invalid_argument("ERROR: variable " + name + - " already exists, use GetVariable, in " - "call to DefineVariable\n"); - } - } - - auto itVariableEmplace = - m_Variables.emplace(name, VariablePy(name, shape, start, count, - isConstantDims, m_DebugMode)); - return itVariableEmplace.first->second; -} - -VariablePy &IOPy::GetVariable(const std::string &name) -{ - auto itVariable = m_Variables.find(name); - - if (m_DebugMode) - { - if (itVariable == m_Variables.end()) - { - throw std::invalid_argument("ERROR: variable " + name + - " doesn't exist, in " - "call to GetVariable\n"); - } - } - return itVariable->second; -} - -EnginePy IOPy::Open(const std::string &name, const int openMode) -{ - return EnginePy(m_IO, name, static_cast(openMode), - m_IO.m_MPIComm); -} - -} // end namespace adios diff --git a/bindings/python/IOPy.h b/bindings/python/IOPy.h deleted file mode 100644 index c56255c51b..0000000000 --- a/bindings/python/IOPy.h +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * IOPy.h - * - * Created on: Mar 14, 2017 - * Author: William F Godoy godoywf@ornl.gov - */ - -#ifndef ADIOS2_BINDINGS_PYTHON_SOURCE_IOPY_H_ -#define ADIOS2_BINDINGS_PYTHON_SOURCE_IOPY_H_ - -/// \cond EXCLUDE_FROM_DOXYGEN -#include -/// \endcond - -#include "EnginePy.h" -#include "adiosPyTypes.h" - -namespace adios2 -{ - -class IOPy -{ - -public: - IO &m_IO; - const bool m_DebugMode; - - IOPy(IO &io, const bool debugMode); - - ~IOPy() = default; - - void SetEngine(const std::string engineType); - - void SetParameters(const pyKwargs &kwargs) noexcept; - unsigned int AddTransport(const std::string type, - const pyKwargs &kwargs) noexcept; - - VariablePy &DefineVariable(const std::string &name, const pyList shape, - const pyList start, const pyList count, - const bool isConstantDims); - - VariablePy &GetVariable(const std::string &name); - - EnginePy Open(const std::string &name, const int openMode); - -private: - /** - * Extra map needed as Variables are not created in ADIOS at - * DefineVariable, but until Write when type is known from numpy - */ - std::map m_Variables; -}; - -} // end namespace adios - -#endif /* BINDINGS_PYTHON_SOURCE_IOPY_H_ */ diff --git a/bindings/python/PyADIOS.cpp b/bindings/python/PyADIOS.cpp new file mode 100644 index 0000000000..06f2ea3cab --- /dev/null +++ b/bindings/python/PyADIOS.cpp @@ -0,0 +1,55 @@ +#include + +#include + +#ifdef ADIOS2_HAVE_MPI +#include +#endif + +namespace adios2 +{ +template void GeneratePythonBindings(pybind11::module &m); + +template<> +void GeneratePythonBindings(pybind11::module &m) +{ + pybind11::class_(m, "ADIOS") + .def(pybind11::init([](const bool debug) { + auto a = new ADIOS(debug); + a->m_HostLanguage = "Python"; + return a; + }), + pybind11::arg("debug") = true) + .def(pybind11::init([](const std::string config, const bool debug) { + auto a = new ADIOS(config, debug); + a->m_HostLanguage = "Python"; + return a; + }), + pybind11::arg("config").none(false), + pybind11::arg("debug") = true) +#ifdef ADIOS2_HAVE_MPI + .def(pybind11::init([](pybind11::object comm, const bool debug) { + import_mpi4py__MPI(); + auto a = new ADIOS(*PyMPIComm_Get(comm.ptr()), debug); + a->m_HostLanguage = "Python"; + return a; + }), + pybind11::arg("comm").none(false), + pybind11::arg("debug") = true) + .def(pybind11::init([](pybind11::object comm, const std::string config, + const bool debug) { + import_mpi4py__MPI(); + auto a = new ADIOS(config, *PyMPIComm_Get(comm.ptr()), debug); + a->m_HostLanguage = "Python"; + return a; + }), + pybind11::arg("comm").none(false), + pybind11::arg("config").none(false), + pybind11::arg("debug") = true) +#endif + .def_readonly("HostLanguage", &ADIOS::m_HostLanguage) + .def("DeclareIO", &ADIOS::DeclareIO) + .def("GetIO", &ADIOS::GetIO); +} + +} // end namespace adios2 diff --git a/bindings/python/PyIO.cpp b/bindings/python/PyIO.cpp new file mode 100644 index 0000000000..b7cf1d3e75 --- /dev/null +++ b/bindings/python/PyIO.cpp @@ -0,0 +1,34 @@ +#include + +#include + +namespace adios2 +{ +template void GeneratePythonBindings(pybind11::module &m); + +template<> +void GeneratePythonBindings(pybind11::module &m) +{ + pybind11::class_(m, "IO") + .def_property("EngineType", &adios2::IO::SetEngine, + &adios2::IO::GetEngine) + .def_property("Parameters", &adios2::IO::SetParameters, + &adios2::IO::GetParameters) + .def("SetIOMode", &adios2::IO::SetIOMode) + .def("AddTransport", &adios2::IO::AddTransport) + .def_property_readonly("TransportParameters", + &adios2::IO::GetTransportParameters) + .def("DefineVariable", + [](adios2::IO &io, std::string &name, adios2::Dims &shape, + adios2::Dims &start, adios2::Dims &count, bool constantDims, + int t) { + if (t == 1) + return pybind11::cast(io.DefineVariable( + name, shape, start, count, constantDims)); + else + return pybind11::cast(io.DefineVariable( + name, shape, start, count, constantDims)); + }); +} + +} // end namespace adios2 diff --git a/bindings/python/PyVariable.cpp b/bindings/python/PyVariable.cpp new file mode 100644 index 0000000000..0466822417 --- /dev/null +++ b/bindings/python/PyVariable.cpp @@ -0,0 +1,34 @@ +#include + +#include +#include + +namespace adios2 +{ +template void GeneratePythonBindings(pybind11::module &m); + +template<> +void GeneratePythonBindings(pybind11::module &m) +{ + pybind11::class_(m, "Variable") + .def_readonly("Name", &adios2::VariableBase::m_Name) + .def_readonly("Type", &adios2::VariableBase::m_Type) + .def_readonly("ElementSize", &adios2::VariableBase::m_ElementSize) + .def_readonly("Shape", &adios2::VariableBase::m_Shape) + .def_readonly("Start", &adios2::VariableBase::m_Start) + .def_readonly("Count", &adios2::VariableBase::m_Count) + .def_readonly("IsScalar", &adios2::VariableBase::m_SingleValue) + .def_readonly("HasConstantDims", &adios2::VariableBase::m_ConstantDims) + .def_property_readonly("PayLoadSize", + &adios2::VariableBase::PayLoadSize) + .def_property_readonly("TotalSize", &adios2::VariableBase::TotalSize); +#define pyvar(T, L) \ + pybind11::class_, adios2::VariableBase>( \ + m, "Variable" ADIOS2_STRINGIFY(L)) \ + .def_readwrite("Data", &adios2::Variable::m_Data); + ADIOS2_FOREACH_TYPE_2ARGS(pyvar) +#undef pyvar + +} + +} // end namespace adios2 diff --git a/bindings/python/VariablePy.cpp b/bindings/python/VariablePy.cpp deleted file mode 100644 index 4946db480d..0000000000 --- a/bindings/python/VariablePy.cpp +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * VariablePy.cpp - * - * Created on: Mar 17, 2017 - * Author: William F Godoy godoywf@ornl.gov - */ - -#include "VariablePy.h" - -namespace adios2 -{ - -VariablePy::VariablePy(const std::string &name, const pyList shape, - const pyList start, const pyList count, - const bool isConstantDims, const bool debugMode) -: m_Name(name), m_Shape(shape), m_Start(start), m_Count(count), - m_IsConstantDims(isConstantDims), m_DebugMode(debugMode) -{ -} - -void VariablePy::SetDimensions(const pyList shape, const pyList start, - const pyList count) -{ - if (m_DebugMode) - { - if (m_IsConstantDims) - { - throw std::invalid_argument( - "ERROR: variable " + m_Name + - " dimensions are constant, in call from SetDimensions\n"); - } - } - - m_Shape = shape; - m_Start = start; - m_Count = count; -} - -std::string VariablePy::GetType() const noexcept -{ - return m_VariableBase->m_Type; -} - -} // end namespace adios diff --git a/bindings/python/VariablePy.h b/bindings/python/VariablePy.h deleted file mode 100644 index 0c6bd3635c..0000000000 --- a/bindings/python/VariablePy.h +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * VariablePy.h - * - * Created on: Mar 13, 2017 - * Author: William F Godoy godoywf@ornl.gov - */ - -#ifndef ADIOS2_BINDINGS_PYTHON_SOURCE_VARIABLEPY_H_ -#define ADIOS2_BINDINGS_PYTHON_SOURCE_VARIABLEPY_H_ - -#include - -#include "adiosPyFunctions.h" - -namespace adios2 -{ - -class VariablePy -{ - -public: - const std::string m_Name; - pyList m_Shape; - pyList m_Start; - pyList m_Count; - const bool m_IsConstantDims; - - VariableBase *m_VariableBase = nullptr; - bool m_IsDefined = false; - - VariablePy(const std::string &name, const pyList shape, const pyList start, - const pyList count, const bool isConstantDims, - const bool debugMode); - - ~VariablePy() = default; - - void SetDimensions(const pyList shape, const pyList start, - const pyList count); - - std::string GetType() const noexcept; - -private: - const bool m_DebugMode; -}; - -} // end namespace adios - -#endif /* BINDINGS_PYTHON_SOURCE_VARIABLEPY_H_ */ diff --git a/bindings/python/adios2py.cpp b/bindings/python/adios2py.cpp new file mode 100644 index 0000000000..18db103d40 --- /dev/null +++ b/bindings/python/adios2py.cpp @@ -0,0 +1,16 @@ +#include + +#include + +namespace adios2 +{ + template + void GeneratePythonBindings(pybind11::module &m); +} + +PYBIND11_MODULE(adios2, m) +{ + adios2::GeneratePythonBindings(m); + adios2::GeneratePythonBindings(m); + adios2::GeneratePythonBindings(m); +} diff --git a/bindings/python/adiosPyFunctions.cpp b/bindings/python/adiosPyFunctions.cpp deleted file mode 100644 index 4b4a2f61a5..0000000000 --- a/bindings/python/adiosPyFunctions.cpp +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * adiosPyFunctions.cpp - * - * Created on: Mar 13, 2017 - * Author: William F Godoy godoywf@ornl.gov - */ - -#include "adiosPyFunctions.h" - -namespace adios2 -{ - -Dims PyListToDims(const pyList list) noexcept -{ - const unsigned int length = pybind11::len(list); - Dims dimensions; - dimensions.reserve(length); - - for (unsigned int i = 0; i < length; ++i) - { - dimensions.push_back(pybind11::cast(list[i])); - } - - return dimensions; -} - -Params KwargsToParams(const pyKwargs &kwargs) noexcept -{ - Params parameters; - - for (const auto &pair : kwargs) - { - parameters.emplace(pybind11::cast(pair.first), - pybind11::cast(pair.second)); - } - return parameters; -} - -} // end namespace adios diff --git a/bindings/python/adiosPyFunctions.h b/bindings/python/adiosPyFunctions.h deleted file mode 100644 index 9138deb75e..0000000000 --- a/bindings/python/adiosPyFunctions.h +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * adiosPyFunctions.h - * - * Created on: Mar 13, 2017 - * Author: William F Godoy godoywf@ornl.gov - */ - -#ifndef ADIOS2_BINDINGS_PYTHON_SOURCE_ADIOSPYFUNCTIONS_H_ -#define ADIOS2_BINDINGS_PYTHON_SOURCE_ADIOSPYFUNCTIONS_H_ - -#include -#include -#include - -#include "adios2/ADIOSTypes.h" -#include "adiosPyTypes.h" - -namespace adios2 -{ - -/** - * Python list to vector of dimensions (Dims) - * @param list python list of numbers - * @return adios::Dims - */ -Dims PyListToDims(const pyList list) noexcept; - -/** - * Python dictionary kwargs to adios::Params (std::map) - * @param kwargs dictionary - * @return adios::Params - */ -Params KwargsToParams(const pyKwargs &kwargs) noexcept; - -} // end namespace adios - -#endif /* BINDINGS_PYTHON_SOURCE_ADIOSPYFUNCTIONS_H_ */ diff --git a/bindings/python/adiosPyTypes.h b/bindings/python/adiosPyTypes.h deleted file mode 100644 index e4ba524784..0000000000 --- a/bindings/python/adiosPyTypes.h +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * adiosPyTypes.h - * - * Created on: Jun 7, 2017 - * Author: William F Godoy godoywf@ornl.gov - */ - -#ifndef ADIOS2_BINDINGS_PYTHON_SOURCE_ADIOSPYTYPES_H_ -#define ADIOS2_BINDINGS_PYTHON_SOURCE_ADIOSPYTYPES_H_ - -#include -#include -#include - -namespace adios2 -{ -// pytypes -using pyObject = pybind11::object; -using pyTuple = pybind11::tuple; -using pyDict = pybind11::dict; -using pyKwargs = pybind11::kwargs; -using pyList = pybind11::list; - -// numpy -using pyArray = pybind11::array; -using pyDType = pybind11::dtype; -} - -#endif /* BINDINGS_PYTHON_SOURCE_ADIOSPYTYPES_H_ */ diff --git a/bindings/python/gluePyBind11.cpp b/bindings/python/gluePyBind11.cpp deleted file mode 100644 index 85f5ec2194..0000000000 --- a/bindings/python/gluePyBind11.cpp +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * gluePyBind11.cpp - * - * Created on: Mar 16, 2017 - * Author: William F Godoy godoywf@ornl.gov - */ - -#include - -#include -#include - -#ifdef ADIOS2_HAVE_MPI -#include -#endif - -#include "ADIOSPy.h" -#include "EnginePy.h" -#include "IOPy.h" -#include "VariablePy.h" -#include "adiosPyFunctions.h" -#include "adiosPyTypes.h" - -#ifdef ADIOS2_HAVE_MPI -adios2::ADIOSPy ADIOSPyInitConfig(const std::string configFile, - adios2::pyObject &object, - const bool debugMode) -{ - MPI_Comm *mpiCommPtr = PyMPIComm_Get(object.ptr()); - - if (import_mpi4py() < 0) - { - throw std::runtime_error("ERROR: could not import mpi4py " - "communicator, in call to ADIOS " - "constructor\n"); - } - - if (mpiCommPtr == nullptr) - { - throw std::runtime_error("ERROR: mpi4py communicator is null, in call " - "to ADIOS constructor\n"); - } - return adios2::ADIOSPy(configFile, *mpiCommPtr, debugMode); -} - -adios2::ADIOSPy ADIOSPyInit(adios2::pyObject &object, const bool debugMode) -{ - MPI_Comm *mpiCommPtr = PyMPIComm_Get(object.ptr()); - - if (import_mpi4py() < 0) - { - throw std::runtime_error("ERROR: could not import mpi4py " - "communicator, in call to ADIOS " - "constructor\n"); - } - - if (mpiCommPtr == nullptr) - { - throw std::runtime_error("ERROR: mpi4py communicator is null, in call " - "to ADIOS constructor\n"); - } - return adios2::ADIOSPy(*mpiCommPtr, debugMode); -} -#else -adios2::ADIOSPy ADIOSPyInitConfig(const std::string configFile, - const bool debugMode) -{ - return adios2::ADIOSPy(configFile, debugMode); -} - -adios2::ADIOSPy ADIOSPyInit(const bool debugMode) -{ - return adios2::ADIOSPy(debugMode); -} -#endif - -PYBIND11_PLUGIN(adios2) -{ -#ifdef ADIOS2_HAVE_MPI - if (import_mpi4py() < 0) - { - throw std::runtime_error( - "ERROR: mpi4py not loaded correctly\n"); /* Python 2.X */ - } -#endif - - pybind11::module m("adios2", "ADIOS2 Python bindings using pybind11"); - m.attr("DebugON") = true; - m.attr("DebugOFF") = false; - m.attr("ConstantDims") = true; - m.attr("OpenModeWrite") = static_cast(adios2::OpenMode::Write); - m.attr("OpenModeRead") = static_cast(adios2::OpenMode::Read); - m.attr("OpenModeAppend") = static_cast(adios2::OpenMode::Append); - m.attr("OpenModeReadWrite") = static_cast(adios2::OpenMode::ReadWrite); - m.def("ADIOS", &ADIOSPyInit, "Function that creates an ADIOS class object"); - m.def("ADIOS", &ADIOSPyInitConfig, - "Function that creates an ADIOS class object using a config file"); - - pybind11::class_(m, "ADIOSPy") - .def("DeclareIO", &adios2::ADIOSPy::DeclareIO); - - pybind11::class_(m, "IOPy") - .def("SetEngine", &adios2::IOPy::SetEngine) - .def("SetParameters", &adios2::IOPy::SetParameters) - .def("AddTransport", &adios2::IOPy::AddTransport) - .def("DefineVariable", &adios2::IOPy::DefineVariable, - pybind11::return_value_policy::reference_internal, - pybind11::arg("name"), pybind11::arg("shape") = adios2::pyList(), - pybind11::arg("start") = adios2::pyList(), - pybind11::arg("count") = adios2::pyList(), - pybind11::arg("isConstantDims") = false) - .def("GetVariable", &adios2::IOPy::GetVariable, - pybind11::return_value_policy::reference_internal) - .def("Open", (adios2::EnginePy (adios2::IOPy::*)(const std::string &, - const int)) & - adios2::IOPy::Open); - - pybind11::class_(m, "VariablePy") - .def("SetDimensions", &adios2::VariablePy::SetDimensions); - - pybind11::class_(m, "EnginePy") - .def("Write", &adios2::EnginePy::Write) - .def("Advance", &adios2::EnginePy::Advance, - pybind11::arg("timeoutSeconds") = 0.) - .def("Close", &adios2::EnginePy::Close, - pybind11::arg("transportIndex") = -1); - - return m.ptr(); -} From f91161af3338c52e25f5b5edfcebbc235bbc0664 Mon Sep 17 00:00:00 2001 From: Chuck Atkins Date: Thu, 28 Sep 2017 11:49:52 -0400 Subject: [PATCH 02/24] Add an addition helper macro for preprocessor stringification. --- source/adios2/ADIOSMacros.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/adios2/ADIOSMacros.h b/source/adios2/ADIOSMacros.h index 1f01f9ea60..1d047893ae 100644 --- a/source/adios2/ADIOSMacros.h +++ b/source/adios2/ADIOSMacros.h @@ -11,6 +11,10 @@ #include #include "adios2/ADIOSTypes.h" + +#define ADIOS2_STRINGIFY_HELPER(X) #X +#define ADIOS2_STRINGIFY(X) ADIOS2_STRINGIFY_HELPER(X) + // The ADIOS_FOREACH_TYPE_1ARG macro assumes the given argument is a macro which // takes a single argument that is a type and then inserts the given MACRO for // each of the known primitive types From e3fc2c02bb92285b35df9e374b105f8b5c54190d Mon Sep 17 00:00:00 2001 From: Chuck Atkins Date: Thu, 28 Sep 2017 11:56:51 -0400 Subject: [PATCH 03/24] Add some additional get/set functionality to IO --- source/adios2/core/IO.cpp | 8 +++++++- source/adios2/core/IO.h | 10 +++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/source/adios2/core/IO.cpp b/source/adios2/core/IO.cpp index 1de67f0987..f58435a466 100644 --- a/source/adios2/core/IO.cpp +++ b/source/adios2/core/IO.cpp @@ -45,6 +45,7 @@ IO::IO(const std::string name, MPI_Comm mpiComm, const bool inConfigFile, } void IO::SetEngine(const std::string engineType) { m_EngineType = engineType; } +const std::string IO::GetEngine() const { return m_EngineType; } void IO::SetIOMode(const IOMode ioMode) { m_IOMode = ioMode; }; void IO::SetParameters(const Params ¶meters) { m_Parameters = parameters; } @@ -55,7 +56,7 @@ void IO::SetSingleParameter(const std::string key, m_Parameters[key] = value; } -const Params &IO::GetParameters() const { return m_Parameters; } +Params &IO::GetParameters() { return m_Parameters; } unsigned int IO::AddTransport(const std::string type, const Params ¶meters) { @@ -70,6 +71,11 @@ unsigned int IO::AddTransport(const std::string type, const Params ¶meters) return static_cast(m_TransportsParameters.size() - 1); } +const std::vector &IO::GetTransportParameters() const +{ + return m_TransportsParameters; +} + void IO::SetTransportSingleParameter(const unsigned int transportIndex, const std::string key, const std::string value) diff --git a/source/adios2/core/IO.h b/source/adios2/core/IO.h index f29191dac3..49e1e99a7c 100644 --- a/source/adios2/core/IO.h +++ b/source/adios2/core/IO.h @@ -80,6 +80,12 @@ class IO */ void SetEngine(const std::string engine); + /** + * Gets the engine type for this IO class object + * @param engine + */ + const std::string GetEngine() const; + /** Set the IO mode (collective or independent) * @param IO mode */ void SetIOMode(const IOMode mode); @@ -102,7 +108,7 @@ class IO /** * Retrieve existing parameter set */ - const Params &GetParameters() const; + Params &GetParameters(); /** * Adds a transport and its parameters for the IO Engine @@ -113,6 +119,8 @@ class IO unsigned int AddTransport(const std::string type, const Params ¶ms = Params()); + const std::vector &GetTransportParameters() const; + /** * Set a single parameter to an existing transport identified with a * transportIndex handler from AddTransport. This function overwrites From ead0cdaeb0a44691773e29eb7ab641b3aafe690d Mon Sep 17 00:00:00 2001 From: Chuck Atkins Date: Fri, 29 Sep 2017 12:13:24 -0400 Subject: [PATCH 04/24] More types --- bindings/python/CMakeLists.txt | 6 +-- bindings/python/PyAttribute.cpp | 29 ++++++++++++++ bindings/python/PyEnums.cpp | 67 +++++++++++++++++++++++++++++++++ bindings/python/PyIO.cpp | 59 +++++++++++++++++++---------- bindings/python/PySelection.cpp | 24 ++++++++++++ bindings/python/PyVariable.cpp | 27 ++++++------- bindings/python/adios2py.cpp | 2 + source/adios2/core/IO.cpp | 2 + source/adios2/core/IO.h | 1 + 9 files changed, 181 insertions(+), 36 deletions(-) create mode 100644 bindings/python/PyAttribute.cpp create mode 100644 bindings/python/PyEnums.cpp create mode 100644 bindings/python/PySelection.cpp diff --git a/bindings/python/CMakeLists.txt b/bindings/python/CMakeLists.txt index c1f9f384c8..e5b9231eee 100644 --- a/bindings/python/CMakeLists.txt +++ b/bindings/python/CMakeLists.txt @@ -1,6 +1,8 @@ pybind11_add_module(adios2py MODULE adios2py.cpp + PyAttribute.cpp PyADIOS.cpp + PyEnums.cpp PyIO.cpp PyVariable.cpp # adiosPyFunctions.h @@ -10,10 +12,6 @@ pybind11_add_module(adios2py MODULE # EnginePy.inl # EnginePy.cpp # gluePyBind11.cpp -# IOPy.h -# IOPy.cpp -# VariablePy.h -# VariablePy.cpp ) target_link_libraries(adios2py PRIVATE adios2) if(ADIOS2_HAVE_MPI) diff --git a/bindings/python/PyAttribute.cpp b/bindings/python/PyAttribute.cpp new file mode 100644 index 0000000000..a8de0ec50f --- /dev/null +++ b/bindings/python/PyAttribute.cpp @@ -0,0 +1,29 @@ +#include + +#include +#include + +namespace adios2 +{ +template void GeneratePythonBindings(pybind11::module &m); + +template<> +void GeneratePythonBindings(pybind11::module &m) +{ + pybind11::class_(m, "Attribute") + .def_readonly("Name", &AttributeBase::m_Name) + .def_readonly("Type", &AttributeBase::m_Type) + .def_readonly("Elements", &AttributeBase::m_Elements) + .def_readonly("Name", &AttributeBase::m_IsSingleValue); + +#define pyattr(T, L) \ + pybind11::class_, AttributeBase>( \ + m, "Attribute" ADIOS2_STRINGIFY(L)) \ + .def_readwrite("DataArray", &Attribute::m_DataArray) \ + .def_readwrite("DataSingleValue", &Attribute::m_DataSingleValue); + ADIOS2_FOREACH_TYPE_2ARGS(pyattr) +#undef pyvar + +} + +} // end namespace adios2 diff --git a/bindings/python/PyEnums.cpp b/bindings/python/PyEnums.cpp new file mode 100644 index 0000000000..06616d0228 --- /dev/null +++ b/bindings/python/PyEnums.cpp @@ -0,0 +1,67 @@ +#include + +#include + +namespace adios2 +{ +template void GeneratePythonBindings(pybind11::module &m); + +template<> +void GeneratePythonBindings(pybind11::module &m) +{ + pybind11::enum_(m, "ShapeID") + .value("GlobalValue", ShapeID::GlobalValue) + .value("GlobalArray", ShapeID::GlobalArray) + .value("JoinedArray", ShapeID::JoinedArray) + .value("LocalValue", ShapeID::LocalValue) + .value("LocalArray", ShapeID::LocalArray) + .export_values(); + + pybind11::enum_(m, "IOMode") + .value("Independent", IOMode::Independent) + .value("Collective", IOMode::Collective) + .export_values(); + + pybind11::enum_(m, "OpenMode") + .value("Undefined", OpenMode::Undefined) + .value("Write", OpenMode::Write) + .value("Read", OpenMode::Read) + .value("Append", OpenMode::Append) + .value("ReadWrite", OpenMode::ReadWrite) + .export_values(); + + pybind11::enum_(m, "ReadMultiplexPattern") + .value("GlobalReaders", ReadMultiplexPattern::GlobalReaders) + .value("RoundRobin", ReadMultiplexPattern::RoundRobin) + .value("FirstInFirstOut", ReadMultiplexPattern::FirstInFirstOut) + .value("OpenAllSteps", ReadMultiplexPattern::OpenAllSteps) + .export_values(); + + pybind11::enum_(m, "ReadMode") + .value("NonBlocking", ReadMode::Blocking) + .value("Blocking", ReadMode::Blocking) + .export_values(); + + pybind11::enum_(m, "AdvanceMode") + .value("Append", AdvanceMode::Append) + .value("Update", AdvanceMode::Update) + .value("NextAvailable", AdvanceMode::NextAvailable) + .value("LatestAvailable", AdvanceMode::LatestAvailable) + .export_values(); + + pybind11::enum_(m, "AdvanceStatus") + .value("OK", AdvanceStatus::OK) + .value("StepNotReady", AdvanceStatus::StepNotReady) + .value("EndOfStream", AdvanceStatus::EndOfStream) + .value("OtherError", AdvanceStatus::OtherError) + .export_values(); + + pybind11::enum_(m, "SelectionType") + .value("BoundingBox", SelectionType::BoundingBox) + .value("Points", SelectionType::Points) + .value("WriteBlock", SelectionType::WriteBlock) + .value("Auto", SelectionType::Auto) + .export_values(); +} + +} // end namespace adios2 diff --git a/bindings/python/PyIO.cpp b/bindings/python/PyIO.cpp index b7cf1d3e75..08222485d7 100644 --- a/bindings/python/PyIO.cpp +++ b/bindings/python/PyIO.cpp @@ -1,34 +1,55 @@ #include #include +#include namespace adios2 { -template void GeneratePythonBindings(pybind11::module &m); +template +void GeneratePythonBindings(pybind11::module &m); -template<> +template <> void GeneratePythonBindings(pybind11::module &m) { - pybind11::class_(m, "IO") - .def_property("EngineType", &adios2::IO::SetEngine, - &adios2::IO::GetEngine) - .def_property("Parameters", &adios2::IO::SetParameters, - &adios2::IO::GetParameters) - .def("SetIOMode", &adios2::IO::SetIOMode) - .def("AddTransport", &adios2::IO::AddTransport) + pybind11::class_(m, "IO") + .def_property("EngineType", &IO::SetEngine, + &IO::GetEngine) + .def_property("Parameters", &IO::SetParameters, + (Params& (IO::*)())&IO::GetParameters) + .def("SetIOMode", &IO::SetIOMode) + .def("AddTransport", &IO::AddTransport) .def_property_readonly("TransportParameters", - &adios2::IO::GetTransportParameters) + &IO::GetTransportParameters) .def("DefineVariable", - [](adios2::IO &io, std::string &name, adios2::Dims &shape, - adios2::Dims &start, adios2::Dims &count, bool constantDims, - int t) { - if (t == 1) - return pybind11::cast(io.DefineVariable( - name, shape, start, count, constantDims)); + [](IO &io, std::string &name, Dims &shape, + Dims &start, Dims &count, bool constantDims, + pybind11::dtype type) { + if (type.is_none()) + { + // Delay variable creation + } +#define define_variable(T) \ + else if (type.kind() == pybind11::dtype::of().kind()) \ + { \ + return pybind11::cast( \ + io.DefineVariable(name, shape, start, count, constantDims)); \ + } + ADIOS2_FOREACH_TYPE_1ARG(define_variable) +#undef define_variable else - return pybind11::cast(io.DefineVariable( - name, shape, start, count, constantDims)); - }); + { + throw std::invalid_argument( + "Error: IO::DefineVariable called " + "with unsupported datatype"); + } + }, + pybind11::arg("name").none(false), + pybind11::arg("shape").none(false) = Dims{}, + pybind11::arg("start").none(false) = Dims{}, + pybind11::arg("count").none(false) = Dims{}, + pybind11::arg("constantDims") = false, + pybind11::arg("type").none(false)); +} } } // end namespace adios2 diff --git a/bindings/python/PySelection.cpp b/bindings/python/PySelection.cpp new file mode 100644 index 0000000000..aaccbd26fb --- /dev/null +++ b/bindings/python/PySelection.cpp @@ -0,0 +1,24 @@ +#include + +#include +#include + +namespace adios2 +{ +template void GeneratePythonBindings(pybind11::module &m); + +template<> +void GeneratePythonBindings(pybind11::module &m) +{ + pybind11::class_(m, "Selection") + .def_readonly("Type", &Selection::m_Type); + + pybind11::class_("SelectionBoundingBox") + .def(pybind11::init(), + pybind11::arg("start"), pybind11::arg("count"), + pybind11::arg("debug") = false) + .def_readwrite("Start", &SelectionBoundingBox.m_Start) + .def_readwrite("Count", &SelectionBoundingBox.m_Count); +} + +} // end namespace adios2 diff --git a/bindings/python/PyVariable.cpp b/bindings/python/PyVariable.cpp index 0466822417..e627f1b824 100644 --- a/bindings/python/PyVariable.cpp +++ b/bindings/python/PyVariable.cpp @@ -10,22 +10,23 @@ template void GeneratePythonBindings(pybind11::module &m); template<> void GeneratePythonBindings(pybind11::module &m) { - pybind11::class_(m, "Variable") - .def_readonly("Name", &adios2::VariableBase::m_Name) - .def_readonly("Type", &adios2::VariableBase::m_Type) - .def_readonly("ElementSize", &adios2::VariableBase::m_ElementSize) - .def_readonly("Shape", &adios2::VariableBase::m_Shape) - .def_readonly("Start", &adios2::VariableBase::m_Start) - .def_readonly("Count", &adios2::VariableBase::m_Count) - .def_readonly("IsScalar", &adios2::VariableBase::m_SingleValue) - .def_readonly("HasConstantDims", &adios2::VariableBase::m_ConstantDims) + pybind11::class_(m, "Variable") + .def_readonly("Name", &VariableBase::m_Name) + .def_readonly("Type", &VariableBase::m_Type) + .def_readonly("ElementSize", &VariableBase::m_ElementSize) + .def_readonly("Shape", &VariableBase::m_Shape) + .def_readonly("Start", &VariableBase::m_Start) + .def_readonly("Count", &VariableBase::m_Count) + .def_readonly("IsScalar", &VariableBase::m_SingleValue) + .def_readonly("HasConstantDims", &VariableBase::m_ConstantDims) .def_property_readonly("PayLoadSize", - &adios2::VariableBase::PayLoadSize) - .def_property_readonly("TotalSize", &adios2::VariableBase::TotalSize); + &VariableBase::PayLoadSize) + .def_property_readonly("TotalSize", &VariableBase::TotalSize); + #define pyvar(T, L) \ - pybind11::class_, adios2::VariableBase>( \ + pybind11::class_, VariableBase>( \ m, "Variable" ADIOS2_STRINGIFY(L)) \ - .def_readwrite("Data", &adios2::Variable::m_Data); + .def_readwrite("Data", &Variable::m_Data); ADIOS2_FOREACH_TYPE_2ARGS(pyvar) #undef pyvar diff --git a/bindings/python/adios2py.cpp b/bindings/python/adios2py.cpp index 18db103d40..4af91e7791 100644 --- a/bindings/python/adios2py.cpp +++ b/bindings/python/adios2py.cpp @@ -10,6 +10,8 @@ namespace adios2 PYBIND11_MODULE(adios2, m) { + adios2::GeneratePythonBindings(m); + adios2::GeneratePythonBindings(m); adios2::GeneratePythonBindings(m); adios2::GeneratePythonBindings(m); adios2::GeneratePythonBindings(m); diff --git a/source/adios2/core/IO.cpp b/source/adios2/core/IO.cpp index f58435a466..6304a44bb2 100644 --- a/source/adios2/core/IO.cpp +++ b/source/adios2/core/IO.cpp @@ -58,6 +58,8 @@ void IO::SetSingleParameter(const std::string key, Params &IO::GetParameters() { return m_Parameters; } +const Params &IO::GetParameters() const { return m_Parameters; } + unsigned int IO::AddTransport(const std::string type, const Params ¶meters) { Params parametersMap(parameters); diff --git a/source/adios2/core/IO.h b/source/adios2/core/IO.h index 49e1e99a7c..2ea35ba0c8 100644 --- a/source/adios2/core/IO.h +++ b/source/adios2/core/IO.h @@ -109,6 +109,7 @@ class IO * Retrieve existing parameter set */ Params &GetParameters(); + const Params &GetParameters() const; /** * Adds a transport and its parameters for the IO Engine From e0cc08d3d82b6df815568c9e5a65d3b1ad170782 Mon Sep 17 00:00:00 2001 From: Chuck Atkins Date: Fri, 29 Sep 2017 13:01:39 -0400 Subject: [PATCH 05/24] Fix missing stl include --- bindings/python/PyADIOS.cpp | 14 ++++++-------- bindings/python/PyAttribute.cpp | 20 ++++++++++---------- bindings/python/PyEnums.cpp | 7 ++++--- bindings/python/PyIO.cpp | 14 ++++++-------- bindings/python/PySelection.cpp | 5 +++-- bindings/python/PyVariable.cpp | 9 ++++----- bindings/python/adios2py.cpp | 4 ++-- 7 files changed, 35 insertions(+), 38 deletions(-) diff --git a/bindings/python/PyADIOS.cpp b/bindings/python/PyADIOS.cpp index 06f2ea3cab..be03ba9bdd 100644 --- a/bindings/python/PyADIOS.cpp +++ b/bindings/python/PyADIOS.cpp @@ -8,9 +8,10 @@ namespace adios2 { -template void GeneratePythonBindings(pybind11::module &m); +template +void GeneratePythonBindings(pybind11::module &m); -template<> +template <> void GeneratePythonBindings(pybind11::module &m) { pybind11::class_(m, "ADIOS") @@ -25,8 +26,7 @@ void GeneratePythonBindings(pybind11::module &m) a->m_HostLanguage = "Python"; return a; }), - pybind11::arg("config").none(false), - pybind11::arg("debug") = true) + pybind11::arg("config").none(false), pybind11::arg("debug") = true) #ifdef ADIOS2_HAVE_MPI .def(pybind11::init([](pybind11::object comm, const bool debug) { import_mpi4py__MPI(); @@ -34,8 +34,7 @@ void GeneratePythonBindings(pybind11::module &m) a->m_HostLanguage = "Python"; return a; }), - pybind11::arg("comm").none(false), - pybind11::arg("debug") = true) + pybind11::arg("comm").none(false), pybind11::arg("debug") = true) .def(pybind11::init([](pybind11::object comm, const std::string config, const bool debug) { import_mpi4py__MPI(); @@ -44,8 +43,7 @@ void GeneratePythonBindings(pybind11::module &m) return a; }), pybind11::arg("comm").none(false), - pybind11::arg("config").none(false), - pybind11::arg("debug") = true) + pybind11::arg("config").none(false), pybind11::arg("debug") = true) #endif .def_readonly("HostLanguage", &ADIOS::m_HostLanguage) .def("DeclareIO", &ADIOS::DeclareIO) diff --git a/bindings/python/PyAttribute.cpp b/bindings/python/PyAttribute.cpp index a8de0ec50f..56c6ecbf73 100644 --- a/bindings/python/PyAttribute.cpp +++ b/bindings/python/PyAttribute.cpp @@ -5,25 +5,25 @@ namespace adios2 { -template void GeneratePythonBindings(pybind11::module &m); +template +void GeneratePythonBindings(pybind11::module &m); -template<> +template <> void GeneratePythonBindings(pybind11::module &m) { pybind11::class_(m, "Attribute") - .def_readonly("Name", &AttributeBase::m_Name) - .def_readonly("Type", &AttributeBase::m_Type) - .def_readonly("Elements", &AttributeBase::m_Elements) - .def_readonly("Name", &AttributeBase::m_IsSingleValue); + .def_readonly("Name", &AttributeBase::m_Name) + .def_readonly("Type", &AttributeBase::m_Type) + .def_readonly("Elements", &AttributeBase::m_Elements) + .def_readonly("Name", &AttributeBase::m_IsSingleValue); #define pyattr(T, L) \ - pybind11::class_, AttributeBase>( \ + pybind11::class_, AttributeBase>( \ m, "Attribute" ADIOS2_STRINGIFY(L)) \ - .def_readwrite("DataArray", &Attribute::m_DataArray) \ - .def_readwrite("DataSingleValue", &Attribute::m_DataSingleValue); + .def_readwrite("DataArray", &Attribute::m_DataArray) \ + .def_readwrite("DataSingleValue", &Attribute::m_DataSingleValue); ADIOS2_FOREACH_TYPE_2ARGS(pyattr) #undef pyvar - } } // end namespace adios2 diff --git a/bindings/python/PyEnums.cpp b/bindings/python/PyEnums.cpp index 06616d0228..20c23e90b1 100644 --- a/bindings/python/PyEnums.cpp +++ b/bindings/python/PyEnums.cpp @@ -4,9 +4,10 @@ namespace adios2 { -template void GeneratePythonBindings(pybind11::module &m); +template +void GeneratePythonBindings(pybind11::module &m); -template<> +template <> void GeneratePythonBindings(pybind11::module &m) { pybind11::enum_(m, "ShapeID") @@ -41,7 +42,7 @@ void GeneratePythonBindings(pybind11::module &m) .value("NonBlocking", ReadMode::Blocking) .value("Blocking", ReadMode::Blocking) .export_values(); - + pybind11::enum_(m, "AdvanceMode") .value("Append", AdvanceMode::Append) .value("Update", AdvanceMode::Update) diff --git a/bindings/python/PyIO.cpp b/bindings/python/PyIO.cpp index 08222485d7..3ac3ae8cda 100644 --- a/bindings/python/PyIO.cpp +++ b/bindings/python/PyIO.cpp @@ -1,7 +1,8 @@ #include -#include #include +#include +#include namespace adios2 { @@ -12,18 +13,16 @@ template <> void GeneratePythonBindings(pybind11::module &m) { pybind11::class_(m, "IO") - .def_property("EngineType", &IO::SetEngine, - &IO::GetEngine) + .def_property("EngineType", &IO::SetEngine, &IO::GetEngine) .def_property("Parameters", &IO::SetParameters, - (Params& (IO::*)())&IO::GetParameters) + (Params & (IO::*)()) & IO::GetParameters) .def("SetIOMode", &IO::SetIOMode) .def("AddTransport", &IO::AddTransport) .def_property_readonly("TransportParameters", &IO::GetTransportParameters) .def("DefineVariable", - [](IO &io, std::string &name, Dims &shape, - Dims &start, Dims &count, bool constantDims, - pybind11::dtype type) { + [](IO &io, std::string &name, Dims &shape, Dims &start, + Dims &count, bool constantDims, pybind11::dtype type) { if (type.is_none()) { // Delay variable creation @@ -50,6 +49,5 @@ void GeneratePythonBindings(pybind11::module &m) pybind11::arg("constantDims") = false, pybind11::arg("type").none(false)); } -} } // end namespace adios2 diff --git a/bindings/python/PySelection.cpp b/bindings/python/PySelection.cpp index aaccbd26fb..ec57d03505 100644 --- a/bindings/python/PySelection.cpp +++ b/bindings/python/PySelection.cpp @@ -5,9 +5,10 @@ namespace adios2 { -template void GeneratePythonBindings(pybind11::module &m); +template +void GeneratePythonBindings(pybind11::module &m); -template<> +template <> void GeneratePythonBindings(pybind11::module &m) { pybind11::class_(m, "Selection") diff --git a/bindings/python/PyVariable.cpp b/bindings/python/PyVariable.cpp index e627f1b824..3b1b87932c 100644 --- a/bindings/python/PyVariable.cpp +++ b/bindings/python/PyVariable.cpp @@ -5,9 +5,10 @@ namespace adios2 { -template void GeneratePythonBindings(pybind11::module &m); +template +void GeneratePythonBindings(pybind11::module &m); -template<> +template <> void GeneratePythonBindings(pybind11::module &m) { pybind11::class_(m, "Variable") @@ -19,8 +20,7 @@ void GeneratePythonBindings(pybind11::module &m) .def_readonly("Count", &VariableBase::m_Count) .def_readonly("IsScalar", &VariableBase::m_SingleValue) .def_readonly("HasConstantDims", &VariableBase::m_ConstantDims) - .def_property_readonly("PayLoadSize", - &VariableBase::PayLoadSize) + .def_property_readonly("PayLoadSize", &VariableBase::PayLoadSize) .def_property_readonly("TotalSize", &VariableBase::TotalSize); #define pyvar(T, L) \ @@ -29,7 +29,6 @@ void GeneratePythonBindings(pybind11::module &m) .def_readwrite("Data", &Variable::m_Data); ADIOS2_FOREACH_TYPE_2ARGS(pyvar) #undef pyvar - } } // end namespace adios2 diff --git a/bindings/python/adios2py.cpp b/bindings/python/adios2py.cpp index 4af91e7791..217b5b451e 100644 --- a/bindings/python/adios2py.cpp +++ b/bindings/python/adios2py.cpp @@ -4,8 +4,8 @@ namespace adios2 { - template - void GeneratePythonBindings(pybind11::module &m); +template +void GeneratePythonBindings(pybind11::module &m); } PYBIND11_MODULE(adios2, m) From e24d63399a8d149eddc7627e4edf28bef8fbd7cd Mon Sep 17 00:00:00 2001 From: Chuck Atkins Date: Mon, 2 Oct 2017 09:33:52 -0400 Subject: [PATCH 06/24] Mane python dependency search aware of PYTHON env var --- cmake/FindPythonFull.cmake | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmake/FindPythonFull.cmake b/cmake/FindPythonFull.cmake index 6b70423eaa..2db706b648 100644 --- a/cmake/FindPythonFull.cmake +++ b/cmake/FindPythonFull.cmake @@ -29,6 +29,9 @@ include(CMakeFindDependencyMacro) set(_req_vars) foreach(comp IN LISTS PythonFull_FIND_COMPONENTS) if(comp STREQUAL Interp) + if((NOT PYTHON_EXECUTABLE) AND (NOT "$ENV{PYTHON}" STREQUAL "")) + set(PYTHON_EXECUTABLE "$ENV{PYTHON}" CACHE FILEPATH "") + endif() find_package(PythonInterp) set(PythonFull_${comp}_FOUND ${PYTHONINTERP_FOUND}) list(APPEND _req_vars PYTHON_EXECUTABLE) From 1391392ab05163a6e8e2c94e4a61e96f909c8415 Mon Sep 17 00:00:00 2001 From: Chuck Atkins Date: Fri, 6 Oct 2017 12:05:53 -0400 Subject: [PATCH 07/24] bunches of updates --- bindings/python/CMakeLists.txt | 8 +- bindings/python/PyADIOS.cpp | 4 +- bindings/python/PyAttribute.cpp | 4 +- bindings/python/PyEngine.cpp | 108 +++++++++++++++ bindings/python/PyEnums.cpp | 4 +- bindings/python/PyIO.cpp | 28 ++-- bindings/python/PyVariable.cpp | 38 +++--- bindings/python/adios2py.cpp | 5 +- source/adios2/ADIOSMacros.h | 2 + source/adios2/core/Engine.cpp | 11 +- source/adios2/core/Engine.h | 2 +- source/adios2/core/Engine.tcc | 124 ------------------ source/adios2/core/IO.cpp | 2 + source/adios2/core/IO.h | 5 + source/adios2/helper/adiosType.h | 8 ++ source/adios2/helper/adiosType.inl | 11 ++ .../pybind11/include/pybind11/numpy.h | 5 + 17 files changed, 203 insertions(+), 166 deletions(-) create mode 100644 bindings/python/PyEngine.cpp diff --git a/bindings/python/CMakeLists.txt b/bindings/python/CMakeLists.txt index e5b9231eee..7e7db72ee8 100644 --- a/bindings/python/CMakeLists.txt +++ b/bindings/python/CMakeLists.txt @@ -3,15 +3,9 @@ pybind11_add_module(adios2py MODULE PyAttribute.cpp PyADIOS.cpp PyEnums.cpp + PyEngine.cpp PyIO.cpp PyVariable.cpp -# adiosPyFunctions.h -# adiosPyFunctions.cpp -# adiosPyTypes.h -# EnginePy.h -# EnginePy.inl -# EnginePy.cpp -# gluePyBind11.cpp ) target_link_libraries(adios2py PRIVATE adios2) if(ADIOS2_HAVE_MPI) diff --git a/bindings/python/PyADIOS.cpp b/bindings/python/PyADIOS.cpp index be03ba9bdd..c4ab8b5e52 100644 --- a/bindings/python/PyADIOS.cpp +++ b/bindings/python/PyADIOS.cpp @@ -1,7 +1,7 @@ -#include - #include +#include + #ifdef ADIOS2_HAVE_MPI #include #endif diff --git a/bindings/python/PyAttribute.cpp b/bindings/python/PyAttribute.cpp index 56c6ecbf73..a6dc79e942 100644 --- a/bindings/python/PyAttribute.cpp +++ b/bindings/python/PyAttribute.cpp @@ -1,8 +1,8 @@ -#include - #include #include +#include + namespace adios2 { template diff --git a/bindings/python/PyEngine.cpp b/bindings/python/PyEngine.cpp new file mode 100644 index 0000000000..15901170d6 --- /dev/null +++ b/bindings/python/PyEngine.cpp @@ -0,0 +1,108 @@ +#include +#include +#include + +#include + +#include + +namespace adios2 +{ +template +void GeneratePythonBindings(pybind11::module &m); + +template <> +void GeneratePythonBindings(pybind11::module &m) +{ + // Wrapping for internal Engine classes + pybind11::class_>(m, "EngineBase") + .def("Write", + [](Engine &e, VariableBase &variable, pybind11::array values) { + e.Write(variable, values.data()); + }) + .def("Write", + [](Engine &e, const std::string name, pybind11::array values) { + e.Write(name, values.data()); + }) + .def("InquireVariable", + [](Engine &e, const std::string name) -> VariableBase * { + VariableBase *var; +#define inquire(T) \ + if (var = e.InquireVariable(name, false)) \ + { \ + return var; \ + } + ADIOS2_FOREACH_TYPE_1ARG(inquire) +#undef inquire + return nullptr; + }) + .def("Close", &Engine::Close); + + + // + // Wrappings to extend Engine in Python + // + + // This first base class used to translate the protected template functions + // into public members that can be overridden in python + class PyEngineBase : public Engine + { + public: + PyEngineBase(const std::string engineType, IO &io, + const std::string &name, const OpenMode openMode) + : Engine(engineType, io, name, openMode, io.m_MPIComm) + { + } + + virtual ~PyEngineBase() = default; + + using Engine::Init; + virtual void DoWrite(VariableBase &var, pybind11::array values) = 0; + using Engine::Close; + + protected: + // The C++ code calls this implementation which will redirect to the + // python implementation. +#define define_dowrite(T) \ + void DoWrite(Variable &var, const T *values) override \ + { \ + DoWrite(var, \ + pybind11::array_t(std::accumulate(var.m_Count.begin(), \ + var.m_Count.end(), 0), \ + values)); \ + } + ADIOS2_FOREACH_TYPE_1ARG(define_dowrite) +#undef define_dowrite + }; + + + // This is the trampoline class used by pybind11 to implement the + // inheritance model and virtual springboard + class PyEngine : public PyEngineBase + { + public: + using PyEngineBase::PyEngineBase; + void Init() override + { + PYBIND11_OVERLOAD(void, PyEngineBase, Init,); + } + void DoWrite(VariableBase &var, pybind11::array values) override + { + PYBIND11_OVERLOAD_PURE(void, PyEngineBase, DoWrite, var, values); + } + void Close(const int transportIndex) + { + PYBIND11_OVERLOAD_PURE(void, PyEngineBase, Close, transportIndex); + } + }; + + pybind11::class_(m, "Engine") + .def(pybind11::init()) + .def("Init", &PyEngineBase::Init) + .def("DoWrite", + (void (PyEngineBase::*)(VariableBase &, pybind11::array)) & + PyEngineBase::DoWrite) + .def("Close", &PyEngineBase::Close); +} +} // end namespace adios2 diff --git a/bindings/python/PyEnums.cpp b/bindings/python/PyEnums.cpp index 20c23e90b1..35322e1733 100644 --- a/bindings/python/PyEnums.cpp +++ b/bindings/python/PyEnums.cpp @@ -1,7 +1,7 @@ -#include - #include +#include + namespace adios2 { template diff --git a/bindings/python/PyIO.cpp b/bindings/python/PyIO.cpp index 3ac3ae8cda..d34a478074 100644 --- a/bindings/python/PyIO.cpp +++ b/bindings/python/PyIO.cpp @@ -1,9 +1,9 @@ -#include - -#include #include +#include #include +#include + namespace adios2 { template @@ -13,22 +13,22 @@ template <> void GeneratePythonBindings(pybind11::module &m) { pybind11::class_(m, "IO") - .def_property("EngineType", &IO::SetEngine, &IO::GetEngine) - .def_property("Parameters", &IO::SetParameters, - (Params & (IO::*)()) & IO::GetParameters) - .def("SetIOMode", &IO::SetIOMode) + .def_property("EngineType", &IO::GetEngine, &IO::SetEngine) + .def_property("Parameters", (Params & (IO::*)()) & IO::GetParameters, + &IO::SetParameters) .def("AddTransport", &IO::AddTransport) .def_property_readonly("TransportParameters", &IO::GetTransportParameters) .def("DefineVariable", [](IO &io, std::string &name, Dims &shape, Dims &start, - Dims &count, bool constantDims, pybind11::dtype type) { + Dims &count, bool constantDims, pybind11::object type) { + pybind11::dtype dt = pybind11::dtype::from_args(type); if (type.is_none()) { // Delay variable creation } #define define_variable(T) \ - else if (type.kind() == pybind11::dtype::of().kind()) \ + else if (dt.type() == pybind11::dtype::of().type()) \ { \ return pybind11::cast( \ io.DefineVariable(name, shape, start, count, constantDims)); \ @@ -47,7 +47,15 @@ void GeneratePythonBindings(pybind11::module &m) pybind11::arg("start").none(false) = Dims{}, pybind11::arg("count").none(false) = Dims{}, pybind11::arg("constantDims") = false, - pybind11::arg("type").none(false)); + pybind11::arg("type").none(false)) + .def("RemoveVariable", &IO::RemoveVariable, + pybind11::arg("name").none(false)) + .def("GetVariable", &IO::GetVariableBase, + pybind11::arg("name").none(false)) + .def("Open", [](IO &io, const std::string &name, + const OpenMode mode) { + return io.Open(name, mode); + }, pybind11::return_value_policy::reference); } } // end namespace adios2 diff --git a/bindings/python/PyVariable.cpp b/bindings/python/PyVariable.cpp index 3b1b87932c..118062404b 100644 --- a/bindings/python/PyVariable.cpp +++ b/bindings/python/PyVariable.cpp @@ -1,8 +1,9 @@ -#include - #include +#include #include +#include + namespace adios2 { template @@ -11,21 +12,28 @@ void GeneratePythonBindings(pybind11::module &m); template <> void GeneratePythonBindings(pybind11::module &m) { - pybind11::class_(m, "Variable") - .def_readonly("Name", &VariableBase::m_Name) - .def_readonly("Type", &VariableBase::m_Type) - .def_readonly("ElementSize", &VariableBase::m_ElementSize) - .def_readonly("Shape", &VariableBase::m_Shape) - .def_readonly("Start", &VariableBase::m_Start) - .def_readonly("Count", &VariableBase::m_Count) - .def_readonly("IsScalar", &VariableBase::m_SingleValue) - .def_readonly("HasConstantDims", &VariableBase::m_ConstantDims) - .def_property_readonly("PayLoadSize", &VariableBase::PayLoadSize) - .def_property_readonly("TotalSize", &VariableBase::TotalSize); + class PyVariableBase : public VariableBase + { + public: + using VariableBase::VariableBase; + pybind11::dtype m_DType; + }; + + pybind11::class_ varBase(m, "Variable"); + varBase.def_readonly("Name", &VariableBase::m_Name); + varBase.def_readonly("ElementSize", &VariableBase::m_ElementSize); + varBase.def_readonly("Shape", &VariableBase::m_Shape); + varBase.def_readonly("Start", &VariableBase::m_Start); + varBase.def_readonly("Count", &VariableBase::m_Count); + varBase.def_readonly("IsScalar", &VariableBase::m_SingleValue); + varBase.def_readonly("HasConstantDims", &VariableBase::m_ConstantDims); + varBase.def_property_readonly("PayLoadSize", &VariableBase::PayLoadSize); + varBase.def_property_readonly("TotalSize", &VariableBase::TotalSize); + varBase.def_property_readonly( + "Type", [](PyVariableBase &self) { return self.m_DType; }); #define pyvar(T, L) \ - pybind11::class_, VariableBase>( \ - m, "Variable" ADIOS2_STRINGIFY(L)) \ + pybind11::class_>(m, "Variable" ADIOS2_STRINGIFY(L), varBase) \ .def_readwrite("Data", &Variable::m_Data); ADIOS2_FOREACH_TYPE_2ARGS(pyvar) #undef pyvar diff --git a/bindings/python/adios2py.cpp b/bindings/python/adios2py.cpp index 217b5b451e..8b3d275e14 100644 --- a/bindings/python/adios2py.cpp +++ b/bindings/python/adios2py.cpp @@ -1,7 +1,7 @@ -#include - #include +#include + namespace adios2 { template @@ -13,6 +13,7 @@ PYBIND11_MODULE(adios2, m) adios2::GeneratePythonBindings(m); adios2::GeneratePythonBindings(m); adios2::GeneratePythonBindings(m); + adios2::GeneratePythonBindings(m); adios2::GeneratePythonBindings(m); adios2::GeneratePythonBindings(m); } diff --git a/source/adios2/ADIOSMacros.h b/source/adios2/ADIOSMacros.h index 1d047893ae..63eced280b 100644 --- a/source/adios2/ADIOSMacros.h +++ b/source/adios2/ADIOSMacros.h @@ -112,6 +112,7 @@ // #define ADIOS2_FOREACH_TYPE_2ARGS(MACRO) \ MACRO(char, Char) \ + MACRO(signed char, SChar) \ MACRO(unsigned char, UChar) \ MACRO(short, Short) \ MACRO(unsigned short, UShort) \ @@ -130,6 +131,7 @@ #define ADIOS2_FOREACH_PRIMITIVE_TYPE_2ARGS(MACRO) \ MACRO(char, Char) \ + MACRO(signed char, SChar) \ MACRO(unsigned char, UChar) \ MACRO(short, Short) \ MACRO(unsigned short, UShort) \ diff --git a/source/adios2/core/Engine.cpp b/source/adios2/core/Engine.cpp index 9d19e892b7..84f8ad08ba 100644 --- a/source/adios2/core/Engine.cpp +++ b/source/adios2/core/Engine.cpp @@ -18,6 +18,8 @@ #include "adios2/helper/adiosFunctions.h" //GetType +#include + namespace adios2 { @@ -130,6 +132,12 @@ VariableBase *Engine::InquireVariableUnknown(const std::string &name, } #define define(T, L) \ + template <> \ + Variable *Engine::InquireVariable(const std::string &variableName, \ + const bool readIn) \ + { \ + return InquireVariable##L(variableName, readIn); \ + } \ Variable *Engine::InquireVariable##L(const std::string &name, \ const bool readIn) \ { \ @@ -170,4 +178,5 @@ void Engine::ThrowUp(const std::string function) const ADIOS2_FOREACH_TYPE_1ARG(declare_template_instantiation) #undef declare_template_instantiation -} // end namespace adios2 + +} // end namespace adios diff --git a/source/adios2/core/Engine.h b/source/adios2/core/Engine.h index 3a405452f0..94c4823d77 100644 --- a/source/adios2/core/Engine.h +++ b/source/adios2/core/Engine.h @@ -294,7 +294,7 @@ class Engine * reading * @return a vector of strings */ - std::vector VariableNames(); + std::vector VariableNames() const; /** * Closes a particular transport, or all if -1 diff --git a/source/adios2/core/Engine.tcc b/source/adios2/core/Engine.tcc index 74666c7324..bdaaba3b14 100644 --- a/source/adios2/core/Engine.tcc +++ b/source/adios2/core/Engine.tcc @@ -47,130 +47,6 @@ void Engine::Write(const std::string &variableName, const T values) Write(m_IO.GetVariable(variableName), &values); } -template <> -Variable *Engine::InquireVariable(const std::string &variableName, - const bool readIn) -{ - return InquireVariableChar(variableName, readIn); -} - -template <> -Variable * -Engine::InquireVariable(const std::string &variableName, - const bool readIn) -{ - return InquireVariableUChar(variableName, readIn); -} - -template <> -Variable *Engine::InquireVariable(const std::string &variableName, - const bool readIn) -{ - return InquireVariableShort(variableName, readIn); -} - -template <> -Variable * -Engine::InquireVariable(const std::string &variableName, - const bool readIn) -{ - return InquireVariableUShort(variableName, readIn); -} - -template <> -Variable *Engine::InquireVariable(const std::string &variableName, - const bool readIn) -{ - return InquireVariableInt(variableName, readIn); -} - -template <> -Variable * -Engine::InquireVariable(const std::string &variableName, - const bool readIn) -{ - return InquireVariableUInt(variableName, readIn); -} - -template <> -Variable * -Engine::InquireVariable(const std::string &variableName, - const bool readIn) -{ - return InquireVariableLInt(variableName, readIn); -} - -template <> -Variable * -Engine::InquireVariable(const std::string &variableName, - const bool readIn) -{ - return InquireVariableLLInt(variableName, readIn); -} - -template <> -Variable * -Engine::InquireVariable(const std::string &variableName, - const bool readIn) -{ - return InquireVariableULInt(variableName, readIn); -} - -template <> -Variable * -Engine::InquireVariable(const std::string &variableName, - const bool readIn) -{ - return InquireVariableULLInt(variableName, readIn); -} - -template <> -Variable *Engine::InquireVariable(const std::string &variableName, - const bool readIn) -{ - return InquireVariableFloat(variableName, readIn); -} - -template <> -Variable * -Engine::InquireVariable(const std::string &variableName, - const bool readIn) -{ - return InquireVariableDouble(variableName, readIn); -} - -template <> -Variable * -Engine::InquireVariable(const std::string &variableName, - const bool readIn) -{ - return InquireVariableLDouble(variableName, readIn); -} - -template <> -Variable * -Engine::InquireVariable(const std::string &variableName, - const bool readIn) -{ - return InquireVariableCFloat(variableName, readIn); -} - -template <> -Variable * -Engine::InquireVariable(const std::string &variableName, - const bool readIn) -{ - return InquireVariableCDouble(variableName, readIn); -} - -template <> -Variable * -Engine::InquireVariable(const std::string &variableName, - const bool readIn) -{ - return InquireVariableCLDouble(variableName, readIn); -} - } // end namespace adios #endif /** ADIOS2_CORE_ENGINE_TCC_ */ diff --git a/source/adios2/core/IO.cpp b/source/adios2/core/IO.cpp index 6304a44bb2..744524b240 100644 --- a/source/adios2/core/IO.cpp +++ b/source/adios2/core/IO.cpp @@ -46,7 +46,9 @@ IO::IO(const std::string name, MPI_Comm mpiComm, const bool inConfigFile, void IO::SetEngine(const std::string engineType) { m_EngineType = engineType; } const std::string IO::GetEngine() const { return m_EngineType; } + void IO::SetIOMode(const IOMode ioMode) { m_IOMode = ioMode; }; +const IOMode IO::GetIOMode() const { return m_IOMode; } void IO::SetParameters(const Params ¶meters) { m_Parameters = parameters; } diff --git a/source/adios2/core/IO.h b/source/adios2/core/IO.h index 2ea35ba0c8..9e81a97374 100644 --- a/source/adios2/core/IO.h +++ b/source/adios2/core/IO.h @@ -90,6 +90,11 @@ class IO * @param IO mode */ void SetIOMode(const IOMode mode); + /** + * Get the IO mode (collective or independent) + */ + const IOMode GetIOMode() const; + /** * Version that passes a map to fill out parameters * initializer list = { "param1", "value1" }, {"param2", "value2"}, diff --git a/source/adios2/helper/adiosType.h b/source/adios2/helper/adiosType.h index 529ab97eed..6dc051d439 100644 --- a/source/adios2/helper/adiosType.h +++ b/source/adios2/helper/adiosType.h @@ -30,6 +30,14 @@ namespace adios2 template inline std::string GetType() noexcept; +/** + * Gets type description from template parameter T + * @return string with type description, sutable for use in variable and class + * names + */ +template +inline std::string GetTypeDescription() noexcept; + /** * Check in types set if "type" is one of the aliases for a certain type, * (e.g. if type = integer is an accepted alias for "int", returning true) diff --git a/source/adios2/helper/adiosType.inl b/source/adios2/helper/adiosType.inl index b81bb59818..0a6b1a6e0a 100644 --- a/source/adios2/helper/adiosType.inl +++ b/source/adios2/helper/adiosType.inl @@ -14,6 +14,8 @@ #error "Inline file should only be included from it's header, never on it's own" #endif +#include "adios2/ADIOSMacros.h" + namespace adios2 { @@ -120,6 +122,15 @@ inline std::string GetType>() noexcept return "long double complex"; } +#define define_impl(T, L) \ + template <> \ + inline std::string GetTypeDescription() noexcept \ + { \ + return ADIOS2_STRINGIFY(L); \ + } +ADIOS2_FOREACH_TYPE_2ARGS(define_impl) +#undef define_impl + template bool IsTypeAlias( const std::string type, diff --git a/thirdparty/pybind11/pybind11/include/pybind11/numpy.h b/thirdparty/pybind11/pybind11/include/pybind11/numpy.h index 55bb816984..abf389163b 100644 --- a/thirdparty/pybind11/pybind11/include/pybind11/numpy.h +++ b/thirdparty/pybind11/pybind11/include/pybind11/numpy.h @@ -471,6 +471,11 @@ class dtype : public object { return detail::array_descriptor_proxy(m_ptr)->kind; } + /// + char type() const { + return detail::array_descriptor_proxy(m_ptr)->type; + } + private: static object _dtype_from_pep3118() { static PyObject *obj = module::import("numpy.core._internal") From 249c8778497837d53636cd576fb3edae6014f516 Mon Sep 17 00:00:00 2001 From: Scott Wittenburg Date: Fri, 6 Oct 2017 12:18:41 -0600 Subject: [PATCH 08/24] Get the python wrapping test working again --- .../bindings/python/TestBPWriteTypes_nompi.py | 39 +++++++++++-------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/testing/adios2/bindings/python/TestBPWriteTypes_nompi.py b/testing/adios2/bindings/python/TestBPWriteTypes_nompi.py index 842adacd8c..0fe407efd6 100644 --- a/testing/adios2/bindings/python/TestBPWriteTypes_nompi.py +++ b/testing/adios2/bindings/python/TestBPWriteTypes_nompi.py @@ -16,51 +16,56 @@ # Test data data = SmallTestData() -adios = adios2.ADIOS(adios2.DebugON) +adios = adios2.ADIOS() bpIO = adios.DeclareIO("NPTypes") # ADIOS Variable name, shape, start, offset, constant dims # All local variables varI8 = bpIO.DefineVariable( - "varI8", [], [], [data.I8.size], adios2.ConstantDims) + "varI8", [], [], [data.I8.size], True, data.I8.dtype) varI16 = bpIO.DefineVariable( - "varI16", [], [], [data.I16.size], adios2.ConstantDims) + "varI16", [], [], [data.I16.size], True, data.I16.dtype) varI32 = bpIO.DefineVariable( - "varI32", [], [], [data.I32.size], adios2.ConstantDims) -varI64 = bpIO.DefineVariable( - "varI64", [], [], [data.I64.size], adios2.ConstantDims) + "varI32", [], [], [data.I32.size], True, data.I32.dtype) +# FIXME: Uncomment below when 64 bit ints work properly +# varI64 = bpIO.DefineVariable( +# "varI64", [], [], [data.I64.size], True, data.I64.dtype) varU8 = bpIO.DefineVariable( - "varUI8", [], [], [data.U8.size], adios2.ConstantDims) + "varUI8", [], [], [data.U8.size], True, data.U8.dtype) varU16 = bpIO.DefineVariable( - "varUI16", [], [], [data.U16.size], adios2.ConstantDims) + "varUI16", [], [], [data.U16.size], True, data.U16.dtype) varU32 = bpIO.DefineVariable( - "varUI32", [], [], [data.U32.size], adios2.ConstantDims) -varU64 = bpIO.DefineVariable( - "varUI64", [], [], [data.U64.size], adios2.ConstantDims) + "varUI32", [], [], [data.U32.size], True, data.U32.dtype) +# FIXME: Uncomment below when 64 bit ints work properly +# varU64 = bpIO.DefineVariable( +# "varUI64", [], [], [data.U64.size], True, data.U64.dtype) varR32 = bpIO.DefineVariable( - "varR32", [], [], [data.R32.size], adios2.ConstantDims) + "varR32", [], [], [data.R32.size], True, data.R32.dtype) varR64 = bpIO.DefineVariable( - "varR64", [], [], [data.R64.size], adios2.ConstantDims) + "varR64", [], [], [data.R64.size], True, data.R64.dtype) # ADIOS Engine -bpFileWriter = bpIO.Open("npTypes.bp", adios2.OpenModeWrite) +bpFileWriter = bpIO.Open("npTypes.bp", adios2.OpenMode.Write) bpFileWriter.Write(varI8, data.I8) bpFileWriter.Write(varI16, data.I16) bpFileWriter.Write(varI32, data.I32) -bpFileWriter.Write(varI64, data.I64) +# FIXME: Uncomment below when 64 bit ints work properly +# bpFileWriter.Write(varI64, data.I64) bpFileWriter.Write(varU8, data.U8) bpFileWriter.Write(varU16, data.U16) bpFileWriter.Write(varU32, data.U32) -bpFileWriter.Write(varU64, data.U64) +# FIXME: Uncomment below when 64 bit ints work properly +# bpFileWriter.Write(varU64, data.U64) bpFileWriter.Write(varR32, data.R32) bpFileWriter.Write(varR64, data.R64) -bpFileWriter.Close() +# FIXME: Pass transportIndex until the default args works properly +bpFileWriter.Close(-1) From 197799a97a3c66ce805c700bff302ab5a9b3ef70 Mon Sep 17 00:00:00 2001 From: Scott Wittenburg Date: Fri, 6 Oct 2017 12:49:37 -0600 Subject: [PATCH 09/24] Add a python interpreter, an instance helper, and a new engine type --- source/adios2/CMakeLists.txt | 12 ++ source/adios2/core/IO.cpp | 13 ++ .../adios2/engine/pyengine/PythonEngine.cpp | 183 ++++++++++++++++++ source/adios2/engine/pyengine/PythonEngine.h | 82 ++++++++ source/adios2/helper/PythonInstanceBuilder.h | 58 ++++++ source/adios2/helper/PythonInterpreter.cpp | 66 +++++++ source/adios2/helper/PythonInterpreter.h | 49 +++++ 7 files changed, 463 insertions(+) create mode 100644 source/adios2/engine/pyengine/PythonEngine.cpp create mode 100644 source/adios2/engine/pyengine/PythonEngine.h create mode 100644 source/adios2/helper/PythonInstanceBuilder.h create mode 100644 source/adios2/helper/PythonInterpreter.cpp create mode 100644 source/adios2/helper/PythonInterpreter.h diff --git a/source/adios2/CMakeLists.txt b/source/adios2/CMakeLists.txt index ab0ddc0279..0ddf3b1e30 100644 --- a/source/adios2/CMakeLists.txt +++ b/source/adios2/CMakeLists.txt @@ -135,6 +135,18 @@ if(ADIOS2_HAVE_HDF5) target_link_libraries(adios2 PRIVATE ${HDF5_C_LIBRARIES}) endif() +if(ADIOS2_HAVE_Python) + target_sources(adios2 PRIVATE + helper/PythonInterpreter.h + helper/PythonInterpreter.cpp + helper/PythonInstanceBuilder.h + engine/pyengine/PythonEngine.h + engine/pyengine/PythonEngine.cpp + ) + target_include_directories(adios2 PRIVATE ${PYBIND11_INCLUDE_DIR} ${PYTHON_INCLUDE_DIRS}) + target_link_libraries(adios2 PRIVATE pybind11::embed) +endif() + # Set library version information set_target_properties(adios2 PROPERTIES VERSION ${ADIOS2_VERSION} diff --git a/source/adios2/core/IO.cpp b/source/adios2/core/IO.cpp index 744524b240..34148b2d34 100644 --- a/source/adios2/core/IO.cpp +++ b/source/adios2/core/IO.cpp @@ -34,6 +34,10 @@ #endif #endif +#ifdef ADIOS2_HAVE_PYTHON +#include "adios2/engine/pyengine/PythonEngine.h" +#endif + namespace adios2 { @@ -320,6 +324,15 @@ std::shared_ptr IO::Open(const std::string &name, { engine = std::make_shared(*this, name, openMode, mpiComm); } + else if (m_EngineType == "PythonEngine") + { +#ifdef ADIOS2_HAVE_PYTHON + engine = std::make_shared(*this, name, openMode, mpiComm); +#else + throw std::invalid_argument("ERROR: this version didn't compile with " + "Python enabled, can't use PythonEngine\n"); +#endif + } else { if (m_DebugMode) diff --git a/source/adios2/engine/pyengine/PythonEngine.cpp b/source/adios2/engine/pyengine/PythonEngine.cpp new file mode 100644 index 0000000000..ab833ee6c0 --- /dev/null +++ b/source/adios2/engine/pyengine/PythonEngine.cpp @@ -0,0 +1,183 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * PythonEngine.cpp + * + * Created on: Sept 21, 2017 + * Author: Scott Wittenburg + */ + +#include "PythonEngine.h" + +#include +#include +#include +#include +#include +#include + +#include "adios2/helper/PythonInterpreter.h" +#include "adios2/helper/PythonInstanceBuilder.h" + +#include +#include + +namespace adios2 +{ + +/******************************************************************************/ + +struct PythonEngine::Impl +{ + std::string m_PluginName; + pybind11::object enginePyObject; +}; + + +/******************************************************************************/ + +PythonEngine::PythonEngine(IO &io, const std::string &name, + const OpenMode openMode, MPI_Comm mpiComm) +: Engine("Plugin", io, name, openMode, mpiComm), m_Impl(new Impl) +{ + Init(); + // m_Impl->m_Plugin = + // m_Impl->m_HandleCreate(io, m_Impl->m_PluginName, openMode, mpiComm); + std::cout << "PythonEngine::PythonEngine" << std::endl; +} + +PythonEngine::~PythonEngine() { + // m_Impl->m_HandleDestroy(m_Impl->m_Plugin); + std::cout << "PythonEngine::~PythonEngine" << std::endl; +} + +void PythonEngine::PerformReads(ReadMode mode) +{ + // m_Impl->m_Plugin->PerformReads(mode); + std::cout << "PythonEngine::PerformReads" << std::endl; +} + +void PythonEngine::Release() { + // m_Impl->m_Plugin->Release(); + std::cout << "PythonEngine::Release" << std::endl; +} + +void PythonEngine::Advance(const float timeoutSeconds) +{ + // m_Impl->m_Plugin->Advance(timeoutSeconds); + std::cout << "PythonEngine::Advance(timeoutSeconds)" + << std::endl; +} + +void PythonEngine::Advance(const AdvanceMode mode, const float timeoutSeconds) +{ + // m_Impl->m_Plugin->Advance(mode, timeoutSeconds); + std::cout << "PythonEngine::Advance(mode, timeoutSeconds)" + << std::endl; +} + +void PythonEngine::AdvanceAsync(const AdvanceMode mode, + AdvanceAsyncCallback callback) +{ + // m_Impl->m_Plugin->AdvanceAsync(mode, callback); + std::cout << "PythonEngine::AdvanceAsync" << std::endl; +} + +void PythonEngine::SetCallBack( + std::function)> + callback) +{ + // m_Impl->m_Plugin->SetCallBack(callback); + std::cout << "PythonEngine::SetCallBack" << std::endl; +} + +void PythonEngine::Close(const int transportIndex) +{ + // m_Impl->m_Plugin->Close(transportIndex); + std::cout << "PythonEngine::Close" << std::endl; +} + +void PythonEngine::Init() +{ + auto paramPluginNameIt = m_IO.m_Parameters.find("PluginName"); + if (paramPluginNameIt == m_IO.m_Parameters.end()) + { + throw std::invalid_argument("PythonEngine: PluginName must be " + "specified in engine parameters"); + } + m_Impl->m_PluginName = paramPluginNameIt->second; + + // Get the python engine plugin module name, if provided + std::string* pluginModuleName = nullptr; + auto paramPluginModuleIt = m_IO.m_Parameters.find("PluginModule"); + if (paramPluginModuleIt != m_IO.m_Parameters.end()) + { + pluginModuleName = &(paramPluginModuleIt->second); + } + + // Get the python engine plugin class name + auto paramPluginClassIt = m_IO.m_Parameters.find("PluginClass"); + if (paramPluginClassIt == m_IO.m_Parameters.end()) + { + throw std::invalid_argument("PythonEngine: PluginClass must be " + "specified in engine parameters"); + } + std::string pluginClassName = paramPluginClassIt->second; + + // Initialize python interpreter if it's not already running + adios2::PythonInterpreter::instance().initialize(); + + m_Impl->enginePyObject = + adios2::PythonInstanceBuilder::BuildInstance(pluginClassName, + pluginModuleName); +} + +#define define(T) \ + void PythonEngine::DoWrite(Variable &variable, const T *values) \ + { \ + std::cout << "PythonEngine::DoWrite(var, vals), " \ + << "type: " << typeid(T).name() << std::endl; \ + m_Impl->enginePyObject.attr("Writef")(); \ + } \ + void PythonEngine::DoScheduleRead(Variable &variable, \ + const T *values) \ + { \ + std::cout << "PythonEngine::DoScheduledRead(var, vals), " \ + << "type: " << typeid(T).name() << std::endl; \ + } \ + void PythonEngine::DoScheduleRead(const std::string &variableName, \ + const T *values) \ + { \ + std::cout << "PythonEngine::DoScheduledRead(varName, vals), " \ + << "type: " << typeid(T).name() << std::endl; \ + } +ADIOS2_FOREACH_TYPE_1ARG(define) +#undef define + +void PythonEngine::DoWrite(VariableCompound &variable, + const void *values) +{ + // m_Impl->m_Plugin->DoWrite(variable, values); + std::cout << "PythonEngine::DoWrite" << std::endl; +} + +#define define(T, L) \ + Variable *PythonEngine::InquireVariable##L( \ + const std::string &name, const bool readIn) \ + { \ + std::cout << "PythonEngine::InquireVariable##L" << std::endl; \ + return InquireVariable##L(name, readIn); \ + } +ADIOS2_FOREACH_TYPE_2ARGS(define) +#undef define + +VariableBase *PythonEngine::InquireVariableUnknown( + const std::string &name, const bool readIn) +{ + std::cout << "PythonEngine::InquireVariableUnknown" << std::endl; + return InquireVariableUnknown(name, readIn); +} + +} // end namespace adios2 diff --git a/source/adios2/engine/pyengine/PythonEngine.h b/source/adios2/engine/pyengine/PythonEngine.h new file mode 100644 index 0000000000..a253b39783 --- /dev/null +++ b/source/adios2/engine/pyengine/PythonEngine.h @@ -0,0 +1,82 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * PythonEngine.h Support for an engine implemented in python + * + * Created on: Sept 21, 2017 + * Author: Scott Wittenburg + */ + +#ifndef ADIOS2_ENGINE_PYENGINE_PYTHONENGINE_H_ +#define ADIOS2_ENGINE_PYENGINE_PYTHONENGINE_H_ + + +#include // for function +#include // for unique_ptr +#include // for string +#include // for add_pointer +#include // for vector + +#include "adios2/ADIOSMPICommOnly.h" +#include "adios2/ADIOSMacros.h" +#include "adios2/ADIOSTypes.h" +#include "adios2/core/Engine.h" +#include "adios2/core/IO.h" +#include "adios2/core/Variable.h" +#include "adios2/core/VariableCompound.h" + +namespace adios2 +{ + +/** A front-end wrapper for an engine implemented in python */ +class PythonEngine : public Engine +{ + +public: + PythonEngine(IO &io, const std::string &name, const OpenMode openMode, + MPI_Comm mpiComm); + virtual ~PythonEngine(); + + void PerformReads(ReadMode mode) override; + void Release() override; + void Advance(const float timeoutSeconds = 0.0) override; + void Advance(const AdvanceMode mode, + const float timeoutSeconds = 0.0) override; + void AdvanceAsync(const AdvanceMode mode, + AdvanceAsyncCallback callback) override; + + void SetCallBack(std::function)> + callback) override; + + void Close(const int transportIndex = -1) override; + +protected: + void Init() override; + +#define declare(T) \ + void DoWrite(Variable &variable, const T *values) override; \ + void DoScheduleRead(Variable &variable, const T *values) override; \ + void DoScheduleRead(const std::string &variableName, const T *values) \ + override; + ADIOS2_FOREACH_TYPE_1ARG(declare) +#undef declare + void DoWrite(VariableCompound &variable, const void *values) override; + +#define declare(T, L) \ + Variable *InquireVariable##L(const std::string &name, \ + const bool readIn) override; + ADIOS2_FOREACH_TYPE_2ARGS(declare) +#undef declare + VariableBase *InquireVariableUnknown(const std::string &name, + const bool readIn) override; + +private: + struct Impl; + std::unique_ptr m_Impl; +}; + +} // end namespace adios + +#endif /* ADIOS2_ENGINE_PYENGINE_PYTHONENGINE_H_ */ diff --git a/source/adios2/helper/PythonInstanceBuilder.h b/source/adios2/helper/PythonInstanceBuilder.h new file mode 100644 index 0000000000..29636c1f30 --- /dev/null +++ b/source/adios2/helper/PythonInstanceBuilder.h @@ -0,0 +1,58 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * PythonInstanceBuilder.h Instantiate some arbitrary python class + * + * Created on: Oct 02, 2017 + * Author: Scott Wittenburg + */ + +#ifndef ADIOS2_HELPER_PYTHONINSTANCEBUILDER_H_ +#define ADIOS2_HELPER_PYTHONINSTANCEBUILDER_H_ + +#include +#include + +namespace adios2 +{ +namespace PythonInstanceBuilder +{ + pybind11::object BuildInstance(const std::string& className, + const std::string* moduleName = nullptr) + { + pybind11::dict globals = pybind11::globals(); + + if (moduleName != nullptr) + { + pybind11::object moduleObject; + + if (globals.contains((*moduleName).c_str())) + { + moduleObject = globals[(*moduleName).c_str()]; + } + else + { + moduleObject = pybind11::module::import((*moduleName).c_str()); + } + + pybind11::object constructor = + moduleObject.attr(className.c_str()); + return constructor(); + } + else if (globals.contains(className.c_str())) + { + pybind11::object constructor = globals[className.c_str()]; + return constructor(); + } + + // Unable to instantiate the object in this case + throw std::runtime_error("PythonInstanceBuilder: Specified class was " + "not present in main module, nor was a " + "module name provided in the parameters. " + "Unable to instantiate python class."); + } +} +} + +#endif /* ADIOS2_HELPER_PYTHONINSTANCEBUILDER_H_ */ diff --git a/source/adios2/helper/PythonInterpreter.cpp b/source/adios2/helper/PythonInterpreter.cpp new file mode 100644 index 0000000000..3c88241ac4 --- /dev/null +++ b/source/adios2/helper/PythonInterpreter.cpp @@ -0,0 +1,66 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * PythonInterpreter.cpp + * + * Created on: Sept 27, 2017 + * Author: Scott Wittenburg + */ + +#include "PythonInterpreter.h" + +#include + +#if WIN32 +#include +extern __declspec(dllimport) int Py_NoSiteFlag; +#endif + +namespace adios2 +{ + +PythonInterpreter PythonInterpreter::m_instance; + +PythonInterpreter& PythonInterpreter::instance() +{ + return PythonInterpreter::m_instance; +} + +PythonInterpreter::PythonInterpreter() + : m_embedded(false) +{ + this->initialize(); +} + +PythonInterpreter::~PythonInterpreter() +{ + this->finalize(); +} + +bool PythonInterpreter::isInitialized() const +{ + return Py_IsInitialized() != 0; +} + +void PythonInterpreter::initialize() +{ + if (this->isInitialized()) + { + return; + } + + m_embedded = true; + Py_NoSiteFlag = 1; + pybind11::initialize_interpreter(); +} + +void PythonInterpreter::finalize() +{ + if (this->isInitialized()) + { + pybind11::finalize_interpreter(); + m_embedded = false; + } +} +} \ No newline at end of file diff --git a/source/adios2/helper/PythonInterpreter.h b/source/adios2/helper/PythonInterpreter.h new file mode 100644 index 0000000000..ef4d8a4b37 --- /dev/null +++ b/source/adios2/helper/PythonInterpreter.h @@ -0,0 +1,49 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * PythonInterpreter.h Singleton python interpreter functionality + * + * Created on: Sept 27, 2017 + * Author: Scott Wittenburg + */ + +#ifndef ADIOS2_HELPER_PYTHONINTERPRETER_H_ +#define ADIOS2_HELPER_PYTHONINTERPRETER_H_ + +namespace adios2 +{ + +/** @brief Singleton class for encapsulating the embedded interpreter. + Ensures that the interpreter is initialized/finalized at static + initialization/deinitialization. It is left to the user to ensure + that their python modules can be found on the python path. + */ +class PythonInterpreter +{ +public: + static PythonInterpreter& instance(); + + // Check if python is initialized. + bool isInitialized() const; + + // Initialize the embedded python + void initialize(); + + // Finalize the embedded python. + void finalize(); + + // Returns true if the embedded python session has been initialized. + bool isEmbedded() const { return m_embedded; } + +private: + PythonInterpreter(); + virtual ~PythonInterpreter(); + + static PythonInterpreter m_instance; + + bool m_embedded; +}; +} + +#endif /* ADIOS2_HELPER_PYTHONINTERPRETER_H_ */ \ No newline at end of file From 89f1a551f07ee6fc584d3bb54a82a8a96d1e3e57 Mon Sep 17 00:00:00 2001 From: Scott Wittenburg Date: Fri, 6 Oct 2017 17:16:08 -0600 Subject: [PATCH 10/24] Lots of changes to get things working, and we're still not there --- bindings/python/PyEngine.cpp | 24 ++-- cmake/ADIOSFunctions.cmake | 21 ++- examples/hello/bpWriter/CMakeLists.txt | 6 + .../hello/bpWriter/examplePythonPlugin.cpp | 73 ++++++++++ source/adios2/core/Engine.h | 6 + .../adios2/engine/pyengine/PythonEngine.cpp | 22 +-- source/adios2/helper/PythonInstanceBuilder.h | 4 +- .../bindings/python/Adios2PythonTestBase.py | 15 ++ testing/adios2/bindings/python/CMakeLists.txt | 26 +++- .../bindings/python/TestBPWriteTypes.py | 129 ++++++++++-------- .../bindings/python/TestBPWriteTypes_nompi.py | 71 ---------- .../bindings/python/TestPythonEngine.py | 28 ++++ .../python/TestPythonEngineFromCpp.cpp | 92 +++++++++++++ 13 files changed, 363 insertions(+), 154 deletions(-) create mode 100644 examples/hello/bpWriter/examplePythonPlugin.cpp create mode 100644 testing/adios2/bindings/python/Adios2PythonTestBase.py delete mode 100644 testing/adios2/bindings/python/TestBPWriteTypes_nompi.py create mode 100644 testing/adios2/bindings/python/TestPythonEngine.py create mode 100644 testing/adios2/bindings/python/TestPythonEngineFromCpp.cpp diff --git a/bindings/python/PyEngine.cpp b/bindings/python/PyEngine.cpp index 15901170d6..3214b5463c 100644 --- a/bindings/python/PyEngine.cpp +++ b/bindings/python/PyEngine.cpp @@ -15,16 +15,16 @@ template <> void GeneratePythonBindings(pybind11::module &m) { // Wrapping for internal Engine classes - pybind11::class_>(m, "EngineBase") - .def("Write", + pybind11::class_> engine(m, "EngineBase"); + engine.def("Write", [](Engine &e, VariableBase &variable, pybind11::array values) { e.Write(variable, values.data()); - }) - .def("Write", + }); + engine.def("Write", [](Engine &e, const std::string name, pybind11::array values) { e.Write(name, values.data()); - }) - .def("InquireVariable", + }); + engine.def("InquireVariable", [](Engine &e, const std::string name) -> VariableBase * { VariableBase *var; #define inquire(T) \ @@ -35,8 +35,8 @@ void GeneratePythonBindings(pybind11::module &m) ADIOS2_FOREACH_TYPE_1ARG(inquire) #undef inquire return nullptr; - }) - .def("Close", &Engine::Close); + }); + engine.def("Close", &Engine::Close); // @@ -66,6 +66,7 @@ void GeneratePythonBindings(pybind11::module &m) #define define_dowrite(T) \ void DoWrite(Variable &var, const T *values) override \ { \ + std::cout << "BINGO!" << std::endl; \ DoWrite(var, \ pybind11::array_t(std::accumulate(var.m_Count.begin(), \ var.m_Count.end(), 0), \ @@ -84,19 +85,22 @@ void GeneratePythonBindings(pybind11::module &m) using PyEngineBase::PyEngineBase; void Init() override { - PYBIND11_OVERLOAD(void, PyEngineBase, Init,); + std::cout << "PyEngine Init() trampoline" << std::endl; + PYBIND11_OVERLOAD(void, PyEngineBase, Init,); } void DoWrite(VariableBase &var, pybind11::array values) override { + std::cout << "PyEngine DoWrite() trampoline" << std::endl; PYBIND11_OVERLOAD_PURE(void, PyEngineBase, DoWrite, var, values); } void Close(const int transportIndex) { + std::cout << "PyEngine Close() trampoline" << std::endl; PYBIND11_OVERLOAD_PURE(void, PyEngineBase, Close, transportIndex); } }; - pybind11::class_(m, "Engine") + pybind11::class_(m, "Engine", engine) .def(pybind11::init()) .def("Init", &PyEngineBase::Init) diff --git a/cmake/ADIOSFunctions.cmake b/cmake/ADIOSFunctions.cmake index 8864a202fd..9d39948402 100644 --- a/cmake/ADIOSFunctions.cmake +++ b/cmake/ADIOSFunctions.cmake @@ -24,8 +24,13 @@ endfunction() function(python_add_test) set(options) + # NAME: test name + # ADDTOPYPATH: Provide this if python modules need to be found in specific directories + # USE_MPI_FOR_PYTHON_TESTS: Set to anything for mpi mode set(oneValueArgs NAME + ADDTOPYPATH + USE_MPI_FOR_PYTHON_TESTS ) # EXEC_WRAPPER: Any extra arguments to pass on the command line before test case # SCRIPT: Script name and corresponding comand line inputs @@ -34,8 +39,20 @@ function(python_add_test) add_test(NAME ${ARGS_NAME} COMMAND ${ARGS_EXEC_WRAPPER} ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/${ARGS_SCRIPT} ) - set_property(TEST ${ARGS_NAME} PROPERTY - ENVIRONMENT "PYTHONPATH=${ADIOS2_BINARY_DIR}/${CMAKE_INSTALL_PYTHONDIR}:$ENV{PYTHONPATH}" + + set(testsPythonPath "${ADIOS2_BINARY_DIR}/${CMAKE_INSTALL_PYTHONDIR}:$ENV{PYTHONPATH}") + + if(DEFINED ARGS_ADDTOPYPATH) + set(testsPythonPath "${ARGS_ADDTOPYPATH}:${testsPythonPath}") + endif() + + if(ARGS_USE_MPI_FOR_PYTHON_TESTS) + set(mpi_flag "ADIOS2_PYTHON_TESTS_USE_MPI=${ARGS_USE_MPI_FOR_PYTHON_TESTS}") + endif() + + set_property(TEST ${ARGS_NAME} PROPERTY ENVIRONMENT + "PYTHONPATH=${testsPythonPath}" + ${mpi_flag} ) endfunction() diff --git a/examples/hello/bpWriter/CMakeLists.txt b/examples/hello/bpWriter/CMakeLists.txt index 3f3f19f4ad..c1aed3a77d 100644 --- a/examples/hello/bpWriter/CMakeLists.txt +++ b/examples/hello/bpWriter/CMakeLists.txt @@ -27,3 +27,9 @@ endif() target_link_libraries(hello_bpWriter adios2) target_link_libraries(hello_bpWriter_c adios2) + + + if(ADIOS2_HAVE_Python) + add_executable(examplePythonPlugin examplePythonPlugin.cpp) + target_link_libraries(examplePythonPlugin PUBLIC adios2) + endif() \ No newline at end of file diff --git a/examples/hello/bpWriter/examplePythonPlugin.cpp b/examples/hello/bpWriter/examplePythonPlugin.cpp new file mode 100644 index 0000000000..cf4c0a1993 --- /dev/null +++ b/examples/hello/bpWriter/examplePythonPlugin.cpp @@ -0,0 +1,73 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * examplePythonPlugin.cpp Example of using an Engine defined in python + * + * Created on: Sept 21, 2017 + * Author: Scott Wittenburg //std::ios_base::failure +#include //std::cout +#include //std::invalid_argument std::exception +#include + +#include + +int main(int argc, char *argv[]) +{ + /** Application variable */ + std::vector myFloats = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; + const std::size_t Nx = myFloats.size(); + + try + { + /** ADIOS class factory of IO class objects, DebugON is recommended */ + adios2::ADIOS adios(adios2::DebugON); + + /*** IO class object: settings and factory of Settings: Variables, + * Parameters, Transports, and Execution: Engines */ + adios2::IO &io = adios.DeclareIO("PythonPluginIO"); + + /** global array: name, { shape (total dimensions) }, { start (local) }, + * { count (local) }, all are constant dimensions */ + adios2::Variable &var = io.DefineVariable( + "data", {}, {}, {Nx}, adios2::ConstantDims); + + /** Engine derived class, spawned to start IO operations */ + io.SetEngine("PythonEngine"); + io.SetParameters({{"PluginName", "FirstPythonPlugin"}, + {"PluginModule", "TestPythonEngine"}, + {"PluginClass", "TestPythonEngine"}}); + auto writer = io.Open("TestPythonPlugin", adios2::OpenMode::Write); + + if (!writer) + { + throw std::ios_base::failure("ERROR: writer not created at Open\n"); + } + + /** Write variable for buffering */ + writer->Write(var, myFloats.data()); + + /** Create bp file, engine becomes unreachable after this*/ + writer->Close(); + } + catch (std::invalid_argument &e) + { + std::cout << "Invalid argument exception, STOPPING PROGRAM\n"; + std::cout << e.what() << "\n"; + } + catch (std::ios_base::failure &e) + { + std::cout << "IO System base failure exception, STOPPING PROGRAM\n"; + std::cout << e.what() << "\n"; + } + catch (std::exception &e) + { + std::cout << "Exception, STOPPING PROGRAM from rank\n"; + std::cout << e.what() << "\n"; + } + + return 0; +} \ No newline at end of file diff --git a/source/adios2/core/Engine.h b/source/adios2/core/Engine.h index 94c4823d77..80f6693c2e 100644 --- a/source/adios2/core/Engine.h +++ b/source/adios2/core/Engine.h @@ -37,6 +37,12 @@ namespace adios2 * Close */ class Engine { + + // PythonEngine not only inherits from Engine, but also holds a shared_ptr + // to an instance of Engine, so it can proxy method calls to it. Hence, + // PythonEngine needs access to everything in here. + friend class PythonEngine; + public: using AdvanceAsyncCallback = std::function)>; diff --git a/source/adios2/engine/pyengine/PythonEngine.cpp b/source/adios2/engine/pyengine/PythonEngine.cpp index ab833ee6c0..81480ed5fd 100644 --- a/source/adios2/engine/pyengine/PythonEngine.cpp +++ b/source/adios2/engine/pyengine/PythonEngine.cpp @@ -31,6 +31,7 @@ namespace adios2 struct PythonEngine::Impl { std::string m_PluginName; + pybind11::object enginePyClass; pybind11::object enginePyObject; }; @@ -39,12 +40,13 @@ struct PythonEngine::Impl PythonEngine::PythonEngine(IO &io, const std::string &name, const OpenMode openMode, MPI_Comm mpiComm) -: Engine("Plugin", io, name, openMode, mpiComm), m_Impl(new Impl) +: Engine("PythonEngine", io, name, openMode, mpiComm), m_Impl(new Impl) { + std::cout << "PythonEngine::PythonEngine" << std::endl; Init(); // m_Impl->m_Plugin = // m_Impl->m_HandleCreate(io, m_Impl->m_PluginName, openMode, mpiComm); - std::cout << "PythonEngine::PythonEngine" << std::endl; + m_Impl->enginePyObject = m_Impl->enginePyClass("PythonEngine", io, name, openMode); } PythonEngine::~PythonEngine() { @@ -129,18 +131,20 @@ void PythonEngine::Init() // Initialize python interpreter if it's not already running adios2::PythonInterpreter::instance().initialize(); - m_Impl->enginePyObject = + m_Impl->enginePyClass = adios2::PythonInstanceBuilder::BuildInstance(pluginClassName, pluginModuleName); } -#define define(T) \ +#define define(T) \ void PythonEngine::DoWrite(Variable &variable, const T *values) \ - { \ - std::cout << "PythonEngine::DoWrite(var, vals), " \ - << "type: " << typeid(T).name() << std::endl; \ - m_Impl->enginePyObject.attr("Writef")(); \ - } \ + { \ + std::shared_ptr eng = m_Impl->enginePyObject.cast>(); \ + std::cout << "PythonEngine::DoWrite(var, vals), " \ + << "type: " << typeid(T).name() << std::endl; \ + eng->DoWrite(variable, values); \ + } \ + \ void PythonEngine::DoScheduleRead(Variable &variable, \ const T *values) \ { \ diff --git a/source/adios2/helper/PythonInstanceBuilder.h b/source/adios2/helper/PythonInstanceBuilder.h index 29636c1f30..8ece50d907 100644 --- a/source/adios2/helper/PythonInstanceBuilder.h +++ b/source/adios2/helper/PythonInstanceBuilder.h @@ -38,12 +38,12 @@ namespace PythonInstanceBuilder pybind11::object constructor = moduleObject.attr(className.c_str()); - return constructor(); + return constructor; } else if (globals.contains(className.c_str())) { pybind11::object constructor = globals[className.c_str()]; - return constructor(); + return constructor; } // Unable to instantiate the object in this case diff --git a/testing/adios2/bindings/python/Adios2PythonTestBase.py b/testing/adios2/bindings/python/Adios2PythonTestBase.py new file mode 100644 index 0000000000..10143cb874 --- /dev/null +++ b/testing/adios2/bindings/python/Adios2PythonTestBase.py @@ -0,0 +1,15 @@ +import unittest +import os + +class Adios2PythonTestBase(unittest.TestCase): + ### Convenience method to let python tests know if MPI is enabled + @staticmethod + def isUsingMpi(): + if 'ADIOS2_PYTHON_TESTS_USE_MPI' in os.environ: + return True + return False + + ### Expose main method so python tests don't need to import unittest + @staticmethod + def main(): + unittest.main() diff --git a/testing/adios2/bindings/python/CMakeLists.txt b/testing/adios2/bindings/python/CMakeLists.txt index 6286000b81..f6e3102529 100644 --- a/testing/adios2/bindings/python/CMakeLists.txt +++ b/testing/adios2/bindings/python/CMakeLists.txt @@ -3,13 +3,27 @@ # accompanying file Copyright.txt for details. #------------------------------------------------------------------------------# -if(NOT ADIOS2_HAVE_MPI) - python_add_test(NAME PythonBPWrite SCRIPT TestBPWriteTypes_nompi.py) -endif() - if(ADIOS2_HAVE_MPI) + set(extra_test_args EXEC_WRAPPER ${MPIEXEC_COMMAND}) set(test_parameters EXEC_WRAPPER - ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS}) - python_add_test(NAME PythonBPWrite ${test_parameters} SCRIPT TestBPWriteTypes.py) + ${MPIEXEC} -oversubscribe ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS}) + set(use_mpi USE_MPI_FOR_PYTHON_TESTS "TRUE") endif() + +python_add_test(NAME PythonBPWrite SCRIPT TestBPWriteTypes.py + ${test_parameters} ${use_mpi}) +# python_add_test(NAME TestOpenPythonEngineFromPython SCRIPT TestOpenPythonEngineFromPython.py +# ${test_parameters} ${use_mpi} +# ADDTOPYPATH ${CMAKE_CURRENT_SOURCE_DIR}) +# python_add_test(NAME TestOpenInlinePythonEngineFromPython SCRIPT TestOpenInlinePythonEngineFromPython.py +# ${test_parameters} ${use_mpi}) + +add_executable(TestPythonEngineFromCpp TestPythonEngineFromCpp.cpp) +target_link_libraries(TestPythonEngineFromCpp adios2 gtest gtest_main) + +gtest_add_tests(TARGET TestPythonEngineFromCpp ${extra_test_args} TEST_LIST pyEngineTests) + +set_property(TEST ${pyEngineTests} PROPERTY + ENVIRONMENT "PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR}:${ADIOS2_BINARY_DIR}/${CMAKE_INSTALL_PYTHONDIR}:$ENV{PYTHONPATH}" +) diff --git a/testing/adios2/bindings/python/TestBPWriteTypes.py b/testing/adios2/bindings/python/TestBPWriteTypes.py index 36cfe7243a..0c20202314 100644 --- a/testing/adios2/bindings/python/TestBPWriteTypes.py +++ b/testing/adios2/bindings/python/TestBPWriteTypes.py @@ -8,61 +8,82 @@ # Created on: Feb 2, 2017 # Author: William F Godoy godoywf@ornl.gov +from Adios2PythonTestBase import Adios2PythonTestBase + +if Adios2PythonTestBase.isUsingMpi(): + from mpi4py import MPI from adios2NPTypes import SmallTestData -from mpi4py import MPI import adios2 - -# Test data -data = SmallTestData() - -comm = MPI.COMM_WORLD -adios = adios2.ADIOS(comm, adios2.DebugON) - -bpIO = adios.DeclareIO("NPTypes") - -# ADIOS Variable name, shape, start, offset, constant dims -# All local variables -varI8 = bpIO.DefineVariable( - "varI8", [], [], [data.I8.size], adios2.ConstantDims) -varI16 = bpIO.DefineVariable( - "varI16", [], [], [data.I16.size], adios2.ConstantDims) -varI32 = bpIO.DefineVariable( - "varI32", [], [], [data.I32.size], adios2.ConstantDims) -varI64 = bpIO.DefineVariable( - "varI64", [], [], [data.I64.size], adios2.ConstantDims) - -varU8 = bpIO.DefineVariable( - "varUI8", [], [], [data.U8.size], adios2.ConstantDims) -varU16 = bpIO.DefineVariable( - "varUI16", [], [], [data.U16.size], adios2.ConstantDims) -varU32 = bpIO.DefineVariable( - "varUI32", [], [], [data.U32.size], adios2.ConstantDims) -varU64 = bpIO.DefineVariable( - "varUI64", [], [], [data.U64.size], adios2.ConstantDims) - -varR32 = bpIO.DefineVariable( - "varR32", [], [], [data.R32.size], adios2.ConstantDims) - -varR64 = bpIO.DefineVariable( - "varR64", [], [], [data.R64.size], adios2.ConstantDims) - - -# ADIOS Engine -bpFileWriter = bpIO.Open("npTypes.bp", adios2.OpenModeWrite) - -bpFileWriter.Write(varI8, data.I8) -bpFileWriter.Write(varI16, data.I16) -bpFileWriter.Write(varI32, data.I32) -bpFileWriter.Write(varI64, data.I64) - -bpFileWriter.Write(varU8, data.U8) -bpFileWriter.Write(varU16, data.U16) -bpFileWriter.Write(varU32, data.U32) -bpFileWriter.Write(varU64, data.U64) - -bpFileWriter.Write(varR32, data.R32) -bpFileWriter.Write(varR64, data.R64) - -bpFileWriter.Close() +### +### Create a testcase class with some tests +### +class TestBPWriteTypes(Adios2PythonTestBase): + def testWriteTypes(self): + # Test data + data = SmallTestData() + + if self.isUsingMpi(): + comm = MPI.COMM_WORLD + adios = adios2.ADIOS(comm) + else: + adios = adios2.ADIOS() + + bpIO = adios.DeclareIO("NPTypes") + + # ADIOS Variable name, shape, start, offset, constant dims + # All local variables + varI8 = bpIO.DefineVariable( + "varI8", [], [], [data.I8.size], True, data.I8.dtype) + varI16 = bpIO.DefineVariable( + "varI16", [], [], [data.I16.size], True, data.I16.dtype) + varI32 = bpIO.DefineVariable( + "varI32", [], [], [data.I32.size], True, data.I32.dtype) + # FIXME: Uncomment below when 64 bit ints work properly + # varI64 = bpIO.DefineVariable( + # "varI64", [], [], [data.I64.size], True, data.I64.dtype) + + varU8 = bpIO.DefineVariable( + "varUI8", [], [], [data.U8.size], True, data.U8.dtype) + varU16 = bpIO.DefineVariable( + "varUI16", [], [], [data.U16.size], True, data.U16.dtype) + varU32 = bpIO.DefineVariable( + "varUI32", [], [], [data.U32.size], True, data.U32.dtype) + # FIXME: Uncomment below when 64 bit ints work properly + # varU64 = bpIO.DefineVariable( + # "varUI64", [], [], [data.U64.size], True, data.U64.dtype) + + varR32 = bpIO.DefineVariable( + "varR32", [], [], [data.R32.size], True, data.R32.dtype) + + varR64 = bpIO.DefineVariable( + "varR64", [], [], [data.R64.size], True, data.R64.dtype) + + + # ADIOS Engine + bpFileWriter = bpIO.Open("npTypes.bp", adios2.OpenMode.Write) + + bpFileWriter.Write(varI8, data.I8) + bpFileWriter.Write(varI16, data.I16) + bpFileWriter.Write(varI32, data.I32) + # FIXME: Uncomment below when 64 bit ints work properly + # bpFileWriter.Write(varI64, data.I64) + + bpFileWriter.Write(varU8, data.U8) + bpFileWriter.Write(varU16, data.U16) + bpFileWriter.Write(varU32, data.U32) + # FIXME: Uncomment below when 64 bit ints work properly + # bpFileWriter.Write(varU64, data.U64) + + bpFileWriter.Write(varR32, data.R32) + bpFileWriter.Write(varR64, data.R64) + + # FIXME: Pass transportIndex until the default args works properly + bpFileWriter.Close(-1) + +### +### Trigger the tests +### +if __name__ == '__main__': + Adios2PythonTestBase.main() \ No newline at end of file diff --git a/testing/adios2/bindings/python/TestBPWriteTypes_nompi.py b/testing/adios2/bindings/python/TestBPWriteTypes_nompi.py deleted file mode 100644 index 0fe407efd6..0000000000 --- a/testing/adios2/bindings/python/TestBPWriteTypes_nompi.py +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env python - -# -# Distributed under the OSI-approved Apache License, Version 2.0. See -# accompanying file Copyright.txt for details. -# -# TestBPWriteTypes.py: test Python numpy types in ADIOS2 File Write -# Created on: Feb 2, 2017 -# Author: William F Godoy godoywf@ornl.gov - - -from adios2NPTypes import SmallTestData -import adios2 - - -# Test data -data = SmallTestData() - -adios = adios2.ADIOS() - -bpIO = adios.DeclareIO("NPTypes") - -# ADIOS Variable name, shape, start, offset, constant dims -# All local variables -varI8 = bpIO.DefineVariable( - "varI8", [], [], [data.I8.size], True, data.I8.dtype) -varI16 = bpIO.DefineVariable( - "varI16", [], [], [data.I16.size], True, data.I16.dtype) -varI32 = bpIO.DefineVariable( - "varI32", [], [], [data.I32.size], True, data.I32.dtype) -# FIXME: Uncomment below when 64 bit ints work properly -# varI64 = bpIO.DefineVariable( -# "varI64", [], [], [data.I64.size], True, data.I64.dtype) - -varU8 = bpIO.DefineVariable( - "varUI8", [], [], [data.U8.size], True, data.U8.dtype) -varU16 = bpIO.DefineVariable( - "varUI16", [], [], [data.U16.size], True, data.U16.dtype) -varU32 = bpIO.DefineVariable( - "varUI32", [], [], [data.U32.size], True, data.U32.dtype) -# FIXME: Uncomment below when 64 bit ints work properly -# varU64 = bpIO.DefineVariable( -# "varUI64", [], [], [data.U64.size], True, data.U64.dtype) - -varR32 = bpIO.DefineVariable( - "varR32", [], [], [data.R32.size], True, data.R32.dtype) - -varR64 = bpIO.DefineVariable( - "varR64", [], [], [data.R64.size], True, data.R64.dtype) - - -# ADIOS Engine -bpFileWriter = bpIO.Open("npTypes.bp", adios2.OpenMode.Write) - -bpFileWriter.Write(varI8, data.I8) -bpFileWriter.Write(varI16, data.I16) -bpFileWriter.Write(varI32, data.I32) -# FIXME: Uncomment below when 64 bit ints work properly -# bpFileWriter.Write(varI64, data.I64) - -bpFileWriter.Write(varU8, data.U8) -bpFileWriter.Write(varU16, data.U16) -bpFileWriter.Write(varU32, data.U32) -# FIXME: Uncomment below when 64 bit ints work properly -# bpFileWriter.Write(varU64, data.U64) - -bpFileWriter.Write(varR32, data.R32) -bpFileWriter.Write(varR64, data.R64) - -# FIXME: Pass transportIndex until the default args works properly -bpFileWriter.Close(-1) diff --git a/testing/adios2/bindings/python/TestPythonEngine.py b/testing/adios2/bindings/python/TestPythonEngine.py new file mode 100644 index 0000000000..2ba8c30ba2 --- /dev/null +++ b/testing/adios2/bindings/python/TestPythonEngine.py @@ -0,0 +1,28 @@ + +import sys + +# When importing from C++, this path is missing, and thus numpy is +# unable to import the __future__ module. This is a temporary +# workaround to allow that to happen. +sys.path.append('/usr/lib/python2.7') + +import adios2 +import numpy # Required when importing this module from C++ + +print('Inside TestPythonEngine module') + +class TestPythonEngine(adios2.Engine): + def __init__(self, engineType, io, name, openMode): + adios2.Engine.__init__(self, engineType, io, name, openMode) + print('Inside TestPythonEngine python ctor, just called parent ctor') + + def DoWrite(self, variable, values): + print('Inside TestPythonEngine.DoWrite') + print(variable) + print(values) + + def Advance(self, timeoutSeconds): + print('Inside TestPythonEngine.Advance') + + def Close(self, transportIndex): + print('Inside TestPythonEngine.Close') diff --git a/testing/adios2/bindings/python/TestPythonEngineFromCpp.cpp b/testing/adios2/bindings/python/TestPythonEngineFromCpp.cpp new file mode 100644 index 0000000000..f2ceba6cce --- /dev/null +++ b/testing/adios2/bindings/python/TestPythonEngineFromCpp.cpp @@ -0,0 +1,92 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + */ + +#include +#include + +/** + * Test that with the proper plugin module and class specified, we + * can successfully create the python engine from C++. + */ +TEST(PythonEngineTest, CreatePythonEngineFromCPlusPlus) +{ + int mpiRank = 0, mpiSize = 1; + +#ifdef ADIOS2_HAVE_MPI + MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); + MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + adios2::ADIOS adios(MPI_COMM_WORLD, adios2::DebugON); +#else + adios2::ADIOS adios(adios2::DebugON); +#endif + + adios2::IO &io = adios.DeclareIO("PythonPluginIO"); + + io.SetEngine("PythonEngine"); + io.SetParameters({{"PluginName", "TestPythonPlugin"}, + {"PluginModule", "TestPythonEngine"}, + {"PluginClass", "TestPythonEngine"}}); + + std::shared_ptr writer; + EXPECT_NO_THROW({ + writer = io.Open("TestPythonPlugin", adios2::OpenMode::Write); + }); + + ASSERT_NE(writer.get(), nullptr); + + writer->Close(); +} + +/** + * Test that without the proper plugin module specified, IO fails to + * instantiate the engine and throws the correct exception type. + */ +TEST(PythonEngineTest, CreatePythonEngineFromCPlusPlusExpectImportError) +{ + int mpiRank = 0, mpiSize = 1; + +#ifdef ADIOS2_HAVE_MPI + MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); + MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + adios2::ADIOS adios(MPI_COMM_WORLD, adios2::DebugON); +#else + adios2::ADIOS adios(adios2::DebugON); +#endif + + adios2::IO &io = adios.DeclareIO("PythonPluginIO"); + + io.SetEngine("PythonEngine"); + + // By not specifying "PluginModule" parameter, we should fail during + // the "Open" call... + io.SetParameters({{"PluginName", "TestPythonPlugin"}, + {"PluginClass", "TestPythonEngine"}}); + + std::shared_ptr writer; + + ASSERT_THROW({ + writer = io.Open("TestPythonPlugin", adios2::OpenMode::Write); + }, std::runtime_error); +} + +//****************************************************************************** +// main +//****************************************************************************** + +int main(int argc, char **argv) +{ +#ifdef ADIOS2_HAVE_MPI + MPI_Init(nullptr, nullptr); +#endif + + ::testing::InitGoogleTest(&argc, argv); + int result = RUN_ALL_TESTS(); + +#ifdef ADIOS2_HAVE_MPI + MPI_Finalize(); +#endif + + return result; +} From 457ae76d13eb69c62f22d8e13ce40d42292a5360 Mon Sep 17 00:00:00 2001 From: Scott Wittenburg Date: Mon, 9 Oct 2017 15:45:32 -0600 Subject: [PATCH 11/24] Remove Engine friendship, move PyEngineBase into adios2, make DoWrite work --- bindings/python/CMakeLists.txt | 1 + bindings/python/PyEngine.cpp | 45 ++------------ source/adios2/CMakeLists.txt | 1 + source/adios2/core/Engine.cpp | 4 +- source/adios2/core/Engine.h | 6 -- source/adios2/engine/pyengine/PyEngineBase.h | 60 +++++++++++++++++++ .../adios2/engine/pyengine/PythonEngine.cpp | 9 +-- .../bindings/python/TestPythonEngine.py | 6 ++ 8 files changed, 80 insertions(+), 52 deletions(-) create mode 100644 source/adios2/engine/pyengine/PyEngineBase.h diff --git a/bindings/python/CMakeLists.txt b/bindings/python/CMakeLists.txt index 7e7db72ee8..ea68de426f 100644 --- a/bindings/python/CMakeLists.txt +++ b/bindings/python/CMakeLists.txt @@ -7,6 +7,7 @@ pybind11_add_module(adios2py MODULE PyIO.cpp PyVariable.cpp ) +target_include_directories(adios2py PRIVATE ${ADIOS2_SOURCE_DIR}/source/adios2/engine/pyengine) target_link_libraries(adios2py PRIVATE adios2) if(ADIOS2_HAVE_MPI) target_link_libraries(adios2py PRIVATE PythonModule::mpi4py) diff --git a/bindings/python/PyEngine.cpp b/bindings/python/PyEngine.cpp index 3214b5463c..c0520b303a 100644 --- a/bindings/python/PyEngine.cpp +++ b/bindings/python/PyEngine.cpp @@ -5,6 +5,7 @@ #include #include +#include "PyEngineBase.h" namespace adios2 { @@ -38,45 +39,10 @@ void GeneratePythonBindings(pybind11::module &m) }); engine.def("Close", &Engine::Close); - // // Wrappings to extend Engine in Python // - // This first base class used to translate the protected template functions - // into public members that can be overridden in python - class PyEngineBase : public Engine - { - public: - PyEngineBase(const std::string engineType, IO &io, - const std::string &name, const OpenMode openMode) - : Engine(engineType, io, name, openMode, io.m_MPIComm) - { - } - - virtual ~PyEngineBase() = default; - - using Engine::Init; - virtual void DoWrite(VariableBase &var, pybind11::array values) = 0; - using Engine::Close; - - protected: - // The C++ code calls this implementation which will redirect to the - // python implementation. -#define define_dowrite(T) \ - void DoWrite(Variable &var, const T *values) override \ - { \ - std::cout << "BINGO!" << std::endl; \ - DoWrite(var, \ - pybind11::array_t(std::accumulate(var.m_Count.begin(), \ - var.m_Count.end(), 0), \ - values)); \ - } - ADIOS2_FOREACH_TYPE_1ARG(define_dowrite) -#undef define_dowrite - }; - - // This is the trampoline class used by pybind11 to implement the // inheritance model and virtual springboard class PyEngine : public PyEngineBase @@ -85,27 +51,24 @@ void GeneratePythonBindings(pybind11::module &m) using PyEngineBase::PyEngineBase; void Init() override { - std::cout << "PyEngine Init() trampoline" << std::endl; PYBIND11_OVERLOAD(void, PyEngineBase, Init,); } - void DoWrite(VariableBase &var, pybind11::array values) override + void DoWrite(VariableBase *var, pybind11::array values) override { - std::cout << "PyEngine DoWrite() trampoline" << std::endl; PYBIND11_OVERLOAD_PURE(void, PyEngineBase, DoWrite, var, values); } void Close(const int transportIndex) { - std::cout << "PyEngine Close() trampoline" << std::endl; PYBIND11_OVERLOAD_PURE(void, PyEngineBase, Close, transportIndex); } }; - pybind11::class_(m, "Engine", engine) + pybind11::class_>(m, "Engine", engine) .def(pybind11::init()) .def("Init", &PyEngineBase::Init) .def("DoWrite", - (void (PyEngineBase::*)(VariableBase &, pybind11::array)) & + (void (PyEngineBase::*)(VariableBase *, pybind11::array)) & PyEngineBase::DoWrite) .def("Close", &PyEngineBase::Close); } diff --git a/source/adios2/CMakeLists.txt b/source/adios2/CMakeLists.txt index 0ddf3b1e30..9528499d84 100644 --- a/source/adios2/CMakeLists.txt +++ b/source/adios2/CMakeLists.txt @@ -142,6 +142,7 @@ if(ADIOS2_HAVE_Python) helper/PythonInstanceBuilder.h engine/pyengine/PythonEngine.h engine/pyengine/PythonEngine.cpp + engine/pyengine/PyEngineBase.h ) target_include_directories(adios2 PRIVATE ${PYBIND11_INCLUDE_DIR} ${PYTHON_INCLUDE_DIRS}) target_link_libraries(adios2 PRIVATE pybind11::embed) diff --git a/source/adios2/core/Engine.cpp b/source/adios2/core/Engine.cpp index 84f8ad08ba..814c6bf85d 100644 --- a/source/adios2/core/Engine.cpp +++ b/source/adios2/core/Engine.cpp @@ -64,7 +64,9 @@ void Engine::Release() {} void Engine::PerformReads(ReadMode /*mode*/){}; // PROTECTED -void Engine::Init() {} +void Engine::Init() { + std::cout << "Inside Engine::Init()" << std::endl; +} void Engine::InitParameters() {} diff --git a/source/adios2/core/Engine.h b/source/adios2/core/Engine.h index 80f6693c2e..94c4823d77 100644 --- a/source/adios2/core/Engine.h +++ b/source/adios2/core/Engine.h @@ -37,12 +37,6 @@ namespace adios2 * Close */ class Engine { - - // PythonEngine not only inherits from Engine, but also holds a shared_ptr - // to an instance of Engine, so it can proxy method calls to it. Hence, - // PythonEngine needs access to everything in here. - friend class PythonEngine; - public: using AdvanceAsyncCallback = std::function)>; diff --git a/source/adios2/engine/pyengine/PyEngineBase.h b/source/adios2/engine/pyengine/PyEngineBase.h new file mode 100644 index 0000000000..07c04f5b91 --- /dev/null +++ b/source/adios2/engine/pyengine/PyEngineBase.h @@ -0,0 +1,60 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * PyEngineBase.h: This is the base class used to translate the protected + * template functions into public members that can be overridden in python. + * + * Created on: Oct 9, 2017 + * Author: Chuck Atkins + */ + +#include +#include +// #include + +// #include "adios2/core/Engine.h" +// #include "adios2/core/IO.h" +// #include "adios2/core/Variable.h" + +#ifndef ADIOS2_ENGINE_PYENGINE_PYENGINEBASE_H_ +#define ADIOS2_ENGINE_PYENGINE_PYENGINEBASE_H_ + +namespace adios2 +{ + +class PyEngineBase : public Engine +{ + // Make sure PythonEngine can forward all its calls here + friend class PythonEngine; + +public: + PyEngineBase(const std::string engineType, IO &io, + const std::string &name, const OpenMode openMode) + : Engine(engineType, io, name, openMode, io.m_MPIComm) + { + } + + virtual ~PyEngineBase() = default; + + using Engine::Init; + virtual void DoWrite(VariableBase *var, pybind11::array values) = 0; + using Engine::Close; + +protected: + // The C++ code calls this implementation which will redirect to the + // python implementation. +#define define_dowrite(T) \ + void DoWrite(Variable &var, const T *values) override \ + { \ + DoWrite(&var, \ + pybind11::array_t(std::accumulate(var.m_Count.begin(), \ + var.m_Count.end(), 0), \ + values)); \ + } + ADIOS2_FOREACH_TYPE_1ARG(define_dowrite) +#undef define_dowrite +}; +} + +#endif /* ADIOS2_ENGINE_PYENGINE_PYENGINEBASE_H_ */ diff --git a/source/adios2/engine/pyengine/PythonEngine.cpp b/source/adios2/engine/pyengine/PythonEngine.cpp index 81480ed5fd..b7332a437a 100644 --- a/source/adios2/engine/pyengine/PythonEngine.cpp +++ b/source/adios2/engine/pyengine/PythonEngine.cpp @@ -9,6 +9,7 @@ */ #include "PythonEngine.h" +#include "PyEngineBase.h" #include #include @@ -33,6 +34,7 @@ struct PythonEngine::Impl std::string m_PluginName; pybind11::object enginePyClass; pybind11::object enginePyObject; + std::shared_ptr eng; }; @@ -44,9 +46,8 @@ PythonEngine::PythonEngine(IO &io, const std::string &name, { std::cout << "PythonEngine::PythonEngine" << std::endl; Init(); - // m_Impl->m_Plugin = - // m_Impl->m_HandleCreate(io, m_Impl->m_PluginName, openMode, mpiComm); m_Impl->enginePyObject = m_Impl->enginePyClass("PythonEngine", io, name, openMode); + m_Impl->eng = m_Impl->enginePyObject.cast>(); } PythonEngine::~PythonEngine() { @@ -99,6 +100,7 @@ void PythonEngine::Close(const int transportIndex) { // m_Impl->m_Plugin->Close(transportIndex); std::cout << "PythonEngine::Close" << std::endl; + m_Impl->eng->Close(); } void PythonEngine::Init() @@ -139,10 +141,9 @@ void PythonEngine::Init() #define define(T) \ void PythonEngine::DoWrite(Variable &variable, const T *values) \ { \ - std::shared_ptr eng = m_Impl->enginePyObject.cast>(); \ std::cout << "PythonEngine::DoWrite(var, vals), " \ << "type: " << typeid(T).name() << std::endl; \ - eng->DoWrite(variable, values); \ + m_Impl->eng->DoWrite(variable, values); \ } \ \ void PythonEngine::DoScheduleRead(Variable &variable, \ diff --git a/testing/adios2/bindings/python/TestPythonEngine.py b/testing/adios2/bindings/python/TestPythonEngine.py index 2ba8c30ba2..ec32a34bb1 100644 --- a/testing/adios2/bindings/python/TestPythonEngine.py +++ b/testing/adios2/bindings/python/TestPythonEngine.py @@ -15,6 +15,12 @@ class TestPythonEngine(adios2.Engine): def __init__(self, engineType, io, name, openMode): adios2.Engine.__init__(self, engineType, io, name, openMode) print('Inside TestPythonEngine python ctor, just called parent ctor') + + # Calls Engine::Init() if no Init() method is defined here + self.Init() + + # def Init(self): + # print('Inside TestPythonEngine Init()') def DoWrite(self, variable, values): print('Inside TestPythonEngine.DoWrite') From bcfc4da4f0c94e7eefb691e8d39141826575c2e5 Mon Sep 17 00:00:00 2001 From: Scott Wittenburg Date: Mon, 9 Oct 2017 17:19:19 -0600 Subject: [PATCH 12/24] Forward the rest of the Engine api methods to the python instance --- .../adios2/engine/pyengine/PythonEngine.cpp | 65 +++++++------------ 1 file changed, 23 insertions(+), 42 deletions(-) diff --git a/source/adios2/engine/pyengine/PythonEngine.cpp b/source/adios2/engine/pyengine/PythonEngine.cpp index b7332a437a..cb2d1fce67 100644 --- a/source/adios2/engine/pyengine/PythonEngine.cpp +++ b/source/adios2/engine/pyengine/PythonEngine.cpp @@ -44,47 +44,36 @@ PythonEngine::PythonEngine(IO &io, const std::string &name, const OpenMode openMode, MPI_Comm mpiComm) : Engine("PythonEngine", io, name, openMode, mpiComm), m_Impl(new Impl) { - std::cout << "PythonEngine::PythonEngine" << std::endl; Init(); m_Impl->enginePyObject = m_Impl->enginePyClass("PythonEngine", io, name, openMode); m_Impl->eng = m_Impl->enginePyObject.cast>(); } -PythonEngine::~PythonEngine() { - // m_Impl->m_HandleDestroy(m_Impl->m_Plugin); - std::cout << "PythonEngine::~PythonEngine" << std::endl; -} +PythonEngine::~PythonEngine() {} void PythonEngine::PerformReads(ReadMode mode) { - // m_Impl->m_Plugin->PerformReads(mode); - std::cout << "PythonEngine::PerformReads" << std::endl; + m_Impl->eng->PerformReads(mode); } void PythonEngine::Release() { - // m_Impl->m_Plugin->Release(); - std::cout << "PythonEngine::Release" << std::endl; + m_Impl->eng->Release(); } void PythonEngine::Advance(const float timeoutSeconds) { - // m_Impl->m_Plugin->Advance(timeoutSeconds); - std::cout << "PythonEngine::Advance(timeoutSeconds)" - << std::endl; + m_Impl->eng->Advance(timeoutSeconds); } void PythonEngine::Advance(const AdvanceMode mode, const float timeoutSeconds) { - // m_Impl->m_Plugin->Advance(mode, timeoutSeconds); - std::cout << "PythonEngine::Advance(mode, timeoutSeconds)" - << std::endl; + m_Impl->eng->Advance(mode, timeoutSeconds); } void PythonEngine::AdvanceAsync(const AdvanceMode mode, AdvanceAsyncCallback callback) { - // m_Impl->m_Plugin->AdvanceAsync(mode, callback); - std::cout << "PythonEngine::AdvanceAsync" << std::endl; + m_Impl->eng->AdvanceAsync(mode, callback); } void PythonEngine::SetCallBack( @@ -92,14 +81,11 @@ void PythonEngine::SetCallBack( std::vector)> callback) { - // m_Impl->m_Plugin->SetCallBack(callback); - std::cout << "PythonEngine::SetCallBack" << std::endl; + m_Impl->eng->SetCallBack(callback); } void PythonEngine::Close(const int transportIndex) { - // m_Impl->m_Plugin->Close(transportIndex); - std::cout << "PythonEngine::Close" << std::endl; m_Impl->eng->Close(); } @@ -141,22 +127,18 @@ void PythonEngine::Init() #define define(T) \ void PythonEngine::DoWrite(Variable &variable, const T *values) \ { \ - std::cout << "PythonEngine::DoWrite(var, vals), " \ - << "type: " << typeid(T).name() << std::endl; \ - m_Impl->eng->DoWrite(variable, values); \ + m_Impl->eng->DoWrite(variable, values); \ } \ \ void PythonEngine::DoScheduleRead(Variable &variable, \ - const T *values) \ - { \ - std::cout << "PythonEngine::DoScheduledRead(var, vals), " \ - << "type: " << typeid(T).name() << std::endl; \ - } \ + const T *values) \ + { \ + m_Impl->eng->DoScheduleRead(variable, values); \ + } \ void PythonEngine::DoScheduleRead(const std::string &variableName, \ - const T *values) \ - { \ - std::cout << "PythonEngine::DoScheduledRead(varName, vals), " \ - << "type: " << typeid(T).name() << std::endl; \ + const T *values) \ + { \ + m_Impl->eng->DoScheduleRead(variableName, values); \ } ADIOS2_FOREACH_TYPE_1ARG(define) #undef define @@ -164,16 +146,16 @@ ADIOS2_FOREACH_TYPE_1ARG(define) void PythonEngine::DoWrite(VariableCompound &variable, const void *values) { - // m_Impl->m_Plugin->DoWrite(variable, values); - std::cout << "PythonEngine::DoWrite" << std::endl; + std::cout << "PythonEngine::DoWrite(VariableCompound &variable, " + << "const void *values) is not yet implemented" << std::endl; + // m_Impl->eng->DoWrite(variable, values); } -#define define(T, L) \ +#define define(T, L) \ Variable *PythonEngine::InquireVariable##L( \ - const std::string &name, const bool readIn) \ - { \ - std::cout << "PythonEngine::InquireVariable##L" << std::endl; \ - return InquireVariable##L(name, readIn); \ + const std::string &name, const bool readIn) \ + { \ + return m_Impl->eng->InquireVariable##L(name, readIn); \ } ADIOS2_FOREACH_TYPE_2ARGS(define) #undef define @@ -181,8 +163,7 @@ ADIOS2_FOREACH_TYPE_2ARGS(define) VariableBase *PythonEngine::InquireVariableUnknown( const std::string &name, const bool readIn) { - std::cout << "PythonEngine::InquireVariableUnknown" << std::endl; - return InquireVariableUnknown(name, readIn); + return m_Impl->eng->InquireVariableUnknown(name, readIn); } } // end namespace adios2 From 0b88c03373147420b9dc818e30a4da12c6ac316a Mon Sep 17 00:00:00 2001 From: Scott Wittenburg Date: Tue, 10 Oct 2017 10:53:35 -0600 Subject: [PATCH 13/24] Put back the python tests, make Engine::Close default arg work, etc... --- bindings/python/PyEngine.cpp | 2 +- testing/adios2/bindings/python/CMakeLists.txt | 10 +- .../bindings/python/TestBPWriteTypes.py | 3 +- .../TestOpenInlinePythonEngineFromPython.py | 110 ++++++++++++++++++ .../python/TestOpenPythonEngineFromPython.py | 59 ++++++++++ .../bindings/python/TestPythonEngine.py | 6 + .../python/TestPythonEngineFromCpp.cpp | 10 ++ 7 files changed, 192 insertions(+), 8 deletions(-) create mode 100644 testing/adios2/bindings/python/TestOpenInlinePythonEngineFromPython.py create mode 100644 testing/adios2/bindings/python/TestOpenPythonEngineFromPython.py diff --git a/bindings/python/PyEngine.cpp b/bindings/python/PyEngine.cpp index c0520b303a..27ebdb2a4e 100644 --- a/bindings/python/PyEngine.cpp +++ b/bindings/python/PyEngine.cpp @@ -37,7 +37,7 @@ void GeneratePythonBindings(pybind11::module &m) #undef inquire return nullptr; }); - engine.def("Close", &Engine::Close); + engine.def("Close", &Engine::Close, pybind11::arg("transportIndex") = -1); // // Wrappings to extend Engine in Python diff --git a/testing/adios2/bindings/python/CMakeLists.txt b/testing/adios2/bindings/python/CMakeLists.txt index f6e3102529..ca9c6c8a59 100644 --- a/testing/adios2/bindings/python/CMakeLists.txt +++ b/testing/adios2/bindings/python/CMakeLists.txt @@ -13,11 +13,11 @@ endif() python_add_test(NAME PythonBPWrite SCRIPT TestBPWriteTypes.py ${test_parameters} ${use_mpi}) -# python_add_test(NAME TestOpenPythonEngineFromPython SCRIPT TestOpenPythonEngineFromPython.py -# ${test_parameters} ${use_mpi} -# ADDTOPYPATH ${CMAKE_CURRENT_SOURCE_DIR}) -# python_add_test(NAME TestOpenInlinePythonEngineFromPython SCRIPT TestOpenInlinePythonEngineFromPython.py -# ${test_parameters} ${use_mpi}) +python_add_test(NAME TestOpenPythonEngineFromPython SCRIPT TestOpenPythonEngineFromPython.py + ${test_parameters} ${use_mpi} + ADDTOPYPATH ${CMAKE_CURRENT_SOURCE_DIR}) +python_add_test(NAME TestOpenInlinePythonEngineFromPython SCRIPT TestOpenInlinePythonEngineFromPython.py + ${test_parameters} ${use_mpi}) add_executable(TestPythonEngineFromCpp TestPythonEngineFromCpp.cpp) target_link_libraries(TestPythonEngineFromCpp adios2 gtest gtest_main) diff --git a/testing/adios2/bindings/python/TestBPWriteTypes.py b/testing/adios2/bindings/python/TestBPWriteTypes.py index 0c20202314..713200e9a5 100644 --- a/testing/adios2/bindings/python/TestBPWriteTypes.py +++ b/testing/adios2/bindings/python/TestBPWriteTypes.py @@ -79,8 +79,7 @@ def testWriteTypes(self): bpFileWriter.Write(varR32, data.R32) bpFileWriter.Write(varR64, data.R64) - # FIXME: Pass transportIndex until the default args works properly - bpFileWriter.Close(-1) + bpFileWriter.Close() ### ### Trigger the tests diff --git a/testing/adios2/bindings/python/TestOpenInlinePythonEngineFromPython.py b/testing/adios2/bindings/python/TestOpenInlinePythonEngineFromPython.py new file mode 100644 index 0000000000..fd877ef721 --- /dev/null +++ b/testing/adios2/bindings/python/TestOpenInlinePythonEngineFromPython.py @@ -0,0 +1,110 @@ +from Adios2PythonTestBase import Adios2PythonTestBase + +if Adios2PythonTestBase.isUsingMpi(): + from mpi4py import MPI + +import numpy +import adios2 + +MODULE_LEVEL_VARIABLE = 0 + +### +### Define a new engine type in python +### +class TestInlinePythonEngine(adios2.Engine): + def __init__(self, engineType, io, name, openMode): + adios2.Engine.__init__(self, engineType, io, name, openMode) + print('Inside TestInlinePythonEngine python ctor, just called parent ctor') + + # Calls Engine::Init() if no Init() method is defined here + self.Init() + + # def Init(self): + # print('Inside TestPythonEngine Init()') + + def DoWrite(self, variable, values): + global MODULE_LEVEL_VARIABLE + MODULE_LEVEL_VARIABLE += 1 + print('Inside TestInlinePythonEngine.DoWrite') + print(variable) + print(values) + + def Advance(self, timeoutSeconds): + print('Inside TestInlinePythonEngine.Advance') + + def Close(self, transportIndex): + print('Inside TestInlinePythonEngine.Close') + +### +### Create a testcase class with some tests +### +class TestOpenInlinePythonEngineFromPython(Adios2PythonTestBase): + def testCreateEngine(self): + global MODULE_LEVEL_VARIABLE + self.assertEqual(MODULE_LEVEL_VARIABLE, 0) + MODULE_LEVEL_VARIABLE += 1 + self.assertEqual(MODULE_LEVEL_VARIABLE, 1) + + rank = 0 + size = 1 + + if self.isUsingMpi(): + # MPI + comm = MPI.COMM_WORLD + rank = comm.Get_rank() + size = comm.Get_size() + adios = adios2.ADIOS(comm) + else: + adios = adios2.ADIOS() + + # User data + myArray = numpy.array([0., 1., 2., 3., 4., 5., 6., 7., 8., 9.]) + Nx = myArray.size + + # ADIOS IO + bpIO = adios.DeclareIO("BPFile_N2N") + + # ADIOS Variable name, shape, start, offset, constant dims + ioArray = bpIO.DefineVariable( + "bpArray", [size * Nx], [rank * Nx], [Nx], True, myArray.dtype) + + # Engine derived class, spawned to start IO operations + bpIO.EngineType = "PythonEngine" + bpIO.Parameters = { + "PluginName": "FirstPythonPlugin", + "PluginClass": "TestInlinePythonEngine" + } + + # ADIOS Engine + bpFileWriter = bpIO.Open("npArray.bp", adios2.OpenMode.Write) + bpFileWriter.Write(ioArray, myArray) + + self.assertEqual(MODULE_LEVEL_VARIABLE, 2) + + bpFileWriter.Close() + + def testCreateEngineWithWrongName(self): + if self.isUsingMpi(): + # MPI + comm = MPI.COMM_WORLD + adios = adios2.ADIOS(comm) + else: + adios = adios2.ADIOS() + + bpIO = adios.DeclareIO("BPFile_N2N") + + bpIO.EngineType = "PythonEngine" + bpIO.Parameters = { + "PluginName": "FirstPythonPlugin", + "PluginClass": "MissingEngine" + } + + # Make sure exception is raised due to wrong engine name + with self.assertRaises(RuntimeError): + bpFileWriter = bpIO.Open("npArray.bp", adios2.OpenMode.Write) + +### +### Trigger the tests +### +if __name__ == '__main__': + Adios2PythonTestBase.main() diff --git a/testing/adios2/bindings/python/TestOpenPythonEngineFromPython.py b/testing/adios2/bindings/python/TestOpenPythonEngineFromPython.py new file mode 100644 index 0000000000..60a6093c6a --- /dev/null +++ b/testing/adios2/bindings/python/TestOpenPythonEngineFromPython.py @@ -0,0 +1,59 @@ +from Adios2PythonTestBase import Adios2PythonTestBase + +if Adios2PythonTestBase.isUsingMpi(): + from mpi4py import MPI + +import numpy +import adios2 + +from TestPythonEngine import TestPythonEngine + +### +### Create a testcase class with some tests +### +class TestOpenPythonEngineFromPython(Adios2PythonTestBase): + def testCreateEngine(self): + rank = 0 + size = 1 + + if self.isUsingMpi(): + # MPI + comm = MPI.COMM_WORLD + rank = comm.Get_rank() + size = comm.Get_size() + adios = adios2.ADIOS(comm) + else: + adios = adios2.ADIOS() + + # User data + myArray = numpy.array([0., 1., 2., 3., 4., 5., 6., 7., 8., 9.]) + Nx = myArray.size + + # ADIOS IO + bpIO = adios.DeclareIO("BPFile_N2N") + + # ADIOS Variable name, shape, start, offset, constant dims + ioArray = bpIO.DefineVariable( + "bpArray", [size * Nx], [rank * Nx], [Nx], True, myArray.dtype) + + # Engine derived class, spawned to start IO operations + bpIO.EngineType = "PythonEngine" + + # Since we import the TestPythonEngine class directly, we should not need + # to provide a "PluginModule" parameter + bpIO.Parameters = { + "PluginName": "DoNotReallyCare", + "PluginClass": "TestPythonEngine" + } + + # ADIOS Engine + bpFileWriter = bpIO.Open("npArray.bp", adios2.OpenMode.Write) + bpFileWriter.Write(ioArray, myArray) + + bpFileWriter.Close() + +### +### Trigger the tests +### +if __name__ == '__main__': + Adios2PythonTestBase.main() diff --git a/testing/adios2/bindings/python/TestPythonEngine.py b/testing/adios2/bindings/python/TestPythonEngine.py index ec32a34bb1..604f9273c3 100644 --- a/testing/adios2/bindings/python/TestPythonEngine.py +++ b/testing/adios2/bindings/python/TestPythonEngine.py @@ -6,6 +6,12 @@ # workaround to allow that to happen. sys.path.append('/usr/lib/python2.7') +# When importing from C++, this path is also missing. This lives in the +# virtual environment directory and is part of the path automatically +# when running the virtual env python. +sys.path.append('/data/scott/projects/adios2/adios-env/lib/python2.7/site-packages') + + import adios2 import numpy # Required when importing this module from C++ diff --git a/testing/adios2/bindings/python/TestPythonEngineFromCpp.cpp b/testing/adios2/bindings/python/TestPythonEngineFromCpp.cpp index f2ceba6cce..c82bca57d7 100644 --- a/testing/adios2/bindings/python/TestPythonEngineFromCpp.cpp +++ b/testing/adios2/bindings/python/TestPythonEngineFromCpp.cpp @@ -14,6 +14,10 @@ TEST(PythonEngineTest, CreatePythonEngineFromCPlusPlus) { int mpiRank = 0, mpiSize = 1; + // Application variable + std::vector myDoubles = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; + const std::size_t Nx = myDoubles.size(); + #ifdef ADIOS2_HAVE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); @@ -24,6 +28,9 @@ TEST(PythonEngineTest, CreatePythonEngineFromCPlusPlus) adios2::IO &io = adios.DeclareIO("PythonPluginIO"); + adios2::Variable &var = io.DefineVariable( + "data", {mpiSize * Nx}, {mpiRank * Nx}, {Nx}, adios2::ConstantDims); + io.SetEngine("PythonEngine"); io.SetParameters({{"PluginName", "TestPythonPlugin"}, {"PluginModule", "TestPythonEngine"}, @@ -36,6 +43,9 @@ TEST(PythonEngineTest, CreatePythonEngineFromCPlusPlus) ASSERT_NE(writer.get(), nullptr); + /** Write variable for buffering */ + writer->Write(var, myDoubles.data()); + writer->Close(); } From 6b415f2b191ac10aba040ff728c00b58f6f8a692 Mon Sep 17 00:00:00 2001 From: Scott Wittenburg Date: Tue, 10 Oct 2017 11:12:51 -0600 Subject: [PATCH 14/24] Rename PythonInstanceBuilder to PythonModuleHelper --- source/adios2/CMakeLists.txt | 2 +- source/adios2/engine/pyengine/PythonEngine.cpp | 6 +++--- ...thonInstanceBuilder.h => PythonModuleHelper.h} | 15 ++++++++------- 3 files changed, 12 insertions(+), 11 deletions(-) rename source/adios2/helper/{PythonInstanceBuilder.h => PythonModuleHelper.h} (76%) diff --git a/source/adios2/CMakeLists.txt b/source/adios2/CMakeLists.txt index 9528499d84..7f206828e4 100644 --- a/source/adios2/CMakeLists.txt +++ b/source/adios2/CMakeLists.txt @@ -139,7 +139,7 @@ if(ADIOS2_HAVE_Python) target_sources(adios2 PRIVATE helper/PythonInterpreter.h helper/PythonInterpreter.cpp - helper/PythonInstanceBuilder.h + helper/PythonModuleHelper.h engine/pyengine/PythonEngine.h engine/pyengine/PythonEngine.cpp engine/pyengine/PyEngineBase.h diff --git a/source/adios2/engine/pyengine/PythonEngine.cpp b/source/adios2/engine/pyengine/PythonEngine.cpp index cb2d1fce67..ba5791a115 100644 --- a/source/adios2/engine/pyengine/PythonEngine.cpp +++ b/source/adios2/engine/pyengine/PythonEngine.cpp @@ -19,7 +19,7 @@ #include #include "adios2/helper/PythonInterpreter.h" -#include "adios2/helper/PythonInstanceBuilder.h" +#include "adios2/helper/PythonModuleHelper.h" #include #include @@ -120,8 +120,8 @@ void PythonEngine::Init() adios2::PythonInterpreter::instance().initialize(); m_Impl->enginePyClass = - adios2::PythonInstanceBuilder::BuildInstance(pluginClassName, - pluginModuleName); + adios2::PythonModuleHelper::FindPythonClass(pluginClassName, + pluginModuleName); } #define define(T) \ diff --git a/source/adios2/helper/PythonInstanceBuilder.h b/source/adios2/helper/PythonModuleHelper.h similarity index 76% rename from source/adios2/helper/PythonInstanceBuilder.h rename to source/adios2/helper/PythonModuleHelper.h index 8ece50d907..964052cf34 100644 --- a/source/adios2/helper/PythonInstanceBuilder.h +++ b/source/adios2/helper/PythonModuleHelper.h @@ -2,24 +2,25 @@ * Distributed under the OSI-approved Apache License, Version 2.0. See * accompanying file Copyright.txt for details. * - * PythonInstanceBuilder.h Instantiate some arbitrary python class + * PythonModuleHelper.h Helper method to find a python class, importing + * it if necessary. * * Created on: Oct 02, 2017 * Author: Scott Wittenburg */ -#ifndef ADIOS2_HELPER_PYTHONINSTANCEBUILDER_H_ -#define ADIOS2_HELPER_PYTHONINSTANCEBUILDER_H_ +#ifndef ADIOS2_HELPER_PYTHONMODULEHELPER_H_ +#define ADIOS2_HELPER_PYTHONMODULEHELPER_H_ #include #include namespace adios2 { -namespace PythonInstanceBuilder +namespace PythonModuleHelper { - pybind11::object BuildInstance(const std::string& className, - const std::string* moduleName = nullptr) + pybind11::object FindPythonClass(const std::string& className, + const std::string* moduleName = nullptr) { pybind11::dict globals = pybind11::globals(); @@ -55,4 +56,4 @@ namespace PythonInstanceBuilder } } -#endif /* ADIOS2_HELPER_PYTHONINSTANCEBUILDER_H_ */ +#endif /* ADIOS2_HELPER_PYTHONMODULEHELPER_H_ */ From 59cf9de46793f97a1a5477f975996d8c5e71d419 Mon Sep 17 00:00:00 2001 From: Scott Wittenburg Date: Tue, 10 Oct 2017 16:58:54 -0600 Subject: [PATCH 15/24] Address issues importing modules within the embedded interpreter --- cmake/DetectOptions.cmake | 8 ++++ source/adios2/helper/PythonInterpreter.cpp | 40 +++++++++++++++++++ .../bindings/python/TestPythonEngine.py | 15 +------ 3 files changed, 49 insertions(+), 14 deletions(-) diff --git a/cmake/DetectOptions.cmake b/cmake/DetectOptions.cmake index 2562e77a2b..c3420f1934 100644 --- a/cmake/DetectOptions.cmake +++ b/cmake/DetectOptions.cmake @@ -119,6 +119,14 @@ if(ADIOS2_USE_Python) endif() if(PythonFull_FOUND) set(ADIOS2_HAVE_Python ON) + message("PYTHONLIBS_FOUND = ${PYTHONLIBS_FOUND}") + message("PYTHON_PREFIX = ${PYTHON_PREFIX}") + message("PYTHON_LIBRARIES = ${PYTHON_LIBRARIES}") + message("PYTHON_INCLUDE_DIRS = ${PYTHON_INCLUDE_DIRS}") + message("PYTHON_MODULE_EXTENSION = ${PYTHON_MODULE_EXTENSION}") + message("PYTHON_MODULE_PREFIX = ${PYTHON_MODULE_PREFIX}") + message("PYTHON_SITE_PACKAGES = ${PYTHON_SITE_PACKAGES}") + message("PYTHON_IS_DEBUG = ${PYTHON_IS_DEBUG}") endif() #SysV IPC diff --git a/source/adios2/helper/PythonInterpreter.cpp b/source/adios2/helper/PythonInterpreter.cpp index 3c88241ac4..32cad4a496 100644 --- a/source/adios2/helper/PythonInterpreter.cpp +++ b/source/adios2/helper/PythonInterpreter.cpp @@ -10,13 +10,27 @@ #include "PythonInterpreter.h" +#include +#include +#include #include +#include "adios2//ADIOSConfig.h" + #if WIN32 #include extern __declspec(dllimport) int Py_NoSiteFlag; #endif +#if defined(__linux) +# ifndef _GNU_SOURCE +# define _GNU_SOURCE +# endif +# include +#elif defined(__unix) +# include +#endif + namespace adios2 { @@ -50,9 +64,35 @@ void PythonInterpreter::initialize() return; } +#if defined(__linux) || defined(__unix) + { + // Find the actual library file where the symbol below is defined, + // then use that path to help find Python run-time libraries. + void *handle = dlsym(RTLD_NEXT, "Py_SetProgramName"); + if(handle) + { + Dl_info di; + int ret = dladdr(handle, &di); + if (ret != 0 && di.dli_saddr && di.dli_fname) + { + Py_SetProgramName(const_cast(di.dli_fname)); + } + } + } +#endif + m_embedded = true; Py_NoSiteFlag = 1; pybind11::initialize_interpreter(); + + // If, at configure time, we found PYTHON_SITE_PACKAGES, then we + // use it now. This allows us to find modules in your virtual + // environment site-packages directory, or extra modules you may + // have installed with your system python. +#ifdef ADIOS2_PYTHON_SITE_PACKAGES_DIRECTORY + pybind11::module sys = pybind11::module::import("sys"); + sys.attr("path").attr("append")(ADIOS2_PYTHON_SITE_PACKAGES_DIRECTORY); +#endif } void PythonInterpreter::finalize() diff --git a/testing/adios2/bindings/python/TestPythonEngine.py b/testing/adios2/bindings/python/TestPythonEngine.py index 604f9273c3..0536d94539 100644 --- a/testing/adios2/bindings/python/TestPythonEngine.py +++ b/testing/adios2/bindings/python/TestPythonEngine.py @@ -1,19 +1,6 @@ -import sys - -# When importing from C++, this path is missing, and thus numpy is -# unable to import the __future__ module. This is a temporary -# workaround to allow that to happen. -sys.path.append('/usr/lib/python2.7') - -# When importing from C++, this path is also missing. This lives in the -# virtual environment directory and is part of the path automatically -# when running the virtual env python. -sys.path.append('/data/scott/projects/adios2/adios-env/lib/python2.7/site-packages') - - import adios2 -import numpy # Required when importing this module from C++ +import numpy as np print('Inside TestPythonEngine module') From 4681c74e723e7231e2c419fae7138d2ff650a55e Mon Sep 17 00:00:00 2001 From: Scott Wittenburg Date: Tue, 10 Oct 2017 20:42:07 -0600 Subject: [PATCH 16/24] If PYTHONHOME is set at run time, use it instead of dladdr approach --- source/adios2/helper/PythonInterpreter.cpp | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/source/adios2/helper/PythonInterpreter.cpp b/source/adios2/helper/PythonInterpreter.cpp index 32cad4a496..3ca4224521 100644 --- a/source/adios2/helper/PythonInterpreter.cpp +++ b/source/adios2/helper/PythonInterpreter.cpp @@ -15,6 +15,8 @@ #include #include +#include "adios2sys/SystemTools.hxx" + #include "adios2//ADIOSConfig.h" #if WIN32 @@ -64,7 +66,16 @@ void PythonInterpreter::initialize() return; } + if (adios2sys::SystemTools::HasEnv("PYTHONHOME")) + { + char* pyHomeVar = + const_cast(adios2sys::SystemTools::GetEnv("PYTHONHOME")); + std::cout << "Will use PYTHONHOME (" << pyHomeVar << ") as argument to " + << "Py_SetProgramName" << std::endl; + Py_SetProgramName(pyHomeVar); + } #if defined(__linux) || defined(__unix) + else { // Find the actual library file where the symbol below is defined, // then use that path to help find Python run-time libraries. @@ -75,7 +86,10 @@ void PythonInterpreter::initialize() int ret = dladdr(handle, &di); if (ret != 0 && di.dli_saddr && di.dli_fname) { - Py_SetProgramName(const_cast(di.dli_fname)); + char* pyHome = const_cast(di.dli_fname); + std::cout << "Will use location of 'Py_SetProgramName' (" << pyHome + << ") as argument to Py_SetProgramName" << std::endl; + Py_SetProgramName(); } } } From cc4236f98a3b999e61dfc2ded9afc61d95b97f63 Mon Sep 17 00:00:00 2001 From: Scott Wittenburg Date: Tue, 10 Oct 2017 20:42:38 -0600 Subject: [PATCH 17/24] Make sure mpi tests run on osx, fix bpwritetypes mpi-mode test --- testing/CMakeLists.txt | 2 +- testing/adios2/bindings/C/CMakeLists.txt | 5 +-- testing/adios2/bindings/python/CMakeLists.txt | 11 ++--- .../bindings/python/TestBPWriteTypes.py | 44 ++++++++++--------- 4 files changed, 30 insertions(+), 32 deletions(-) diff --git a/testing/CMakeLists.txt b/testing/CMakeLists.txt index 17740cd079..f9af5252df 100644 --- a/testing/CMakeLists.txt +++ b/testing/CMakeLists.txt @@ -7,7 +7,7 @@ include(GoogleTest) if(ADIOS2_HAVE_MPI) set(MPIEXEC_COMMAND - ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} + ${MPIEXEC_EXECUTABLE} -oversubscribe ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ) endif() add_subdirectory(adios2) diff --git a/testing/adios2/bindings/C/CMakeLists.txt b/testing/adios2/bindings/C/CMakeLists.txt index 3e8f220fea..ab680174a5 100644 --- a/testing/adios2/bindings/C/CMakeLists.txt +++ b/testing/adios2/bindings/C/CMakeLists.txt @@ -8,10 +8,7 @@ target_link_libraries(TestBPWriteTypes_c adios2 gtest) if(ADIOS2_HAVE_MPI) target_link_libraries(TestBPWriteTypes_c MPI::MPI_C) - set(extra_test_args - EXEC_WRAPPER - ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} - ) + set(extra_test_args EXEC_WRAPPER ${MPIEXEC_COMMAND}) endif() gtest_add_tests(TARGET TestBPWriteTypes_c ${extra_test_args}) diff --git a/testing/adios2/bindings/python/CMakeLists.txt b/testing/adios2/bindings/python/CMakeLists.txt index ca9c6c8a59..5ebf5e6482 100644 --- a/testing/adios2/bindings/python/CMakeLists.txt +++ b/testing/adios2/bindings/python/CMakeLists.txt @@ -5,25 +5,22 @@ if(ADIOS2_HAVE_MPI) set(extra_test_args EXEC_WRAPPER ${MPIEXEC_COMMAND}) - set(test_parameters - EXEC_WRAPPER - ${MPIEXEC} -oversubscribe ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS}) set(use_mpi USE_MPI_FOR_PYTHON_TESTS "TRUE") endif() python_add_test(NAME PythonBPWrite SCRIPT TestBPWriteTypes.py - ${test_parameters} ${use_mpi}) + ${extra_test_args} ${use_mpi}) python_add_test(NAME TestOpenPythonEngineFromPython SCRIPT TestOpenPythonEngineFromPython.py - ${test_parameters} ${use_mpi} + ${extra_test_args} ${use_mpi} ADDTOPYPATH ${CMAKE_CURRENT_SOURCE_DIR}) python_add_test(NAME TestOpenInlinePythonEngineFromPython SCRIPT TestOpenInlinePythonEngineFromPython.py - ${test_parameters} ${use_mpi}) + ${extra_test_args} ${use_mpi}) add_executable(TestPythonEngineFromCpp TestPythonEngineFromCpp.cpp) target_link_libraries(TestPythonEngineFromCpp adios2 gtest gtest_main) gtest_add_tests(TARGET TestPythonEngineFromCpp ${extra_test_args} TEST_LIST pyEngineTests) -set_property(TEST ${pyEngineTests} PROPERTY +set_property(TEST ${pyEngineTests} PROPERTY ENVIRONMENT "PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR}:${ADIOS2_BINARY_DIR}/${CMAKE_INSTALL_PYTHONDIR}:$ENV{PYTHONPATH}" ) diff --git a/testing/adios2/bindings/python/TestBPWriteTypes.py b/testing/adios2/bindings/python/TestBPWriteTypes.py index 713200e9a5..c41dbf6e98 100644 --- a/testing/adios2/bindings/python/TestBPWriteTypes.py +++ b/testing/adios2/bindings/python/TestBPWriteTypes.py @@ -23,62 +23,66 @@ class TestBPWriteTypes(Adios2PythonTestBase): def testWriteTypes(self): # Test data data = SmallTestData() - + rank = 0 + size = 0 + if self.isUsingMpi(): comm = MPI.COMM_WORLD + rank = comm.Get_rank() + size = comm.Get_size() adios = adios2.ADIOS(comm) else: adios = adios2.ADIOS() - + bpIO = adios.DeclareIO("NPTypes") - + # ADIOS Variable name, shape, start, offset, constant dims # All local variables varI8 = bpIO.DefineVariable( - "varI8", [], [], [data.I8.size], True, data.I8.dtype) + "varI8", [size * data.I8.size], [rank * data.I8.size], [data.I8.size], True, data.I8.dtype) varI16 = bpIO.DefineVariable( - "varI16", [], [], [data.I16.size], True, data.I16.dtype) + "varI16", [size * data.I16.size], [rank * data.I16.size], [data.I16.size], True, data.I16.dtype) varI32 = bpIO.DefineVariable( - "varI32", [], [], [data.I32.size], True, data.I32.dtype) + "varI32", [size * data.I32.size], [rank * data.I32.size], [data.I32.size], True, data.I32.dtype) # FIXME: Uncomment below when 64 bit ints work properly # varI64 = bpIO.DefineVariable( # "varI64", [], [], [data.I64.size], True, data.I64.dtype) - + varU8 = bpIO.DefineVariable( - "varUI8", [], [], [data.U8.size], True, data.U8.dtype) + "varUI8", [size * ], [rank * ], [data.U8.size], True, data.U8.dtype) varU16 = bpIO.DefineVariable( - "varUI16", [], [], [data.U16.size], True, data.U16.dtype) + "varUI16", [size * data.U16.size], [rank * data.U16.size], [data.U16.size], True, data.U16.dtype) varU32 = bpIO.DefineVariable( - "varUI32", [], [], [data.U32.size], True, data.U32.dtype) + "varUI32", [size * data.U32.size], [rank * data.U32.size], [data.U32.size], True, data.U32.dtype) # FIXME: Uncomment below when 64 bit ints work properly # varU64 = bpIO.DefineVariable( # "varUI64", [], [], [data.U64.size], True, data.U64.dtype) - + varR32 = bpIO.DefineVariable( - "varR32", [], [], [data.R32.size], True, data.R32.dtype) - + "varR32", [size * data.R32.size], [rank * data.R32.size], [data.R32.size], True, data.R32.dtype) + varR64 = bpIO.DefineVariable( - "varR64", [], [], [data.R64.size], True, data.R64.dtype) - - + "varR64", [size * data.R64.size], [rank * data.R64.size], [data.R64.size], True, data.R64.dtype) + + # ADIOS Engine bpFileWriter = bpIO.Open("npTypes.bp", adios2.OpenMode.Write) - + bpFileWriter.Write(varI8, data.I8) bpFileWriter.Write(varI16, data.I16) bpFileWriter.Write(varI32, data.I32) # FIXME: Uncomment below when 64 bit ints work properly # bpFileWriter.Write(varI64, data.I64) - + bpFileWriter.Write(varU8, data.U8) bpFileWriter.Write(varU16, data.U16) bpFileWriter.Write(varU32, data.U32) # FIXME: Uncomment below when 64 bit ints work properly # bpFileWriter.Write(varU64, data.U64) - + bpFileWriter.Write(varR32, data.R32) bpFileWriter.Write(varR64, data.R64) - + bpFileWriter.Close() ### From f1f2979ce399cc76d9f1122883f47ca9d3c9c218 Mon Sep 17 00:00:00 2001 From: Scott Wittenburg Date: Wed, 11 Oct 2017 09:47:19 -0600 Subject: [PATCH 18/24] Add one more python test, remove some cmake messages, etc... --- cmake/DetectOptions.cmake | 8 --- source/adios2/helper/PythonInterpreter.cpp | 2 +- source/adios2/helper/PythonModuleHelper.h | 9 +-- testing/adios2/bindings/python/CMakeLists.txt | 3 + .../python/TestImportFromCppOpenFromPython.py | 59 +++++++++++++++++++ 5 files changed, 68 insertions(+), 13 deletions(-) create mode 100644 testing/adios2/bindings/python/TestImportFromCppOpenFromPython.py diff --git a/cmake/DetectOptions.cmake b/cmake/DetectOptions.cmake index c3420f1934..2562e77a2b 100644 --- a/cmake/DetectOptions.cmake +++ b/cmake/DetectOptions.cmake @@ -119,14 +119,6 @@ if(ADIOS2_USE_Python) endif() if(PythonFull_FOUND) set(ADIOS2_HAVE_Python ON) - message("PYTHONLIBS_FOUND = ${PYTHONLIBS_FOUND}") - message("PYTHON_PREFIX = ${PYTHON_PREFIX}") - message("PYTHON_LIBRARIES = ${PYTHON_LIBRARIES}") - message("PYTHON_INCLUDE_DIRS = ${PYTHON_INCLUDE_DIRS}") - message("PYTHON_MODULE_EXTENSION = ${PYTHON_MODULE_EXTENSION}") - message("PYTHON_MODULE_PREFIX = ${PYTHON_MODULE_PREFIX}") - message("PYTHON_SITE_PACKAGES = ${PYTHON_SITE_PACKAGES}") - message("PYTHON_IS_DEBUG = ${PYTHON_IS_DEBUG}") endif() #SysV IPC diff --git a/source/adios2/helper/PythonInterpreter.cpp b/source/adios2/helper/PythonInterpreter.cpp index 3ca4224521..d08c88088c 100644 --- a/source/adios2/helper/PythonInterpreter.cpp +++ b/source/adios2/helper/PythonInterpreter.cpp @@ -89,7 +89,7 @@ void PythonInterpreter::initialize() char* pyHome = const_cast(di.dli_fname); std::cout << "Will use location of 'Py_SetProgramName' (" << pyHome << ") as argument to Py_SetProgramName" << std::endl; - Py_SetProgramName(); + Py_SetProgramName(pyHome); } } } diff --git a/source/adios2/helper/PythonModuleHelper.h b/source/adios2/helper/PythonModuleHelper.h index 964052cf34..b24a5b32b8 100644 --- a/source/adios2/helper/PythonModuleHelper.h +++ b/source/adios2/helper/PythonModuleHelper.h @@ -48,10 +48,11 @@ namespace PythonModuleHelper } // Unable to instantiate the object in this case - throw std::runtime_error("PythonInstanceBuilder: Specified class was " - "not present in main module, nor was a " - "module name provided in the parameters. " - "Unable to instantiate python class."); + throw std::runtime_error("PythonModuleHelper::FindPythonClass " + "Specified class was not present in main " + "module, nor was a module name provided in " + "the parameters. Unable to instantiate " + "python class."); } } } diff --git a/testing/adios2/bindings/python/CMakeLists.txt b/testing/adios2/bindings/python/CMakeLists.txt index 5ebf5e6482..1a67599786 100644 --- a/testing/adios2/bindings/python/CMakeLists.txt +++ b/testing/adios2/bindings/python/CMakeLists.txt @@ -13,6 +13,9 @@ python_add_test(NAME PythonBPWrite SCRIPT TestBPWriteTypes.py python_add_test(NAME TestOpenPythonEngineFromPython SCRIPT TestOpenPythonEngineFromPython.py ${extra_test_args} ${use_mpi} ADDTOPYPATH ${CMAKE_CURRENT_SOURCE_DIR}) +python_add_test(NAME TestImportFromCppOpenFromPython SCRIPT TestImportFromCppOpenFromPython.py + ${extra_test_args} ${use_mpi} + ADDTOPYPATH ${CMAKE_CURRENT_SOURCE_DIR}) python_add_test(NAME TestOpenInlinePythonEngineFromPython SCRIPT TestOpenInlinePythonEngineFromPython.py ${extra_test_args} ${use_mpi}) diff --git a/testing/adios2/bindings/python/TestImportFromCppOpenFromPython.py b/testing/adios2/bindings/python/TestImportFromCppOpenFromPython.py new file mode 100644 index 0000000000..814dfd594f --- /dev/null +++ b/testing/adios2/bindings/python/TestImportFromCppOpenFromPython.py @@ -0,0 +1,59 @@ +from Adios2PythonTestBase import Adios2PythonTestBase + +if Adios2PythonTestBase.isUsingMpi(): + from mpi4py import MPI + +import numpy as np +import adios2 + +### +### Create a testcase class with some tests +### +class TestOpenPythonEngineFromPython(Adios2PythonTestBase): + def testCreateEngine(self): + rank = 0 + size = 1 + + if self.isUsingMpi(): + # MPI + comm = MPI.COMM_WORLD + rank = comm.Get_rank() + size = comm.Get_size() + adios = adios2.ADIOS(comm) + else: + adios = adios2.ADIOS() + + # User data + myArray = np.array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9], dtype=np.uint8) + Nx = myArray.size + + # ADIOS IO + bpIO = adios.DeclareIO("BPFile_N2N") + + # ADIOS Variable name, shape, start, offset, constant dims + ioArray = bpIO.DefineVariable( + "bpArray", [size * Nx], [rank * Nx], [Nx], True, myArray.dtype) + + # Engine derived class, spawned to start IO operations + bpIO.EngineType = "PythonEngine" + + # Since we did not import the TestPythonEngine class directly, we must + # provide the "PluginModule" parameter indicating the module containing + # the engine class we want. + bpIO.Parameters = { + "PluginName": "DoNotReallyCare", + "PluginModule": "TestPythonEngine", + "PluginClass": "TestPythonEngine" + } + + # ADIOS Engine + bpFileWriter = bpIO.Open("npArray.bp", adios2.OpenMode.Write) + bpFileWriter.Write(ioArray, myArray) + + bpFileWriter.Close() + +### +### Trigger the tests +### +if __name__ == '__main__': + Adios2PythonTestBase.main() From d2e8bd240451cd67f416fab3bbe6cc10cb3dcd61 Mon Sep 17 00:00:00 2001 From: Scott Wittenburg Date: Wed, 11 Oct 2017 10:27:42 -0600 Subject: [PATCH 19/24] Fix broken test --- testing/adios2/bindings/python/TestBPWriteTypes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testing/adios2/bindings/python/TestBPWriteTypes.py b/testing/adios2/bindings/python/TestBPWriteTypes.py index c41dbf6e98..60d1731562 100644 --- a/testing/adios2/bindings/python/TestBPWriteTypes.py +++ b/testing/adios2/bindings/python/TestBPWriteTypes.py @@ -24,7 +24,7 @@ def testWriteTypes(self): # Test data data = SmallTestData() rank = 0 - size = 0 + size = 1 if self.isUsingMpi(): comm = MPI.COMM_WORLD @@ -49,7 +49,7 @@ def testWriteTypes(self): # "varI64", [], [], [data.I64.size], True, data.I64.dtype) varU8 = bpIO.DefineVariable( - "varUI8", [size * ], [rank * ], [data.U8.size], True, data.U8.dtype) + "varUI8", [size * data.U8.size], [rank * data.U8.size], [data.U8.size], True, data.U8.dtype) varU16 = bpIO.DefineVariable( "varUI16", [size * data.U16.size], [rank * data.U16.size], [data.U16.size], True, data.U16.dtype) varU32 = bpIO.DefineVariable( From 3c825467f401692b7c7c970ebecb3304be596828 Mon Sep 17 00:00:00 2001 From: Scott Wittenburg Date: Wed, 11 Oct 2017 10:27:59 -0600 Subject: [PATCH 20/24] Add site-packages define to the generate function --- cmake/ADIOSFunctions.cmake | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cmake/ADIOSFunctions.cmake b/cmake/ADIOSFunctions.cmake index 9d39948402..9f8e27c810 100644 --- a/cmake/ADIOSFunctions.cmake +++ b/cmake/ADIOSFunctions.cmake @@ -71,6 +71,13 @@ function(GenerateADIOSHeaderConfig) endif() endforeach() + if(ADIOS2_HAVE_Python) + string(APPEND ADIOS2_CONFIG_DEFINES " +/* Python site-packages directory discovered at configure time */ +#define ADIOS2_PYTHON_SITE_PACKAGES_DIRECTORY \"@PYTHON_SITE_PACKAGES@\" +") + endif() + configure_file( ${ADIOS2_SOURCE_DIR}/source/adios2/ADIOSConfig.h.in ${ADIOS2_BINARY_DIR}/source/adios2/ADIOSConfig.h.in From 5ad83c3b07878815047eadcd3d3f59aa668847e6 Mon Sep 17 00:00:00 2001 From: Scott Wittenburg Date: Wed, 11 Oct 2017 11:23:01 -0600 Subject: [PATCH 21/24] Clang formatting --- bindings/python/PyEngine.cpp | 34 +++-- bindings/python/PyIO.cpp | 7 +- bindings/python/PyVariable.cpp | 2 +- source/adios2/core/Engine.cpp | 5 +- source/adios2/engine/pyengine/PyEngineBase.h | 21 ++-- .../adios2/engine/pyengine/PythonEngine.cpp | 67 +++++----- source/adios2/engine/pyengine/PythonEngine.h | 1 - source/adios2/helper/PythonInterpreter.cpp | 119 +++++++++--------- source/adios2/helper/PythonInterpreter.h | 28 ++--- source/adios2/helper/PythonModuleHelper.h | 53 ++++---- .../python/TestPythonEngineFromCpp.cpp | 13 +- 11 files changed, 163 insertions(+), 187 deletions(-) diff --git a/bindings/python/PyEngine.cpp b/bindings/python/PyEngine.cpp index 27ebdb2a4e..30f750837b 100644 --- a/bindings/python/PyEngine.cpp +++ b/bindings/python/PyEngine.cpp @@ -4,8 +4,8 @@ #include -#include #include "PyEngineBase.h" +#include namespace adios2 { @@ -18,28 +18,28 @@ void GeneratePythonBindings(pybind11::module &m) // Wrapping for internal Engine classes pybind11::class_> engine(m, "EngineBase"); engine.def("Write", - [](Engine &e, VariableBase &variable, pybind11::array values) { - e.Write(variable, values.data()); - }); + [](Engine &e, VariableBase &variable, pybind11::array values) { + e.Write(variable, values.data()); + }); engine.def("Write", - [](Engine &e, const std::string name, pybind11::array values) { - e.Write(name, values.data()); - }); + [](Engine &e, const std::string name, pybind11::array values) { + e.Write(name, values.data()); + }); engine.def("InquireVariable", - [](Engine &e, const std::string name) -> VariableBase * { - VariableBase *var; + [](Engine &e, const std::string name) -> VariableBase * { + VariableBase *var; #define inquire(T) \ if (var = e.InquireVariable(name, false)) \ { \ return var; \ } - ADIOS2_FOREACH_TYPE_1ARG(inquire) + ADIOS2_FOREACH_TYPE_1ARG(inquire) #undef inquire - return nullptr; - }); + return nullptr; + }); engine.def("Close", &Engine::Close, pybind11::arg("transportIndex") = -1); - // + // // Wrappings to extend Engine in Python // @@ -49,10 +49,7 @@ void GeneratePythonBindings(pybind11::module &m) { public: using PyEngineBase::PyEngineBase; - void Init() override - { - PYBIND11_OVERLOAD(void, PyEngineBase, Init,); - } + void Init() override { PYBIND11_OVERLOAD(void, PyEngineBase, Init, ); } void DoWrite(VariableBase *var, pybind11::array values) override { PYBIND11_OVERLOAD_PURE(void, PyEngineBase, DoWrite, var, values); @@ -63,7 +60,8 @@ void GeneratePythonBindings(pybind11::module &m) } }; - pybind11::class_>(m, "Engine", engine) + pybind11::class_>( + m, "Engine", engine) .def(pybind11::init()) .def("Init", &PyEngineBase::Init) diff --git a/bindings/python/PyIO.cpp b/bindings/python/PyIO.cpp index d34a478074..d798200f93 100644 --- a/bindings/python/PyIO.cpp +++ b/bindings/python/PyIO.cpp @@ -1,5 +1,5 @@ -#include #include +#include #include #include @@ -53,9 +53,8 @@ void GeneratePythonBindings(pybind11::module &m) .def("GetVariable", &IO::GetVariableBase, pybind11::arg("name").none(false)) .def("Open", [](IO &io, const std::string &name, - const OpenMode mode) { - return io.Open(name, mode); - }, pybind11::return_value_policy::reference); + const OpenMode mode) { return io.Open(name, mode); }, + pybind11::return_value_policy::reference); } } // end namespace adios2 diff --git a/bindings/python/PyVariable.cpp b/bindings/python/PyVariable.cpp index 118062404b..19d65d993f 100644 --- a/bindings/python/PyVariable.cpp +++ b/bindings/python/PyVariable.cpp @@ -1,5 +1,5 @@ -#include #include +#include #include #include diff --git a/source/adios2/core/Engine.cpp b/source/adios2/core/Engine.cpp index 814c6bf85d..bb70fd8369 100644 --- a/source/adios2/core/Engine.cpp +++ b/source/adios2/core/Engine.cpp @@ -64,9 +64,7 @@ void Engine::Release() {} void Engine::PerformReads(ReadMode /*mode*/){}; // PROTECTED -void Engine::Init() { - std::cout << "Inside Engine::Init()" << std::endl; -} +void Engine::Init() { std::cout << "Inside Engine::Init()" << std::endl; } void Engine::InitParameters() {} @@ -180,5 +178,4 @@ void Engine::ThrowUp(const std::string function) const ADIOS2_FOREACH_TYPE_1ARG(declare_template_instantiation) #undef declare_template_instantiation - } // end namespace adios diff --git a/source/adios2/engine/pyengine/PyEngineBase.h b/source/adios2/engine/pyengine/PyEngineBase.h index 07c04f5b91..83ea04415d 100644 --- a/source/adios2/engine/pyengine/PyEngineBase.h +++ b/source/adios2/engine/pyengine/PyEngineBase.h @@ -9,16 +9,13 @@ * Author: Chuck Atkins */ +#ifndef ADIOS2_ENGINE_PYENGINE_PYENGINEBASE_H_ +#define ADIOS2_ENGINE_PYENGINE_PYENGINEBASE_H_ + #include #include -// #include -// #include "adios2/core/Engine.h" -// #include "adios2/core/IO.h" -// #include "adios2/core/Variable.h" - -#ifndef ADIOS2_ENGINE_PYENGINE_PYENGINEBASE_H_ -#define ADIOS2_ENGINE_PYENGINE_PYENGINEBASE_H_ +#include "adios2/core/Engine.h" namespace adios2 { @@ -29,8 +26,8 @@ class PyEngineBase : public Engine friend class PythonEngine; public: - PyEngineBase(const std::string engineType, IO &io, - const std::string &name, const OpenMode openMode) + PyEngineBase(const std::string engineType, IO &io, const std::string &name, + const OpenMode openMode) : Engine(engineType, io, name, openMode, io.m_MPIComm) { } @@ -42,8 +39,8 @@ class PyEngineBase : public Engine using Engine::Close; protected: - // The C++ code calls this implementation which will redirect to the - // python implementation. +// The C++ code calls this implementation which will redirect to the +// python implementation. #define define_dowrite(T) \ void DoWrite(Variable &var, const T *values) override \ { \ @@ -52,7 +49,7 @@ class PyEngineBase : public Engine var.m_Count.end(), 0), \ values)); \ } - ADIOS2_FOREACH_TYPE_1ARG(define_dowrite) + ADIOS2_FOREACH_TYPE_1ARG(define_dowrite) #undef define_dowrite }; } diff --git a/source/adios2/engine/pyengine/PythonEngine.cpp b/source/adios2/engine/pyengine/PythonEngine.cpp index ba5791a115..b15a6be1d4 100644 --- a/source/adios2/engine/pyengine/PythonEngine.cpp +++ b/source/adios2/engine/pyengine/PythonEngine.cpp @@ -12,11 +12,11 @@ #include "PyEngineBase.h" #include +#include #include #include #include #include -#include #include "adios2/helper/PythonInterpreter.h" #include "adios2/helper/PythonModuleHelper.h" @@ -37,7 +37,6 @@ struct PythonEngine::Impl std::shared_ptr eng; }; - /******************************************************************************/ PythonEngine::PythonEngine(IO &io, const std::string &name, @@ -45,7 +44,8 @@ PythonEngine::PythonEngine(IO &io, const std::string &name, : Engine("PythonEngine", io, name, openMode, mpiComm), m_Impl(new Impl) { Init(); - m_Impl->enginePyObject = m_Impl->enginePyClass("PythonEngine", io, name, openMode); + m_Impl->enginePyObject = + m_Impl->enginePyClass("PythonEngine", io, name, openMode); m_Impl->eng = m_Impl->enginePyObject.cast>(); } @@ -56,9 +56,7 @@ void PythonEngine::PerformReads(ReadMode mode) m_Impl->eng->PerformReads(mode); } -void PythonEngine::Release() { - m_Impl->eng->Release(); -} +void PythonEngine::Release() { m_Impl->eng->Release(); } void PythonEngine::Advance(const float timeoutSeconds) { @@ -84,10 +82,7 @@ void PythonEngine::SetCallBack( m_Impl->eng->SetCallBack(callback); } -void PythonEngine::Close(const int transportIndex) -{ - m_Impl->eng->Close(); -} +void PythonEngine::Close(const int transportIndex) { m_Impl->eng->Close(); } void PythonEngine::Init() { @@ -100,7 +95,7 @@ void PythonEngine::Init() m_Impl->m_PluginName = paramPluginNameIt->second; // Get the python engine plugin module name, if provided - std::string* pluginModuleName = nullptr; + std::string *pluginModuleName = nullptr; auto paramPluginModuleIt = m_IO.m_Parameters.find("PluginModule"); if (paramPluginModuleIt != m_IO.m_Parameters.end()) { @@ -119,49 +114,45 @@ void PythonEngine::Init() // Initialize python interpreter if it's not already running adios2::PythonInterpreter::instance().initialize(); - m_Impl->enginePyClass = - adios2::PythonModuleHelper::FindPythonClass(pluginClassName, - pluginModuleName); + m_Impl->enginePyClass = adios2::PythonModuleHelper::FindPythonClass( + pluginClassName, pluginModuleName); } -#define define(T) \ - void PythonEngine::DoWrite(Variable &variable, const T *values) \ - { \ - m_Impl->eng->DoWrite(variable, values); \ - } \ - \ - void PythonEngine::DoScheduleRead(Variable &variable, \ - const T *values) \ - { \ - m_Impl->eng->DoScheduleRead(variable, values); \ - } \ - void PythonEngine::DoScheduleRead(const std::string &variableName, \ - const T *values) \ - { \ - m_Impl->eng->DoScheduleRead(variableName, values); \ +#define define(T) \ + void PythonEngine::DoWrite(Variable &variable, const T *values) \ + { \ + m_Impl->eng->DoWrite(variable, values); \ + } \ + void PythonEngine::DoScheduleRead(Variable &variable, const T *values) \ + { \ + m_Impl->eng->DoScheduleRead(variable, values); \ + } \ + void PythonEngine::DoScheduleRead(const std::string &variableName, \ + const T *values) \ + { \ + m_Impl->eng->DoScheduleRead(variableName, values); \ } ADIOS2_FOREACH_TYPE_1ARG(define) #undef define -void PythonEngine::DoWrite(VariableCompound &variable, - const void *values) +void PythonEngine::DoWrite(VariableCompound &variable, const void *values) { std::cout << "PythonEngine::DoWrite(VariableCompound &variable, " << "const void *values) is not yet implemented" << std::endl; // m_Impl->eng->DoWrite(variable, values); } -#define define(T, L) \ - Variable *PythonEngine::InquireVariable##L( \ - const std::string &name, const bool readIn) \ - { \ - return m_Impl->eng->InquireVariable##L(name, readIn); \ +#define define(T, L) \ + Variable *PythonEngine::InquireVariable##L(const std::string &name, \ + const bool readIn) \ + { \ + return m_Impl->eng->InquireVariable##L(name, readIn); \ } ADIOS2_FOREACH_TYPE_2ARGS(define) #undef define -VariableBase *PythonEngine::InquireVariableUnknown( - const std::string &name, const bool readIn) +VariableBase *PythonEngine::InquireVariableUnknown(const std::string &name, + const bool readIn) { return m_Impl->eng->InquireVariableUnknown(name, readIn); } diff --git a/source/adios2/engine/pyengine/PythonEngine.h b/source/adios2/engine/pyengine/PythonEngine.h index a253b39783..16d033324e 100644 --- a/source/adios2/engine/pyengine/PythonEngine.h +++ b/source/adios2/engine/pyengine/PythonEngine.h @@ -11,7 +11,6 @@ #ifndef ADIOS2_ENGINE_PYENGINE_PYTHONENGINE_H_ #define ADIOS2_ENGINE_PYENGINE_PYTHONENGINE_H_ - #include // for function #include // for unique_ptr #include // for string diff --git a/source/adios2/helper/PythonInterpreter.cpp b/source/adios2/helper/PythonInterpreter.cpp index d08c88088c..2dcc9e5cc4 100644 --- a/source/adios2/helper/PythonInterpreter.cpp +++ b/source/adios2/helper/PythonInterpreter.cpp @@ -11,13 +11,13 @@ #include "PythonInterpreter.h" #include +#include #include #include -#include #include "adios2sys/SystemTools.hxx" -#include "adios2//ADIOSConfig.h" +#include "adios2/ADIOSConfig.h" #if WIN32 #include @@ -25,12 +25,12 @@ extern __declspec(dllimport) int Py_NoSiteFlag; #endif #if defined(__linux) -# ifndef _GNU_SOURCE -# define _GNU_SOURCE -# endif -# include +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif +#include #elif defined(__unix) -# include +#include #endif namespace adios2 @@ -38,83 +38,80 @@ namespace adios2 PythonInterpreter PythonInterpreter::m_instance; -PythonInterpreter& PythonInterpreter::instance() +PythonInterpreter &PythonInterpreter::instance() { - return PythonInterpreter::m_instance; + return PythonInterpreter::m_instance; } -PythonInterpreter::PythonInterpreter() - : m_embedded(false) +PythonInterpreter::PythonInterpreter() : m_embedded(false) { - this->initialize(); + this->initialize(); } -PythonInterpreter::~PythonInterpreter() -{ - this->finalize(); -} +PythonInterpreter::~PythonInterpreter() { this->finalize(); } bool PythonInterpreter::isInitialized() const { - return Py_IsInitialized() != 0; + return Py_IsInitialized() != 0; } void PythonInterpreter::initialize() { - if (this->isInitialized()) - { - return; - } - - if (adios2sys::SystemTools::HasEnv("PYTHONHOME")) - { - char* pyHomeVar = - const_cast(adios2sys::SystemTools::GetEnv("PYTHONHOME")); - std::cout << "Will use PYTHONHOME (" << pyHomeVar << ") as argument to " - << "Py_SetProgramName" << std::endl; - Py_SetProgramName(pyHomeVar); - } + if (this->isInitialized()) + { + return; + } + + if (adios2sys::SystemTools::HasEnv("PYTHONHOME")) + { + char *pyHomeVar = + const_cast(adios2sys::SystemTools::GetEnv("PYTHONHOME")); + std::cout << "Will use PYTHONHOME (" << pyHomeVar << ") as argument to " + << "Py_SetProgramName" << std::endl; + Py_SetProgramName(pyHomeVar); + } #if defined(__linux) || defined(__unix) - else - { - // Find the actual library file where the symbol below is defined, - // then use that path to help find Python run-time libraries. - void *handle = dlsym(RTLD_NEXT, "Py_SetProgramName"); - if(handle) + else { - Dl_info di; - int ret = dladdr(handle, &di); - if (ret != 0 && di.dli_saddr && di.dli_fname) - { - char* pyHome = const_cast(di.dli_fname); - std::cout << "Will use location of 'Py_SetProgramName' (" << pyHome - << ") as argument to Py_SetProgramName" << std::endl; - Py_SetProgramName(pyHome); - } + // Find the actual library file where the symbol below is defined, + // then use that path to help find Python run-time libraries. + void *handle = dlsym(RTLD_NEXT, "Py_SetProgramName"); + if (handle) + { + Dl_info di; + int ret = dladdr(handle, &di); + if (ret != 0 && di.dli_saddr && di.dli_fname) + { + char *pyHome = const_cast(di.dli_fname); + std::cout << "Will use location of 'Py_SetProgramName' (" + << pyHome << ") as argument to Py_SetProgramName" + << std::endl; + Py_SetProgramName(pyHome); + } + } } - } #endif - m_embedded = true; - Py_NoSiteFlag = 1; - pybind11::initialize_interpreter(); + m_embedded = true; + Py_NoSiteFlag = 1; + pybind11::initialize_interpreter(); - // If, at configure time, we found PYTHON_SITE_PACKAGES, then we - // use it now. This allows us to find modules in your virtual - // environment site-packages directory, or extra modules you may - // have installed with your system python. +// If, at configure time, we found PYTHON_SITE_PACKAGES, then we +// use it now. This allows us to find modules in your virtual +// environment site-packages directory, or extra modules you may +// have installed with your system python. #ifdef ADIOS2_PYTHON_SITE_PACKAGES_DIRECTORY - pybind11::module sys = pybind11::module::import("sys"); - sys.attr("path").attr("append")(ADIOS2_PYTHON_SITE_PACKAGES_DIRECTORY); + pybind11::module sys = pybind11::module::import("sys"); + sys.attr("path").attr("append")(ADIOS2_PYTHON_SITE_PACKAGES_DIRECTORY); #endif } void PythonInterpreter::finalize() { - if (this->isInitialized()) - { - pybind11::finalize_interpreter(); - m_embedded = false; - } + if (this->isInitialized()) + { + pybind11::finalize_interpreter(); + m_embedded = false; + } +} } -} \ No newline at end of file diff --git a/source/adios2/helper/PythonInterpreter.h b/source/adios2/helper/PythonInterpreter.h index ef4d8a4b37..f3f155eb54 100644 --- a/source/adios2/helper/PythonInterpreter.h +++ b/source/adios2/helper/PythonInterpreter.h @@ -22,28 +22,28 @@ namespace adios2 class PythonInterpreter { public: - static PythonInterpreter& instance(); + static PythonInterpreter &instance(); - // Check if python is initialized. - bool isInitialized() const; + // Check if python is initialized. + bool isInitialized() const; - // Initialize the embedded python - void initialize(); + // Initialize the embedded python + void initialize(); - // Finalize the embedded python. - void finalize(); + // Finalize the embedded python. + void finalize(); - // Returns true if the embedded python session has been initialized. - bool isEmbedded() const { return m_embedded; } + // Returns true if the embedded python session has been initialized. + bool isEmbedded() const { return m_embedded; } private: - PythonInterpreter(); - virtual ~PythonInterpreter(); + PythonInterpreter(); + virtual ~PythonInterpreter(); - static PythonInterpreter m_instance; + static PythonInterpreter m_instance; - bool m_embedded; + bool m_embedded; }; } -#endif /* ADIOS2_HELPER_PYTHONINTERPRETER_H_ */ \ No newline at end of file +#endif /* ADIOS2_HELPER_PYTHONINTERPRETER_H_ */ diff --git a/source/adios2/helper/PythonModuleHelper.h b/source/adios2/helper/PythonModuleHelper.h index b24a5b32b8..8d86d30bb4 100644 --- a/source/adios2/helper/PythonModuleHelper.h +++ b/source/adios2/helper/PythonModuleHelper.h @@ -19,41 +19,40 @@ namespace adios2 { namespace PythonModuleHelper { - pybind11::object FindPythonClass(const std::string& className, - const std::string* moduleName = nullptr) +pybind11::object FindPythonClass(const std::string &className, + const std::string *moduleName = nullptr) +{ + pybind11::dict globals = pybind11::globals(); + + if (moduleName != nullptr) { - pybind11::dict globals = pybind11::globals(); + pybind11::object moduleObject; - if (moduleName != nullptr) + if (globals.contains((*moduleName).c_str())) { - pybind11::object moduleObject; - - if (globals.contains((*moduleName).c_str())) - { - moduleObject = globals[(*moduleName).c_str()]; - } - else - { - moduleObject = pybind11::module::import((*moduleName).c_str()); - } - - pybind11::object constructor = - moduleObject.attr(className.c_str()); - return constructor; + moduleObject = globals[(*moduleName).c_str()]; } - else if (globals.contains(className.c_str())) + else { - pybind11::object constructor = globals[className.c_str()]; - return constructor; + moduleObject = pybind11::module::import((*moduleName).c_str()); } - // Unable to instantiate the object in this case - throw std::runtime_error("PythonModuleHelper::FindPythonClass " - "Specified class was not present in main " - "module, nor was a module name provided in " - "the parameters. Unable to instantiate " - "python class."); + pybind11::object constructor = moduleObject.attr(className.c_str()); + return constructor; + } + else if (globals.contains(className.c_str())) + { + pybind11::object constructor = globals[className.c_str()]; + return constructor; } + + // Unable to instantiate the object in this case + throw std::runtime_error("PythonModuleHelper::FindPythonClass " + "Specified class was not present in main " + "module, nor was a module name provided in " + "the parameters. Unable to instantiate " + "python class."); +} } } diff --git a/testing/adios2/bindings/python/TestPythonEngineFromCpp.cpp b/testing/adios2/bindings/python/TestPythonEngineFromCpp.cpp index c82bca57d7..3c5cd8bef6 100644 --- a/testing/adios2/bindings/python/TestPythonEngineFromCpp.cpp +++ b/testing/adios2/bindings/python/TestPythonEngineFromCpp.cpp @@ -29,7 +29,7 @@ TEST(PythonEngineTest, CreatePythonEngineFromCPlusPlus) adios2::IO &io = adios.DeclareIO("PythonPluginIO"); adios2::Variable &var = io.DefineVariable( - "data", {mpiSize * Nx}, {mpiRank * Nx}, {Nx}, adios2::ConstantDims); + "data", {mpiSize * Nx}, {mpiRank * Nx}, {Nx}, adios2::ConstantDims); io.SetEngine("PythonEngine"); io.SetParameters({{"PluginName", "TestPythonPlugin"}, @@ -37,9 +37,8 @@ TEST(PythonEngineTest, CreatePythonEngineFromCPlusPlus) {"PluginClass", "TestPythonEngine"}}); std::shared_ptr writer; - EXPECT_NO_THROW({ - writer = io.Open("TestPythonPlugin", adios2::OpenMode::Write); - }); + EXPECT_NO_THROW( + { writer = io.Open("TestPythonPlugin", adios2::OpenMode::Write); }); ASSERT_NE(writer.get(), nullptr); @@ -76,9 +75,9 @@ TEST(PythonEngineTest, CreatePythonEngineFromCPlusPlusExpectImportError) std::shared_ptr writer; - ASSERT_THROW({ - writer = io.Open("TestPythonPlugin", adios2::OpenMode::Write); - }, std::runtime_error); + ASSERT_THROW( + { writer = io.Open("TestPythonPlugin", adios2::OpenMode::Write); }, + std::runtime_error); } //****************************************************************************** From 05a23f139e4e08ba65fe81c7e4d4b24d89b08a54 Mon Sep 17 00:00:00 2001 From: Scott Wittenburg Date: Wed, 11 Oct 2017 12:15:39 -0600 Subject: [PATCH 22/24] Flake8 style fixes --- flake8.cfg | 2 +- .../bindings/python/Adios2PythonTestBase.py | 21 ++++----- .../bindings/python/TestBPWriteTypes.py | 44 +++++++++++-------- .../python/TestImportFromCppOpenFromPython.py | 24 +++++----- .../TestOpenInlinePythonEngineFromPython.py | 37 +++++++++------- .../python/TestOpenPythonEngineFromPython.py | 29 ++++++------ .../bindings/python/TestPythonEngine.py | 3 +- 7 files changed, 91 insertions(+), 69 deletions(-) diff --git a/flake8.cfg b/flake8.cfg index 48317219e0..700efc45d5 100644 --- a/flake8.cfg +++ b/flake8.cfg @@ -2,5 +2,5 @@ max-line-length = 80 max-complexity = 14 format = pylint -ignore = F403,F405,F999 +ignore = F401,F403,F405,F999 exclude = thirdparty/ diff --git a/testing/adios2/bindings/python/Adios2PythonTestBase.py b/testing/adios2/bindings/python/Adios2PythonTestBase.py index 10143cb874..8776048dde 100644 --- a/testing/adios2/bindings/python/Adios2PythonTestBase.py +++ b/testing/adios2/bindings/python/Adios2PythonTestBase.py @@ -1,15 +1,16 @@ import unittest import os + class Adios2PythonTestBase(unittest.TestCase): - ### Convenience method to let python tests know if MPI is enabled - @staticmethod - def isUsingMpi(): - if 'ADIOS2_PYTHON_TESTS_USE_MPI' in os.environ: - return True - return False + # Convenience method to let python tests know if MPI is enabled + @staticmethod + def isUsingMpi(): + if 'ADIOS2_PYTHON_TESTS_USE_MPI' in os.environ: + return True + return False - ### Expose main method so python tests don't need to import unittest - @staticmethod - def main(): - unittest.main() + # Expose main method so python tests don't need to import unittest + @staticmethod + def main(): + unittest.main() diff --git a/testing/adios2/bindings/python/TestBPWriteTypes.py b/testing/adios2/bindings/python/TestBPWriteTypes.py index 60d1731562..2053a5148e 100644 --- a/testing/adios2/bindings/python/TestBPWriteTypes.py +++ b/testing/adios2/bindings/python/TestBPWriteTypes.py @@ -9,16 +9,16 @@ # Author: William F Godoy godoywf@ornl.gov from Adios2PythonTestBase import Adios2PythonTestBase +from adios2NPTypes import SmallTestData +import adios2 if Adios2PythonTestBase.isUsingMpi(): from mpi4py import MPI -from adios2NPTypes import SmallTestData -import adios2 -### -### Create a testcase class with some tests -### +# +# Create a testcase class with some tests +# class TestBPWriteTypes(Adios2PythonTestBase): def testWriteTypes(self): # Test data @@ -39,31 +39,38 @@ def testWriteTypes(self): # ADIOS Variable name, shape, start, offset, constant dims # All local variables varI8 = bpIO.DefineVariable( - "varI8", [size * data.I8.size], [rank * data.I8.size], [data.I8.size], True, data.I8.dtype) + "varI8", [size * data.I8.size], [rank * data.I8.size], + [data.I8.size], True, data.I8.dtype) varI16 = bpIO.DefineVariable( - "varI16", [size * data.I16.size], [rank * data.I16.size], [data.I16.size], True, data.I16.dtype) + "varI16", [size * data.I16.size], [rank * data.I16.size], + [data.I16.size], True, data.I16.dtype) varI32 = bpIO.DefineVariable( - "varI32", [size * data.I32.size], [rank * data.I32.size], [data.I32.size], True, data.I32.dtype) + "varI32", [size * data.I32.size], [rank * data.I32.size], + [data.I32.size], True, data.I32.dtype) # FIXME: Uncomment below when 64 bit ints work properly # varI64 = bpIO.DefineVariable( # "varI64", [], [], [data.I64.size], True, data.I64.dtype) varU8 = bpIO.DefineVariable( - "varUI8", [size * data.U8.size], [rank * data.U8.size], [data.U8.size], True, data.U8.dtype) + "varUI8", [size * data.U8.size], [rank * data.U8.size], + [data.U8.size], True, data.U8.dtype) varU16 = bpIO.DefineVariable( - "varUI16", [size * data.U16.size], [rank * data.U16.size], [data.U16.size], True, data.U16.dtype) + "varUI16", [size * data.U16.size], [rank * data.U16.size], + [data.U16.size], True, data.U16.dtype) varU32 = bpIO.DefineVariable( - "varUI32", [size * data.U32.size], [rank * data.U32.size], [data.U32.size], True, data.U32.dtype) + "varUI32", [size * data.U32.size], [rank * data.U32.size], + [data.U32.size], True, data.U32.dtype) # FIXME: Uncomment below when 64 bit ints work properly # varU64 = bpIO.DefineVariable( # "varUI64", [], [], [data.U64.size], True, data.U64.dtype) varR32 = bpIO.DefineVariable( - "varR32", [size * data.R32.size], [rank * data.R32.size], [data.R32.size], True, data.R32.dtype) + "varR32", [size * data.R32.size], [rank * data.R32.size], + [data.R32.size], True, data.R32.dtype) varR64 = bpIO.DefineVariable( - "varR64", [size * data.R64.size], [rank * data.R64.size], [data.R64.size], True, data.R64.dtype) - + "varR64", [size * data.R64.size], [rank * data.R64.size], + [data.R64.size], True, data.R64.dtype) # ADIOS Engine bpFileWriter = bpIO.Open("npTypes.bp", adios2.OpenMode.Write) @@ -85,8 +92,9 @@ def testWriteTypes(self): bpFileWriter.Close() -### -### Trigger the tests -### + +# +# Trigger the tests +# if __name__ == '__main__': - Adios2PythonTestBase.main() \ No newline at end of file + Adios2PythonTestBase.main() diff --git a/testing/adios2/bindings/python/TestImportFromCppOpenFromPython.py b/testing/adios2/bindings/python/TestImportFromCppOpenFromPython.py index 814dfd594f..af38d6a326 100644 --- a/testing/adios2/bindings/python/TestImportFromCppOpenFromPython.py +++ b/testing/adios2/bindings/python/TestImportFromCppOpenFromPython.py @@ -1,14 +1,15 @@ from Adios2PythonTestBase import Adios2PythonTestBase +import numpy as np +import adios2 + if Adios2PythonTestBase.isUsingMpi(): from mpi4py import MPI -import numpy as np -import adios2 -### -### Create a testcase class with some tests -### +# +# Create a testcase class with some tests +# class TestOpenPythonEngineFromPython(Adios2PythonTestBase): def testCreateEngine(self): rank = 0 @@ -41,9 +42,9 @@ def testCreateEngine(self): # provide the "PluginModule" parameter indicating the module containing # the engine class we want. bpIO.Parameters = { - "PluginName": "DoNotReallyCare", - "PluginModule": "TestPythonEngine", - "PluginClass": "TestPythonEngine" + "PluginName": "DoNotReallyCare", + "PluginModule": "TestPythonEngine", + "PluginClass": "TestPythonEngine" } # ADIOS Engine @@ -52,8 +53,9 @@ def testCreateEngine(self): bpFileWriter.Close() -### -### Trigger the tests -### + +# +# Trigger the tests +# if __name__ == '__main__': Adios2PythonTestBase.main() diff --git a/testing/adios2/bindings/python/TestOpenInlinePythonEngineFromPython.py b/testing/adios2/bindings/python/TestOpenInlinePythonEngineFromPython.py index fd877ef721..aca1c0af31 100644 --- a/testing/adios2/bindings/python/TestOpenInlinePythonEngineFromPython.py +++ b/testing/adios2/bindings/python/TestOpenInlinePythonEngineFromPython.py @@ -1,27 +1,27 @@ from Adios2PythonTestBase import Adios2PythonTestBase -if Adios2PythonTestBase.isUsingMpi(): - from mpi4py import MPI - import numpy import adios2 +if Adios2PythonTestBase.isUsingMpi(): + from mpi4py import MPI + MODULE_LEVEL_VARIABLE = 0 -### -### Define a new engine type in python -### + +# +# Define a new engine type in python +# class TestInlinePythonEngine(adios2.Engine): def __init__(self, engineType, io, name, openMode): adios2.Engine.__init__(self, engineType, io, name, openMode) - print('Inside TestInlinePythonEngine python ctor, just called parent ctor') # Calls Engine::Init() if no Init() method is defined here self.Init() # def Init(self): # print('Inside TestPythonEngine Init()') - + def DoWrite(self, variable, values): global MODULE_LEVEL_VARIABLE MODULE_LEVEL_VARIABLE += 1 @@ -35,9 +35,10 @@ def Advance(self, timeoutSeconds): def Close(self, transportIndex): print('Inside TestInlinePythonEngine.Close') -### -### Create a testcase class with some tests -### + +# +# Create a testcase class with some tests +# class TestOpenInlinePythonEngineFromPython(Adios2PythonTestBase): def testCreateEngine(self): global MODULE_LEVEL_VARIABLE @@ -95,16 +96,22 @@ def testCreateEngineWithWrongName(self): bpIO.EngineType = "PythonEngine" bpIO.Parameters = { - "PluginName": "FirstPythonPlugin", + "PluginName": "FirstPythonPlugin", "PluginClass": "MissingEngine" } + bpFileWriter = None + # Make sure exception is raised due to wrong engine name with self.assertRaises(RuntimeError): bpFileWriter = bpIO.Open("npArray.bp", adios2.OpenMode.Write) -### -### Trigger the tests -### + if bpFileWriter: + bpFileWriter.Close() + + +# +# Trigger the tests +# if __name__ == '__main__': Adios2PythonTestBase.main() diff --git a/testing/adios2/bindings/python/TestOpenPythonEngineFromPython.py b/testing/adios2/bindings/python/TestOpenPythonEngineFromPython.py index 60a6093c6a..d389599422 100644 --- a/testing/adios2/bindings/python/TestOpenPythonEngineFromPython.py +++ b/testing/adios2/bindings/python/TestOpenPythonEngineFromPython.py @@ -1,16 +1,18 @@ from Adios2PythonTestBase import Adios2PythonTestBase -if Adios2PythonTestBase.isUsingMpi(): - from mpi4py import MPI - import numpy import adios2 +# Import this module so the IO factory can find the engine class it needs from TestPythonEngine import TestPythonEngine -### -### Create a testcase class with some tests -### +if Adios2PythonTestBase.isUsingMpi(): + from mpi4py import MPI + + +# +# Create a testcase class with some tests +# class TestOpenPythonEngineFromPython(Adios2PythonTestBase): def testCreateEngine(self): rank = 0 @@ -39,11 +41,11 @@ def testCreateEngine(self): # Engine derived class, spawned to start IO operations bpIO.EngineType = "PythonEngine" - # Since we import the TestPythonEngine class directly, we should not need - # to provide a "PluginModule" parameter + # Since we import the TestPythonEngine class directly, we should not + # need to provide a "PluginModule" parameter bpIO.Parameters = { - "PluginName": "DoNotReallyCare", - "PluginClass": "TestPythonEngine" + "PluginName": "DoNotReallyCare", + "PluginClass": "TestPythonEngine" } # ADIOS Engine @@ -52,8 +54,9 @@ def testCreateEngine(self): bpFileWriter.Close() -### -### Trigger the tests -### + +# +# Trigger the tests +# if __name__ == '__main__': Adios2PythonTestBase.main() diff --git a/testing/adios2/bindings/python/TestPythonEngine.py b/testing/adios2/bindings/python/TestPythonEngine.py index 0536d94539..7c25e29805 100644 --- a/testing/adios2/bindings/python/TestPythonEngine.py +++ b/testing/adios2/bindings/python/TestPythonEngine.py @@ -4,6 +4,7 @@ print('Inside TestPythonEngine module') + class TestPythonEngine(adios2.Engine): def __init__(self, engineType, io, name, openMode): adios2.Engine.__init__(self, engineType, io, name, openMode) @@ -14,7 +15,7 @@ def __init__(self, engineType, io, name, openMode): # def Init(self): # print('Inside TestPythonEngine Init()') - + def DoWrite(self, variable, values): print('Inside TestPythonEngine.DoWrite') print(variable) From f498d7463b45ebb74ac52d09d1c3148df56a9788 Mon Sep 17 00:00:00 2001 From: Scott Wittenburg Date: Wed, 11 Oct 2017 12:58:28 -0600 Subject: [PATCH 23/24] Couple of minor tweaks --- examples/hello/bpWriter/CMakeLists.txt | 11 +++++------ source/adios2/core/Engine.cpp | 2 +- source/adios2/engine/pyengine/PythonEngine.h | 2 +- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/examples/hello/bpWriter/CMakeLists.txt b/examples/hello/bpWriter/CMakeLists.txt index c1aed3a77d..d4b9ee1728 100644 --- a/examples/hello/bpWriter/CMakeLists.txt +++ b/examples/hello/bpWriter/CMakeLists.txt @@ -6,15 +6,15 @@ if(ADIOS2_HAVE_MPI) add_executable(hello_bpWriter helloBPWriter.cpp) target_link_libraries(hello_bpWriter MPI::MPI_C) - + add_executable(hello_bpWriter_c helloBPWriter.c) target_link_libraries(hello_bpWriter_c MPI::MPI_C) - + if(ADIOS2_HAVE_Fortran) add_executable(hello_bpWriter_f helloBPWriter.f90) target_link_libraries(hello_bpWriter_f MPI::MPI_Fortran adios2_f) endif() - + else() add_executable(hello_bpWriter helloBPWriter_nompi.cpp) add_executable(hello_bpWriter_c helloBPWriter_nompi.c) @@ -28,8 +28,7 @@ endif() target_link_libraries(hello_bpWriter adios2) target_link_libraries(hello_bpWriter_c adios2) - - if(ADIOS2_HAVE_Python) +if(ADIOS2_HAVE_Python) add_executable(examplePythonPlugin examplePythonPlugin.cpp) target_link_libraries(examplePythonPlugin PUBLIC adios2) - endif() \ No newline at end of file +endif() diff --git a/source/adios2/core/Engine.cpp b/source/adios2/core/Engine.cpp index bb70fd8369..9131c98a17 100644 --- a/source/adios2/core/Engine.cpp +++ b/source/adios2/core/Engine.cpp @@ -178,4 +178,4 @@ void Engine::ThrowUp(const std::string function) const ADIOS2_FOREACH_TYPE_1ARG(declare_template_instantiation) #undef declare_template_instantiation -} // end namespace adios +} // end namespace adios2 diff --git a/source/adios2/engine/pyengine/PythonEngine.h b/source/adios2/engine/pyengine/PythonEngine.h index 16d033324e..6350b711fd 100644 --- a/source/adios2/engine/pyengine/PythonEngine.h +++ b/source/adios2/engine/pyengine/PythonEngine.h @@ -76,6 +76,6 @@ class PythonEngine : public Engine std::unique_ptr m_Impl; }; -} // end namespace adios +} // end namespace adios2 #endif /* ADIOS2_ENGINE_PYENGINE_PYTHONENGINE_H_ */ From dd8179ae36665ba93b36f5ca15b943740bcccd3f Mon Sep 17 00:00:00 2001 From: Scott Wittenburg Date: Tue, 17 Oct 2017 09:31:39 -0600 Subject: [PATCH 24/24] Try to get python enabled on Windows (VS2017 only) --- bindings/python/CMakeLists.txt | 8 ++++--- cmake/ADIOSFunctions.cmake | 11 +++++++-- .../bindings/python/TestBPWriteTypes.py | 24 ++++++++++++------- 3 files changed, 30 insertions(+), 13 deletions(-) diff --git a/bindings/python/CMakeLists.txt b/bindings/python/CMakeLists.txt index ea68de426f..bb4245170f 100644 --- a/bindings/python/CMakeLists.txt +++ b/bindings/python/CMakeLists.txt @@ -13,17 +13,19 @@ if(ADIOS2_HAVE_MPI) target_link_libraries(adios2py PRIVATE PythonModule::mpi4py) endif() -string(REGEX REPLACE - "^${PYTHON_PREFIX}/[^/]*/python" "${CMAKE_INSTALL_LIBDIR}/python" - CMAKE_INSTALL_PYTHONDIR "${PYTHON_SITE_PACKAGES}" +set(PY_VERSION_STRING "${PYTHON_VERSION_MAJOR}.${PYTHON_VERSION_MINOR}.${PYTHON_VERSION_PATCH}") +set(CMAKE_INSTALL_PYTHONDIR + "${CMAKE_INSTALL_LIBDIR}/python${PY_VERSION_STRING}/site-packages" ) set(CMAKE_INSTALL_PYTHONDIR "${CMAKE_INSTALL_PYTHONDIR}" CACHE INTERNAL "" FORCE ) +message(" ******** adios2py output directory: ${ADIOS2_BINARY_DIR}/${CMAKE_INSTALL_PYTHONDIR}") set_target_properties(adios2py PROPERTIES OUTPUT_NAME adios2 LIBRARY_OUTPUT_DIRECTORY ${ADIOS2_BINARY_DIR}/${CMAKE_INSTALL_PYTHONDIR} RUNTIME_OUTPUT_DIRECTORY ${ADIOS2_BINARY_DIR}/${CMAKE_INSTALL_PYTHONDIR} + ARCHIVE_OUTPUT_DIRECTORY ${ADIOS2_BINARY_DIR}/${CMAKE_INSTALL_PYTHONDIR} ) install(TARGETS adios2py DESTINATION ${CMAKE_INSTALL_PYTHONDIR} diff --git a/cmake/ADIOSFunctions.cmake b/cmake/ADIOSFunctions.cmake index 9f8e27c810..ce9c701acc 100644 --- a/cmake/ADIOSFunctions.cmake +++ b/cmake/ADIOSFunctions.cmake @@ -40,19 +40,26 @@ function(python_add_test) COMMAND ${ARGS_EXEC_WRAPPER} ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/${ARGS_SCRIPT} ) - set(testsPythonPath "${ADIOS2_BINARY_DIR}/${CMAKE_INSTALL_PYTHONDIR}:$ENV{PYTHONPATH}") + set(testsPythonPath "${ADIOS2_BINARY_DIR}/${CMAKE_INSTALL_PYTHONDIR};$ENV{PYTHONPATH}") if(DEFINED ARGS_ADDTOPYPATH) - set(testsPythonPath "${ARGS_ADDTOPYPATH}:${testsPythonPath}") + set(testsPythonPath "${ARGS_ADDTOPYPATH};${testsPythonPath}") endif() if(ARGS_USE_MPI_FOR_PYTHON_TESTS) set(mpi_flag "ADIOS2_PYTHON_TESTS_USE_MPI=${ARGS_USE_MPI_FOR_PYTHON_TESTS}") endif() + if(WIN32) + set(path_var "PATH=${ADIOS2_BINARY_DIR}/bin;$ENV{PATH}") + else() + string(REPLACE ";" ":" testsPythonPath ${testsPythonPath}) + endif() + set_property(TEST ${ARGS_NAME} PROPERTY ENVIRONMENT "PYTHONPATH=${testsPythonPath}" ${mpi_flag} + ${path_var} ) endfunction() diff --git a/testing/adios2/bindings/python/TestBPWriteTypes.py b/testing/adios2/bindings/python/TestBPWriteTypes.py index 2053a5148e..d1fa56f2ee 100644 --- a/testing/adios2/bindings/python/TestBPWriteTypes.py +++ b/testing/adios2/bindings/python/TestBPWriteTypes.py @@ -44,9 +44,11 @@ def testWriteTypes(self): varI16 = bpIO.DefineVariable( "varI16", [size * data.I16.size], [rank * data.I16.size], [data.I16.size], True, data.I16.dtype) - varI32 = bpIO.DefineVariable( - "varI32", [size * data.I32.size], [rank * data.I32.size], - [data.I32.size], True, data.I32.dtype) + + # varI32 = bpIO.DefineVariable( + # "varI32", [size * data.I32.size], [rank * data.I32.size], + # [data.I32.size], True, data.I32.dtype) + # FIXME: Uncomment below when 64 bit ints work properly # varI64 = bpIO.DefineVariable( # "varI64", [], [], [data.I64.size], True, data.I64.dtype) @@ -57,9 +59,11 @@ def testWriteTypes(self): varU16 = bpIO.DefineVariable( "varUI16", [size * data.U16.size], [rank * data.U16.size], [data.U16.size], True, data.U16.dtype) - varU32 = bpIO.DefineVariable( - "varUI32", [size * data.U32.size], [rank * data.U32.size], - [data.U32.size], True, data.U32.dtype) + + # varU32 = bpIO.DefineVariable( + # "varUI32", [size * data.U32.size], [rank * data.U32.size], + # [data.U32.size], True, data.U32.dtype) + # FIXME: Uncomment below when 64 bit ints work properly # varU64 = bpIO.DefineVariable( # "varUI64", [], [], [data.U64.size], True, data.U64.dtype) @@ -77,13 +81,17 @@ def testWriteTypes(self): bpFileWriter.Write(varI8, data.I8) bpFileWriter.Write(varI16, data.I16) - bpFileWriter.Write(varI32, data.I32) + + # bpFileWriter.Write(varI32, data.I32) + # FIXME: Uncomment below when 64 bit ints work properly # bpFileWriter.Write(varI64, data.I64) bpFileWriter.Write(varU8, data.U8) bpFileWriter.Write(varU16, data.U16) - bpFileWriter.Write(varU32, data.U32) + + # bpFileWriter.Write(varU32, data.U32) + # FIXME: Uncomment below when 64 bit ints work properly # bpFileWriter.Write(varU64, data.U64)