Skip to content

Commit

Permalink
Merge pull request #4059 from vicentebolea/backports-from-master
Browse files Browse the repository at this point in the history 
backports from master
  • Loading branch information
@vicentebolea
vicentebolea authored Feb 29, 2024
2 parents 7914949 + 37bd41a commit 1f50ea2
Show file tree
Hide file tree
Showing 79 changed files with 1,618 additions and 1,365 deletions.
8 changes: 7 additions & 1 deletion .github/workflows/everything.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -339,7 +339,7 @@ jobs:

runs-on: ${{ matrix.image }}
env:
GH_YML_JOBNAME: ${{ matrix.os }}-${{ matrix.compiler }}-${{ matrix.parallel }}
GH_YML_JOBNAME: ${{ matrix.os }}-${{ matrix.compiler }}${{ matrix.shared == 'static' && '-static' || ''}}-${{ matrix.parallel }}
GH_YML_BASE_OS: Windows
GH_YML_MATRIX_OS: ${{ matrix.os }}
GH_YML_MATRIX_COMPILER: ${{ matrix.compiler }}
Expand All @@ -349,6 +349,7 @@ jobs:
fail-fast: false
matrix:
os: [win2019, win2022]
shared: [shared]
parallel: [serial, ompi]
include:
- os: win2019
Expand All @@ -357,6 +358,11 @@ jobs:
- os: win2022
image: windows-2022
compiler: vs2022
- os: win2022
image: windows-2022
shared: static
compiler: vs2022
parallel: serial

defaults:
run:
Expand Down
6 changes: 3 additions & 3 deletions CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -187,6 +187,9 @@ adios_option(Campaign "Enable support for Campaigns (requires SQLite3 and ZLIB
adios_option(AWSSDK "Enable support for S3 compatible storage using AWS SDK's S3 module" OFF)
adios_option(Derived_Variable "Enable support for derived variables" OFF)
adios_option(PIP "Enable support for pip packaging" OFF)

option(ADIOS2_Blosc2_PREFER_SHARED "Prefer shared Blosc2 libraries" ON)
mark_as_advanced(ADIOS2_Blosc2_PREFER_SHARED)
mark_as_advanced(ADIOS2_USE_PIP)
include(${PROJECT_SOURCE_DIR}/cmake/DetectOptions.cmake)

Expand Down Expand Up @@ -217,9 +220,6 @@ if(ADIOS2_HAVE_MPI)
add_definitions(-DOMPI_SKIP_MPICXX -DMPICH_SKIP_MPICXX)
endif()

cmake_dependent_option(ADIOS2_Blosc2_PREFER_SHARED "Prefer shared Blosc2 libraries"
ON "Blosc2_shlib_available" OFF)

#------------------------------------------------------------------------------#
# POSIX O_DIRECT is only working for Unix in adios for now
#------------------------------------------------------------------------------#
Expand Down
48 changes: 44 additions & 4 deletions bindings/Python/py11Engine.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,12 +70,44 @@ void Engine::Put(Variable variable, const pybind11::array &array, const Mode lau
else
{
throw std::invalid_argument("ERROR: for variable " + variable.Name() +
" numpy array type is not supported or "
" numpy array type " + variable.Type() +
" is not supported (found type " + ToString(type) +
") or "
"is not memory contiguous "
", in call to Put\n");
}
}

void Engine::Put(Variable variable, const std::vector<int64_t> &ints, const Mode launch)
{
helper::CheckForNullptr(m_Engine, "in call to Engine::Put list of ints");
helper::CheckForNullptr(variable.m_VariableBase,
"for variable, in call to Engine::Put list of ints");

m_Engine->Put(*dynamic_cast<core::Variable<int64_t> *>(variable.m_VariableBase),
reinterpret_cast<const int64_t *>(ints.data()), launch);
}

void Engine::Put(Variable variable, const std::vector<double> &floats, const Mode launch)
{
helper::CheckForNullptr(m_Engine, "in call to Engine::Put list of floats");
helper::CheckForNullptr(variable.m_VariableBase,
"for variable, in call to Engine::Put list of floats");

m_Engine->Put(*dynamic_cast<core::Variable<double> *>(variable.m_VariableBase),
reinterpret_cast<const double *>(floats.data()), launch);
}

void Engine::Put(Variable variable, const std::vector<std::complex<double>> &complexes,
const Mode launch)
{
helper::CheckForNullptr(m_Engine, "in call to Engine::Put list of complexes");
helper::CheckForNullptr(variable.m_VariableBase,
"for variable, in call to Engine::Put list of complexes");
m_Engine->Put(*dynamic_cast<core::Variable<std::complex<double>> *>(variable.m_VariableBase),
reinterpret_cast<const std::complex<double> *>(complexes.data()), launch);
}

void Engine::Put(Variable variable, const std::string &string)
{
helper::CheckForNullptr(m_Engine, "for engine, in call to Engine::Put string");
Expand Down Expand Up @@ -235,12 +267,18 @@ std::vector<std::map<std::string, std::string>> Engine::BlocksInfo(std::string &
const size_t step) const
{
std::vector<std::map<std::string, std::string>> rv;
auto &varMap = m_Engine->m_IO.GetVariables();
auto itVariable = varMap.find(var_name);
if (itVariable == varMap.end())
{
return rv;
}

// Grab the specified variable object and get its type string
adios2::DataType var_type = m_Engine->GetIO().InquireVariableType(var_name);

MinVarInfo *minBlocksInfo = nullptr;
auto itVariable = m_Engine->m_IO.GetVariables().find(var_name);

auto Variable = itVariable->second.get();
minBlocksInfo = m_Engine->MinBlocksInfo(*Variable, 0);
if (minBlocksInfo)
Expand All @@ -261,7 +299,8 @@ std::vector<std::map<std::string, std::string>> Engine::BlocksInfo(std::string &
{
start_ss << ",";
}
start_ss << info.Start[i];
start_ss << (minBlocksInfo->WasLocalValue ? reinterpret_cast<size_t>(info.Start)
: info.Start[i]);
}
}
info_map["Start"] = start_ss.str();
Expand All @@ -278,7 +317,8 @@ std::vector<std::map<std::string, std::string>> Engine::BlocksInfo(std::string &
{
count_ss << ",";
}
count_ss << info.Count[i];
count_ss << (minBlocksInfo->WasLocalValue ? reinterpret_cast<size_t>(info.Count)
: info.Count[i]);
}
}
info_map["Count"] = count_ss.str();
Expand Down
6 changes: 6 additions & 0 deletions bindings/Python/py11Engine.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,12 @@ class Engine
StepStatus BeginStep();

void Put(Variable variable, const pybind11::array &array, const Mode launch = Mode::Deferred);
void Put(Variable variable, const std::vector<int64_t> &ints,
const Mode launch = Mode::Deferred);
void Put(Variable variable, const std::vector<double> &doubles,
const Mode launch = Mode::Deferred);
void Put(Variable variable, const std::vector<std::complex<double>> &complexes,
const Mode launch = Mode::Deferred);
void Put(Variable variable, const std::string &string);
void PerformPuts();
void PerformDataWrite();
Expand Down
121 changes: 116 additions & 5 deletions bindings/Python/py11IO.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,51 @@ Variable IO::DefineVariable(const std::string &name, const pybind11::array &arra
return Variable(variable);
}

Variable IO::DefineVariable(const std::string &name, const pybind11::object &value,
const Dims &shape, const Dims &start, const Dims &count,
const bool isConstantDims)
{
helper::CheckForNullptr(m_IO, "for variable " + name + ", in call to IO::DefineVariable");
core::VariableBase *variable = nullptr;
const auto t = value.get_type();
const auto ts = pybind11::str(t);
const auto tss = pybind11::cast<std::string>(ts);
if (pybind11::isinstance<pybind11::str>(value))
{
variable = &m_IO->DefineVariable<std::string>(name);
}
else if (pybind11::isinstance<pybind11::int_>(value))
{
variable = &m_IO->DefineVariable<int64_t>(name, shape, start, count, isConstantDims);
}
else if (pybind11::isinstance<pybind11::float_>(value))
{
variable = &m_IO->DefineVariable<double>(name, shape, start, count, isConstantDims);
}
else if (tss == "<class 'complex'>")
{
variable =
&m_IO->DefineVariable<std::complex<double>>(name, shape, start, count, isConstantDims);
}
else if (tss == "<class 'numpy.complex64'>")
{
variable =
&m_IO->DefineVariable<std::complex<float>>(name, shape, start, count, isConstantDims);
}
else if (tss == "<class 'numpy.complex128'>")
{
variable =
&m_IO->DefineVariable<std::complex<double>>(name, shape, start, count, isConstantDims);
}
else
{
throw std::invalid_argument("ERROR: variable " + name +
" can't be defined with an object with type " + tss +
", in call to DefineVariable\n");
}
return Variable(variable);
}

Variable IO::InquireVariable(const std::string &name)
{
helper::CheckForNullptr(m_IO, "for variable " + name + ", in call to IO::InquireVariable");
Expand All @@ -126,7 +171,6 @@ Attribute IO::DefineAttribute(const std::string &name, const pybind11::array &ar
const std::string &variableName, const std::string separator)
{
helper::CheckForNullptr(m_IO, "for attribute " + name + ", in call to IO::DefineAttribute");

core::AttributeBase *attribute = nullptr;

if (false)
Expand Down Expand Up @@ -156,7 +200,6 @@ Attribute IO::DefineAttribute(const std::string &name, const std::string &string
const std::string &variableName, const std::string separator)
{
helper::CheckForNullptr(m_IO, "for attribute " + name + ", in call to IO::DefineAttribute");

return Attribute(
&m_IO->DefineAttribute<std::string>(name, stringValue, variableName, separator));
}
Expand All @@ -165,11 +208,78 @@ Attribute IO::DefineAttribute(const std::string &name, const std::vector<std::st
const std::string &variableName, const std::string separator)
{
helper::CheckForNullptr(m_IO, "for attribute " + name + ", in call to IO::DefineAttribute");

return Attribute(&m_IO->DefineAttribute<std::string>(name, strings.data(), strings.size(),
variableName, separator));
}

Attribute IO::DefineAttribute(const std::string &name, const std::vector<int> &ints,
const std::string &variableName, const std::string separator)
{
helper::CheckForNullptr(m_IO, "for attribute " + name + ", in call to IO::DefineAttribute");
return Attribute(
&m_IO->DefineAttribute<int>(name, ints.data(), ints.size(), variableName, separator));
}

Attribute IO::DefineAttribute(const std::string &name, const std::vector<double> &doubles,
const std::string &variableName, const std::string separator)
{
helper::CheckForNullptr(m_IO, "for attribute " + name + ", in call to IO::DefineAttribute");
return Attribute(&m_IO->DefineAttribute<double>(name, doubles.data(), doubles.size(),
variableName, separator));
}

Attribute IO::DefineAttribute(const std::string &name,
const std::vector<std::complex<double>> &complexdoubles,
const std::string &variableName, const std::string separator)
{
helper::CheckForNullptr(m_IO, "for attribute " + name + ", in call to IO::DefineAttribute");
return Attribute(&m_IO->DefineAttribute<std::complex<double>>(
name, complexdoubles.data(), complexdoubles.size(), variableName, separator));
}

Attribute IO::DefineAttribute(const std::string &name, const pybind11::object &value,
const std::string &variableName, const std::string separator)
{
helper::CheckForNullptr(m_IO, "for attribute " + name + ", in call to IO::DefineAttribute");

core::AttributeBase *attribute = nullptr;
const auto t = value.get_type();
const auto ts = pybind11::str(t);
const auto tss = pybind11::cast<std::string>(ts);
if (pybind11::isinstance<pybind11::int_>(value))
{
auto v = pybind11::cast<const int64_t>(value);
attribute = &m_IO->DefineAttribute(name, v, variableName, separator);
}
else if (pybind11::isinstance<pybind11::float_>(value))
{
auto v = pybind11::cast<const double>(value);
attribute = &m_IO->DefineAttribute(name, v, variableName, separator);
}
else if (tss == "<class 'complex'>")
{
auto v = pybind11::cast<const std::complex<double>>(value);
attribute = &m_IO->DefineAttribute(name, v, variableName, separator);
}
else if (tss == "<class 'numpy.complex64'>")
{
auto v = pybind11::cast<const std::complex<float>>(value);
attribute = &m_IO->DefineAttribute(name, v, variableName, separator);
}
else if (tss == "<class 'numpy.complex128'>")
{
auto v = pybind11::cast<const std::complex<double>>(value);
attribute = &m_IO->DefineAttribute(name, v, variableName, separator);
}
else
{
throw std::invalid_argument("ERROR: attribute " + name +
" can't be defined with an object with type " + tss +
", in call to DefineAttribute\n");
}
return Attribute(attribute);
}

Attribute IO::InquireAttribute(const std::string &name, const std::string &variableName,
const std::string separator)
{
Expand Down Expand Up @@ -234,10 +344,11 @@ std::map<std::string, Params> IO::AvailableVariables()
return m_IO->GetAvailableVariables();
}

std::map<std::string, Params> IO::AvailableAttributes()
std::map<std::string, Params> IO::AvailableAttributes(const std::string &varname,
const std::string &separator)
{
helper::CheckForNullptr(m_IO, "in call to IO::AvailableAttributes");
return m_IO->GetAvailableAttributes();
return m_IO->GetAvailableAttributes(varname, separator);
}

std::string IO::VariableType(const std::string &name) const
Expand Down
24 changes: 23 additions & 1 deletion bindings/Python/py11IO.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@

#include <pybind11/numpy.h>

#include <complex>
#include <string>

#include "py11Attribute.h"
Expand Down Expand Up @@ -57,6 +58,10 @@ class IO
const Dims &shape, const Dims &start, const Dims &count,
const bool isConstantDims);

Variable DefineVariable(const std::string &name, const pybind11::object &value,
const Dims &shape, const Dims &start, const Dims &count,
const bool isConstantDims);

Variable InquireVariable(const std::string &name);

Attribute DefineAttribute(const std::string &name, const pybind11::array &array,
Expand All @@ -71,6 +76,22 @@ class IO
const std::string &variableName = "",
const std::string separator = "/");

Attribute DefineAttribute(const std::string &name, const std::vector<int> &ints,
const std::string &variableName = "",
const std::string separator = "/");

Attribute DefineAttribute(const std::string &name, const std::vector<double> &doubles,
const std::string &variableName = "",
const std::string separator = "/");

Attribute DefineAttribute(const std::string &name,
const std::vector<std::complex<double>> &complexdoubles,
const std::string &variableName = "",
const std::string separator = "/");

Attribute DefineAttribute(const std::string &name, const pybind11::object &value,
const std::string &variableName, const std::string separator);

Attribute InquireAttribute(const std::string &name, const std::string &variableName = "",
const std::string separator = "/");

Expand All @@ -92,7 +113,8 @@ class IO

std::map<std::string, Params> AvailableVariables();

std::map<std::string, Params> AvailableAttributes();
std::map<std::string, Params> AvailableAttributes(const std::string &varname = "",
const std::string &separator = "/");

std::string VariableType(const std::string &name) const;

Expand Down
Loading

0 comments on commit 1f50ea2

Please sign in to comment.