Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

BP5 JoinedArray support #3488

Merged
merged 3 commits into from
Feb 20, 2023
Merged

BP5 JoinedArray support #3488

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
7b91b42
BP5 JoinedArray support
eisenhauer Feb 20, 2023
e2cf8e7
test
eisenhauer Feb 20, 2023
a4d4fa6
Remove diagnostics
eisenhauer Feb 20, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions source/adios2/engine/bp5/BP5Writer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1756,6 +1756,11 @@ void BP5Writer::PutCommon(VariableBase &variable, const void *values, bool sync)
{
Count = variable.m_Count.data();
}
else if (variable.m_ShapeID == ShapeID::JoinedArray)
{
Count = variable.m_Count.data();
Shape = variable.m_Shape.data();
}

size_t ObjSize;
if (variable.m_Type == DataType::Struct)
Expand Down
6 changes: 6 additions & 0 deletions source/adios2/engine/bp5/BP5Writer.tcc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,12 @@ void BP5Writer::PutCommonSpan(Variable<T> &variable,
Start = variable.m_Start.data();
Count = variable.m_Count.data();
}
else if (variable.m_ShapeID == ShapeID::JoinedArray)
{
Shape = variable.m_Shape.data();
DimCount = variable.m_Count.size();
Count = variable.m_Count.data();
}
else if (variable.m_ShapeID == ShapeID::LocalArray)
{
DimCount = variable.m_Count.size();
Expand Down
153 changes: 146 additions & 7 deletions source/adios2/toolkit/format/bp5/BP5Deserializer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -592,9 +592,16 @@ void BP5Deserializer::SetupForStep(size_t Step, size_t WriterCount)
{
m_Engine->m_IO.RemoveVariable(RecPair.second->VarName);
RecPair.second->Variable = NULL;
if (RecPair.second->OrigShapeID == ShapeID::JoinedArray)
{
auto VarRec = RecPair.second;
VarRec->JoinedDimen = SIZE_MAX;
VarRec->LastJoinedOffset = NULL;
VarRec->LastJoinedShape = NULL;
}
}
m_CurrentWriterCohortSize = WriterCount;
}
m_CurrentWriterCohortSize = WriterCount;
}

size_t BP5Deserializer::WriterCohortSize(size_t Step) const
Expand Down Expand Up @@ -693,6 +700,15 @@ void BP5Deserializer::InstallMetaData(void *MetadataBlock, size_t BlockLen,
MetadataBaseArray[Step] = m_MetadataBaseAddrs;
m_FreeableMBA = nullptr;
}

JoinedDimArray.resize(Step + 1);
if (JoinedDimArray[Step] == nullptr)
{
m_JoinedDimenOffsetArrays = new std::vector<void *>();
m_JoinedDimenOffsetArrays->resize(writerCohortSize);
JoinedDimArray[Step] = m_JoinedDimenOffsetArrays;
m_FreeableMBA = nullptr;
}
}
else
{
Expand All @@ -705,9 +721,56 @@ void BP5Deserializer::InstallMetaData(void *MetadataBlock, size_t BlockLen,
{
m_MetadataBaseAddrs->resize(writerCohortSize);
}

if (!m_JoinedDimenOffsetArrays)
{
m_JoinedDimenOffsetArrays = new std::vector<void *>();
m_FreeableMBA = m_JoinedDimenOffsetArrays;
}
if (writerCohortSize > m_JoinedDimenOffsetArrays->size())
{
m_JoinedDimenOffsetArrays->resize(writerCohortSize);
}
}
(*m_MetadataBaseAddrs)[WriterRank] = BaseData;

size_t JoinedDimenTotal = 0;
for (int i = 0; i < Control->ControlCount; i++)
{
size_t FieldOffset = ControlFields[i].FieldOffset;
BP5VarRec *VarRec = ControlFields[i].VarRec;
void *field_data = (char *)BaseData + FieldOffset;
if (!BP5BitfieldTest((BP5MetadataInfoStruct *)BaseData, i))
{
continue;
}
if (ControlFields[i].OrigShapeID == ShapeID::JoinedArray)
{
MetaArrayRec *meta_base = (MetaArrayRec *)field_data;
JoinedDimenTotal += meta_base->DBCount;
if (VarRec->JoinedDimen == SIZE_MAX)
{
for (size_t i = 0; i < meta_base->Dims; i++)
{
if (meta_base->Shape[i] == JoinedDim)
{
VarRec->JoinedDimen = i;
}
}
}
}
}

// Allocate memory to hold new offset values for Joined Arrays
size_t CurJoinedDimenOffset = 0;
size_t *JoinedDimenOffsetArray = NULL;
if (JoinedDimenTotal)
JoinedDimenOffsetArray = (size_t *)malloc(
JoinedDimenTotal * writerCohortSize * sizeof(size_t));

// store this away so it can be deallocated later
(*m_JoinedDimenOffsetArrays)[WriterRank] = JoinedDimenOffsetArray;

for (int i = 0; i < Control->ControlCount; i++)
{
size_t FieldOffset = ControlFields[i].FieldOffset;
Expand Down Expand Up @@ -735,7 +798,8 @@ void BP5Deserializer::InstallMetaData(void *MetadataBlock, size_t BlockLen,
}
}
if ((ControlFields[i].OrigShapeID == ShapeID::GlobalArray) ||
(ControlFields[i].OrigShapeID == ShapeID::LocalArray))
(ControlFields[i].OrigShapeID == ShapeID::LocalArray) ||
(ControlFields[i].OrigShapeID == ShapeID::JoinedArray))
{
MetaArrayRec *meta_base = (MetaArrayRec *)field_data;
size_t BlockCount =
Expand All @@ -747,18 +811,68 @@ void BP5Deserializer::InstallMetaData(void *MetadataBlock, size_t BlockLen,
* switcheroo */
ReverseDimensions(meta_base->Count, meta_base->Dims,
BlockCount);
if (ControlFields[i].OrigShapeID == ShapeID::GlobalArray)
if ((ControlFields[i].OrigShapeID == ShapeID::GlobalArray) ||
(ControlFields[i].OrigShapeID == ShapeID::JoinedArray))
{
ReverseDimensions(meta_base->Shape, meta_base->Dims, 1);
ReverseDimensions(meta_base->Offsets, meta_base->Dims,
BlockCount);
if (ControlFields[i].OrigShapeID == ShapeID::GlobalArray)
{
ReverseDimensions(meta_base->Offsets, meta_base->Dims,
BlockCount);
}
}
}
if ((WriterRank == 0) || (VarRec->GlobalDims == NULL))
{
// use the shape from rank 0 (or first non-NULL)
VarRec->GlobalDims = meta_base->Shape;
}
if (ControlFields[i].OrigShapeID == ShapeID::JoinedArray)
{
// setup Offsets
meta_base->Offsets =
&JoinedDimenOffsetArray[CurJoinedDimenOffset];
CurJoinedDimenOffset += meta_base->DBCount;

for (size_t b = 0; b < BlockCount; b++)
{
size_t PreviousJoinedOffset = 0;
if (VarRec->LastJoinedShape != NULL)
{
// Offset of this new block is the prior total size,
// stored in Shape
PreviousJoinedOffset =
VarRec->LastJoinedShape[VarRec->JoinedDimen];
}
else
{
// We're going to track the accumulated Joined Shape in
// whatever Shape metadata entry we selected for
// GlobalDims (might be rank 0, might be other)
VarRec->LastJoinedShape = VarRec->GlobalDims;
// overwrite the JoinedDimen value in that entry
VarRec->LastJoinedShape[VarRec->JoinedDimen] = 0;
}
VarRec->LastJoinedShape[VarRec->JoinedDimen] +=
meta_base->Count[(b * meta_base->Dims) +
VarRec->JoinedDimen];
for (size_t i = 0; i < meta_base->Dims; i++)
{
if (i == VarRec->JoinedDimen)
{
meta_base->Offsets[(b * meta_base->Dims) + i] =
PreviousJoinedOffset;
}
else
{
meta_base->Offsets[(b * meta_base->Dims) + i] = 0;
}
}
VarRec->LastJoinedOffset =
&meta_base->Offsets[(b * meta_base->Dims)];
}
}

if (!VarRec->Variable)
{
VarRec->Variable = ArrayVarSetup(
Expand Down Expand Up @@ -834,6 +948,27 @@ void BP5Deserializer::InstallMetaData(void *MetadataBlock, size_t BlockLen,
VarRec->LastTSAdded = Step;
}
}
if (WriterRank == (m_CurrentWriterCohortSize - 1))
{
// do step finalization procedures
if (!m_RandomAccessMode)
{
for (auto RecPair : VarByKey)
{
if (RecPair.second->Variable != NULL)
if (RecPair.second->OrigShapeID == ShapeID::JoinedArray)
{
auto VarRec = RecPair.second;
VariableBase *Var =
static_cast<VariableBase *>(VarRec->Variable);
for (size_t i = 0; i < VarRec->DimCount; i++)
{
Var->m_Shape[i] = VarRec->GlobalDims[i];
}
}
}
}
}
}

void BP5Deserializer::InstallAttributeData(void *AttributeBlock,
Expand Down Expand Up @@ -1249,7 +1384,8 @@ bool BP5Deserializer::QueueGetSingle(core::VariableBase &variable,
}
MemorySpace MemSpace = variable.GetMemorySpace(DestData);
if ((variable.m_SelectionType == adios2::SelectionType::BoundingBox) &&
(variable.m_ShapeID == ShapeID::GlobalArray))
((variable.m_ShapeID == ShapeID::GlobalArray) ||
(variable.m_ShapeID == ShapeID::JoinedArray)))
{
BP5ArrayRequest Req;
Req.VarRec = VarRec;
Expand Down Expand Up @@ -2112,7 +2248,8 @@ bool BP5Deserializer::VarShape(const VariableBase &Var, const size_t RelStep,
Dims &Shape) const
{
BP5VarRec *VarRec = LookupVarByKey((void *)&Var);
if (VarRec->OrigShapeID != ShapeID::GlobalArray)
if (!((VarRec->OrigShapeID == ShapeID::GlobalArray) ||
(VarRec->OrigShapeID == ShapeID::JoinedArray)))
{
return false;
}
Expand Down Expand Up @@ -2151,6 +2288,7 @@ bool BP5Deserializer::VariableMinMax(const VariableBase &Var, const size_t Step,
{
BP5VarRec *VarRec = LookupVarByKey((void *)&Var);
if ((VarRec->OrigShapeID == ShapeID::LocalArray) ||
(VarRec->OrigShapeID == ShapeID::JoinedArray) ||
(VarRec->OrigShapeID == ShapeID::GlobalArray))
{
if (VarRec->MinMaxOffset == SIZE_MAX)
Expand All @@ -2174,6 +2312,7 @@ bool BP5Deserializer::VariableMinMax(const VariableBase &Var, const size_t Step,
for (size_t RelStep = StartStep; RelStep < StopStep; RelStep++)
{
if ((VarRec->OrigShapeID == ShapeID::LocalArray) ||
(VarRec->OrigShapeID == ShapeID::JoinedArray) ||
(VarRec->OrigShapeID == ShapeID::GlobalArray))
{
for (size_t WriterRank = 0; WriterRank < writerCohortSize;
Expand Down
8 changes: 8 additions & 0 deletions source/adios2/toolkit/format/bp5/BP5Deserializer.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,9 @@ class BP5Deserializer : virtual public BP5Base
void *Variable = NULL;
char *VarName = NULL;
size_t DimCount = 0;
size_t JoinedDimen = SIZE_MAX;
size_t *LastJoinedOffset = NULL;
size_t *LastJoinedShape = NULL;
ShapeID OrigShapeID;
core::StructDefinition *Def = nullptr;
core::StructDefinition *ReaderDef = nullptr;
Expand Down Expand Up @@ -170,12 +173,17 @@ class BP5Deserializer : virtual public BP5Base
nullptr; // may be a pointer into MetadataBaseArray or m_FreeableMBA
std::vector<void *> *m_FreeableMBA = nullptr;

std::vector<void *> *m_JoinedDimenOffsetArrays = nullptr;

// for random access mode, for each timestep, for each writerrank, what
// metameta info applies to the metadata
std::vector<std::vector<ControlInfo *>> m_ControlArray;
// for random access mode, for each timestep, for each writerrank, base
// address of the metadata
std::vector<std::vector<void *> *> MetadataBaseArray;
// for random access mode, for each timestep, for each writerrank, base
// address of the joined dim arrays
std::vector<std::vector<void *> *> JoinedDimArray;

ControlInfo *ControlBlocks = nullptr;
ControlInfo *GetPriorControl(FMFormat Format);
Expand Down
2 changes: 0 additions & 2 deletions testing/adios2/engine/staging-common/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -235,8 +235,6 @@ endif()
# BP5 tests
if(ADIOS2_HAVE_BP5)
set (BP5_TESTS ${ALL_SIMPLE_TESTS} ${SPECIAL_TESTS})
# BP5 doesn't do Joined
list (FILTER BP5_TESTS EXCLUDE REGEX "Joined.*")
# Delayed reader not worth testing on file engines
list (FILTER BP5_TESTS EXCLUDE REGEX "DelayedReader")
# Discard not a feature of BP5
Expand Down
1 change: 1 addition & 0 deletions testing/adios2/engine/staging-common/TestReadJoined.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,7 @@ TEST_F(CommonReadTest, ADIOS2CommonRead1D8)

int Nrows;
reader.Get(rows_var, Nrows);
std::cout << "Reader expecting " << Nrows << std::endl;
EXPECT_EQ(var.Shape()[0], Nrows);
EXPECT_EQ(var.Shape()[1], Ncols);

Expand Down
2 changes: 2 additions & 0 deletions testing/adios2/engine/staging-common/TestWriteJoined.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,8 @@ TEST_F(CommonWriteTest, ADIOS2CommonWrite)
}
if (mpiRank == 0)
{
std::cout << "Writer Generating " << nTotalRows[step] << " in total"
<< std::endl;
writer.Put(rows_var, nTotalRows[step]);
}
for (int block = 0; block < nblocks; ++block)
Expand Down