Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

make tagging routines EB_aware #2962

Merged
merged 5 commits into from
Sep 25, 2022
Merged

make tagging routines EB_aware #2962

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
290f482
make tagging routines EB_aware
asalmgren Sep 25, 2022
1b55caf
make this work if AMREX_USE_EB defined but the MultiFab not build wit…
asalmgren Sep 25, 2022
9aedeeb
fix compiler errors
asalmgren Sep 25, 2022
9dc3d1d
fix compiler complaints
asalmgren Sep 25, 2022
23ea81c
remove unused variable
asalmgren Sep 25, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
220 changes: 180 additions & 40 deletions Src/AmrCore/AMReX_ErrorList.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -300,78 +300,218 @@ AMRErrorTag::operator() (TagBoxArray& tba,

if (m_test == GRAD)
{
ParallelFor(tba, [=] AMREX_GPU_DEVICE (int bi, int i, int j, int k) noexcept
#ifdef AMREX_USE_EB
if (mf->hasEBFabFactory()) {
auto const& ebfact =
dynamic_cast<amrex::EBFArrayBoxFactory const&>(mf->Factory());
auto const& flags = ebfact.getMultiEBCellFlagFab().arrays();
ParallelFor(tba, [=] AMREX_GPU_DEVICE (int bi, int i, int j, int k) noexcept
{
auto const& dat = datma[bi];
auto const& flag = flags[bi];

Real ax = 0.; Real ay = 0.;
if (flag(i,j,k).isConnected(1,0,0)) {
ax = amrex::max(ax,amrex::Math::abs(dat(i+1,j,k) - dat(i,j,k)));
}
if (flag(i,j,k).isConnected(-1,0,0)) {
ax = amrex::max(ax,amrex::Math::abs(dat(i,j,k) - dat(i-1,j,k)));
}
if (flag(i,j,k).isConnected(0,1,0)) {
ay = amrex::max(ay,amrex::Math::abs(dat(i,j+1,k) - dat(i,j,k)));
}
if (flag(i,j,k).isConnected(0,-1,0)) {
ay = amrex::max(ay,amrex::Math::abs(dat(i,j,k) - dat(i,j-1,k)));
}
#if AMREX_SPACEDIM > 2
Real az = 0.;
if (flag(i,j,k).isConnected(0,0,1)) {
az = amrex::max(az,amrex::Math::abs(dat(i,j,k+1) - dat(i,j,k)));
}
if (flag(i,j,k).isConnected(0,0,-1)) {
az = amrex::max(az,amrex::Math::abs(dat(i,j,k) - dat(i,j,k-1)));
}
#endif
if (amrex::max(AMREX_D_DECL(ax,ay,az)) >= threshold) {
tagma[bi](i,j,k) = tag_update;
}
});
} else
#endif
{
auto const& dat = datma[bi];
auto ax = amrex::Math::abs(dat(i+1,j,k) - dat(i,j,k));
ax = amrex::max(ax,amrex::Math::abs(dat(i,j,k) - dat(i-1,j,k)));
ParallelFor(tba, [=] AMREX_GPU_DEVICE (int bi, int i, int j, int k) noexcept
{
auto const& dat = datma[bi];

Real ax = 0.;
ax = amrex::Math::abs(dat(i+1,j,k) - dat(i,j,k));
ax = amrex::max(ax,amrex::Math::abs(dat(i,j,k) - dat(i-1,j,k)));
#if AMREX_SPACEDIM == 1
if (ax >= threshold) { tagma[bi](i,j,k) = tag_update;}
if (ax >= threshold) { tagma[bi](i,j,k) = tag_update;}
#else
auto ay = amrex::Math::abs(dat(i,j+1,k) - dat(i,j,k));
ay = amrex::max(ay,amrex::Math::abs(dat(i,j,k) - dat(i,j-1,k)));
Real ay = 0.;
ay = amrex::Math::abs(dat(i,j+1,k) - dat(i,j,k));
ay = amrex::max(ay,amrex::Math::abs(dat(i,j,k) - dat(i,j-1,k)));
#if AMREX_SPACEDIM > 2
auto az = amrex::Math::abs(dat(i,j,k+1) - dat(i,j,k));
az = amrex::max(az,amrex::Math::abs(dat(i,j,k) - dat(i,j,k-1)));
#endif
if (amrex::max(AMREX_D_DECL(ax,ay,az)) >= threshold) {
tagma[bi](i,j,k) = tag_update;
}
#endif
});
Real az = 0.;
az = amrex::Math::abs(dat(i,j,k+1) - dat(i,j,k));
az = amrex::max(az,amrex::Math::abs(dat(i,j,k) - dat(i,j,k-1)));
#endif // DIM > 2
if (amrex::max(AMREX_D_DECL(ax,ay,az)) >= threshold) {
tagma[bi](i,j,k) = tag_update;
}
#endif // DIM > 1
});
}
}
else if (m_test == RELGRAD)
{
ParallelFor(tba, [=] AMREX_GPU_DEVICE (int bi, int i, int j, int k) noexcept
#ifdef AMREX_USE_EB
if (mf->hasEBFabFactory()) {
auto const& ebfact =
dynamic_cast<amrex::EBFArrayBoxFactory const&>(mf->Factory());
auto const& flags = ebfact.getMultiEBCellFlagFab().arrays();
ParallelFor(tba, [=] AMREX_GPU_DEVICE (int bi, int i, int j, int k) noexcept
{
auto const& dat = datma[bi];
auto const& flag = flags[bi];

Real ax = 0.; Real ay = 0.;

if (flag(i,j,k).isConnected(1,0,0)) {
ax = amrex::max(ax,amrex::Math::abs(dat(i+1,j,k) - dat(i,j,k)));
}
if (flag(i,j,k).isConnected(-1,0,0)) {
ax = amrex::max(ax,amrex::Math::abs(dat(i,j,k) - dat(i-1,j,k)));
}
if (flag(i,j,k).isConnected(0,1,0)) {
ay = amrex::max(ay,amrex::Math::abs(dat(i,j+1,k) - dat(i,j,k)));
}
if (flag(i,j,k).isConnected(0,-1,0)) {
ay = amrex::max(ay,amrex::Math::abs(dat(i,j,k) - dat(i,j-1,k)));
}
#if AMREX_SPACEDIM > 2
Real az = 0.;
if (flag(i,j,k).isConnected(0,0,1)) {
az = amrex::max(az,amrex::Math::abs(dat(i,j,k+1) - dat(i,j,k)));
}
if (flag(i,j,k).isConnected(0,0,-1)) {
az = amrex::max(az,amrex::Math::abs(dat(i,j,k) - dat(i,j,k-1)));
}
#endif // DIM > 2
if (amrex::max(AMREX_D_DECL(ax,ay,az))
>= threshold * amrex::Math::abs(dat(i,j,k))) {
tagma[bi](i,j,k) = tag_update;
}
});
} else
#endif
{
auto const& dat = datma[bi];
auto ax = amrex::Math::abs(dat(i+1,j,k) - dat(i,j,k));
ax = amrex::max(ax,amrex::Math::abs(dat(i,j,k) - dat(i-1,j,k)));
ParallelFor(tba, [=] AMREX_GPU_DEVICE (int bi, int i, int j, int k) noexcept
{
auto const& dat = datma[bi];

Real ax = amrex::Math::abs(dat(i+1,j,k) - dat(i,j,k));
ax = amrex::max(ax,amrex::Math::abs(dat(i,j,k) - dat(i-1,j,k)));
#if AMREX_SPACEDIM == 1
if (ax >= threshold * amrex::Math::abs(dat(i,j,k))) { tagma[bi](i,j,k) = tag_update;}
if (ax >= threshold * amrex::Math::abs(dat(i,j,k))) { tagma[bi](i,j,k) = tag_update;}
#else
auto ay = amrex::Math::abs(dat(i,j+1,k) - dat(i,j,k));
ay = amrex::max(ay,amrex::Math::abs(dat(i,j,k) - dat(i,j-1,k)));
Real ay = amrex::Math::abs(dat(i,j+1,k) - dat(i,j,k));
ay = amrex::max(ay,amrex::Math::abs(dat(i,j,k) - dat(i,j-1,k)));
#if AMREX_SPACEDIM > 2
auto az = amrex::Math::abs(dat(i,j,k+1) - dat(i,j,k));
az = amrex::max(az,amrex::Math::abs(dat(i,j,k) - dat(i,j,k-1)));
#endif
if (amrex::max(AMREX_D_DECL(ax,ay,az))
>= threshold * amrex::Math::abs(dat(i,j,k))) {
tagma[bi](i,j,k) = tag_update;
}
#endif
});
Real az = amrex::Math::abs(dat(i,j,k+1) - dat(i,j,k));
az = amrex::max(az,amrex::Math::abs(dat(i,j,k) - dat(i,j,k-1)));
#endif // DIM > 2
if (amrex::max(AMREX_D_DECL(ax,ay,az))
>= threshold * amrex::Math::abs(dat(i,j,k))) {
tagma[bi](i,j,k) = tag_update;
}
#endif // DIM > 1
});
}
}
else if (m_test == LESS)
{
ParallelFor(tba, [=] AMREX_GPU_DEVICE (int bi, int i, int j, int k) noexcept
#ifdef AMREX_USE_EB
if (mf->hasEBFabFactory()) {
auto const& ebfact =
dynamic_cast<amrex::EBFArrayBoxFactory const&>(mf->Factory());
auto const& flags = ebfact.getMultiEBCellFlagFab().arrays();
ParallelFor(tba, [=] AMREX_GPU_DEVICE (int bi, int i, int j, int k) noexcept
{
Real vol = volume_weighting ? Geometry::Volume(IntVect{AMREX_D_DECL(i,j,k)}, geomdata) : 1.0_rt;
auto const& flag = flags[bi];
if (!flag(i,j,k).isCovered()) {
if (datma[bi](i,j,k) * vol <= threshold) {
tagma[bi](i,j,k) = tag_update;
}
}
});
} else
#endif
{
ParallelFor(tba, [=] AMREX_GPU_DEVICE (int bi, int i, int j, int k) noexcept {
Real vol = volume_weighting ? Geometry::Volume(IntVect{AMREX_D_DECL(i,j,k)}, geomdata) : 1.0_rt;
if (datma[bi](i,j,k) * vol <= threshold) {
tagma[bi](i,j,k) = tag_update;
}
});
}
}
else if (m_test == GREATER)
{
#ifdef AMREX_USE_EB
if (mf->hasEBFabFactory()) {
auto const& ebfact =
dynamic_cast<amrex::EBFArrayBoxFactory const&>(mf->Factory());
auto const& flags = ebfact.getMultiEBCellFlagFab().arrays();
ParallelFor(tba, [=] AMREX_GPU_DEVICE (int bi, int i, int j, int k) noexcept
{
Real vol = volume_weighting ? Geometry::Volume(IntVect{AMREX_D_DECL(i,j,k)}, geomdata) : 1.0_rt;
auto const& flag = flags[bi];
if (!flag(i,j,k).isCovered()) {
if (datma[bi](i,j,k) * vol >= threshold) {
tagma[bi](i,j,k) = tag_update;
}
}
});
} else
#endif
ParallelFor(tba, [=] AMREX_GPU_DEVICE (int bi, int i, int j, int k) noexcept
{
Real vol = volume_weighting ? Geometry::Volume(IntVect{AMREX_D_DECL(i,j,k)}, geomdata) : 1.0_rt;
if (datma[bi](i,j,k) * vol >= threshold) {
tagma[bi](i,j,k) = tag_update;
}
if (datma[bi](i,j,k) * vol >= threshold) {
tagma[bi](i,j,k) = tag_update;
}
});
}
else if (m_test == VORT)
{
const Real fac = threshold * Real(std::pow(2,level));
ParallelFor(tba, [=] AMREX_GPU_DEVICE (int bi, int i, int j, int k) noexcept
#ifdef AMREX_USE_EB
if (mf->hasEBFabFactory()) {
auto const& ebfact =
dynamic_cast<amrex::EBFArrayBoxFactory const&>(mf->Factory());
auto const& flags = ebfact.getMultiEBCellFlagFab().arrays();
ParallelFor(tba, [=] AMREX_GPU_DEVICE (int bi, int i, int j, int k) noexcept
{
auto const& flag = flags[bi];
if (!flag(i,j,k).isCovered()) {
if (datma[bi](i,j,k) >= fac) {
tagma[bi](i,j,k) = tag_update;
}
}
});
} else
#endif
{
if (datma[bi](i,j,k) >= fac) {
tagma[bi](i,j,k) = tag_update;
}
});
ParallelFor(tba, [=] AMREX_GPU_DEVICE (int bi, int i, int j, int k) noexcept
{
if (datma[bi](i,j,k) >= fac) {
tagma[bi](i,j,k) = tag_update;
}
});
}
}
else
{
Expand Down
2 changes: 0 additions & 2 deletions Src/Base/AMReX_VisMF.H
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -638,7 +638,6 @@ Read (FabArray<FAB>& fa, const std::string& name)
}

int totalioreqs = nboxes;
int messtotal = 0;
int reqspending = 0;
int iopfileindex;
std::deque<int> iopreads;
Expand Down Expand Up @@ -669,7 +668,6 @@ Read (FabArray<FAB>& fa, const std::string& name)
}
} else {
ParallelDescriptor::Send(vreads, tryproc, readtag);
++messtotal;
++reqspending;
}
availablefiles.erase(afilesiter);
Expand Down
4 changes: 0 additions & 4 deletions Src/Particle/AMReX_ParticleInit.H
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1022,8 +1022,6 @@ InitRandom (Long icount,

ParticleLocData pld;

int cnt = 0;

Vector<std::map<std::pair<int, int>, Gpu::HostVector<ParticleType> > > host_particles;
host_particles.reserve(15);
host_particles.resize(finestLevel()+1);
Expand Down Expand Up @@ -1079,8 +1077,6 @@ InitRandom (Long icount,
for (int i = 0; i < NArrayInt; i++) {
host_int_attribs[pld.m_lev][ind][i].push_back(pdata.int_array_data[i]);
}

cnt++;
}
}

Expand Down