Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Replace mfiter loops with the MF parallel for versions #1069

Draft
wants to merge 1 commit into
base: main
Choose a base branch
from
Draft
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions amr-wind/diffusion/incflo_diffusion.cpp
Original file line number Diff line number Diff line change
@@ -221,6 +221,7 @@ void viscosity_to_uniform_space(
repo.get_mesh_mapping_det_j(amr_wind::FieldLoc::ZFACE);

// beta accounted for mesh mapping (x-face) = J/fac^2 * mu
// needs openmp pragma?
for (amrex::MFIter mfi(b[0]); mfi.isValid(); ++mfi) {
amrex::Array4<amrex::Real> const& mu = b[0].array(mfi);
amrex::Array4<amrex::Real const> const& fac =
@@ -235,6 +236,7 @@ void viscosity_to_uniform_space(
});
}
// beta accounted for mesh mapping (y-face) = J/fac^2 * mu
// needs openmp pragma?
for (amrex::MFIter mfi(b[1]); mfi.isValid(); ++mfi) {
amrex::Array4<amrex::Real> const& mu = b[1].array(mfi);
amrex::Array4<amrex::Real const> const& fac =
@@ -249,6 +251,7 @@ void viscosity_to_uniform_space(
});
}
// beta accounted for mesh mapping (z-face) = J/fac^2 * mu
// needs openmp pragma?
for (amrex::MFIter mfi(b[2]); mfi.isValid(); ++mfi) {
amrex::Array4<amrex::Real> const& mu = b[2].array(mfi);
amrex::Array4<amrex::Real const> const& fac =
2 changes: 2 additions & 0 deletions amr-wind/equation_systems/CompRHSOps.H
Original file line number Diff line number Diff line change
@@ -79,6 +79,7 @@ struct ComputeRHSOp
#ifdef AMREX_USE_OMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
// this one
for (amrex::MFIter mfi(field(lev)); mfi.isValid(); ++mfi) {
const auto& bx = mfi.tilebox();
auto fld = field(lev).array(mfi);
@@ -199,6 +200,7 @@ struct ComputeRHSOp
#ifdef AMREX_USE_OMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
// this one
for (amrex::MFIter mfi(field(lev)); mfi.isValid(); ++mfi) {
const auto& bx = mfi.tilebox();
auto fld = field(lev).array(mfi);
1 change: 1 addition & 0 deletions amr-wind/equation_systems/DiffusionOps.cpp
Original file line number Diff line number Diff line change
@@ -128,6 +128,7 @@ void DiffSolverIface<LinOp>::linsys_solve_impl()
#ifdef AMREX_USE_OMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
// this one
for (amrex::MFIter mfi(rhs, amrex::TilingIfNotGPU()); mfi.isValid();
++mfi) {
const auto& bx = mfi.tilebox();
1 change: 1 addition & 0 deletions amr-wind/equation_systems/PDEOps.H
Original file line number Diff line number Diff line change
@@ -100,6 +100,7 @@ struct SrcTermOpBase
#ifdef AMREX_USE_OMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
// this one
for (amrex::MFIter mfi(src_term, amrex::TilingIfNotGPU());
mfi.isValid(); ++mfi) {
const auto& bx = mfi.tilebox();
2 changes: 2 additions & 0 deletions amr-wind/equation_systems/density/density_ops.H
Original file line number Diff line number Diff line change
@@ -30,6 +30,7 @@ struct ComputeRHSOp<Density, Scheme>
#ifdef AMREX_USE_OMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
// this one
for (amrex::MFIter mfi(field(lev)); mfi.isValid(); ++mfi) {
const auto& bx = mfi.tilebox();
auto rho = field(lev).array(mfi);
@@ -62,6 +63,7 @@ struct ComputeRHSOp<Density, Scheme>
#ifdef AMREX_USE_OMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
// this one
for (amrex::MFIter mfi(field(lev)); mfi.isValid(); ++mfi) {
const auto& bx = mfi.tilebox();
auto rho = field(lev).array(mfi);
6 changes: 6 additions & 0 deletions amr-wind/equation_systems/icns/icns_advection.cpp
Original file line number Diff line number Diff line change
@@ -399,6 +399,8 @@ void MacProjOp::mac_proj_to_uniform_space(
// scale U^mac to accommodate for mesh mapping -> U^bar = J/fac *
// U^mac beta accounted for mesh mapping = J/fac^2 * 1/rho construct
// rho and mesh map u_mac on x-face
// this one
// needs openmp pragma?
for (amrex::MFIter mfi(*(rho_face[0])); mfi.isValid(); ++mfi) {
amrex::Array4<amrex::Real> const& u = u_mac(lev).array(mfi);
amrex::Array4<amrex::Real> const& rho = rho_face[0]->array(mfi);
@@ -415,6 +417,8 @@ void MacProjOp::mac_proj_to_uniform_space(
});
}
// construct rho on y-face
// this one
// needs openmp pragma?
for (amrex::MFIter mfi(*(rho_face[1])); mfi.isValid(); ++mfi) {
amrex::Array4<amrex::Real> const& v = v_mac(lev).array(mfi);
amrex::Array4<amrex::Real> const& rho = rho_face[1]->array(mfi);
@@ -431,6 +435,8 @@ void MacProjOp::mac_proj_to_uniform_space(
});
}
// construct rho on z-face
// this one
// needs openmp pragma?
for (amrex::MFIter mfi(*(rho_face[2])); mfi.isValid(); ++mfi) {
amrex::Array4<amrex::Real> const& w = w_mac(lev).array(mfi);
amrex::Array4<amrex::Real> const& rho = rho_face[2]->array(mfi);
4 changes: 4 additions & 0 deletions amr-wind/equation_systems/icns/icns_diffusion.H
Original file line number Diff line number Diff line change
@@ -169,6 +169,7 @@ public:
#ifdef AMREX_USE_OMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
// this one
for (amrex::MFIter mfi(divtau(lev), amrex::TilingIfNotGPU());
mfi.isValid(); ++mfi) {
const auto& bx = mfi.tilebox();
@@ -240,6 +241,7 @@ public:
#ifdef AMREX_USE_OMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
// this one
for (amrex::MFIter mfi(rhs, amrex::TilingIfNotGPU()); mfi.isValid();
++mfi) {
const auto& bx = mfi.tilebox();
@@ -401,6 +403,7 @@ public:
#ifdef AMREX_USE_OMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
// this one
for (amrex::MFIter mfi(divtau(lev), amrex::TilingIfNotGPU());
mfi.isValid(); ++mfi) {
const auto& bx = mfi.tilebox();
@@ -483,6 +486,7 @@ public:
#ifdef AMREX_USE_OMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
// this one
for (amrex::MFIter mfi(rhs, amrex::TilingIfNotGPU()); mfi.isValid();
++mfi) {
const auto& bx = mfi.tilebox();
1 change: 1 addition & 0 deletions amr-wind/equation_systems/icns/icns_ops.H
Original file line number Diff line number Diff line change
@@ -93,6 +93,7 @@ struct SrcTermOp<ICNS> : SrcTermOpBase<ICNS>
#ifdef AMREX_USE_OMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
// this one
for (amrex::MFIter mfi(src_term, amrex::TilingIfNotGPU());
mfi.isValid(); ++mfi) {
const auto& bx = mfi.tilebox();
2 changes: 2 additions & 0 deletions amr-wind/equation_systems/levelset/levelset_ops.H
Original file line number Diff line number Diff line change
@@ -69,6 +69,7 @@ struct ComputeRHSOp<Levelset, Scheme>
#ifdef AMREX_USE_OMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
// this one
for (amrex::MFIter mfi(field(lev)); mfi.isValid(); ++mfi) {
const auto& bx = mfi.tilebox();
auto phi = field(lev).array(mfi);
@@ -101,6 +102,7 @@ struct ComputeRHSOp<Levelset, Scheme>
#ifdef AMREX_USE_OMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
// this one
for (amrex::MFIter mfi(field(lev)); mfi.isValid(); ++mfi) {
const auto& bx = mfi.tilebox();
auto phi = field(lev).array(mfi);
1 change: 1 addition & 0 deletions amr-wind/equation_systems/tke/tke_ops.H
Original file line number Diff line number Diff line change
@@ -42,6 +42,7 @@ struct PostSolveOp<TKE>
const int nlevels = repo.num_active_levels();
const auto clip_value = m_clip_value;
for (int lev = 0; lev < nlevels; ++lev) {
// this one
for (amrex::MFIter mfi(field(lev)); mfi.isValid(); ++mfi) {
const auto& bx = mfi.tilebox();
const auto& field_arr = field(lev).array(mfi);
1 change: 1 addition & 0 deletions amr-wind/equation_systems/vof/vof_hybridsolver_ops.H
Original file line number Diff line number Diff line change
@@ -19,6 +19,7 @@ static void replace_masked_vof(
auto& vof = f_vof(lev);
const auto& vof_new = f_vof_new(lev);

// this one
for (amrex::MFIter mfi(iblank); mfi.isValid(); ++mfi) {
const auto& gbx = mfi.growntilebox();
const amrex::Array4<const int>& native_flag =
2 changes: 2 additions & 0 deletions amr-wind/immersed_boundary/bluff_body/bluff_body_ops.cpp
Original file line number Diff line number Diff line change
@@ -47,6 +47,7 @@ void apply_mms_vel(CFDSim& sim)
const auto& dx = geom[lev].CellSizeArray();
const auto& problo = geom[lev].ProbLoArray();

// this one
for (amrex::MFIter mfi(levelset(lev)); mfi.isValid(); ++mfi) {
const auto& bx = mfi.growntilebox();
const auto& phi = levelset(lev).const_array(mfi);
@@ -90,6 +91,7 @@ void apply_dirichlet_vel(CFDSim& sim, const amrex::Vector<amrex::Real>& vel_bc)
// Defining the "ghost-cell" band distance
amrex::Real phi_b = std::cbrt(dx[0] * dx[1] * dx[2]);

// this one
for (amrex::MFIter mfi(levelset(lev)); mfi.isValid(); ++mfi) {
const auto& bx = mfi.tilebox();
const auto& varr = velocity(lev).array(mfi);
1 change: 1 addition & 0 deletions amr-wind/immersed_boundary/bluff_body/box_ops.H
Original file line number Diff line number Diff line change
@@ -59,6 +59,7 @@ struct InitDataOp<Box>
for (int lev = 0; lev < nlevels; ++lev) {
const auto& problo = geom[lev].ProbLoArray();
const auto& dx = geom[lev].CellSizeArray();
// this one
for (amrex::MFIter mfi(levelset(lev)); mfi.isValid(); ++mfi) {
const auto& bx = mfi.growntilebox();
const auto& epsilon_node = mask_node(lev).array(mfi);
1 change: 1 addition & 0 deletions amr-wind/immersed_boundary/bluff_body/cylinder_ops.H
Original file line number Diff line number Diff line change
@@ -58,6 +58,7 @@ struct InitDataOp<Cylinder>
const auto& problo = geom[lev].ProbLoArray();
const auto& dx = geom[lev].CellSizeArray();

// this one
for (amrex::MFIter mfi(levelset(lev)); mfi.isValid(); ++mfi) {
const auto& phi = levelset(lev).array(mfi);

1 change: 1 addition & 0 deletions amr-wind/immersed_boundary/bluff_body/sphere_ops.H
Original file line number Diff line number Diff line change
@@ -56,6 +56,7 @@ struct InitDataOp<Sphere>
const auto& problo = geom[lev].ProbLoArray();
const auto& dx = geom[lev].CellSizeArray();

// this one
for (amrex::MFIter mfi(levelset(lev)); mfi.isValid(); ++mfi) {
const auto& bx = mfi.growntilebox();
const auto& phi = levelset(lev).array(mfi);
5 changes: 5 additions & 0 deletions amr-wind/mesh_mapping_models/ChannelFlowMap.cpp
Original file line number Diff line number Diff line change
@@ -70,6 +70,7 @@ void ChannelFlowMap::create_cell_node_map(int lev, const amrex::Geometry& geom)
prob_hi[0] - prob_lo[0], prob_hi[1] - prob_lo[1],
prob_hi[2] - prob_lo[2]};

// this one
for (amrex::MFIter mfi((*m_mesh_scale_fac_cc)(lev)); mfi.isValid(); ++mfi) {

const auto& bx = mfi.growntilebox();
@@ -149,6 +150,7 @@ void ChannelFlowMap::create_face_map(int lev, const amrex::Geometry& geom)
prob_hi[0] - prob_lo[0], prob_hi[1] - prob_lo[1],
prob_hi[2] - prob_lo[2]};

// this one
for (amrex::MFIter mfi((*m_mesh_scale_fac_xf)(lev)); mfi.isValid(); ++mfi) {

const auto& bx = mfi.growntilebox();
@@ -181,6 +183,7 @@ void ChannelFlowMap::create_face_map(int lev, const amrex::Geometry& geom)
});
}

// this one
for (amrex::MFIter mfi((*m_mesh_scale_fac_yf)(lev)); mfi.isValid(); ++mfi) {

const auto& bx = mfi.growntilebox();
@@ -213,6 +216,7 @@ void ChannelFlowMap::create_face_map(int lev, const amrex::Geometry& geom)
});
}

// this one
for (amrex::MFIter mfi((*m_mesh_scale_fac_zf)(lev)); mfi.isValid(); ++mfi) {

const auto& bx = mfi.growntilebox();
@@ -277,6 +281,7 @@ void ChannelFlowMap::create_non_uniform_mesh(
probhi_physical[0] - prob_lo[0], probhi_physical[1] - prob_lo[1],
probhi_physical[2] - prob_lo[2]};

// this one
for (amrex::MFIter mfi((*m_non_uniform_coord_cc)(lev)); mfi.isValid();
++mfi) {

5 changes: 5 additions & 0 deletions amr-wind/mesh_mapping_models/ConstantMap.cpp
Original file line number Diff line number Diff line change
@@ -27,6 +27,7 @@ void ConstantMap::create_cell_node_map(int lev)
amrex::Real fac_y = m_fac[1];
amrex::Real fac_z = m_fac[2];

// this one
for (amrex::MFIter mfi((*m_mesh_scale_fac_cc)(lev)); mfi.isValid(); ++mfi) {

const auto& bx = mfi.growntilebox();
@@ -69,6 +70,7 @@ void ConstantMap::create_face_map(int lev)
amrex::Real fac_y = m_fac[1];
amrex::Real fac_z = m_fac[2];

// this one
for (amrex::MFIter mfi((*m_mesh_scale_fac_xf)(lev)); mfi.isValid(); ++mfi) {
const auto& bx = mfi.growntilebox();
amrex::Array4<amrex::Real> const& scale_fac_xf =
@@ -86,6 +88,7 @@ void ConstantMap::create_face_map(int lev)
});
}

// this one
for (amrex::MFIter mfi((*m_mesh_scale_fac_yf)(lev)); mfi.isValid(); ++mfi) {
const auto& bx = mfi.growntilebox();
amrex::Array4<amrex::Real> const& scale_fac_yf =
@@ -103,6 +106,7 @@ void ConstantMap::create_face_map(int lev)
});
}

// this one
for (amrex::MFIter mfi((*m_mesh_scale_fac_zf)(lev)); mfi.isValid(); ++mfi) {
const auto& bx = mfi.growntilebox();
amrex::Array4<amrex::Real> const& scale_fac_zf =
@@ -129,6 +133,7 @@ void ConstantMap::create_non_uniform_mesh(int lev, const amrex::Geometry& geom)
const auto& problo = geom.ProbLoArray();
const auto& dx = geom.CellSizeArray();

// this one
for (amrex::MFIter mfi((*m_non_uniform_coord_cc)(lev)); mfi.isValid();
++mfi) {

4 changes: 4 additions & 0 deletions amr-wind/ocean_waves/boundary_ops/OceanWavesBoundary.cpp
Original file line number Diff line number Diff line change
@@ -96,6 +96,7 @@ void OceanWavesBoundary::set_velocity(
auto shift_to_interior =
amrex::IntVect::TheDimensionVector(idir) * (ori.isLow() ? 1 : -1);

// this one
for (amrex::MFIter mfi(mfab); mfi.isValid(); ++mfi) {
auto gbx = amrex::grow(mfi.validbox(), nghost);
amrex::IntVect shift_to_cc = {0, 0, 0};
@@ -166,6 +167,7 @@ void OceanWavesBoundary::set_vof(
const auto& dbx = ori.isLow() ? amrex::adjCellLo(domain, idir, nghost)
: amrex::adjCellHi(domain, idir, nghost);

// this one
for (amrex::MFIter mfi(mfab); mfi.isValid(); ++mfi) {
auto gbx = amrex::grow(mfi.validbox(), nghost);
const auto& bx =
@@ -216,6 +218,7 @@ void OceanWavesBoundary::set_density(
const auto& dbx = ori.isLow() ? amrex::adjCellLo(domain, idir, nghost)
: amrex::adjCellHi(domain, idir, nghost);

// this one
for (amrex::MFIter mfi(mfab); mfi.isValid(); ++mfi) {
auto gbx = amrex::grow(mfi.validbox(), nghost);
const auto& bx =
@@ -287,6 +290,7 @@ void OceanWavesBoundary::set_inflow_sibling_velocity(
#ifdef AMREX_USE_OMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
// this one
for (amrex::MFIter mfi(mfab); mfi.isValid(); ++mfi) {
const auto& vbx = mfi.validbox();
const auto& bx = vbx & dbx;
1 change: 1 addition & 0 deletions amr-wind/overset/OversetOps.cpp
Original file line number Diff line number Diff line change
@@ -140,6 +140,7 @@ void OversetOps::update_gradp()
#ifdef AMREX_USE_OMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
// this one
for (MFIter mfi(grad_p(lev), TilingIfNotGPU()); mfi.isValid(); ++mfi) {
Box const& tbx = mfi.tilebox();
Array4<Real> const& gp_lev = grad_p(lev).array(mfi);
1 change: 1 addition & 0 deletions amr-wind/physics/BoussinesqBubble.cpp
Original file line number Diff line number Diff line change
@@ -32,6 +32,7 @@ void BoussinesqBubble::initialize_fields(int level, const amrex::Geometry& geom)
auto& density = m_density(level);
auto& scalars = (*m_temperature)(level);

// needs openmp pragma?
for (amrex::MFIter mfi(density); mfi.isValid(); ++mfi) {
const auto& vbx = mfi.validbox();

3 changes: 2 additions & 1 deletion amr-wind/physics/BurggrafFlow.cpp
Original file line number Diff line number Diff line change
@@ -63,6 +63,7 @@ void BurggrafFlow::initialize_fields(int level, const amrex::Geometry& geom)

UExact u_exact;
VExact v_exact;
// this one
for (amrex::MFIter mfi(velocity); mfi.isValid(); ++mfi) {
const auto& vbx = mfi.validbox();

@@ -181,4 +182,4 @@ void BurggrafFlow::post_init_actions() { output_error(); }

void BurggrafFlow::post_advance_work() { output_error(); }

} // namespace amr_wind::burggraf
} // namespace amr_wind::burggraf
2 changes: 2 additions & 0 deletions amr-wind/physics/ChannelFlow.cpp
Original file line number Diff line number Diff line change
@@ -148,6 +148,7 @@ void ChannelFlow::initialize_fields(
const auto C0 = m_C0;
const auto C1 = m_C1;
const auto dpdx = m_dpdx;
// this one
for (amrex::MFIter mfi(velocity); mfi.isValid(); ++mfi) {
const auto& vbx = mfi.validbox();
auto vel = velocity.array(mfi);
@@ -181,6 +182,7 @@ void ChannelFlow::initialize_fields(
}
auto& walldist = m_repo.get_field("wall_dist")(level);

// this one
for (amrex::MFIter mfi(velocity); mfi.isValid(); ++mfi) {
const auto& vbx = mfi.validbox();
auto vel = velocity.array(mfi);
2 changes: 2 additions & 0 deletions amr-wind/physics/ConvectingTaylorVortex.cpp
Original file line number Diff line number Diff line change
@@ -156,6 +156,7 @@ void ConvectingTaylorVortex::initialize_fields(
GpyExact gpy_exact;
GpzExact gpz_exact;

// this one
for (amrex::MFIter mfi(velocity); mfi.isValid(); ++mfi) {
const auto& vbx = mfi.validbox();

@@ -240,6 +241,7 @@ amrex::Real ConvectingTaylorVortex::compute_error(const Field& field)
}

if (m_sim.has_overset()) {
// this one
for (amrex::MFIter mfi(field(lev)); mfi.isValid(); ++mfi) {
const auto& vbx = mfi.validbox();

1 change: 1 addition & 0 deletions amr-wind/physics/EkmanSpiral.cpp
Original file line number Diff line number Diff line change
@@ -111,6 +111,7 @@ void EkmanSpiral::initialize_fields(int level, const amrex::Geometry& geom)
UExact u_exact;
VExact v_exact;

// this one
for (amrex::MFIter mfi(velocity); mfi.isValid(); ++mfi) {
const auto& vbx = mfi.validbox();

Loading