From 457e73b2156816fd50927125e96139f6cbacb95c Mon Sep 17 00:00:00 2001 From: James Wright Date: Mon, 19 Jun 2023 10:54:06 -0600 Subject: [PATCH 1/3] fluids: Compatability for PETSc w/ 64bit ints --- examples/fluids/navierstokes.c | 2 +- examples/fluids/navierstokes.h | 3 + examples/fluids/problems/blasius.c | 14 ++-- examples/fluids/problems/sgs_dd_model.c | 6 +- examples/fluids/problems/stg_shur14.c | 16 ++--- examples/fluids/src/differential_filter.c | 9 ++- examples/fluids/src/grid_anisotropy_tensor.c | 18 ++--- examples/fluids/src/misc.c | 66 +++++++++++++++++-- examples/fluids/src/setuplibceed.c | 17 +++-- examples/fluids/src/setupts.c | 23 ++++--- examples/fluids/src/turb_spanstats.c | 12 ++-- .../fluids/src/velocity_gradient_projection.c | 3 +- 12 files changed, 130 insertions(+), 59 deletions(-) diff --git a/examples/fluids/navierstokes.c b/examples/fluids/navierstokes.c index 8aaca1925c..78a34f3c58 100644 --- a/examples/fluids/navierstokes.c +++ b/examples/fluids/navierstokes.c @@ -280,7 +280,7 @@ int main(int argc, char **argv) { } // Mesh const PetscInt num_comp_q = 5; - CeedInt glob_dofs, owned_dofs; + PetscInt glob_dofs, owned_dofs; PetscInt glob_nodes, local_nodes; const CeedInt num_P = app_ctx->degree + 1, num_Q = num_P + app_ctx->q_extra; // -- Get global size diff --git a/examples/fluids/navierstokes.h b/examples/fluids/navierstokes.h index 5f2ea7ac4b..c1bc03aa02 100644 --- a/examples/fluids/navierstokes.h +++ b/examples/fluids/navierstokes.h @@ -415,6 +415,9 @@ PetscErrorCode PHASTADatFileGetNRows(const MPI_Comm comm, const char path[PETSC_ PetscErrorCode PHASTADatFileReadToArrayReal(const MPI_Comm comm, const char path[PETSC_MAX_PATH_LEN], PetscReal array[]); +PetscErrorCode IntArrayC2P(PetscInt num_entries, CeedInt **array_ceed, PetscInt **array_petsc); +PetscErrorCode IntArrayP2C(PetscInt num_entries, PetscInt **array_petsc, CeedInt **array_ceed); + // ----------------------------------------------------------------------------- // Turbulence Statistics Collection Functions // ----------------------------------------------------------------------------- diff --git a/examples/fluids/problems/blasius.c b/examples/fluids/problems/blasius.c index 28e2e681e6..f1c5c504f9 100644 --- a/examples/fluids/problems/blasius.c +++ b/examples/fluids/problems/blasius.c @@ -111,7 +111,7 @@ PetscErrorCode ComputeChebyshevCoefficients(BlasiusContext blasius) { } static PetscErrorCode GetYNodeLocs(const MPI_Comm comm, const char path[PETSC_MAX_PATH_LEN], PetscReal **pynodes, PetscInt *nynodes) { - PetscInt ndims, dims[2]; + int ndims, dims[2]; FILE *fp; const PetscInt char_array_len = 512; char line[char_array_len]; @@ -132,8 +132,7 @@ static PetscErrorCode GetYNodeLocs(const MPI_Comm comm, const char path[PETSC_MA PetscCall(PetscSynchronizedFGets(comm, fp, char_array_len, line)); PetscCall(PetscStrToArray(line, ' ', &ndims, &array)); PetscCheck(ndims == dims[1], comm, PETSC_ERR_FILE_UNEXPECTED, - "Line %" PetscInt_FMT " of %s does not contain correct number of columns (%" PetscInt_FMT " instead of %" PetscInt_FMT ")", i, path, - ndims, dims[1]); + "Line %" PetscInt_FMT " of %s does not contain correct number of columns (%d instead of %d)", i, path, ndims, dims[1]); node_locs[i] = (PetscReal)atof(array[0]); } @@ -207,11 +206,12 @@ static PetscErrorCode ModifyMesh(MPI_Comm comm, DM dm, PetscInt dim, PetscReal g *node_locs = temp_node_locs; } else { PetscCheck(*num_node_locs >= faces[1] + 1, comm, PETSC_ERR_FILE_UNEXPECTED, - "The y_node_locs_path has too few locations; There are %d + 1 nodes, but only %d locations given", faces[1] + 1, *num_node_locs); + "The y_node_locs_path has too few locations; There are %" PetscInt_FMT " + 1 nodes, but only %" PetscInt_FMT " locations given", + faces[1] + 1, *num_node_locs); if (*num_node_locs > faces[1] + 1) { PetscCall(PetscPrintf(comm, - "WARNING: y_node_locs_path has more locations (%d) " - "than the mesh has nodes (%d). This maybe unintended.\n", + "WARNING: y_node_locs_path has more locations (%" PetscInt_FMT ") " + "than the mesh has nodes (%" PetscInt_FMT "). This maybe unintended.\n", *num_node_locs, faces[1] + 1)); } PetscScalar max_y = (*node_locs)[faces[1]]; @@ -252,7 +252,7 @@ PetscErrorCode NS_BLASIUS(ProblemData *problem, DM dm, void *ctx, SimpleBC bc) { CeedScalar T_wall = 288.; // K CeedScalar delta0 = 4.2e-3; // m CeedScalar P0 = 1.01e5; // Pa - CeedInt N = 20; // Number of Chebyshev terms + PetscInt N = 20; // Number of Chebyshev terms PetscBool weakT = PETSC_FALSE; // weak density or temperature PetscReal mesh_refine_height = 5.9e-4; // m PetscReal mesh_growth = 1.08; // [-] diff --git a/examples/fluids/problems/sgs_dd_model.c b/examples/fluids/problems/sgs_dd_model.c index ad8e98efea..b97f20685e 100644 --- a/examples/fluids/problems/sgs_dd_model.c +++ b/examples/fluids/problems/sgs_dd_model.c @@ -66,7 +66,8 @@ PetscErrorCode SGS_DD_ModelSetupNodalEvaluation(Ceed ceed, User user, CeedData c SGS_DD_Data sgs_dd_data = user->sgs_dd_data; CeedQFunction qf_multiplicity, qf_sgs_dd_nodal; CeedOperator op_multiplicity, op_sgs_dd_nodal; - CeedInt num_elem, elem_size, num_comp_q, dim, num_qpts_1d, num_comp_grad_velo, num_comp_x, num_comp_grid_aniso; + CeedInt num_elem, elem_size, num_comp_q, num_qpts_1d, num_comp_grad_velo, num_comp_x, num_comp_grid_aniso; + PetscInt dim; CeedVector multiplicity, inv_multiplicity; CeedElemRestriction elem_restr_inv_multiplicity, elem_restr_grad_velo, elem_restr_sgs; @@ -160,7 +161,8 @@ PetscErrorCode SGS_DD_ModelSetupNodalEvaluation(Ceed ceed, User user, CeedData c // @brief Create CeedOperator to compute SGS contribution to the residual PetscErrorCode SGS_ModelSetupNodalIFunction(Ceed ceed, User user, CeedData ceed_data, SGS_DD_ModelSetupData sgs_dd_setup_data) { SGS_DD_Data sgs_dd_data = user->sgs_dd_data; - CeedInt dim, num_comp_q, num_comp_qd, num_comp_x, num_qpts_1d, num_nodes_1d; + CeedInt num_comp_q, num_comp_qd, num_comp_x, num_qpts_1d, num_nodes_1d; + PetscInt dim; CeedQFunction qf_sgs_apply; CeedOperator op_sgs_apply; CeedBasis basis_sgs; diff --git a/examples/fluids/problems/stg_shur14.c b/examples/fluids/problems/stg_shur14.c index e4cf1bb429..06f26dd2a4 100644 --- a/examples/fluids/problems/stg_shur14.c +++ b/examples/fluids/problems/stg_shur14.c @@ -61,7 +61,8 @@ PetscErrorCode CalcCholeskyDecomp(MPI_Comm comm, PetscInt nprofs, const CeedScal * @param[in,out] stg_ctx STGShur14Context where the data will be loaded into */ static PetscErrorCode ReadSTGInflow(const MPI_Comm comm, const char path[PETSC_MAX_PATH_LEN], STGShur14Context stg_ctx) { - PetscInt ndims, dims[2]; + PetscInt dims[2]; + int ndims; FILE *fp; const PetscInt char_array_len = 512; char line[char_array_len]; @@ -81,8 +82,7 @@ static PetscErrorCode ReadSTGInflow(const MPI_Comm comm, const char path[PETSC_M PetscCall(PetscSynchronizedFGets(comm, fp, char_array_len, line)); PetscCall(PetscStrToArray(line, ' ', &ndims, &array)); PetscCheck(ndims == dims[1], comm, PETSC_ERR_FILE_UNEXPECTED, - "Line %" PetscInt_FMT " of %s does not have correct number of columns (%" PetscInt_FMT " instead of %" PetscInt_FMT ")", i, path, - ndims, dims[1]); + "Line %" PetscInt_FMT " of %s does not have correct number of columns (%d instead of %" PetscInt_FMT ")", i, path, ndims, dims[1]); wall_dist[i] = (CeedScalar)atof(array[0]); ubar[0][i] = (CeedScalar)atof(array[1]); @@ -119,7 +119,8 @@ static PetscErrorCode ReadSTGInflow(const MPI_Comm comm, const char path[PETSC_M * @param[in,out] stg_ctx STGShur14Context where the data will be loaded into */ static PetscErrorCode ReadSTGRand(const MPI_Comm comm, const char path[PETSC_MAX_PATH_LEN], STGShur14Context stg_ctx) { - PetscInt ndims, dims[2]; + PetscInt dims[2]; + int ndims; FILE *fp; const PetscInt char_array_len = 512; char line[char_array_len]; @@ -136,8 +137,7 @@ static PetscErrorCode ReadSTGRand(const MPI_Comm comm, const char path[PETSC_MAX PetscCall(PetscSynchronizedFGets(comm, fp, char_array_len, line)); PetscCall(PetscStrToArray(line, ' ', &ndims, &array)); PetscCheck(ndims == dims[1], comm, PETSC_ERR_FILE_UNEXPECTED, - "Line %" PetscInt_FMT " of %s does not have correct number of columns (%" PetscInt_FMT " instead of %" PetscInt_FMT ")", i, path, - ndims, dims[1]); + "Line %" PetscInt_FMT " of %s does not have correct number of columns (%d instead of %" PetscInt_FMT ")", i, path, ndims, dims[1]); d[0][i] = (CeedScalar)atof(array[0]); d[1][i] = (CeedScalar)atof(array[1]); @@ -173,8 +173,8 @@ PetscErrorCode GetSTGContextData(const MPI_Comm comm, const DM dm, char stg_infl PetscCall(PHASTADatFileGetNRows(comm, stg_rand_path, &nmodes)); PetscCall(PHASTADatFileGetNRows(comm, stg_inflow_path, &nprofs)); PetscCheck(nmodes < STG_NMODES_MAX, comm, PETSC_ERR_SUP, - "Number of wavemodes in %s (%" PetscInt_FMT ") exceeds STG_NMODES_MAX (%" PetscInt_FMT "). Change size of STG_NMODES_MAX and recompile", - stg_rand_path, nmodes, STG_NMODES_MAX); + "Number of wavemodes in %s (%" PetscInt_FMT ") exceeds STG_NMODES_MAX (%d). Change size of STG_NMODES_MAX and recompile", stg_rand_path, + nmodes, STG_NMODES_MAX); { STGShur14Context temp_ctx; diff --git a/examples/fluids/src/differential_filter.c b/examples/fluids/src/differential_filter.c index 0cd4671f2b..7a05ff5d19 100644 --- a/examples/fluids/src/differential_filter.c +++ b/examples/fluids/src/differential_filter.c @@ -17,7 +17,8 @@ PetscErrorCode DifferentialFilterCreateOperators(Ceed ceed, User user, CeedData ceed_data, CeedQFunctionContext diff_filter_qfctx) { DiffFilterData diff_filter = user->diff_filter; DM dm_filter = diff_filter->dm_filter; - CeedInt num_comp_q, num_comp_qd, dim, num_qpts_1d, num_nodes_1d, num_comp_x; + CeedInt num_comp_q, num_comp_qd, num_qpts_1d, num_nodes_1d, num_comp_x; + PetscInt dim; PetscFunctionBeginUser; PetscCall(DMGetDimension(user->dm, &dim)); @@ -81,7 +82,8 @@ PetscErrorCode DifferentialFilterCreateOperators(Ceed ceed, User user, CeedData CeedOperator op_lhs; OperatorApplyContext mat_ctx; Mat mat_lhs; - CeedInt num_comp_qd, dim, num_comp_grid_aniso; + CeedInt num_comp_qd; + PetscInt dim, num_comp_grid_aniso; CeedElemRestriction elem_restr_grid_aniso; CeedVector grid_aniso_ceed; @@ -113,7 +115,8 @@ PetscErrorCode DifferentialFilterCreateOperators(Ceed ceed, User user, CeedData CeedQFunctionCreateInterior(ceed, 1, DifferentialFilter_LHS_11, DifferentialFilter_LHS_11_loc, &qf_lhs); break; default: - SETERRQ(PetscObjectComm((PetscObject)user->dm), PETSC_ERR_SUP, "Differential filtering not available for (%d) components", num_comp_filter); + SETERRQ(PetscObjectComm((PetscObject)user->dm), PETSC_ERR_SUP, "Differential filtering not available for (%" PetscInt_FMT ") components", + num_comp_filter); } CeedQFunctionSetContext(qf_lhs, diff_filter_qfctx); diff --git a/examples/fluids/src/grid_anisotropy_tensor.c b/examples/fluids/src/grid_anisotropy_tensor.c index ecc32844b4..6c38866cd5 100644 --- a/examples/fluids/src/grid_anisotropy_tensor.c +++ b/examples/fluids/src/grid_anisotropy_tensor.c @@ -18,7 +18,8 @@ PetscErrorCode GridAnisotropyTensorProjectionSetupApply(Ceed ceed, User user, Ce CeedOperator op_rhs_assemble, op_mass; CeedQFunction qf_rhs_assemble, qf_mass; CeedBasis basis_grid_aniso; - PetscInt dim, q_data_size, num_qpts_1d, num_nodes_1d; + PetscInt dim; + CeedInt num_qpts_1d, num_nodes_1d, q_data_size; MPI_Comm comm = PetscObjectComm((PetscObject)user->dm); KSP ksp; @@ -133,33 +134,34 @@ PetscErrorCode GridAnisotropyTensorProjectionSetupApply(Ceed ceed, User user, Ce } PetscErrorCode GridAnisotropyTensorCalculateCollocatedVector(Ceed ceed, User user, CeedData ceed_data, CeedElemRestriction *elem_restr_grid_aniso, - CeedVector *aniso_colloc_ceed, PetscInt *num_comp_grid_aniso) { - PetscInt dim, q_data_size, num_qpts_1d, num_nodes_1d, loc_num_elem; + CeedVector *aniso_colloc_ceed, PetscInt *num_comp_aniso) { + PetscInt dim; + CeedInt q_data_size, num_qpts_1d, num_nodes_1d, loc_num_elem; CeedQFunction qf_colloc; CeedOperator op_colloc; CeedBasis basis_grid_aniso; PetscFunctionBeginUser; // -- Get Pre-requisite things - *num_comp_grid_aniso = 7; + *num_comp_aniso = 7; PetscCall(DMGetDimension(user->dm, &dim)); CeedBasisGetNumQuadraturePoints1D(ceed_data->basis_q, &num_qpts_1d); CeedBasisGetNumNodes1D(ceed_data->basis_q, &num_nodes_1d); CeedElemRestrictionGetNumComponents(ceed_data->elem_restr_qd_i, &q_data_size); - PetscCall(GetRestrictionForDomain(ceed, user->dm, 0, 0, 0, 0, num_qpts_1d, *num_comp_grid_aniso, NULL, NULL, elem_restr_grid_aniso)); + PetscCall(GetRestrictionForDomain(ceed, user->dm, 0, 0, 0, 0, num_qpts_1d, *num_comp_aniso, NULL, NULL, elem_restr_grid_aniso)); CeedInt Q_dim = CeedIntPow(num_qpts_1d, dim); CeedElemRestrictionGetNumElements(ceed_data->elem_restr_q, &loc_num_elem); - CeedElemRestrictionCreateStrided(ceed, loc_num_elem, Q_dim, *num_comp_grid_aniso, *num_comp_grid_aniso * loc_num_elem * Q_dim, CEED_STRIDES_BACKEND, + CeedElemRestrictionCreateStrided(ceed, loc_num_elem, Q_dim, *num_comp_aniso, *num_comp_aniso * loc_num_elem * Q_dim, CEED_STRIDES_BACKEND, elem_restr_grid_aniso); - CeedBasisCreateTensorH1Lagrange(ceed, dim, *num_comp_grid_aniso, num_nodes_1d, num_qpts_1d, CEED_GAUSS, &basis_grid_aniso); + CeedBasisCreateTensorH1Lagrange(ceed, dim, *num_comp_aniso, num_nodes_1d, num_qpts_1d, CEED_GAUSS, &basis_grid_aniso); // -- Build collocation operator CeedQFunctionCreateInterior(ceed, 1, AnisotropyTensorCollocate, AnisotropyTensorCollocate_loc, &qf_colloc); CeedQFunctionAddInput(qf_colloc, "qdata", q_data_size, CEED_EVAL_NONE); - CeedQFunctionAddOutput(qf_colloc, "v", *num_comp_grid_aniso, CEED_EVAL_NONE); + CeedQFunctionAddOutput(qf_colloc, "v", *num_comp_aniso, CEED_EVAL_NONE); CeedOperatorCreate(ceed, qf_colloc, NULL, NULL, &op_colloc); CeedOperatorSetField(op_colloc, "qdata", ceed_data->elem_restr_qd_i, CEED_BASIS_COLLOCATED, ceed_data->q_data); diff --git a/examples/fluids/src/misc.c b/examples/fluids/src/misc.c index 07063845cf..aa18537c66 100644 --- a/examples/fluids/src/misc.c +++ b/examples/fluids/src/misc.c @@ -330,15 +330,15 @@ PetscErrorCode NodalProjectionDataDestroy(NodalProjectionData context) { */ PetscErrorCode PHASTADatFileOpen(const MPI_Comm comm, const char path[PETSC_MAX_PATH_LEN], const PetscInt char_array_len, PetscInt dims[2], FILE **fp) { - PetscInt ndims; - char line[char_array_len]; - char **array; + int ndims; + char line[char_array_len]; + char **array; PetscFunctionBeginUser; PetscCall(PetscFOpen(comm, path, "r", fp)); PetscCall(PetscSynchronizedFGets(comm, *fp, char_array_len, line)); PetscCall(PetscStrToArray(line, ' ', &ndims, &array)); - PetscCheck(ndims == 2, comm, PETSC_ERR_FILE_UNEXPECTED, "Found %" PetscInt_FMT " dimensions instead of 2 on the first line of %s", ndims, path); + PetscCheck(ndims == 2, comm, PETSC_ERR_FILE_UNEXPECTED, "Found %d dimensions instead of 2 on the first line of %s", ndims, path); for (PetscInt i = 0; i < ndims; i++) dims[i] = atoi(array[i]); PetscCall(PetscStrToArrayDestroy(ndims, array)); @@ -369,7 +369,8 @@ PetscErrorCode PHASTADatFileGetNRows(const MPI_Comm comm, const char path[PETSC_ } PetscErrorCode PHASTADatFileReadToArrayReal(MPI_Comm comm, const char path[PETSC_MAX_PATH_LEN], PetscReal array[]) { - PetscInt ndims, dims[2]; + PetscInt dims[2]; + int ndims; FILE *fp; const PetscInt char_array_len = 512; char line[char_array_len]; @@ -382,8 +383,7 @@ PetscErrorCode PHASTADatFileReadToArrayReal(MPI_Comm comm, const char path[PETSC PetscCall(PetscSynchronizedFGets(comm, fp, char_array_len, line)); PetscCall(PetscStrToArray(line, ' ', &ndims, &row_array)); PetscCheck(ndims == dims[1], comm, PETSC_ERR_FILE_UNEXPECTED, - "Line %" PetscInt_FMT " of %s does not contain enough columns (%" PetscInt_FMT " instead of %" PetscInt_FMT ")", i, path, ndims, - dims[1]); + "Line %" PetscInt_FMT " of %s does not contain enough columns (%d instead of %" PetscInt_FMT ")", i, path, ndims, dims[1]); for (PetscInt j = 0; j < dims[1]; j++) { array[i * dims[1] + j] = (PetscReal)atof(row_array[j]); @@ -410,3 +410,55 @@ PetscErrorCode RegisterLogEvents() { PetscCall(PetscLogEventRegister("CeedOpAsmPBD", libCEED_classid, &FLUIDS_CeedOperatorAssemblePointBlockDiagonal)); PetscFunctionReturn(PETSC_SUCCESS); } + +/** + @brief Translate array of CeedInt to PetscInt. + If the types differ, `array_ceed` is freed with `free()` and `array_petsc` is allocated with `malloc()`. + Caller is responsible for freeing `array_petsc` with `free()`. + + @param[in] num_entries Number of array entries + @param[in,out] array_ceed Array of CeedInts + @param[out] array_petsc Array of PetscInts +**/ +PetscErrorCode IntArrayC2P(PetscInt num_entries, CeedInt **array_ceed, PetscInt **array_petsc) { + CeedInt int_c = 0; + PetscInt int_p = 0; + + PetscFunctionBeginUser; + if (sizeof(int_c) == sizeof(int_p)) { + *array_petsc = (PetscInt *)*array_ceed; + } else { + *array_petsc = malloc(num_entries * sizeof(PetscInt)); + for (PetscInt i = 0; i < num_entries; i++) (*array_petsc)[i] = (*array_ceed)[i]; + free(*array_ceed); + } + *array_ceed = NULL; + + PetscFunctionReturn(PETSC_SUCCESS); +} + +/** + @brief Translate array of PetscInt to CeedInt. + If the types differ, `array_petsc` is freed with `PetscFree()` and `array_ceed` is allocated with `PetscMalloc1()`. + Caller is responsible for freeing `array_ceed` with `PetscFree()`. + + @param[in] num_entries Number of array entries + @param[in,out] array_petsc Array of PetscInts + @param[out] array_ceed Array of CeedInts +**/ +PetscErrorCode IntArrayP2C(PetscInt num_entries, PetscInt **array_petsc, CeedInt **array_ceed) { + CeedInt int_c = 0; + PetscInt int_p = 0; + + PetscFunctionBeginUser; + if (sizeof(int_c) == sizeof(int_p)) { + *array_ceed = (CeedInt *)*array_petsc; + } else { + PetscCall(PetscMalloc1(num_entries, array_ceed)); + for (PetscInt i = 0; i < num_entries; i++) (*array_ceed)[i] = (*array_petsc)[i]; + PetscCall(PetscFree(*array_petsc)); + } + *array_petsc = NULL; + + PetscFunctionReturn(PETSC_SUCCESS); +} diff --git a/examples/fluids/src/setuplibceed.c b/examples/fluids/src/setuplibceed.c index bf53e82a51..bdcd918955 100644 --- a/examples/fluids/src/setuplibceed.c +++ b/examples/fluids/src/setuplibceed.c @@ -19,13 +19,16 @@ PetscInt Involute(PetscInt i) { return i >= 0 ? i : -(i + 1); } // Utility function to create local CEED restriction PetscErrorCode CreateRestrictionFromPlex(Ceed ceed, DM dm, CeedInt height, DMLabel domain_label, CeedInt label_value, PetscInt dm_field, CeedElemRestriction *elem_restr) { - PetscInt num_elem, elem_size, num_dof, num_comp, *elem_restr_offsets; + PetscInt num_elem, elem_size, num_dof, num_comp, *elem_restr_offsets_petsc; + CeedInt *elem_restr_offsets_ceed; PetscFunctionBeginUser; - PetscCall(DMPlexGetLocalOffsets(dm, domain_label, label_value, height, dm_field, &num_elem, &elem_size, &num_comp, &num_dof, &elem_restr_offsets)); + PetscCall( + DMPlexGetLocalOffsets(dm, domain_label, label_value, height, dm_field, &num_elem, &elem_size, &num_comp, &num_dof, &elem_restr_offsets_petsc)); - CeedElemRestrictionCreate(ceed, num_elem, elem_size, num_comp, 1, num_dof, CEED_MEM_HOST, CEED_COPY_VALUES, elem_restr_offsets, elem_restr); - PetscCall(PetscFree(elem_restr_offsets)); + PetscCall(IntArrayP2C(num_elem * elem_size, &elem_restr_offsets_petsc, &elem_restr_offsets_ceed)); + CeedElemRestrictionCreate(ceed, num_elem, elem_size, num_comp, 1, num_dof, CEED_MEM_HOST, CEED_COPY_VALUES, elem_restr_offsets_ceed, elem_restr); + PetscCall(PetscFree(elem_restr_offsets_ceed)); PetscFunctionReturn(PETSC_SUCCESS); } @@ -35,8 +38,8 @@ PetscErrorCode GetRestrictionForDomain(Ceed ceed, DM dm, CeedInt height, DMLabel CeedInt q_data_size, CeedElemRestriction *elem_restr_q, CeedElemRestriction *elem_restr_x, CeedElemRestriction *elem_restr_qd_i) { DM dm_coord; - CeedInt dim, loc_num_elem; - CeedInt Q_dim; + CeedInt Q_dim, loc_num_elem; + PetscInt dim; CeedElemRestriction elem_restr_tmp; PetscFunctionBeginUser; @@ -87,7 +90,7 @@ PetscErrorCode AddBCSubOperator(Ceed ceed, DM dm, CeedData ceed_data, DMLabel do } // ---- CEED Vector - PetscInt loc_num_elem_sur; + CeedInt loc_num_elem_sur; CeedElemRestrictionGetNumElements(elem_restr_q_sur, &loc_num_elem_sur); CeedVectorCreate(ceed, q_data_size_sur * loc_num_elem_sur * num_qpts_sur, &q_data_sur); diff --git a/examples/fluids/src/setupts.c b/examples/fluids/src/setupts.c index fdf3622059..114a4c4b46 100644 --- a/examples/fluids/src/setupts.c +++ b/examples/fluids/src/setupts.c @@ -217,29 +217,32 @@ PetscErrorCode IFunction_NS(TS ts, PetscReal t, Vec Q, Vec Q_dot, Vec G, void *u static PetscErrorCode FormPreallocation(User user, PetscBool pbdiagonal, Mat J, CeedVector *coo_values) { PetscCount ncoo; - PetscInt *rows, *cols; + PetscInt *rows_petsc, *cols_petsc; PetscFunctionBeginUser; if (pbdiagonal) { CeedSize l_size; CeedOperatorGetActiveVectorLengths(user->op_ijacobian, &l_size, NULL); - ncoo = l_size * 5; - rows = malloc(ncoo * sizeof(rows[0])); - cols = malloc(ncoo * sizeof(cols[0])); + ncoo = l_size * 5; + rows_petsc = malloc(ncoo * sizeof(rows_petsc)); + cols_petsc = malloc(ncoo * sizeof(cols_petsc)); for (PetscCount n = 0; n < l_size / 5; n++) { for (PetscInt i = 0; i < 5; i++) { for (PetscInt j = 0; j < 5; j++) { - rows[(n * 5 + i) * 5 + j] = n * 5 + i; - cols[(n * 5 + i) * 5 + j] = n * 5 + j; + rows_petsc[(n * 5 + i) * 5 + j] = n * 5 + i; + cols_petsc[(n * 5 + i) * 5 + j] = n * 5 + j; } } } } else { - PetscCall(CeedOperatorLinearAssembleSymbolic(user->op_ijacobian, &ncoo, &rows, &cols)); + CeedInt *rows_ceed, *cols_ceed; + PetscCall(CeedOperatorLinearAssembleSymbolic(user->op_ijacobian, &ncoo, &rows_ceed, &cols_ceed)); + PetscCall(IntArrayC2P(ncoo, &rows_ceed, &rows_petsc)); + PetscCall(IntArrayC2P(ncoo, &cols_ceed, &cols_petsc)); } - PetscCall(MatSetPreallocationCOOLocal(J, ncoo, rows, cols)); - free(rows); - free(cols); + PetscCall(MatSetPreallocationCOOLocal(J, ncoo, rows_petsc, cols_petsc)); + free(rows_petsc); + free(cols_petsc); CeedVectorCreate(user->ceed, ncoo, coo_values); PetscFunctionReturn(PETSC_SUCCESS); } diff --git a/examples/fluids/src/turb_spanstats.c b/examples/fluids/src/turb_spanstats.c index 036eebc570..934d458d09 100644 --- a/examples/fluids/src/turb_spanstats.c +++ b/examples/fluids/src/turb_spanstats.c @@ -144,7 +144,7 @@ PetscErrorCode GetQuadratureCoords(Ceed ceed, DM dm, CeedElemRestriction elem_re CeedElemRestriction elem_restr_qx; CeedQFunction qf_quad_coords; CeedOperator op_quad_coords; - PetscInt num_comp_x, loc_num_elem, num_elem_qpts; + CeedInt num_comp_x, loc_num_elem, num_elem_qpts; OperatorApplyContext op_quad_coords_ctx; PetscFunctionBeginUser; @@ -176,9 +176,10 @@ PetscErrorCode GetQuadratureCoords(Ceed ceed, DM dm, CeedElemRestriction elem_re } PetscErrorCode SpanStatsSetupDataCreate(Ceed ceed, User user, CeedData ceed_data, ProblemData *problem, SpanStatsSetupData *stats_data) { - DM dm = user->spanstats.dm; - CeedInt dim, P, Q, num_comp_x, num_comp_stats = user->spanstats.num_comp_stats; - Vec X_loc; + DM dm = user->spanstats.dm; + PetscInt dim; + CeedInt P, Q, num_comp_x, num_comp_stats = user->spanstats.num_comp_stats; + Vec X_loc; PetscFunctionBeginUser; PetscCall(PetscNew(stats_data)); @@ -236,7 +237,8 @@ PetscErrorCode SpanStatsSetupDataDestroy(SpanStatsSetupData data) { // Create PetscSF for child-to-parent communication PetscErrorCode CreateStatsSF(Ceed ceed, CeedData ceed_data, SpanStatsSetupData stats_data, DM parentdm, DM childdm, PetscSF *statssf) { - PetscInt child_num_qpnts, parent_num_qpnts, num_comp_x; + PetscInt child_num_qpnts, parent_num_qpnts; + CeedInt num_comp_x; Vec Child_qx_coords, Parent_qx_coords; PetscFunctionBeginUser; diff --git a/examples/fluids/src/velocity_gradient_projection.c b/examples/fluids/src/velocity_gradient_projection.c index dcfe733915..7a84fd2763 100644 --- a/examples/fluids/src/velocity_gradient_projection.c +++ b/examples/fluids/src/velocity_gradient_projection.c @@ -53,7 +53,8 @@ PetscErrorCode VelocityGradientProjectionSetup(Ceed ceed, User user, CeedData ce CeedQFunction qf_rhs_assemble, qf_mass; CeedBasis basis_grad_velo; CeedElemRestriction elem_restr_grad_velo; - PetscInt dim, num_comp_x, num_comp_q, q_data_size, num_qpts_1d, num_nodes_1d; + PetscInt dim; + CeedInt num_comp_x, num_comp_q, q_data_size, num_qpts_1d, num_nodes_1d; PetscFunctionBeginUser; PetscCall(PetscNew(&user->grad_velo_proj)); From 8213d101c4b98f0f1613e6f4ff63a274069549a8 Mon Sep 17 00:00:00 2001 From: James Wright Date: Mon, 19 Jun 2023 16:03:12 -0600 Subject: [PATCH 2/3] fluids: Remove problem->bc function pointers - Not needed, as essential boundary conditions go through `DMPlexInsertBoundaryValues_NS` or `DMPlexInsertBoundaryValues_StrongBC` --- examples/fluids/navierstokes.h | 4 +--- examples/fluids/problems/advection.c | 2 -- examples/fluids/problems/advection2d.c | 2 -- examples/fluids/problems/eulervortex.c | 2 -- examples/fluids/problems/newtonian.c | 2 -- examples/fluids/problems/shocktube.c | 2 -- examples/fluids/src/setupdm.c | 12 ++++-------- 7 files changed, 5 insertions(+), 21 deletions(-) diff --git a/examples/fluids/navierstokes.h b/examples/fluids/navierstokes.h index c1bc03aa02..8a2058a0e7 100644 --- a/examples/fluids/navierstokes.h +++ b/examples/fluids/navierstokes.h @@ -277,9 +277,7 @@ struct ProblemData_private { CeedScalar dm_scale; ProblemQFunctionSpec setup_vol, setup_sur, ics, apply_vol_rhs, apply_vol_ifunction, apply_vol_ijacobian, apply_inflow, apply_outflow, apply_freestream, apply_inflow_jacobian, apply_outflow_jacobian, apply_freestream_jacobian; - bool non_zero_time; - PetscErrorCode (*bc)(PetscInt, PetscReal, const PetscReal[], PetscInt, PetscScalar[], void *); - void *bc_ctx; + bool non_zero_time; PetscBool bc_from_ics, use_strong_bc_ceed; PetscErrorCode (*print_info)(ProblemData *, AppCtx); }; diff --git a/examples/fluids/problems/advection.c b/examples/fluids/problems/advection.c index b8dcda66f6..df9fbd7f8e 100644 --- a/examples/fluids/problems/advection.c +++ b/examples/fluids/problems/advection.c @@ -51,8 +51,6 @@ PetscErrorCode NS_ADVECTION(ProblemData *problem, DM dm, void *ctx, SimpleBC bc) problem->apply_vol_ifunction.qfunction_loc = IFunction_Advection_loc; problem->apply_inflow.qfunction = Advection_InOutFlow; problem->apply_inflow.qfunction_loc = Advection_InOutFlow_loc; - problem->bc = Exact_Advection; - problem->bc_ctx = setup_context; problem->non_zero_time = PETSC_FALSE; problem->print_info = PRINT_ADVECTION; diff --git a/examples/fluids/problems/advection2d.c b/examples/fluids/problems/advection2d.c index f153cb7195..b074a5ae8d 100644 --- a/examples/fluids/problems/advection2d.c +++ b/examples/fluids/problems/advection2d.c @@ -49,8 +49,6 @@ PetscErrorCode NS_ADVECTION2D(ProblemData *problem, DM dm, void *ctx, SimpleBC b problem->apply_vol_ifunction.qfunction_loc = IFunction_Advection2d_loc; problem->apply_inflow.qfunction = Advection2d_InOutFlow; problem->apply_inflow.qfunction_loc = Advection2d_InOutFlow_loc; - problem->bc = Exact_Advection2d; - problem->bc_ctx = setup_context; problem->non_zero_time = PETSC_TRUE; problem->print_info = PRINT_ADVECTION2D; diff --git a/examples/fluids/problems/eulervortex.c b/examples/fluids/problems/eulervortex.c index a17305c74c..ff432054ae 100644 --- a/examples/fluids/problems/eulervortex.c +++ b/examples/fluids/problems/eulervortex.c @@ -50,8 +50,6 @@ PetscErrorCode NS_EULER_VORTEX(ProblemData *problem, DM dm, void *ctx, SimpleBC problem->apply_inflow.qfunction_loc = TravelingVortex_Inflow_loc; problem->apply_outflow.qfunction = Euler_Outflow; problem->apply_outflow.qfunction_loc = Euler_Outflow_loc; - problem->bc = Exact_Euler; - problem->bc_ctx = euler_ctx; problem->non_zero_time = PETSC_TRUE; problem->print_info = PRINT_EULER_VORTEX; diff --git a/examples/fluids/problems/newtonian.c b/examples/fluids/problems/newtonian.c index e487a71865..92379df3d0 100644 --- a/examples/fluids/problems/newtonian.c +++ b/examples/fluids/problems/newtonian.c @@ -91,8 +91,6 @@ PetscErrorCode NS_NEWTONIAN_IG(ProblemData *problem, DM dm, void *ctx, SimpleBC problem->setup_vol.qfunction_loc = Setup_loc; problem->setup_sur.qfunction = SetupBoundary; problem->setup_sur.qfunction_loc = SetupBoundary_loc; - problem->bc = NULL; - problem->bc_ctx = NULL; problem->non_zero_time = PETSC_FALSE; problem->print_info = PRINT_NEWTONIAN; diff --git a/examples/fluids/problems/shocktube.c b/examples/fluids/problems/shocktube.c index fb8cea543d..974f281b67 100644 --- a/examples/fluids/problems/shocktube.c +++ b/examples/fluids/problems/shocktube.c @@ -47,8 +47,6 @@ PetscErrorCode NS_SHOCKTUBE(ProblemData *problem, DM dm, void *ctx, SimpleBC bc) problem->apply_vol_rhs.qfunction_loc = EulerShockTube_loc; problem->apply_vol_ifunction.qfunction = NULL; problem->apply_vol_ifunction.qfunction_loc = NULL; - problem->bc = Exact_ShockTube; - problem->bc_ctx = setup_context; problem->non_zero_time = PETSC_FALSE; problem->print_info = PRINT_SHOCKTUBE; diff --git a/examples/fluids/src/setupdm.c b/examples/fluids/src/setupdm.c index 7298982787..aa9912fb5d 100644 --- a/examples/fluids/src/setupdm.c +++ b/examples/fluids/src/setupdm.c @@ -53,26 +53,22 @@ PetscErrorCode SetUpDM(DM dm, ProblemData *problem, PetscInt degree, SimpleBC bc PetscCall(DMPlexLabelComplete(dm, label)); // Set wall BCs if (bc->num_wall > 0) { - PetscCall(DMAddBoundary(dm, DM_BC_ESSENTIAL, "wall", label, bc->num_wall, bc->walls, 0, bc->num_comps, bc->wall_comps, - (void (*)(void))problem->bc, NULL, problem->bc_ctx, NULL)); + PetscCall(DMAddBoundary(dm, DM_BC_ESSENTIAL, "wall", label, bc->num_wall, bc->walls, 0, bc->num_comps, bc->wall_comps, NULL, NULL, NULL, NULL)); } // Set slip BCs in the x direction if (bc->num_slip[0] > 0) { PetscInt comps[1] = {1}; - PetscCall(DMAddBoundary(dm, DM_BC_ESSENTIAL, "slipx", label, bc->num_slip[0], bc->slips[0], 0, 1, comps, (void (*)(void))NULL, NULL, - problem->bc_ctx, NULL)); + PetscCall(DMAddBoundary(dm, DM_BC_ESSENTIAL, "slipx", label, bc->num_slip[0], bc->slips[0], 0, 1, comps, NULL, NULL, NULL, NULL)); } // Set slip BCs in the y direction if (bc->num_slip[1] > 0) { PetscInt comps[1] = {2}; - PetscCall(DMAddBoundary(dm, DM_BC_ESSENTIAL, "slipy", label, bc->num_slip[1], bc->slips[1], 0, 1, comps, (void (*)(void))NULL, NULL, - problem->bc_ctx, NULL)); + PetscCall(DMAddBoundary(dm, DM_BC_ESSENTIAL, "slipy", label, bc->num_slip[1], bc->slips[1], 0, 1, comps, NULL, NULL, NULL, NULL)); } // Set slip BCs in the z direction if (bc->num_slip[2] > 0) { PetscInt comps[1] = {3}; - PetscCall(DMAddBoundary(dm, DM_BC_ESSENTIAL, "slipz", label, bc->num_slip[2], bc->slips[2], 0, 1, comps, (void (*)(void))NULL, NULL, - problem->bc_ctx, NULL)); + PetscCall(DMAddBoundary(dm, DM_BC_ESSENTIAL, "slipz", label, bc->num_slip[2], bc->slips[2], 0, 1, comps, NULL, NULL, NULL, NULL)); } { PetscBool use_strongstg = PETSC_FALSE; From 0de6a49f23d50f23159378ab6ba3748e09d75511 Mon Sep 17 00:00:00 2001 From: James Wright Date: Mon, 19 Jun 2023 14:49:22 -0600 Subject: [PATCH 3/3] fluids: Use 32bit integer for file token - Also code into the token whether PETSc was coded using 32 or 64 bit integers --- examples/fluids/navierstokes.h | 4 +++- examples/fluids/src/misc.c | 15 +++++++++------ examples/fluids/src/setupts.c | 4 ++-- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/examples/fluids/navierstokes.h b/examples/fluids/navierstokes.h index 8a2058a0e7..98b7e2b7c6 100644 --- a/examples/fluids/navierstokes.h +++ b/examples/fluids/navierstokes.h @@ -397,7 +397,9 @@ PetscErrorCode SetupICsFromBinary(MPI_Comm comm, AppCtx app_ctx, Vec Q); PetscErrorCode SetBCsFromICs_NS(DM dm, Vec Q, Vec Q_loc); // Versioning token for binary checkpoints -extern const PetscInt FLUIDS_FILE_TOKEN; +extern const PetscInt32 FLUIDS_FILE_TOKEN; // for backwards compatibility +extern const PetscInt32 FLUIDS_FILE_TOKEN_32; +extern const PetscInt32 FLUIDS_FILE_TOKEN_64; // Create appropriate mass qfunction based on number of components N PetscErrorCode CreateMassQFunction(Ceed ceed, CeedInt N, CeedInt q_data_size, CeedQFunction *qf); diff --git a/examples/fluids/src/misc.c b/examples/fluids/src/misc.c index aa18537c66..9b8c5389d2 100644 --- a/examples/fluids/src/misc.c +++ b/examples/fluids/src/misc.c @@ -92,13 +92,15 @@ PetscErrorCode DMPlexInsertBoundaryValues_NS(DM dm, PetscBool insert_essential, // @brief Load vector from binary file, possibly with embedded solution time and step number PetscErrorCode LoadFluidsBinaryVec(MPI_Comm comm, PetscViewer viewer, Vec Q, PetscReal *time, PetscInt *step_number) { - PetscInt token, file_step_number; - PetscReal file_time; + PetscInt file_step_number; + PetscInt32 token; + PetscReal file_time; PetscFunctionBeginUser; // Attempt - PetscCall(PetscViewerBinaryRead(viewer, &token, 1, NULL, PETSC_INT)); - if (token == FLUIDS_FILE_TOKEN) { // New style format; we're reading a file with step number and time in the header + PetscCall(PetscViewerBinaryRead(viewer, &token, 1, NULL, PETSC_INT32)); + if (token == FLUIDS_FILE_TOKEN_32 || token == FLUIDS_FILE_TOKEN_64 || + token == FLUIDS_FILE_TOKEN) { // New style format; we're reading a file with step number and time in the header PetscCall(PetscViewerBinaryRead(viewer, &file_step_number, 1, NULL, PETSC_INT)); PetscCall(PetscViewerBinaryRead(viewer, &file_time, 1, NULL, PETSC_REAL)); if (time) *time = file_time; @@ -204,11 +206,12 @@ PetscErrorCode PostProcess_NS(TS ts, CeedData ceed_data, DM dm, ProblemData *pro if (user->app_ctx->test_type == TESTTYPE_SOLVER) { PetscCall(RegressionTests_NS(user->app_ctx, Q)); } - PetscFunctionReturn(PETSC_SUCCESS); } -const PetscInt FLUIDS_FILE_TOKEN = 0xceedf00; +const PetscInt32 FLUIDS_FILE_TOKEN = 0xceedf00; // for backwards compatibility +const PetscInt32 FLUIDS_FILE_TOKEN_32 = 0xceedf32; +const PetscInt32 FLUIDS_FILE_TOKEN_64 = 0xceedf64; // Gather initial Q values in case of continuation of simulation PetscErrorCode SetupICsFromBinary(MPI_Comm comm, AppCtx app_ctx, Vec Q) { diff --git a/examples/fluids/src/setupts.c b/examples/fluids/src/setupts.c index 114a4c4b46..f789f58f72 100644 --- a/examples/fluids/src/setupts.c +++ b/examples/fluids/src/setupts.c @@ -365,8 +365,8 @@ PetscErrorCode WriteOutput(User user, Vec Q, PetscInt step_no, PetscScalar time) } PetscCall(PetscViewerBinaryOpen(user->comm, file_path, FILE_MODE_WRITE, &viewer)); - PetscInt token = FLUIDS_FILE_TOKEN; - PetscCall(PetscViewerBinaryWrite(viewer, &token, 1, PETSC_INT)); + PetscInt32 token = PetscDefined(USE_64BIT_INDICES) ? FLUIDS_FILE_TOKEN_64 : FLUIDS_FILE_TOKEN_32; + PetscCall(PetscViewerBinaryWrite(viewer, &token, 1, PETSC_INT32)); PetscCall(PetscViewerBinaryWrite(viewer, &step_no, 1, PETSC_INT)); time /= user->units->second; // Dimensionalize time back PetscCall(PetscViewerBinaryWrite(viewer, &time, 1, PETSC_REAL));