diff --git a/src/api/pdc_region/pdc_region_transfer.c b/src/api/pdc_region/pdc_region_transfer.c index 87f9f3625..1ffb0f84c 100644 --- a/src/api/pdc_region/pdc_region_transfer.c +++ b/src/api/pdc_region/pdc_region_transfer.c @@ -1529,25 +1529,29 @@ release_region_buffer(char *buf, uint64_t *obj_dims, int local_ndim, uint64_t *l if (local_ndim == 2) { if (access_type == PDC_READ) { ptr = new_buf; - for (i = 0; i < local_size[0]; ++i) { - memcpy(buf + ((local_offset[0] + i) * obj_dims[1] + local_offset[1]) * unit, ptr, - local_size[1] * unit); - ptr += local_size[1] * unit; - } + // Tang + memcpy(buf, ptr, local_size[0] * local_size[1] * unit); + /* for (i = 0; i < local_size[0]; ++i) { */ + /* memcpy(buf + ((local_offset[0] + i) * obj_dims[1] + local_offset[1]) * unit, ptr, */ + /* local_size[1] * unit); */ + /* ptr += local_size[1] * unit; */ + /* } */ } } else if (local_ndim == 3) { if (access_type == PDC_READ) { ptr = new_buf; - for (i = 0; i < local_size[0]; ++i) { - for (j = 0; j < local_size[1]; ++j) { - memcpy(buf + ((local_offset[0] + i) * obj_dims[1] * obj_dims[2] + - (local_offset[1] + j) * obj_dims[2] + local_offset[2]) * - unit, - ptr, local_size[2] * unit); - ptr += local_size[2] * unit; - } - } + // Tang + memcpy(buf, ptr, local_size[0] * local_size[1] * local_size[2] * unit); + /* for (i = 0; i < local_size[0]; ++i) { */ + /* for (j = 0; j < local_size[1]; ++j) { */ + /* memcpy(buf + ((local_offset[0] + i) * obj_dims[1] * obj_dims[2] + */ + /* (local_offset[1] + j) * obj_dims[2] + local_offset[2]) * */ + /* unit, */ + /* ptr, local_size[2] * unit); */ + /* ptr += local_size[2] * unit; */ + /* } */ + /* } */ } } if (bulk_buf_ref) { diff --git a/src/server/pdc_server_region/pdc_server_data.c b/src/server/pdc_server_region/pdc_server_data.c index 1a49d6758..86f0ff611 100644 --- a/src/server/pdc_server_region/pdc_server_data.c +++ b/src/server/pdc_server_region/pdc_server_data.c @@ -5229,10 +5229,33 @@ PDC_Server_data_read_from(uint64_t obj_id, struct pdc_region_info *region_info, #ifdef PDC_TIMING start_posix = MPI_Wtime(); #endif - if (pread(region->fd, tmp_buf, overlap_region->data_size, overlap_region->offset) != - (ssize_t)overlap_region->data_size) { - printf("==PDC_SERVER[%d]: pread failed to read enough bytes %d\n", pdc_server_rank_g, - __LINE__); + // Read up to 1GB at a time + uint64_t read_max = 1*1024*1024*1024llu; + if (overlap_region->data_size > read_max) { + uint64_t buf_off = 0; + uint64_t reg_off = overlap_region->offset; + uint64_t leftover = overlap_region->data_size; + ssize_t read_size = read_max; + + while (leftover > 0) { + /* printf("==PDC_SERVER[%d]: pread %llu, leftover %llu\n", pdc_server_rank_g, read_size, leftover); */ + if (pread(region->fd, tmp_buf+buf_off, read_size, reg_off) != read_size) { + printf("==PDC_SERVER[%d]: pread failed to read enough bytes %llu, LINE %d\n", + pdc_server_rank_g, read_size, __LINE__); + } + reg_off += read_size; + buf_off += read_size; + leftover -= read_size; + if (leftover < read_size) + read_size = leftover; + } + } + else { + if (pread(region->fd, tmp_buf, overlap_region->data_size, overlap_region->offset) != + (ssize_t)overlap_region->data_size) { + printf("==PDC_SERVER[%d]: pread failed to read enough bytes %d\n", pdc_server_rank_g, + __LINE__); + } } #ifdef PDC_TIMING pdc_server_timings->PDCdata_server_read_posix += MPI_Wtime() - start_posix; diff --git a/src/tools/pdc_access_eqsim.c b/src/tools/pdc_access_eqsim.c index 4505ccd69..f44459e44 100644 --- a/src/tools/pdc_access_eqsim.c +++ b/src/tools/pdc_access_eqsim.c @@ -18,7 +18,7 @@ main(int argc, char **argv) opensees_size, round = 1; int start_x[4096], start_y[4096]; uint64_t pdc_dims[3], pdc_offset[3], pdc_size[3], pdc_local_offset[3], pdc_local_size[3]; - uint32_t value_size; + psize_t value_size; // 12x, 32x, 32x char * fname, tag_name[128]; uint64_t dims[4] = {4634, 19201, 12801, 1}, chunk_size[4] = {400, 600, 400, 1}; diff --git a/src/tools/pdc_import_eqsim.c b/src/tools/pdc_import_eqsim.c index 4e30b4814..831549fc9 100644 --- a/src/tools/pdc_import_eqsim.c +++ b/src/tools/pdc_import_eqsim.c @@ -20,7 +20,7 @@ main(int argc, char **argv) herr_t status; int i, j, count, total_count, rank, nproc, ssi_downsample, rec_downsample, batchsize, iter, opensees_size; - int start_x[4096], start_y[4096]; + int start_x[4096], start_y[4096], max_start_x=0, max_start_y=0; hsize_t offset[4], size[4], local_offset[4], local_size[4]; hsize_t dims[4] = {4634, 19201, 12801, 1}, chunk_size[4] = {400, 600, 400, 1}; uint64_t pdc_dims[3], pdc_offset[3], pdc_size[3], pdc_local_offset[3], pdc_local_size[3]; @@ -117,8 +117,19 @@ main(int argc, char **argv) // End of HDF5 operations + for (i = 0; i < nproc; i++) { + if (start_x[i] > max_start_x) + max_start_x = start_x[i]; + if (start_y[i] > max_start_y) + max_start_y = start_y[i]; + } + pdc_dims[0] = dims[0]; + pdc_dims[1] = max_start_x + chunk_size[1]; + pdc_dims[2] = max_start_y + chunk_size[2]; + fprintf(stderr, "Rank %d: create obj dims %llu %llu %llu\n", rank, pdc_dims[0], pdc_dims[1], pdc_dims[2]); + + for (i = 0; i < 3; i++) { - pdc_dims[i] = PDC_SIZE_UNLIMITED; pdc_offset[i] = (uint64_t)offset[i]; pdc_size[i] = (uint64_t)size[i]; pdc_local_offset[i] = (uint64_t)local_offset[i];