fix an issue with 3d read segfault

hpc-io · Oct 10, 2024 · 4c9d2bb · 4c9d2bb
1 parent b3f618f
commit 4c9d2bb
Show file tree

Hide file tree

Showing 4 changed files with 59 additions and 21 deletions.
diff --git a/src/api/pdc_region/pdc_region_transfer.c b/src/api/pdc_region/pdc_region_transfer.c
@@ -1529,25 +1529,29 @@ release_region_buffer(char *buf, uint64_t *obj_dims, int local_ndim, uint64_t *l
     if (local_ndim == 2) {
         if (access_type == PDC_READ) {
             ptr = new_buf;
-            for (i = 0; i < local_size[0]; ++i) {
-                memcpy(buf + ((local_offset[0] + i) * obj_dims[1] + local_offset[1]) * unit, ptr,
-                       local_size[1] * unit);
-                ptr += local_size[1] * unit;
-            }
+            // Tang
+            memcpy(buf, ptr, local_size[0] * local_size[1] * unit);
+            /* for (i = 0; i < local_size[0]; ++i) { */
+            /*     memcpy(buf + ((local_offset[0] + i) * obj_dims[1] + local_offset[1]) * unit, ptr, */
+            /*            local_size[1] * unit); */
+            /*     ptr += local_size[1] * unit; */
+            /* } */
         }
     }
     else if (local_ndim == 3) {
         if (access_type == PDC_READ) {
             ptr = new_buf;
-            for (i = 0; i < local_size[0]; ++i) {
-                for (j = 0; j < local_size[1]; ++j) {
-                    memcpy(buf + ((local_offset[0] + i) * obj_dims[1] * obj_dims[2] +
-                                  (local_offset[1] + j) * obj_dims[2] + local_offset[2]) *
-                                     unit,
-                           ptr, local_size[2] * unit);
-                    ptr += local_size[2] * unit;
-                }
-            }
+            // Tang
+            memcpy(buf, ptr, local_size[0] * local_size[1] * local_size[2] * unit);
+            /* for (i = 0; i < local_size[0]; ++i) { */
+            /*     for (j = 0; j < local_size[1]; ++j) { */
+            /*         memcpy(buf + ((local_offset[0] + i) * obj_dims[1] * obj_dims[2] + */
+            /*                       (local_offset[1] + j) * obj_dims[2] + local_offset[2]) * */
+            /*                          unit, */
+            /*                ptr, local_size[2] * unit); */
+            /*         ptr += local_size[2] * unit; */
+            /*     } */
+            /* } */
         }
     }
     if (bulk_buf_ref) {

diff --git a/src/server/pdc_server_region/pdc_server_data.c b/src/server/pdc_server_region/pdc_server_data.c
@@ -5229,10 +5229,33 @@ PDC_Server_data_read_from(uint64_t obj_id, struct pdc_region_info *region_info,
 #ifdef PDC_TIMING
                     start_posix = MPI_Wtime();
 #endif
-                    if (pread(region->fd, tmp_buf, overlap_region->data_size, overlap_region->offset) !=
-                        (ssize_t)overlap_region->data_size) {
-                        printf("==PDC_SERVER[%d]: pread failed to read enough bytes %d\n", pdc_server_rank_g,
-                               __LINE__);
+                    // Read up to 1GB at a time
+                    uint64_t read_max = 1*1024*1024*1024llu;
+                    if (overlap_region->data_size > read_max) {
+                        uint64_t buf_off   = 0;
+                        uint64_t reg_off   = overlap_region->offset;
+                        uint64_t leftover  = overlap_region->data_size;
+                        ssize_t  read_size = read_max;
+
+                        while (leftover > 0) {
+                            /* printf("==PDC_SERVER[%d]: pread %llu, leftover %llu\n", pdc_server_rank_g, read_size, leftover); */
+                            if (pread(region->fd, tmp_buf+buf_off, read_size, reg_off) != read_size) {
+                                printf("==PDC_SERVER[%d]: pread failed to read enough bytes %llu, LINE %d\n", 
+                                        pdc_server_rank_g, read_size, __LINE__);
+                            }
+                            reg_off += read_size;
+                            buf_off += read_size;
+                            leftover -= read_size;
+                            if (leftover < read_size)
+                                read_size = leftover;
+                        }
+                    }
+                    else {
+                        if (pread(region->fd, tmp_buf, overlap_region->data_size, overlap_region->offset) !=
+                            (ssize_t)overlap_region->data_size) {
+                            printf("==PDC_SERVER[%d]: pread failed to read enough bytes %d\n", pdc_server_rank_g,
+                                   __LINE__);
+                        }
                     }
 #ifdef PDC_TIMING
                     pdc_server_timings->PDCdata_server_read_posix += MPI_Wtime() - start_posix;

diff --git a/src/tools/pdc_access_eqsim.c b/src/tools/pdc_access_eqsim.c
@@ -18,7 +18,7 @@ main(int argc, char **argv)
         opensees_size, round = 1;
     int      start_x[4096], start_y[4096];
     uint64_t pdc_dims[3], pdc_offset[3], pdc_size[3], pdc_local_offset[3], pdc_local_size[3];
-    uint32_t value_size;
+    psize_t value_size;
     // 12x, 32x, 32x
     char *   fname, tag_name[128];
     uint64_t dims[4] = {4634, 19201, 12801, 1}, chunk_size[4] = {400, 600, 400, 1};

diff --git a/src/tools/pdc_import_eqsim.c b/src/tools/pdc_import_eqsim.c
@@ -20,7 +20,7 @@ main(int argc, char **argv)
     herr_t status;
 
     int i, j, count, total_count, rank, nproc, ssi_downsample, rec_downsample, batchsize, iter, opensees_size;
-    int start_x[4096], start_y[4096];
+    int start_x[4096], start_y[4096], max_start_x=0, max_start_y=0;
     hsize_t  offset[4], size[4], local_offset[4], local_size[4];
     hsize_t  dims[4] = {4634, 19201, 12801, 1}, chunk_size[4] = {400, 600, 400, 1};
     uint64_t pdc_dims[3], pdc_offset[3], pdc_size[3], pdc_local_offset[3], pdc_local_size[3];
@@ -117,8 +117,19 @@ main(int argc, char **argv)
 
     // End of HDF5 operations
 
+    for (i = 0; i < nproc; i++) {
+        if (start_x[i] > max_start_x)
+            max_start_x = start_x[i];
+        if (start_y[i] > max_start_y)
+            max_start_y = start_y[i];
+    }
+    pdc_dims[0] = dims[0];
+    pdc_dims[1] = max_start_x + chunk_size[1];
+    pdc_dims[2] = max_start_y + chunk_size[2];
+    fprintf(stderr, "Rank %d: create obj dims %llu %llu %llu\n", rank, pdc_dims[0], pdc_dims[1], pdc_dims[2]);
+
+
     for (i = 0; i < 3; i++) {
-        pdc_dims[i]         = PDC_SIZE_UNLIMITED;
         pdc_offset[i]       = (uint64_t)offset[i];
         pdc_size[i]         = (uint64_t)size[i];
         pdc_local_offset[i] = (uint64_t)local_offset[i];