From 8798b005aec992eb24eafce28343c17a8a48cf1d Mon Sep 17 00:00:00 2001 From: kamalchowdhurylbl Date: Tue, 9 Aug 2022 21:17:23 -0500 Subject: [PATCH 01/18] added async vol file --- .DS_Store | Bin 0 -> 6148 bytes src/h5_async_vol.c | 1183 ++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 1132 insertions(+), 51 deletions(-) create mode 100644 .DS_Store diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..e68e8836983384f6deb19cb98b9c994c23fc7a5a GIT binary patch literal 6148 zcmeHK%}T>S5Z-O8O({YS3VK`cTClCP7cU{!7cim+m718M!I&*cY7V84yS|Vw;`2DO zy8(+ii`W_1{pNQ!`$6`HF~;3RIAF|Xj9JhSIVv@R?%GhpBqMShBbx`Y48Zyb<|g*n z0l&S;GM2K3p!oj%ag^o#{wHrVTU*<0t8I0yJMT#rUhe0M%=Ob7v@WHLgG%>d zW5McfGz+wNVt^Q^V*vLD0S(bHSZY*T2XuIS#&`=61$2B%APR$y!BQhcK)5ai)TP`! zF}N-Vzc6`@!BV3xXI#w;bO81@*IPuMjQqG RsvM9m0*Vmoh=E^V;0tT^Nwxq0 literal 0 HcmV?d00001 diff --git a/src/h5_async_vol.c b/src/h5_async_vol.c index 6b5733e..9de87ba 100644 --- a/src/h5_async_vol.c +++ b/src/h5_async_vol.c @@ -1,5 +1,5 @@ /******************************************************************************* -Asynchronous I/O VOL Connector (AsyncVOL) Copyright (c) 2021, The +Asynchronous I/O VOL kamal Connector (AsyncVOL) Copyright (c) 2021, The Regents of the University of California, through Lawrence Berkeley National Laboratory (subject to receipt of any required approvals from the U.S. Dept. of Energy). All rights reserved. @@ -65,7 +65,7 @@ works, and perform publicly and display publicly, and to permit others to do so. /* Whether to display log messge when callback is invoked */ /* (Uncomment to enable) */ /* #define ENABLE_DBG_MSG 1 */ -/* #define PRINT_ERROR_STACK 1 */ + #define PRINT_ERROR_STACK 1 /* #define ENABLE_ASYNC_LOGGING */ #define ASYNC_DBG_MSG_RANK 0 @@ -131,6 +131,7 @@ typedef struct async_task_t { ABT_eventual eventual; int in_abt_pool; int is_done; + int is_merge; ABT_thread abt_thread; hid_t err_stack; int n_dep; @@ -196,17 +197,16 @@ typedef struct async_instance_t { int nfopen; int mpi_size; int mpi_rank; - bool ex_delay; /* Delay background thread execution */ - bool ex_fclose; /* Delay background thread execution until file close */ - bool ex_gclose; /* Delay background thread execution until group close */ - bool ex_dclose; /* Delay background thread execution until dset close */ - bool start_abt_push; /* Start pushing tasks to Argobots pool */ - bool prev_push_state; /* Previous state of start_abt_push before a change*/ - bool pause; /* Pause background thread execution */ - bool disable_implicit_file; /* Disable implicit async execution globally */ - bool disable_implicit; /* Disable implicit async execution for dxpl */ - bool delay_time_env; /* Flag that indicates the delay time is set by env variable */ - bool disable_async_dset_get; /* Disable async execution for dataset get */ + bool ex_delay; /* Delay background thread execution */ + bool ex_fclose; /* Delay background thread execution until file close */ + bool ex_gclose; /* Delay background thread execution until group close */ + bool ex_dclose; /* Delay background thread execution until dset close */ + bool start_abt_push; /* Start pushing tasks to Argobots pool */ + bool prev_push_state; /* Previous state of start_abt_push before a change*/ + bool pause; /* Pause background thread execution */ + bool disable_implicit_file; /* Disable implicit async execution globally */ + bool disable_implicit; /* Disable implicit async execution for dxpl */ + bool delay_time_env; /* Flag that indicates the delay time is set by env variable */ uint64_t delay_time; /* Sleep time before background thread trying to acquire global mutex */ int sleep_time; /* Sleep time between checking the global mutex attemp count */ #ifdef ENABLE_WRITE_MEMCPY @@ -256,8 +256,9 @@ typedef struct async_attr_write_args_t { void * buf; hid_t dxpl_id; void **req; -#ifdef ENABLE_WRITE_MEMCPY + bool free_buf; +#ifdef ENABLE_WRITE_MEMCPY hsize_t data_size; #endif } async_attr_write_args_t; @@ -331,8 +332,9 @@ typedef struct async_dataset_write_args_t { hid_t plist_id; void * buf; void **req; -#ifdef ENABLE_WRITE_MEMCPY + bool free_buf; +#ifdef ENABLE_WRITE_MEMCPY hsize_t data_size; #endif } async_dataset_write_args_t; @@ -1284,21 +1286,20 @@ async_instance_init(int backing_thread_count) } // end for } // end else - aid->pool = pool; - aid->xstreams = progress_xstreams; - aid->num_xstreams = backing_thread_count; - aid->progress_scheds = progress_scheds; - aid->nfopen = 0; - aid->ex_delay = false; - aid->ex_fclose = false; - aid->ex_gclose = false; - aid->ex_dclose = false; - aid->pause = false; - aid->start_abt_push = false; - aid->disable_implicit = false; - aid->disable_implicit_file = false; - aid->delay_time_env = false; - aid->disable_async_dset_get = true; + aid->pool = pool; + aid->xstreams = progress_xstreams; + aid->num_xstreams = backing_thread_count; + aid->progress_scheds = progress_scheds; + aid->nfopen = 0; + aid->ex_delay = false; + aid->ex_fclose = false; + aid->ex_gclose = false; + aid->ex_dclose = false; + aid->pause = false; + aid->start_abt_push = false; + aid->disable_implicit = false; + aid->disable_implicit_file = false; + aid->delay_time_env = false; // Check for delaying operations to file / group / dataset close operations env_var = getenv("HDF5_ASYNC_EXE_FCLOSE"); @@ -1321,10 +1322,6 @@ async_instance_init(int backing_thread_count) if (aid->ex_fclose || aid->ex_gclose || aid->ex_dclose) aid->ex_delay = true; - env_var = getenv("HDF5_ASYNC_DISABLE_DSET_GET"); - if (env_var && *env_var && atoi(env_var) <= 0) - aid->disable_async_dset_get = false; - #ifdef ENABLE_WRITE_MEMCPY // Get max memory allowed for async memcpy env_var = getenv("HDF5_ASYNC_MAX_MEM_MB"); @@ -8863,8 +8860,12 @@ async_dataset_write_fn(void *foo) assert(task); assert(task->async_obj); assert(task->async_obj->magic == ASYNC_MAGIC); - + //fprintf(stderr,"task is_merge=%d\n",task->is_merge); pool_ptr = task->async_obj->pool_ptr; + if (task->is_merge == 1) + goto done; + + func_log(__func__, "trying to aquire global lock"); @@ -8939,7 +8940,7 @@ async_dataset_write_fn(void *foo) usleep(1000); count++; } - + //fprintf(stderr," args mem_type_id=%llu\n",args->mem_type_id); /* Try executing operation, without default error stack handling */ H5E_BEGIN_TRY { @@ -8967,8 +8968,8 @@ async_dataset_write_fn(void *foo) fprintf(fout_g, " [ASYNC ABT ERROR] %s H5VLfree_lib_state failed\n", __func__); task->h5_state = NULL; - if (args->mem_type_id > 0) - H5Tclose(args->mem_type_id); + //if (args->mem_type_id > 0) + // H5Tclose(args->mem_type_id); if (args->mem_space_id > 0) H5Sclose(args->mem_space_id); if (args->file_space_id > 0) @@ -8993,17 +8994,19 @@ async_dataset_write_fn(void *foo) if (async_instance_g && NULL != async_instance_g->qhead.queue && async_instance_g->start_abt_push) push_task_to_abt_pool(&async_instance_g->qhead, *pool_ptr, __func__); -#ifdef ENABLE_WRITE_MEMCPY + if (args->free_buf && args->buf) { free(args->buf); +#ifdef ENABLE_WRITE_MEMCPY async_instance_g->used_mem -= args->data_size; #ifdef ENABLE_DBG_MSG if (async_instance_g && (async_instance_g->mpi_rank == ASYNC_DBG_MSG_RANK || -1 == ASYNC_DBG_MSG_RANK)) fprintf(fout_g, " [ASYNC ABT DBG] %s released dset memcpy\n", __func__); #endif + #endif } -#endif + #ifdef ENABLE_TIMING task->end_time = clock(); @@ -9043,6 +9046,1084 @@ is_contig_memspace(hid_t memspace) } #endif +int +print_dataspace(hid_t mem_space_id) +{ + int ndim; + int num_elements; + hsize_t nblocks; + int N1 = 20; + hsize_t dimsm[1]; + hsize_t *buffer; + + H5S_sel_type type; + herr_t slected_block; + hssize_t numblocks; + async_dataset_write_args_t *args = NULL; + + hsize_t *start_out, *stride_out, *count_out, *block_out; + + herr_t status; + async_task_t * task_iter; + async_task_list_t *task_list_iter; + + if (mem_space_id == H5S_SEL_ALL) + fprintf(stderr, "-----H5S_SEL_ALL----\n"); + else if (mem_space_id == H5S_SEL_NONE) { + fprintf(stderr, "-----H5S_SEL_NONE----\n"); + return 0; + } + + type = H5Sget_select_type(mem_space_id); + + /* if(type==H5S_SEL_ERROR) + fprintf(stderr,"-----Error----\n"); + else if(type==H5S_SEL_NONE) + fprintf(stderr,"-----Empty selection-----\n"); + else if(type==H5S_SEL_POINTS) + fprintf(stderr,"-----Set of points-----\n"); + else if(type==H5S_SEL_HYPERSLABS) + fprintf(stderr,"-----Hyperslab-----\n"); + + else if(type==H5S_SEL_ALL) + fprintf(stderr,"-----Everything-----\n"); + else if(type==H5S_SEL_N) + fprintf(stderr,"-----Sentinel-----cd\n"); + */ + + // fprintf(stderr,"this is after H5Sget_select_type %d\n",type); + if (type == H5S_SEL_POINTS) { + return 0; + } + else if (type == H5S_SEL_HYPERSLABS) { + ndim = H5Sget_simple_extent_ndims(mem_space_id); + // fprintf(stderr, "ndim=%d\n",ndim); + // if (ndim != 1) + // return 0; + + if (H5Sis_regular_hyperslab(mem_space_id)) { + start_out = malloc(ndim * sizeof(hsize_t)); + stride_out = malloc(ndim * sizeof(hsize_t)); + count_out = malloc(ndim * sizeof(hsize_t)); + block_out = malloc(ndim * sizeof(hsize_t)); + + status = H5Sget_regular_hyperslab(mem_space_id, start_out, stride_out, count_out, block_out); + if (ndim == 1) { + + fprintf(stderr, " start = [%llu] \n", (unsigned long long)start_out[0]); + // fprintf(stderr, " stride = [%llu] \n", (unsigned long long)stride_out[0]); + fprintf(stderr, " count = [%llu] \n", (unsigned long long)count_out[0]); + // fprintf(stderr, " block = [%llu] \n", (unsigned long long)block_out[0]); + // fprintf(stderr, "Hyperslab operation on dataspace using start=%llu and + // count=%llu\n",start_out[0],count_out[0]); fflush(stdout); + } + else if (ndim == 2) { + + fprintf(stderr, " start = [%llu, %llu] \n", (unsigned long long)start_out[0], + (unsigned long long)start_out[1]); + // fprintf(stderr, " stride = [%llu, %llu] \n", (unsigned long long)stride_out[0], + // (unsigned long long)stride_out[1]); + fprintf(stderr, " count = [%llu, %llu] \n", (unsigned long long)count_out[0], + (unsigned long long)count_out[1]); + // fprintf(stderr, " block = [%llu, %llu] \n", (unsigned long long)block_out[0], + // (unsigned long long)block_out[1]); + } + else if (ndim == 3) { + + fprintf(stderr, " start = [%llu, %llu, %llu] \n", (unsigned long long)start_out[0], + (unsigned long long)start_out[1], (unsigned long long)start_out[2]); + // fprintf(stderr, " stride = [%llu, %llu, %llu] \n", (unsigned long + // long)stride_out[0], (unsigned long long)stride_out[1], (unsigned long long)stride_out[2]); + fprintf(stderr, " count = [%llu, %llu, %llu] \n", (unsigned long long)count_out[0], + (unsigned long long)count_out[1], (unsigned long long)count_out[2]); + // fprintf(stderr, " block = [%llu, %llu, %llu] \n", (unsigned long + // long)block_out[0], (unsigned long long)block_out[1], (unsigned long long)block_out[2]); + } + + free(start_out); + free(stride_out); + free(count_out); + free(block_out); + } + else { + + nblocks = H5Sget_select_hyper_nblocks(mem_space_id); + num_elements = nblocks * ndim * 2; + buffer = (hsize_t *)malloc(num_elements * sizeof(hsize_t)); + + slected_block = H5Sget_select_hyper_blocklist(mem_space_id, (hsize_t)0, nblocks, buffer); + if (ndim == 1) { + // for(int i=0;iqhead.queue, task_list_iter) + { + DL_FOREACH(task_list_iter->task_list, task_iter) + { + if (task_iter->func == async_dataset_write_fn) { + // checking whether the iterator task is dataset write operation + if (task_iter->parent_obj == parent_obj && task_iter->is_done != 1) { + // checking whether the iterator task is operating on the same dataset of the current task + + iter_args = task_iter->args; + + // fprintf(stderr,"%lld %lld + // %lld\n",task_iter->async_obj->under_object,iter_args->mem_space_id,iter_args->file_space_id); + /* fprintf(stderr,"For iterator task file space:\n"); + print_dataspace(iter_args->file_space_id); + fprintf(stderr,"For current task file space:\n"); + print_dataspace(file_space_id); */ + + hid_t new_memspace; + + ndim = H5Sget_simple_extent_ndims(file_space_id); + dimsm = malloc(ndim * sizeof(hsize_t)); + start = malloc(ndim * sizeof(hsize_t)); + count = malloc(ndim * sizeof(hsize_t)); + mem_start = malloc(ndim * sizeof(hsize_t)); + mem_count = malloc(ndim * sizeof(hsize_t)); + file_start = malloc(ndim * sizeof(hsize_t)); + file_count = malloc(ndim * sizeof(hsize_t)); + iter_mem_start = malloc(ndim * sizeof(hsize_t)); + iter_mem_count = malloc(ndim * sizeof(hsize_t)); + iter_file_start = malloc(ndim * sizeof(hsize_t)); + iter_file_count = malloc(ndim * sizeof(hsize_t)); + + + + + if (check_contiguous(file_space_id, iter_args->file_space_id, start, count) || + check_contiguous(iter_args->file_space_id, file_space_id, start, count)) + // if(check_contiguous(file_space_id,iter_args->file_space_id,start,count)) + { + return_val = 1; + // fprintf(stderr, "\n-------#### contiguous####---------\n"); + + + /* if(prev_task!=NULL){ + continue; + } */ + if (prev_task != NULL) + prev_task->is_merge = 1; + return_task_val = task_iter; + + // fprintf(stderr, "\n-------#### 2 contiguous####---------\n"); + if (ndim == 1) { // push_contiguous(ndim,start,count); + + //fprintf(stderr, "\n start=%llu count=%llu\n", start[0], count[0]); + // H5Sclose(iter_args->file_space_id);// close the file space + + /* if(H5Smodify_select(file_space_id,H5S_SELECT_SET,iter_args->file_space_id)< 0) + {fprintf(stderr,"Error in H5Smodify_select");} */ + + // H5Sclose(iter_args->file_space_id); + new_memspace = H5Screate_simple(ndim, count, NULL); + element_size = H5Tget_size(mem_type_id); + fprintf(stderr,"size of new_buffer= %lld \n",count[0]*element_size); + + new_buffer = malloc(count[0] * element_size); + + + + /* int *buf_temp = buf; + int *new_buffer_temp = new_buffer; */ + + status = H5Sget_regular_hyperslab(mem_space_id, mem_start, NULL, mem_count, NULL); + status = + H5Sget_regular_hyperslab(file_space_id, file_start, NULL, file_count, NULL); + + //fprintf(stderr,"\nmem_start=%lld mem_count=%lld file_start=%lld file_count=%lld element_size=%lld\n",mem_start[0],mem_count[0],file_start[0],file_count[0],element_size); + fprintf(stderr, "\n mem_start=%lld file_start=%lld ", mem_start[0], \ + file_start[0]); + fprintf(stderr, "\n mem_count=%lld file_count=%lld \n", mem_count[0], \ + file_count[0]); + status = H5Sget_regular_hyperslab(iter_args->mem_space_id, iter_mem_start, NULL, + iter_mem_count, NULL); + /* fprintf(stderr,"%d\n",status); + if (H5Sis_regular_hyperslab(iter_args->mem_space_id)) { + fprintf(stderr,"regular hyperslab\n"); + } */ + status = H5Sget_regular_hyperslab(iter_args->file_space_id, iter_file_start, NULL, + iter_file_count, NULL); + // fprintf(stderr,"%d\n",status); + //fprintf(stderr, "\n iter_mem_start=%lld iter_file_start=%lld ", iter_mem_start[0], + // iter_file_start[0]); + //fprintf(stderr, "\n iter_mem_count=%lld iter_file_count=%lld \n", + // iter_mem_count[0], iter_file_count[0]); + if (file_start[0] >= iter_file_start[0]) { + // fprintf(stderr,"file_start[0]>=iter_file_start[0]\n"); + memcpy(new_buffer, buf + iter_file_start[0] * element_size, + iter_file_count[0] * element_size); + memcpy(new_buffer + iter_file_count[0] * element_size, + buf + file_start[0] * element_size, file_count[0] * element_size); + } + else { + // fprintf(stderr,"not file_start[0]>=iter_file_start[0]\n"); + + memcpy(new_buffer, buf + (file_start[0] * element_size), + file_count[0] * element_size); + memcpy(new_buffer + file_count[0] * element_size, + buf + (iter_file_start[0] * element_size), + iter_file_count[0] * element_size); + } + + // memcpy(new_buffer+(file_start[0]*element_size),buf+(mem_start[0]*element_size),mem_count[0]*element_size); + // memcpy(new_buffer,buf,count[0]*element_size); + + /*allocate new_buffer + get the start and count value from memspace + based on the start and count copy the data from buf to new buffer + need to be corect locations + do the same thing for iter_args->buf and iter_args->memspace. + */ + + H5Sclose(iter_args->mem_space_id); + // H5Sclose(iter_args->file_space_id); + + iter_args->mem_space_id = new_memspace; + // iter_args->file_space_id=new_memspace; + + if (iter_args->free_buf == 1) + free(iter_args->buf); + + iter_args->buf = new_buffer; + iter_args->free_buf=1; + + /* if(check_contiguous_overlap(ndim,start,count)) + {fprintf(stderr, "\ncontiguous overlap\n"); + + fprintf(stderr,"\n after contiguous overlap start=%llu + count=%llu\n",start[0],count[0]); + } */ + status = + H5Sselect_hyperslab(file_space_id, H5S_SELECT_SET, start, NULL, count, NULL); + + status = H5Sselect_hyperslab(iter_args->file_space_id, H5S_SELECT_SET, start, + NULL, count, NULL); + + start[0] = 0; + // fprintf(stderr,"\n iter mem space mem_start=%lld mem_count=%lld file_start=%lld + // file_count=%lld + // element_size=%lld\n",start[0],count[0],start[0],count[0],element_size); + + status = H5Sselect_hyperslab(iter_args->mem_space_id, H5S_SELECT_SET, start, NULL, + count, NULL); + /* for (int i = 0; i < 16; i++) + fprintf(stderr, " buf[%d]=%d ", i, buf_temp[i]); + fprintf(stderr, "\n"); + + for (int i = 0; i < 16; i++) + fprintf(stderr, " new_buffer[%d]=%d ", i, new_buffer_temp[i]); + fprintf(stderr, "\n"); */ + //free(new_buffer); + } + else if (ndim == 2) { + + fprintf(stderr, " start=%llux%llu count=%llux%llu\n", start[0], start[1], + count[0], count[1]); + new_memspace = H5Screate_simple(ndim, count, NULL); + element_size = H5Tget_size(mem_type_id); + new_buffer = malloc(count[0] * count[1] * element_size); + int *buf_temp = buf; + int *new_buffer_temp = new_buffer; + + status = H5Sget_regular_hyperslab(mem_space_id, mem_start, NULL, mem_count, NULL); + status = + H5Sget_regular_hyperslab(file_space_id, file_start, NULL, file_count, NULL); + + // fprintf(stderr,"\nmem_start=%lld mem_count=%lld file_start=%lld file_count=%lld + // element_size=%lld\n",mem_start[0],mem_count[0],file_start[0],file_count[0],element_size); + fprintf(stderr, "\n mem_start=%lldx%lld file_start=%lldx%lld ", mem_start[0], + mem_start[1], file_start[0], file_start[1]); + fprintf(stderr, "\n mem_count=%lldx%lld file_count=%lldx%lld \n", mem_count[0], + mem_count[1], file_count[0], file_count[1]); + status = H5Sget_regular_hyperslab(iter_args->mem_space_id, iter_mem_start, NULL, + iter_mem_count, NULL); + + status = H5Sget_regular_hyperslab(iter_args->file_space_id, iter_file_start, NULL, + iter_file_count, NULL); + // fprintf(stderr,"%d\n",status); + fprintf(stderr, "\n iter_mem_start=%lldx%lld iter_file_start=%lldx%lld ", + iter_mem_start[0], iter_mem_start[1], iter_file_start[0], + iter_file_start[1]); + fprintf(stderr, "\n iter_mem_count=%lldx%lld iter_file_count=%lldx%lld \n", + iter_mem_count[0], iter_mem_count[1], iter_file_count[0], + iter_file_count[1]); + + H5Sget_simple_extent_dims(file_space_id, dimsm, NULL); + //fprintf(stderr, "\n dimensions=%lldx%lld \n", dimsm[0], dimsm[1]); + + buffer_count = 0; + feed_buf_count = 0; + if (file_start[0] == iter_file_start[0]) { + if (file_start[1] >= iter_file_start[1]) { + if (file_count[0] == iter_file_count[0]) { + feed_buf_count = dimsm[1] * iter_file_start[0]; + for (int i = 0; i < iter_file_count[0]; i++) { + memcpy(new_buffer + (buffer_count * element_size), + buf + (feed_buf_count + iter_file_start[1]) * element_size, + iter_file_count[1] * element_size); + // fprintf(stderr,"element size=%lld\n",element_size); + buffer_count += iter_file_count[1]; + memcpy(new_buffer + (buffer_count * element_size), + buf + (feed_buf_count + file_start[1]) * element_size, + file_count[1] * element_size); + // fprintf(stderr,"element size=%lld\n",element_size); + buffer_count += file_count[1]; + feed_buf_count += dimsm[1]; + fprintf(stderr,"buffer_count=%d \ + feed_buf_count=%d\n",buffer_count,feed_buf_count); + } + } + } + else { + + if (file_count[0] == iter_file_count[0]) { + feed_buf_count = dimsm[1] * file_start[0]; + for (int i = 0; i < file_count[0]; i++) { + + memcpy(new_buffer + (buffer_count * element_size), + buf + (feed_buf_count + file_start[1]) * element_size, + file_count[1] * element_size); + // fprintf(stderr,"element size=%lld\n",element_size); + buffer_count += file_count[1]; + memcpy(new_buffer + (buffer_count * element_size), + buf + (feed_buf_count + iter_file_start[1]) * element_size, + iter_file_count[1] * element_size); + // fprintf(stderr,"element size=%lld\n",element_size); + buffer_count += iter_file_count[1]; + feed_buf_count += dimsm[1]; + fprintf(stderr,"buffer_count=%d feed_buf_count=%d\n",buffer_count,feed_buf_count); + } + } + } + } + else if (file_start[1] == iter_file_start[1]) { + if (file_start[0] >= iter_file_start[0]) { + if (file_count[1] == iter_file_count[1]) { + feed_buf_count = dimsm[1] * iter_file_start[0]; + for (int i = 0; i < iter_file_count[0]; i++) { + memcpy(new_buffer + (buffer_count * element_size), + buf + (feed_buf_count + iter_file_start[1]) * element_size, + iter_file_count[1] * element_size); + // fprintf(stderr,"element size=%lld\n",element_size); + buffer_count += iter_file_count[1]; + feed_buf_count += dimsm[1]; + // fprintf(stderr,"buffer_count=%d + // feed_buf_count=%d\n",buffer_count,feed_buf_count); + } + for (int i = 0; i < file_count[0]; i++) { + memcpy(new_buffer + (buffer_count * element_size), + buf + (feed_buf_count + file_start[1]) * element_size, + file_count[1] * element_size); + // fprintf(stderr,"element size=%lld\n",element_size); + buffer_count += file_count[1]; + feed_buf_count += dimsm[1]; + // fprintf(stderr,"buffer_count=%d + // feed_buf_count=%d\n",buffer_count,feed_buf_count); + } + } + } + else { + if (file_count[1] == iter_file_count[1]) { + feed_buf_count = dimsm[1] * file_start[0]; + + for (int i = 0; i < file_count[0]; i++) { + memcpy(new_buffer + (buffer_count * element_size), + buf + (feed_buf_count + file_start[1]) * element_size, + file_count[1] * element_size); + // fprintf(stderr,"element size=%lld\n",element_size); + buffer_count += file_count[1]; + feed_buf_count += dimsm[1]; + fprintf(stderr,"buffer_count=%d \ + // feed_buf_count=%d\n",buffer_count,feed_buf_count); + } + for (int i = 0; i < iter_file_count[0]; i++) { + memcpy(new_buffer + (buffer_count * element_size), + buf + (feed_buf_count + iter_file_start[1]) * element_size, + iter_file_count[1] * element_size); + // fprintf(stderr,"element size=%lld\n",element_size); + buffer_count += iter_file_count[1]; + feed_buf_count += dimsm[1]; + fprintf(stderr,"buffer_count=%d \ + feed_buf_count=%d\n",buffer_count,feed_buf_count); + } + } + } + } + + // memcpy(new_buffer+(file_start[0]*element_size),buf+(mem_start[0]*element_size),mem_count[0]*element_size); + // memcpy(new_buffer,buf,count[0]*element_size); + + + // for (int i = 0; i < 16; i++) + // fprintf(stderr, " buf[%d]=%d ", i, buf_temp[i]); + // fprintf(stderr, "\n"); + + // for (int i = 0; i < 16; i++) + // fprintf(stderr, " new_buffer[%d]=%d ", i, new_buffer_temp[i]); + //fprintf(stderr, "\n"); + + + H5Sclose(iter_args->mem_space_id); + //H5Sclose(iter_args->file_space_id); + + iter_args->mem_space_id = new_memspace; + // iter_args->file_space_id=new_memspace; + + iter_args->buf = new_buffer; + iter_args->free_buf=1; + int *iter_buf= iter_args->buf; + for (int i = 0; i < 16; i++) + fprintf(stderr, " iter_args new_buffer[%d]=%d ", i, iter_buf[i]); + fprintf(stderr, "\n"); + + // if(check_contiguous_overlap(ndim,start,count)) + // {fprintf(stderr, "\ncontiguous overlap\n"); + + //fprintf(stderr,"\n after contiguous overlap start=%llu + //count=%llu\n",start[0],count[0]); + //} + status = + H5Sselect_hyperslab(file_space_id, H5S_SELECT_SET, start, NULL, count, NULL); + + status = H5Sselect_hyperslab(iter_args->file_space_id, H5S_SELECT_SET, start, + NULL, count, NULL); + + start[0] = 0; + start[1] = 0; + // fprintf(stderr,"\n iter mem space mem_start=%lld mem_count=%lld file_start=%lld + // file_count=%lld + // element_size=%lld\n",start[0],count[0],start[0],count[0],element_size); + + status = H5Sselect_hyperslab(iter_args->mem_space_id, H5S_SELECT_SET, start, NULL, + count, NULL); + //free(new_buffer); */ + } + else if (ndim == 3) { + //fprintf(stderr," foreach iterator mem_type_id=%llu\n",iter_args->mem_type_id); + fprintf(stderr, " start=%llux%llux%llu count=%llux%llux%llu\n", start[0], + start[1], start[2], count[0], count[1], count[2]); + + new_memspace = H5Screate_simple(ndim, count, NULL); + element_size = H5Tget_size(mem_type_id); + new_buffer = malloc(count[0] * count[1] * count[2] * element_size); + + status = H5Sget_regular_hyperslab(mem_space_id, mem_start, NULL, mem_count, NULL); + status = + H5Sget_regular_hyperslab(file_space_id, file_start, NULL, file_count, NULL); + + /* fprintf(stderr,"\nmem_start=%lld mem_count=%lld file_start=%lld file_count=%lld \ + element_size=%lld\n",mem_start[0],mem_count[0],file_start[0],file_count[0],element_size); + */ + /* fprintf(stderr,"\n mem_start=%lldx%lldx%lld file_start=%lldx%lldx%lld + ",mem_start[0],mem_start[1],mem_start[2],file_start[0],file_start[1],file_start[2]); + fprintf(stderr,"\n mem_count=%lldx%lldx%lld file_count=%lldx%lldx%lld + \n",mem_count[0],mem_count[1],mem_count[2],file_count[0],file_count[1],file_count[2]); */ + status = H5Sget_regular_hyperslab(iter_args->mem_space_id, iter_mem_start, NULL, + iter_mem_count, NULL); + /* fprintf(stderr,"%d\n",status); + if (H5Sis_regular_hyperslab(iter_args->mem_space_id)) { + fprintf(stderr,"regular hyperslab\n"); + } */ + status = H5Sget_regular_hyperslab(iter_args->file_space_id, iter_file_start, NULL, + iter_file_count, NULL); + // fprintf(stderr,"%d\n",status); + fprintf(stderr, "\n file_start=%lldx%lldx%lld iter_file_start=%lldx%lldx%lld ", + file_start[0], file_start[1], file_start[2], iter_file_start[0], + iter_file_start[1], iter_file_start[2]); + fprintf(stderr, "\n file_count=%lldx%lldx%lld iter_file_count=%lldx%lldx%lld \n", + file_count[0], file_count[1], file_count[2], iter_file_count[0], + iter_file_count[1], iter_file_count[2]); + H5Sget_simple_extent_dims(file_space_id, dimsm, NULL); + //fprintf(stderr, "\n dimensions=%lldx%lldx%lld \n", dimsm[0], dimsm[1], dimsm[2]); + int dim_elements = dimsm[1] * dimsm[2]; + buffer_count = 0; + feed_buf_count = 0; + buffer_count = 0; + feed_buf_count = 0; + int *buf_temp = buf; + int *new_buffer_temp = new_buffer; + + + if (file_start[1] == iter_file_start[1]) { + if (file_start[2] >= iter_file_start[2]) { + if (file_count[1] == iter_file_count[1]) { + if (file_count[0] == iter_file_count[0]) { + for (int d = 0; d < file_count[0]; d++) { + + feed_buf_count = + file_start[0]*dim_elements+dim_elements * d + dimsm[2] * iter_file_start[1]; + + for (int i = 0; i < iter_file_count[1]; i++) { + memcpy(new_buffer + (buffer_count * element_size), + buf + (feed_buf_count + iter_file_start[2]) * + element_size, + iter_file_count[2] * element_size); + // fprintf(stderr,"element size=%lld\n",element_size); + buffer_count += iter_file_count[2]; + memcpy(new_buffer + (buffer_count * element_size), + buf + (feed_buf_count + file_start[2]) * + element_size, + file_count[2] * element_size); + // fprintf(stderr,"element size=%lld\n",element_size); + buffer_count += file_count[2]; + feed_buf_count += dimsm[2]; + // fprintf(stderr,"buffer_count=%d \ + //feed_buf_count=%d\n",buffer_count,feed_buf_count); + } + } + } + } + } + else { + + if (file_count[1] == iter_file_count[1]) { + if (file_count[0] == iter_file_count[0]) { + for (int d = 0; d < file_count[0]; d++) { + + feed_buf_count = file_start[0]*dim_elements+dim_elements * d + dimsm[2] * file_start[1]; + // feed_buf_count=dimsm[2]*file_start[1]; + for (int i = 0; i < file_count[1]; i++) { + + memcpy(new_buffer + (buffer_count * element_size), + buf + (feed_buf_count + file_start[2]) * + element_size, + file_count[2] * element_size); + // fprintf(stderr,"element size=%lld\n",element_size); + buffer_count += file_count[2]; + memcpy(new_buffer + (buffer_count * element_size), + buf + (feed_buf_count + iter_file_start[2]) * + element_size, + iter_file_count[2] * element_size); + // fprintf(stderr,"element size=%lld\n",element_size); + buffer_count += iter_file_count[2]; + feed_buf_count += dimsm[2]; + //fprintf(stderr,"buffer_count=%d \ + // feed_buf_count=%d\n",buffer_count,feed_buf_count); + } + } + } + } + } + } + else if (file_start[2] == iter_file_start[2]) { + if (file_start[1] >= iter_file_start[1]) { + if (file_count[2] == iter_file_count[2]) { + if (file_count[0] == iter_file_count[0]) { + for (int d = 0; d < file_count[0]; d++) { + + feed_buf_count = + file_start[0]*dim_elements+dim_elements * d + dimsm[2] * iter_file_start[1]; + + // feed_buf_count=dimsm[2]*iter_file_start[1]; + for (int i = 0; i < iter_file_count[1]; i++) { + memcpy(new_buffer + (buffer_count * element_size), + buf + (feed_buf_count + iter_file_start[2]) * + element_size, + iter_file_count[2] * element_size); + // fprintf(stderr,"element size=%lld\n",element_size); + buffer_count += iter_file_count[2]; + feed_buf_count += dimsm[2]; + //fprintf(stderr, "buffer_count=%d feed_buf_count=%d\n", + // buffer_count, feed_buf_count); + } + for (int i = 0; i < file_count[1]; i++) { + memcpy(new_buffer + (buffer_count * element_size), + buf + (feed_buf_count + file_start[2]) * + element_size, + file_count[2] * element_size); + // fprintf(stderr,"element size=%lld\n",element_size); + buffer_count += file_count[2]; + feed_buf_count += dimsm[2]; + //fprintf(stderr,"buffer_count=%d \ + // feed_buf_count=%d\n",buffer_count,feed_buf_count); + } + } + } + } + } + else { + if (file_count[2] == iter_file_count[2]) { + if (file_count[0] == iter_file_count[0]) { + for (int d = 0; d < file_count[0]; d++) { + + feed_buf_count = file_start[0]*dim_elements+dim_elements * d + dimsm[2] * file_start[1]; + + // feed_buf_count=dimsm[2]*file_start[1]; + + for (int i = 0; i < file_count[1]; i++) { + memcpy(new_buffer + (buffer_count * element_size), + buf + (feed_buf_count + file_start[2]) * + element_size, + file_count[2] * element_size); + // fprintf(stderr,"element size=%lld\n",element_size); + buffer_count += file_count[2]; + feed_buf_count += dimsm[2]; + // fprintf(stderr,"buffer_count=%d \ + // feed_buf_count=%d\n",buffer_count,feed_buf_count); + } + for (int i = 0; i < iter_file_count[1]; i++) { + memcpy(new_buffer + (buffer_count * element_size), + buf + (feed_buf_count + iter_file_start[2]) * + element_size, + iter_file_count[2] * element_size); + // fprintf(stderr,"element size=%lld\n",element_size); + buffer_count += iter_file_count[2]; + feed_buf_count += dimsm[2]; + // fprintf(stderr,"buffer_count=%d \ + // feed_buf_count=%d\n",buffer_count,feed_buf_count); + } + } + } + } + } + } + + // memcpy(new_buffer+(file_start[0]*element_size),buf+(mem_start[0]*element_size),mem_count[0]*element_size); + // memcpy(new_buffer,buf,count[0]*element_size); + + /*allocate new_buffer + get the start and count value from memspace + based on the start and count copy the data from buf to new buffer + need to be corect locations + do the same thing for iter_args->buf and iter_args->memspace. + */ + + /* for (int i = 0; i < 16; i++) + fprintf(stderr, " buf[%d]=%d ", i, buf_temp[i]); + fprintf(stderr, "\n"); + + for (int i = 0; i < 16; i++) + fprintf(stderr, " new_buffer[%d]=%d ", i, new_buffer_temp[i]); + fprintf(stderr, "\n"); */ + + H5Sclose(iter_args->mem_space_id); + // H5Sclose(iter_args->file_space_id); + + iter_args->mem_space_id = new_memspace; + // iter_args->file_space_id=new_memspace; + + iter_args->buf = new_buffer; + iter_args->free_buf=1; + + /* int *iter_buf= iter_args->buf; + for (int i = 0; i < 16; i++) + fprintf(stderr, " iter_args new_buffer[%d]=%d ", i, iter_buf[i]); + fprintf(stderr, "\n"); */ + + + + /* if(check_contiguous_overlap(ndim,start,count)) + {fprintf(stderr, "\ncontiguous overlap\n"); + + fprintf(stderr,"\n after contiguous overlap start=%llu + count=%llu\n",start[0],count[0]); + } */ + status = + H5Sselect_hyperslab(file_space_id, H5S_SELECT_SET, start, NULL, count, NULL); + + status = H5Sselect_hyperslab(iter_args->file_space_id, H5S_SELECT_SET, start, + NULL, count, NULL); + + start[0] = 0; + start[1] = 0; + start[2] = 0; + // fprintf(stderr,"\n iter mem space mem_start=%lld mem_count=%lld file_start=%lld + // file_count=%lld + // element_size=%lld\n",start[0],count[0],start[0],count[0],element_size); + + status = H5Sselect_hyperslab(iter_args->mem_space_id, H5S_SELECT_SET, start, NULL, + count, NULL); + //fprintf(stderr," end of foreach iterator mem_type_id=%llu\n",iter_args->mem_type_id); + + + } + // return 1; + + } + + else { + //free(new_buffer); + // iter_args->buf=buf; + // fprintf(stderr, "-------- Not contiguous---------\n"); + } + free(dimsm); + free(start); + free(count); + free(mem_start); + free(mem_count); + free(file_start); + free(file_count); + free(iter_mem_start); + free(iter_mem_count); + free(iter_file_start); + free(iter_file_count); + + } + } + } + } + //free(new_buffer); + //free(buf); + // return return_val; + return return_task_val; +} +static herr_t +async_dataset_write_merge(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t mem_type_id, + hid_t mem_space_id, hid_t file_space_id, hid_t plist_id, const void *buf) +{ + + int ndim; + int num_elements; + hsize_t nblocks; + int N1 = 20; + hsize_t * dimsm; + void * buffer, *new_buffer; + hid_t memspace, dataspace; + H5S_sel_type type; + herr_t slected_block; + hssize_t numblocks; + async_dataset_write_args_t *args = NULL; + async_dataset_write_args_t *iter_args = NULL; + + hsize_t *start_out, *stride_out, *count_out, *block_out; + hsize_t *start, *count, *mem_start, *mem_count, *file_start, *file_count, *iter_mem_start, + *iter_mem_count, *iter_file_start, *iter_file_count, element_size; + + herr_t status, return_val = 0; + async_task_t * task_iter; + async_task_t * return_task_val; + async_task_list_t *task_list_iter; + int buffer_count, feed_buf_count; + double time1, time2,duration; + double time11,time12,duration1; + double time21,time22,duration2; + + assert(aid); + assert(parent_obj); + assert(parent_obj->magic == ASYNC_MAGIC); + assert(mem_space_id); + + // fprintf(stderr,"this is before H5Sget_select_type\n"); + + // print_dataspace(mem_space_id); + // print_dataspace(file_space_id); + /*if(type==H5S_SEL_HYPERSLABS) + fprintf(stderr,"A hyperslab or compound hyperslab is selected\n"); + else if(type==H5S_SEL_POINTS) + fprintf(stderr,"A sequence of points is selected\n"); + else if(type==H5S_SEL_ALL) + fprintf(stderr,"The entire dataset is selected\n"); + else if(type==H5S_SEL_NONE) + fprintf(stderr,"No selection is defined\n"); + */ + //fprintf(stderr," merged func mem_type_id=%llu\n",mem_type_id); + time1 = MPI_Wtime(); + time11=MPI_Wtime(); + + return_task_val = + foreach_iteration(aid, parent_obj, mem_type_id, mem_space_id, file_space_id, plist_id, buf, NULL); + // return_val=foreach_iteration(aid,parent_obj,mem_type_id,mem_space_id,file_space_id,plist_id,buf,NULL); + time12 = MPI_Wtime(); + duration1 = time12 - time11; + fprintf(stderr,"1st Foreach iter Runtime is %f \n", duration1); + if (return_task_val == NULL) + return_val = 0; + else + return_val = 1; + + for (int i = 0; return_task_val != NULL; i++) + { time21=MPI_Wtime(); + + return_task_val = foreach_iteration(aid, parent_obj, mem_type_id, mem_space_id, file_space_id, + plist_id, buf, return_task_val); + time22=MPI_Wtime(); + + duration2 = time22 - time21; + fprintf(stderr,"in the for loop Runtime is i=%d %f \n", i, duration1); + } + + time2 = MPI_Wtime(); + duration = time2 - time1; + + + + fprintf(stderr,"Foreach Runtime is %f \n", duration); + + return return_val; +} + static herr_t async_dataset_write(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t mem_type_id, hid_t mem_space_id, hid_t file_space_id, hid_t plist_id, const void *buf, void **req) @@ -9053,13 +10134,20 @@ async_dataset_write(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t mem_t bool is_blocking = false; hbool_t acquired = false; unsigned int mutex_count = 1; + herr_t return_val; func_enter(__func__, NULL); assert(aid); assert(parent_obj); assert(parent_obj->magic == ASYNC_MAGIC); + //fprintf(stderr,"mem_type_id=%llu\n",mem_type_id); + + return_val = + async_dataset_write_merge(aid, parent_obj, mem_type_id, mem_space_id, file_space_id, plist_id, buf); + if (return_val == 1) + goto done; async_instance_g->prev_push_state = async_instance_g->start_abt_push; if ((args = (async_dataset_write_args_t *)calloc(1, sizeof(async_dataset_write_args_t))) == NULL) { @@ -9470,8 +10558,7 @@ async_dataset_get(task_list_qtype qtype, async_instance_t *aid, H5VL_async_t *pa args->dxpl_id = H5Pcopy(dxpl_id); args->req = req; - // Temporary fix for data space get that could cause a pthread deadlock - if (req && !aid->disable_async_dset_get) { + if (req) { H5VL_async_t *new_req; if ((new_req = H5VL_async_new_obj(NULL, parent_obj->under_vol_id)) == NULL) { fprintf(fout_g, " [ASYNC VOL ERROR] %s with request object calloc\n", __func__); @@ -20515,15 +21602,9 @@ H5VL_async_str_to_info(const char *str, void **_info) /* Retrieve the underlying VOL connector value and info */ sscanf(str, "under_vol=%u;", &under_vol_value); - under_vol_id = H5VLregister_connector_by_value((H5VL_class_value_t)under_vol_value, H5P_DEFAULT); - if (strstr(str, "[") && strstr(str, "]")) { - under_vol_info_start = strchr(str, '['); - under_vol_info_end = strrchr(str, ']'); - } - else { - under_vol_info_start = strchr(str, '{'); - under_vol_info_end = strrchr(str, '}'); - } + under_vol_id = H5VLregister_connector_by_value((H5VL_class_value_t)under_vol_value, H5P_DEFAULT); + under_vol_info_start = strchr(str, '{'); + under_vol_info_end = strrchr(str, '}'); assert(under_vol_info_end > under_vol_info_start); if (under_vol_info_end != (under_vol_info_start + 1)) { char *under_vol_info_str; From 55dc45976d5c83aaf0b1cdc8298c36f3fae769d3 Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 10 Aug 2022 02:18:07 +0000 Subject: [PATCH 02/18] Committing clang-format changes --- src/h5_async_vol.c | 303 ++++++++++++++++++++++----------------------- 1 file changed, 148 insertions(+), 155 deletions(-) diff --git a/src/h5_async_vol.c b/src/h5_async_vol.c index 9de87ba..ed968e3 100644 --- a/src/h5_async_vol.c +++ b/src/h5_async_vol.c @@ -65,7 +65,7 @@ works, and perform publicly and display publicly, and to permit others to do so. /* Whether to display log messge when callback is invoked */ /* (Uncomment to enable) */ /* #define ENABLE_DBG_MSG 1 */ - #define PRINT_ERROR_STACK 1 +#define PRINT_ERROR_STACK 1 /* #define ENABLE_ASYNC_LOGGING */ #define ASYNC_DBG_MSG_RANK 0 @@ -257,7 +257,7 @@ typedef struct async_attr_write_args_t { hid_t dxpl_id; void **req; - bool free_buf; + bool free_buf; #ifdef ENABLE_WRITE_MEMCPY hsize_t data_size; #endif @@ -333,7 +333,7 @@ typedef struct async_dataset_write_args_t { void * buf; void **req; - bool free_buf; + bool free_buf; #ifdef ENABLE_WRITE_MEMCPY hsize_t data_size; #endif @@ -8860,13 +8860,11 @@ async_dataset_write_fn(void *foo) assert(task); assert(task->async_obj); assert(task->async_obj->magic == ASYNC_MAGIC); - //fprintf(stderr,"task is_merge=%d\n",task->is_merge); + // fprintf(stderr,"task is_merge=%d\n",task->is_merge); pool_ptr = task->async_obj->pool_ptr; if (task->is_merge == 1) goto done; - - func_log(__func__, "trying to aquire global lock"); if ((attempt_count = check_app_acquire_mutex(task, &mutex_count, &acquired)) < 0) @@ -8940,7 +8938,7 @@ async_dataset_write_fn(void *foo) usleep(1000); count++; } - //fprintf(stderr," args mem_type_id=%llu\n",args->mem_type_id); + // fprintf(stderr," args mem_type_id=%llu\n",args->mem_type_id); /* Try executing operation, without default error stack handling */ H5E_BEGIN_TRY { @@ -8968,8 +8966,8 @@ async_dataset_write_fn(void *foo) fprintf(fout_g, " [ASYNC ABT ERROR] %s H5VLfree_lib_state failed\n", __func__); task->h5_state = NULL; - //if (args->mem_type_id > 0) - // H5Tclose(args->mem_type_id); + // if (args->mem_type_id > 0) + // H5Tclose(args->mem_type_id); if (args->mem_space_id > 0) H5Sclose(args->mem_space_id); if (args->file_space_id > 0) @@ -8994,7 +8992,6 @@ async_dataset_write_fn(void *foo) if (async_instance_g && NULL != async_instance_g->qhead.queue && async_instance_g->start_abt_push) push_task_to_abt_pool(&async_instance_g->qhead, *pool_ptr, __func__); - if (args->free_buf && args->buf) { free(args->buf); #ifdef ENABLE_WRITE_MEMCPY @@ -9004,10 +9001,9 @@ async_dataset_write_fn(void *foo) (async_instance_g->mpi_rank == ASYNC_DBG_MSG_RANK || -1 == ASYNC_DBG_MSG_RANK)) fprintf(fout_g, " [ASYNC ABT DBG] %s released dset memcpy\n", __func__); #endif - #endif +#endif } - #ifdef ENABLE_TIMING task->end_time = clock(); #endif @@ -9429,12 +9425,12 @@ foreach_iteration(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t mem_typ DL_FOREACH(async_instance_g->qhead.queue, task_list_iter) { DL_FOREACH(task_list_iter->task_list, task_iter) - { + { if (task_iter->func == async_dataset_write_fn) { // checking whether the iterator task is dataset write operation if (task_iter->parent_obj == parent_obj && task_iter->is_done != 1) { // checking whether the iterator task is operating on the same dataset of the current task - + iter_args = task_iter->args; // fprintf(stderr,"%lld %lld @@ -9459,28 +9455,24 @@ foreach_iteration(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t mem_typ iter_file_start = malloc(ndim * sizeof(hsize_t)); iter_file_count = malloc(ndim * sizeof(hsize_t)); - - - if (check_contiguous(file_space_id, iter_args->file_space_id, start, count) || check_contiguous(iter_args->file_space_id, file_space_id, start, count)) // if(check_contiguous(file_space_id,iter_args->file_space_id,start,count)) - { + { return_val = 1; - // fprintf(stderr, "\n-------#### contiguous####---------\n"); - - + // fprintf(stderr, "\n-------#### contiguous####---------\n"); + /* if(prev_task!=NULL){ continue; - } */ + } */ if (prev_task != NULL) prev_task->is_merge = 1; return_task_val = task_iter; - - // fprintf(stderr, "\n-------#### 2 contiguous####---------\n"); + + // fprintf(stderr, "\n-------#### 2 contiguous####---------\n"); if (ndim == 1) { // push_contiguous(ndim,start,count); - //fprintf(stderr, "\n start=%llu count=%llu\n", start[0], count[0]); + // fprintf(stderr, "\n start=%llu count=%llu\n", start[0], count[0]); // H5Sclose(iter_args->file_space_id);// close the file space /* if(H5Smodify_select(file_space_id,H5S_SELECT_SET,iter_args->file_space_id)< 0) @@ -9489,12 +9481,10 @@ foreach_iteration(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t mem_typ // H5Sclose(iter_args->file_space_id); new_memspace = H5Screate_simple(ndim, count, NULL); element_size = H5Tget_size(mem_type_id); - fprintf(stderr,"size of new_buffer= %lld \n",count[0]*element_size); - - new_buffer = malloc(count[0] * element_size); + fprintf(stderr, "size of new_buffer= %lld \n", count[0] * element_size); + new_buffer = malloc(count[0] * element_size); - /* int *buf_temp = buf; int *new_buffer_temp = new_buffer; */ @@ -9502,11 +9492,12 @@ foreach_iteration(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t mem_typ status = H5Sget_regular_hyperslab(file_space_id, file_start, NULL, file_count, NULL); - //fprintf(stderr,"\nmem_start=%lld mem_count=%lld file_start=%lld file_count=%lld element_size=%lld\n",mem_start[0],mem_count[0],file_start[0],file_count[0],element_size); - fprintf(stderr, "\n mem_start=%lld file_start=%lld ", mem_start[0], \ - file_start[0]); - fprintf(stderr, "\n mem_count=%lld file_count=%lld \n", mem_count[0], \ - file_count[0]); + // fprintf(stderr,"\nmem_start=%lld mem_count=%lld file_start=%lld file_count=%lld + // element_size=%lld\n",mem_start[0],mem_count[0],file_start[0],file_count[0],element_size); + fprintf(stderr, "\n mem_start=%lld file_start=%lld ", mem_start[0], + file_start[0]); + fprintf(stderr, "\n mem_count=%lld file_count=%lld \n", mem_count[0], + file_count[0]); status = H5Sget_regular_hyperslab(iter_args->mem_space_id, iter_mem_start, NULL, iter_mem_count, NULL); /* fprintf(stderr,"%d\n",status); @@ -9516,10 +9507,11 @@ foreach_iteration(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t mem_typ status = H5Sget_regular_hyperslab(iter_args->file_space_id, iter_file_start, NULL, iter_file_count, NULL); // fprintf(stderr,"%d\n",status); - //fprintf(stderr, "\n iter_mem_start=%lld iter_file_start=%lld ", iter_mem_start[0], - // iter_file_start[0]); - //fprintf(stderr, "\n iter_mem_count=%lld iter_file_count=%lld \n", - // iter_mem_count[0], iter_file_count[0]); + // fprintf(stderr, "\n iter_mem_start=%lld iter_file_start=%lld ", + // iter_mem_start[0], + // iter_file_start[0]); + // fprintf(stderr, "\n iter_mem_count=%lld iter_file_count=%lld \n", + // iter_mem_count[0], iter_file_count[0]); if (file_start[0] >= iter_file_start[0]) { // fprintf(stderr,"file_start[0]>=iter_file_start[0]\n"); memcpy(new_buffer, buf + iter_file_start[0] * element_size, @@ -9552,13 +9544,13 @@ foreach_iteration(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t mem_typ iter_args->mem_space_id = new_memspace; // iter_args->file_space_id=new_memspace; - + if (iter_args->free_buf == 1) free(iter_args->buf); - iter_args->buf = new_buffer; - iter_args->free_buf=1; - + iter_args->buf = new_buffer; + iter_args->free_buf = 1; + /* if(check_contiguous_overlap(ndim,start,count)) {fprintf(stderr, "\ncontiguous overlap\n"); @@ -9584,16 +9576,16 @@ foreach_iteration(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t mem_typ for (int i = 0; i < 16; i++) fprintf(stderr, " new_buffer[%d]=%d ", i, new_buffer_temp[i]); - fprintf(stderr, "\n"); */ - //free(new_buffer); + fprintf(stderr, "\n"); */ + // free(new_buffer); } else if (ndim == 2) { - + fprintf(stderr, " start=%llux%llu count=%llux%llu\n", start[0], start[1], count[0], count[1]); - new_memspace = H5Screate_simple(ndim, count, NULL); - element_size = H5Tget_size(mem_type_id); - new_buffer = malloc(count[0] * count[1] * element_size); + new_memspace = H5Screate_simple(ndim, count, NULL); + element_size = H5Tget_size(mem_type_id); + new_buffer = malloc(count[0] * count[1] * element_size); int *buf_temp = buf; int *new_buffer_temp = new_buffer; @@ -9609,7 +9601,7 @@ foreach_iteration(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t mem_typ mem_count[1], file_count[0], file_count[1]); status = H5Sget_regular_hyperslab(iter_args->mem_space_id, iter_mem_start, NULL, iter_mem_count, NULL); - + status = H5Sget_regular_hyperslab(iter_args->file_space_id, iter_file_start, NULL, iter_file_count, NULL); // fprintf(stderr,"%d\n",status); @@ -9621,11 +9613,11 @@ foreach_iteration(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t mem_typ iter_file_count[1]); H5Sget_simple_extent_dims(file_space_id, dimsm, NULL); - //fprintf(stderr, "\n dimensions=%lldx%lld \n", dimsm[0], dimsm[1]); + // fprintf(stderr, "\n dimensions=%lldx%lld \n", dimsm[0], dimsm[1]); buffer_count = 0; feed_buf_count = 0; - if (file_start[0] == iter_file_start[0]) { + if (file_start[0] == iter_file_start[0]) { if (file_start[1] >= iter_file_start[1]) { if (file_count[0] == iter_file_count[0]) { feed_buf_count = dimsm[1] * iter_file_start[0]; @@ -9641,8 +9633,9 @@ foreach_iteration(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t mem_typ // fprintf(stderr,"element size=%lld\n",element_size); buffer_count += file_count[1]; feed_buf_count += dimsm[1]; - fprintf(stderr,"buffer_count=%d \ - feed_buf_count=%d\n",buffer_count,feed_buf_count); + fprintf(stderr, "buffer_count=%d \ + feed_buf_count=%d\n", + buffer_count, feed_buf_count); } } } @@ -9663,7 +9656,8 @@ foreach_iteration(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t mem_typ // fprintf(stderr,"element size=%lld\n",element_size); buffer_count += iter_file_count[1]; feed_buf_count += dimsm[1]; - fprintf(stderr,"buffer_count=%d feed_buf_count=%d\n",buffer_count,feed_buf_count); + fprintf(stderr, "buffer_count=%d feed_buf_count=%d\n", + buffer_count, feed_buf_count); } } } @@ -9705,8 +9699,9 @@ foreach_iteration(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t mem_typ // fprintf(stderr,"element size=%lld\n",element_size); buffer_count += file_count[1]; feed_buf_count += dimsm[1]; - fprintf(stderr,"buffer_count=%d \ - // feed_buf_count=%d\n",buffer_count,feed_buf_count); + fprintf(stderr, "buffer_count=%d \ + // feed_buf_count=%d\n", + buffer_count, feed_buf_count); } for (int i = 0; i < iter_file_count[0]; i++) { memcpy(new_buffer + (buffer_count * element_size), @@ -9715,45 +9710,44 @@ foreach_iteration(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t mem_typ // fprintf(stderr,"element size=%lld\n",element_size); buffer_count += iter_file_count[1]; feed_buf_count += dimsm[1]; - fprintf(stderr,"buffer_count=%d \ - feed_buf_count=%d\n",buffer_count,feed_buf_count); + fprintf(stderr, "buffer_count=%d \ + feed_buf_count=%d\n", + buffer_count, feed_buf_count); } } } - } + } // memcpy(new_buffer+(file_start[0]*element_size),buf+(mem_start[0]*element_size),mem_count[0]*element_size); // memcpy(new_buffer,buf,count[0]*element_size); - // for (int i = 0; i < 16; i++) - // fprintf(stderr, " buf[%d]=%d ", i, buf_temp[i]); - // fprintf(stderr, "\n"); + // fprintf(stderr, " buf[%d]=%d ", i, buf_temp[i]); + // fprintf(stderr, "\n"); - // for (int i = 0; i < 16; i++) + // for (int i = 0; i < 16; i++) // fprintf(stderr, " new_buffer[%d]=%d ", i, new_buffer_temp[i]); - //fprintf(stderr, "\n"); - + // fprintf(stderr, "\n"); H5Sclose(iter_args->mem_space_id); - //H5Sclose(iter_args->file_space_id); + // H5Sclose(iter_args->file_space_id); iter_args->mem_space_id = new_memspace; // iter_args->file_space_id=new_memspace; - iter_args->buf = new_buffer; - iter_args->free_buf=1; - int *iter_buf= iter_args->buf; + iter_args->buf = new_buffer; + iter_args->free_buf = 1; + int *iter_buf = iter_args->buf; for (int i = 0; i < 16; i++) fprintf(stderr, " iter_args new_buffer[%d]=%d ", i, iter_buf[i]); - fprintf(stderr, "\n"); + fprintf(stderr, "\n"); - // if(check_contiguous_overlap(ndim,start,count)) - // {fprintf(stderr, "\ncontiguous overlap\n"); + // if(check_contiguous_overlap(ndim,start,count)) + // {fprintf(stderr, "\ncontiguous overlap\n"); - //fprintf(stderr,"\n after contiguous overlap start=%llu - //count=%llu\n",start[0],count[0]); - //} + // fprintf(stderr,"\n after contiguous overlap start=%llu + // count=%llu\n",start[0],count[0]); + //} status = H5Sselect_hyperslab(file_space_id, H5S_SELECT_SET, start, NULL, count, NULL); @@ -9768,28 +9762,30 @@ foreach_iteration(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t mem_typ status = H5Sselect_hyperslab(iter_args->mem_space_id, H5S_SELECT_SET, start, NULL, count, NULL); - //free(new_buffer); */ + // free(new_buffer); */ } else if (ndim == 3) { - //fprintf(stderr," foreach iterator mem_type_id=%llu\n",iter_args->mem_type_id); + // fprintf(stderr," foreach iterator mem_type_id=%llu\n",iter_args->mem_type_id); fprintf(stderr, " start=%llux%llux%llu count=%llux%llux%llu\n", start[0], start[1], start[2], count[0], count[1], count[2]); new_memspace = H5Screate_simple(ndim, count, NULL); element_size = H5Tget_size(mem_type_id); new_buffer = malloc(count[0] * count[1] * count[2] * element_size); - + status = H5Sget_regular_hyperslab(mem_space_id, mem_start, NULL, mem_count, NULL); status = H5Sget_regular_hyperslab(file_space_id, file_start, NULL, file_count, NULL); - /* fprintf(stderr,"\nmem_start=%lld mem_count=%lld file_start=%lld file_count=%lld \ - element_size=%lld\n",mem_start[0],mem_count[0],file_start[0],file_count[0],element_size); - */ + /* fprintf(stderr,"\nmem_start=%lld mem_count=%lld file_start=%lld file_count=%lld + \ + element_size=%lld\n",mem_start[0],mem_count[0],file_start[0],file_count[0],element_size); + */ /* fprintf(stderr,"\n mem_start=%lldx%lldx%lld file_start=%lldx%lldx%lld ",mem_start[0],mem_start[1],mem_start[2],file_start[0],file_start[1],file_start[2]); fprintf(stderr,"\n mem_count=%lldx%lldx%lld file_count=%lldx%lldx%lld - \n",mem_count[0],mem_count[1],mem_count[2],file_count[0],file_count[1],file_count[2]); */ + \n",mem_count[0],mem_count[1],mem_count[2],file_count[0],file_count[1],file_count[2]); + */ status = H5Sget_regular_hyperslab(iter_args->mem_space_id, iter_mem_start, NULL, iter_mem_count, NULL); /* fprintf(stderr,"%d\n",status); @@ -9806,7 +9802,8 @@ foreach_iteration(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t mem_typ file_count[0], file_count[1], file_count[2], iter_file_count[0], iter_file_count[1], iter_file_count[2]); H5Sget_simple_extent_dims(file_space_id, dimsm, NULL); - //fprintf(stderr, "\n dimensions=%lldx%lldx%lld \n", dimsm[0], dimsm[1], dimsm[2]); + // fprintf(stderr, "\n dimensions=%lldx%lldx%lld \n", dimsm[0], dimsm[1], + // dimsm[2]); int dim_elements = dimsm[1] * dimsm[2]; buffer_count = 0; feed_buf_count = 0; @@ -9814,32 +9811,32 @@ foreach_iteration(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t mem_typ feed_buf_count = 0; int *buf_temp = buf; int *new_buffer_temp = new_buffer; - if (file_start[1] == iter_file_start[1]) { if (file_start[2] >= iter_file_start[2]) { if (file_count[1] == iter_file_count[1]) { if (file_count[0] == iter_file_count[0]) { for (int d = 0; d < file_count[0]; d++) { - - feed_buf_count = - file_start[0]*dim_elements+dim_elements * d + dimsm[2] * iter_file_start[1]; + + feed_buf_count = file_start[0] * dim_elements + + dim_elements * d + + dimsm[2] * iter_file_start[1]; for (int i = 0; i < iter_file_count[1]; i++) { - memcpy(new_buffer + (buffer_count * element_size), + memcpy(new_buffer + (buffer_count * element_size), buf + (feed_buf_count + iter_file_start[2]) * element_size, - iter_file_count[2] * element_size); + iter_file_count[2] * element_size); // fprintf(stderr,"element size=%lld\n",element_size); buffer_count += iter_file_count[2]; - memcpy(new_buffer + (buffer_count * element_size), + memcpy(new_buffer + (buffer_count * element_size), buf + (feed_buf_count + file_start[2]) * element_size, - file_count[2] * element_size); + file_count[2] * element_size); // fprintf(stderr,"element size=%lld\n",element_size); buffer_count += file_count[2]; feed_buf_count += dimsm[2]; - // fprintf(stderr,"buffer_count=%d \ + // fprintf(stderr,"buffer_count=%d \ //feed_buf_count=%d\n",buffer_count,feed_buf_count); } } @@ -9852,20 +9849,21 @@ foreach_iteration(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t mem_typ if (file_count[0] == iter_file_count[0]) { for (int d = 0; d < file_count[0]; d++) { - feed_buf_count = file_start[0]*dim_elements+dim_elements * d + dimsm[2] * file_start[1]; + feed_buf_count = file_start[0] * dim_elements + + dim_elements * d + dimsm[2] * file_start[1]; // feed_buf_count=dimsm[2]*file_start[1]; for (int i = 0; i < file_count[1]; i++) { - memcpy(new_buffer + (buffer_count * element_size), + memcpy(new_buffer + (buffer_count * element_size), buf + (feed_buf_count + file_start[2]) * element_size, - file_count[2] * element_size); + file_count[2] * element_size); // fprintf(stderr,"element size=%lld\n",element_size); buffer_count += file_count[2]; - memcpy(new_buffer + (buffer_count * element_size), + memcpy(new_buffer + (buffer_count * element_size), buf + (feed_buf_count + iter_file_start[2]) * element_size, - iter_file_count[2] * element_size); + iter_file_count[2] * element_size); // fprintf(stderr,"element size=%lld\n",element_size); buffer_count += iter_file_count[2]; feed_buf_count += dimsm[2]; @@ -9883,26 +9881,27 @@ foreach_iteration(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t mem_typ if (file_count[0] == iter_file_count[0]) { for (int d = 0; d < file_count[0]; d++) { - feed_buf_count = - file_start[0]*dim_elements+dim_elements * d + dimsm[2] * iter_file_start[1]; + feed_buf_count = file_start[0] * dim_elements + + dim_elements * d + + dimsm[2] * iter_file_start[1]; // feed_buf_count=dimsm[2]*iter_file_start[1]; for (int i = 0; i < iter_file_count[1]; i++) { - memcpy(new_buffer + (buffer_count * element_size), + memcpy(new_buffer + (buffer_count * element_size), buf + (feed_buf_count + iter_file_start[2]) * element_size, - iter_file_count[2] * element_size); + iter_file_count[2] * element_size); // fprintf(stderr,"element size=%lld\n",element_size); buffer_count += iter_file_count[2]; feed_buf_count += dimsm[2]; - //fprintf(stderr, "buffer_count=%d feed_buf_count=%d\n", - // buffer_count, feed_buf_count); + // fprintf(stderr, "buffer_count=%d feed_buf_count=%d\n", + // buffer_count, feed_buf_count); } for (int i = 0; i < file_count[1]; i++) { - memcpy(new_buffer + (buffer_count * element_size), + memcpy(new_buffer + (buffer_count * element_size), buf + (feed_buf_count + file_start[2]) * element_size, - file_count[2] * element_size); + file_count[2] * element_size); // fprintf(stderr,"element size=%lld\n",element_size); buffer_count += file_count[2]; feed_buf_count += dimsm[2]; @@ -9918,23 +9917,24 @@ foreach_iteration(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t mem_typ if (file_count[0] == iter_file_count[0]) { for (int d = 0; d < file_count[0]; d++) { - feed_buf_count = file_start[0]*dim_elements+dim_elements * d + dimsm[2] * file_start[1]; + feed_buf_count = file_start[0] * dim_elements + + dim_elements * d + dimsm[2] * file_start[1]; // feed_buf_count=dimsm[2]*file_start[1]; for (int i = 0; i < file_count[1]; i++) { - memcpy(new_buffer + (buffer_count * element_size), + memcpy(new_buffer + (buffer_count * element_size), buf + (feed_buf_count + file_start[2]) * element_size, - file_count[2] * element_size); + file_count[2] * element_size); // fprintf(stderr,"element size=%lld\n",element_size); buffer_count += file_count[2]; feed_buf_count += dimsm[2]; - // fprintf(stderr,"buffer_count=%d \ + // fprintf(stderr,"buffer_count=%d \ // feed_buf_count=%d\n",buffer_count,feed_buf_count); } for (int i = 0; i < iter_file_count[1]; i++) { - memcpy(new_buffer + (buffer_count * element_size), + memcpy(new_buffer + (buffer_count * element_size), buf + (feed_buf_count + iter_file_start[2]) * element_size, iter_file_count[2] * element_size); @@ -9974,15 +9974,13 @@ foreach_iteration(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t mem_typ iter_args->mem_space_id = new_memspace; // iter_args->file_space_id=new_memspace; - iter_args->buf = new_buffer; - iter_args->free_buf=1; - - /* int *iter_buf= iter_args->buf; - for (int i = 0; i < 16; i++) - fprintf(stderr, " iter_args new_buffer[%d]=%d ", i, iter_buf[i]); - fprintf(stderr, "\n"); */ + iter_args->buf = new_buffer; + iter_args->free_buf = 1; - + /* int *iter_buf= iter_args->buf; + for (int i = 0; i < 16; i++) + fprintf(stderr, " iter_args new_buffer[%d]=%d ", i, iter_buf[i]); + fprintf(stderr, "\n"); */ /* if(check_contiguous_overlap(ndim,start,count)) {fprintf(stderr, "\ncontiguous overlap\n"); @@ -10005,18 +10003,16 @@ foreach_iteration(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t mem_typ status = H5Sselect_hyperslab(iter_args->mem_space_id, H5S_SELECT_SET, start, NULL, count, NULL); - //fprintf(stderr," end of foreach iterator mem_type_id=%llu\n",iter_args->mem_type_id); - - + // fprintf(stderr," end of foreach iterator + // mem_type_id=%llu\n",iter_args->mem_type_id); } // return 1; - } else { - //free(new_buffer); + // free(new_buffer); // iter_args->buf=buf; - // fprintf(stderr, "-------- Not contiguous---------\n"); + // fprintf(stderr, "-------- Not contiguous---------\n"); } free(dimsm); free(start); @@ -10029,13 +10025,12 @@ foreach_iteration(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t mem_typ free(iter_mem_count); free(iter_file_start); free(iter_file_count); - } } } } - //free(new_buffer); - //free(buf); + // free(new_buffer); + // free(buf); // return return_val; return return_task_val; } @@ -10066,9 +10061,9 @@ async_dataset_write_merge(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t async_task_t * return_task_val; async_task_list_t *task_list_iter; int buffer_count, feed_buf_count; - double time1, time2,duration; - double time11,time12,duration1; - double time21,time22,duration2; + double time1, time2, duration; + double time11, time12, duration1; + double time21, time22, duration2; assert(aid); assert(parent_obj); @@ -10088,39 +10083,37 @@ async_dataset_write_merge(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t else if(type==H5S_SEL_NONE) fprintf(stderr,"No selection is defined\n"); */ - //fprintf(stderr," merged func mem_type_id=%llu\n",mem_type_id); - time1 = MPI_Wtime(); - time11=MPI_Wtime(); + // fprintf(stderr," merged func mem_type_id=%llu\n",mem_type_id); + time1 = MPI_Wtime(); + time11 = MPI_Wtime(); return_task_val = foreach_iteration(aid, parent_obj, mem_type_id, mem_space_id, file_space_id, plist_id, buf, NULL); // return_val=foreach_iteration(aid,parent_obj,mem_type_id,mem_space_id,file_space_id,plist_id,buf,NULL); - time12 = MPI_Wtime(); + time12 = MPI_Wtime(); duration1 = time12 - time11; - fprintf(stderr,"1st Foreach iter Runtime is %f \n", duration1); + fprintf(stderr, "1st Foreach iter Runtime is %f \n", duration1); if (return_task_val == NULL) return_val = 0; else return_val = 1; - - for (int i = 0; return_task_val != NULL; i++) - { time21=MPI_Wtime(); - - return_task_val = foreach_iteration(aid, parent_obj, mem_type_id, mem_space_id, file_space_id, - plist_id, buf, return_task_val); - time22=MPI_Wtime(); - - duration2 = time22 - time21; - fprintf(stderr,"in the for loop Runtime is i=%d %f \n", i, duration1); - } - - time2 = MPI_Wtime(); + + for (int i = 0; return_task_val != NULL; i++) { + time21 = MPI_Wtime(); + + return_task_val = foreach_iteration(aid, parent_obj, mem_type_id, mem_space_id, file_space_id, + plist_id, buf, return_task_val); + time22 = MPI_Wtime(); + + duration2 = time22 - time21; + fprintf(stderr, "in the for loop Runtime is i=%d %f \n", i, duration1); + } + + time2 = MPI_Wtime(); duration = time2 - time1; - - - - fprintf(stderr,"Foreach Runtime is %f \n", duration); - + + fprintf(stderr, "Foreach Runtime is %f \n", duration); + return return_val; } @@ -10141,13 +10134,13 @@ async_dataset_write(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t mem_t assert(aid); assert(parent_obj); assert(parent_obj->magic == ASYNC_MAGIC); - //fprintf(stderr,"mem_type_id=%llu\n",mem_type_id); + // fprintf(stderr,"mem_type_id=%llu\n",mem_type_id); return_val = async_dataset_write_merge(aid, parent_obj, mem_type_id, mem_space_id, file_space_id, plist_id, buf); if (return_val == 1) - goto done; + goto done; async_instance_g->prev_push_state = async_instance_g->start_abt_push; if ((args = (async_dataset_write_args_t *)calloc(1, sizeof(async_dataset_write_args_t))) == NULL) { From 359a229238283a8279545ea41907ec093ccc49ef Mon Sep 17 00:00:00 2001 From: Kamal Chowdhury <106797012+kamalchowdhurylbl@users.noreply.github.com> Date: Tue, 9 Aug 2022 21:22:06 -0500 Subject: [PATCH 03/18] Create 1d_asyn_MPI_write.c --- test/1d_asyn_MPI_write.c | 436 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 436 insertions(+) create mode 100644 test/1d_asyn_MPI_write.c diff --git a/test/1d_asyn_MPI_write.c b/test/1d_asyn_MPI_write.c new file mode 100644 index 0000000..3673865 --- /dev/null +++ b/test/1d_asyn_MPI_write.c @@ -0,0 +1,436 @@ +/************************************************************ + + This example shows how to write and read a hyperslab. It + is derived from the h5_read.c and h5_write.c examples in + the "Introduction to HDF5". It works on the 3 dimensional data. + + ************************************************************/ + +#include "hdf5.h" +#include + +#define FILE "sds.h5" +#define DATASETNAME "IntArray" + +#define RANK 1 // since it will work on 2 dimensional data +#define RANK_OUT 1 + +#define X 40 + +#define X1 40 + +void +drawBorder() +{ + fprintf(stderr, "\n"); + for (int i = 0; i < 60; i++) { + fprintf(stderr, "#"); + } + fprintf(stderr, "\n"); +} + +int +main(int argc, char **argv) +{ + hsize_t dimsf[1]; /* dataset dimensions */ + int data[X]; /* data to write */ + + /* + * Data and output buffer initialization. + */ + hid_t file, dataset; /* handles */ + hid_t dataspace; + hid_t memspace; + hsize_t dimsm[1]; /* memory space dimensions 1D*/ + hsize_t dims_out[1]; /* dataset dimensions 1D */ + herr_t status; + hid_t property_list_id_MPIO; /* property list identifier */ + hid_t data_transfer_propertylist; + + int data_out[X]; // data out 3d is 6x6x3 + int data_out1[X1]; // data out1 is 2x6x3 + + hsize_t count[1]; /* size of the hyperslab in the file */ + hsize_t offset[1]; /* hyperslab offset in the file */ + hsize_t count_out[1]; /* size of the hyperslab in memory */ + hsize_t offset_out[1]; + + int i, j, k, status_n, rank; + int print_dbg_msg = 1; + hbool_t op_failed; + size_t num_in_progress; + hid_t es_id = H5EScreate(); + int provided; + /* + * MPI variables + */ + int mpi_size, mpi_rank; + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Info info = MPI_INFO_NULL; + + /* + * Initialize MPI + */ + // MPI_Init(&argc, &argv); + MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); + MPI_Comm_size(comm, &mpi_size); + MPI_Comm_rank(comm, &mpi_rank); + + property_list_id_MPIO = H5Pcreate(H5P_FILE_ACCESS); + H5Pset_fapl_mpio(property_list_id_MPIO, comm, info); + + data_transfer_propertylist = H5Pcreate(H5P_DATASET_XFER); + H5Pset_dxpl_mpio(data_transfer_propertylist, H5FD_MPIO_COLLECTIVE); + + // 1d data + int l = 0; + + for (i = 0; i < X; i++) { + data[i] = l; + l++; + } + + if (mpi_rank == 0) { + + for (i = 0; i < X; i++) + fprintf(stderr, "%5d", data[i]); + fprintf(stderr, "\n"); + } + + // 1d operations + + file = H5Fcreate_async(FILE, H5F_ACC_TRUNC, H5P_DEFAULT, property_list_id_MPIO, es_id); + + dimsf[0] = X; + dataspace = H5Screate_simple(RANK, dimsf, NULL); + dimsm[0] = X1; + memspace = H5Screate_simple(RANK_OUT, dimsm, NULL); // RANK_OUT=3 + dataset = H5Dcreate_async(file, DATASETNAME, H5T_STD_I32BE, dataspace, H5P_DEFAULT, H5P_DEFAULT, + H5P_DEFAULT, es_id); + + int array[][4][2] = {{{0, 5}, + {5, 3}, + {8, 2}, // testcase 1 + {10, 10}}, + {{0, 5}, + {5, 5}, + {8, 2}, // testcase 2 + {10, 5}}, + {{0, 5}, + {10, 10}, // testcase 3 + {8, 2}, + {5, 3}}, + {{0, 5}, + {10, 10}, // testcase 4 + {5, 3}, + {8, 3}}, + {{0, 5}, + {10, 10}, // testcase 5 + {5, 5}, + {20, 3}}, + {{2, 3}, + {11, 10}, // testcase 6 + {5, 3}, + {8, 3}}, + {{3, 5}, + {11, 10}, // testcase 7 + {5, 3}, + {8, 3}}}; + /* int array[][4][2]={ + {{0,5}, + {5,3}, + {8,2}, //testcase 1 + {10,10} + } + }; */ + + int testcases = (int)sizeof(array) / sizeof(array[0]); + // fprintf(stderr, "%d",testcases); + // testcases=7; + hid_t dataset_array[testcases]; + char datasetname[testcases][12]; + // int arr[4][2]; + + for (int j = 0; j < testcases; j++) { + + snprintf(datasetname[j], 12, "Testcase# %d", j + 1); + // fprintf(stderr,"dataset name= %s \n",datasetname[j]); + dataset_array[j] = H5Dcreate_async(file, datasetname[j], H5T_STD_I32BE, dataspace, H5P_DEFAULT, + H5P_DEFAULT, H5P_DEFAULT, es_id); + } + + if (mpi_rank == 0) { + + for (int j = 0; j < testcases; j++) { + + drawBorder(); + fprintf(stderr, "dataset name= %s \n", datasetname[j]); + drawBorder(); + + for (int i = 0; i < 4; i++) { + + offset[0] = array[j][i][0]; + count[0] = array[j][i][1]; + + status = H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, offset, NULL, count, NULL); + + // printf("MPI rank=%d Hyperslab operation on dataspace using offset %llu and count %llu + // \n",mpi_rank,offset[0],count[0]); + offset_out[0] = array[j][i][0]; + count_out[0] = array[j][i][1]; + // fprintf(stderr,"%lld=%lld %lld=%lld\n",offset[0],offset_out[0],count[0],count_out[0]); + status = H5Sselect_hyperslab(memspace, H5S_SELECT_SET, offset_out, NULL, count_out, NULL); + + // fprintf(stderr,"----------------- on memory space using offset %llu and count %llu + // \n",offset_out[0],count_out[0]); + + status = H5Dwrite_async(dataset_array[j], H5T_NATIVE_INT, memspace, dataspace, + data_transfer_propertylist, data, es_id); + } + } + + /* + offset[0] = 0; + count[0] = 5; + + status = H5Sselect_hyperslab (dataspace, H5S_SELECT_SET, offset, NULL, count, NULL); + + printf("MPI rank=%d Hyperslab operation on dataspace using offset %llu and count %llu + \n",mpi_rank,offset[0],count[0]); offset_out[0] = 0; //offset=0x0 count_out[0] = 5; //count_out=2 X + 2 + + status = H5Sselect_hyperslab (memspace, H5S_SELECT_SET, offset_out, NULL, count_out, NULL); + + fprintf(stderr,"----------------- on memory space using offset %llu and count %llu + \n",offset_out[0],count_out[0]); + + status = H5Dwrite_async (dataset, H5T_NATIVE_INT, memspace, dataspace, data_transfer_propertylist, + data,es_id); + + + + offset[0] = 5; + count[0] = 3; + + status = H5Sselect_hyperslab (dataspace, H5S_SELECT_SET, offset, NULL, count, NULL); + + printf("MPI rank=%d Hyperslab operation on dataspace using offset %llu and count %llu + \n",mpi_rank,offset[0],count[0]); offset_out[0] = 5; //offset=0x0 count_out[0] = 3; //count_out=2 X + 2 + + status = H5Sselect_hyperslab (memspace, H5S_SELECT_SET, offset_out, NULL, count_out, NULL); + + + printf("----------------- on memory space using offset %llu and count %llu + \n",offset_out[0],count_out[0]); + + status = H5Dwrite_async (dataset, H5T_NATIVE_INT, memspace, dataspace, data_transfer_propertylist, + data,es_id); + + + offset[0] = 8; + count[0] = 2; + + status = H5Sselect_hyperslab (dataspace, H5S_SELECT_SET, offset, NULL, count, NULL); + + printf("MPI rank=%d Hyperslab operation on dataspace using offset %llu and count %llu + \n",mpi_rank,offset[0],count[0]); offset_out[0] = 8; //offset=0x0 count_out[0] = 2; //count_out=2 X + 2 + + status = H5Sselect_hyperslab (memspace, H5S_SELECT_SET, offset_out, NULL, count_out, NULL); + + printf("----------------- on memory space using offset %llu and count %llu + \n",offset_out[0],count_out[0]); + + status = H5Dwrite_async (dataset, H5T_NATIVE_INT, memspace, dataspace, data_transfer_propertylist, + data,es_id); + + + + offset[0] = 10; + count[0] = 10; + + status = H5Sselect_hyperslab (dataspace, H5S_SELECT_SET, offset, NULL, count, NULL); + + printf("MPI rank=%d Hyperslab operation on dataspace using offset %llu and count %llu + \n",mpi_rank,offset[0],count[0]); offset_out[0] = 10; //offset=0x0 count_out[0] = 10; //count_out=2 + X 2 + + status = H5Sselect_hyperslab (memspace, H5S_SELECT_SET, offset_out, NULL, count_out, NULL); + + printf("----------------- on memory space using offset %llu and count %llu + \n",offset_out[0],count_out[0]); + + status = H5Dwrite_async (dataset, H5T_NATIVE_INT, memspace, dataspace, data_transfer_propertylist, + data,es_id); + + + */ + } + if (mpi_rank == 1) { + + offset[0] = 20; + + count[0] = 5; + status = H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, offset, NULL, count, NULL); + printf("MPI rank=%d Hyperslab operation on dataspace using offset %llu and count %llu \n", mpi_rank, + offset[0], count[0]); + + offset_out[0] = 20; + + count_out[0] = 5; + status = H5Sselect_hyperslab(memspace, H5S_SELECT_SET, offset_out, NULL, count_out, NULL); + printf("----------------- on memory space using offset %llu and count %llu \n", offset_out[0], + count_out[0]); + + // status = H5Dread_async (dataset, H5T_NATIVE_INT, memspace, dataspace, H5P_DEFAULT, + // data_out2,es_id); + status = H5Dwrite_async(dataset, H5T_NATIVE_INT, memspace, dataspace, data_transfer_propertylist, + data, es_id); + + offset[0] = 25; + + count[0] = 3; + status = H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, offset, NULL, count, NULL); + printf("MPI rank=%d Hyperslab operation on dataspace using offset %llu and count %llu \n", mpi_rank, + offset[0], count[0]); + + offset_out[0] = 25; + + count_out[0] = 3; + status = H5Sselect_hyperslab(memspace, H5S_SELECT_SET, offset_out, NULL, count_out, NULL); + printf("----------------- on memory space using offset %llu and count %llu \n", offset_out[0], + count_out[0]); + + // status = H5Dread_async (dataset, H5T_NATIVE_INT, memspace, dataspace, H5P_DEFAULT, + // data_out2,es_id); + status = H5Dwrite_async(dataset, H5T_NATIVE_INT, memspace, dataspace, data_transfer_propertylist, + data, es_id); + + /* + + offset[0] = 28; + + count[0] = 2; + status = H5Sselect_hyperslab (dataspace, H5S_SELECT_SET, offset, NULL, count, NULL); + printf("MPI rank=%d Hyperslab operation on dataspace using offset %llu and count %llu + \n",mpi_rank,offset[0],count[0]); + + + offset_out[0] = 28; + + count_out[0] = 2; + status = H5Sselect_hyperslab (memspace, H5S_SELECT_SET, offset_out, NULL, count_out, NULL); + printf("----------------- on memory space using offset %llu and count %llu + \n",offset_out[0],count_out[0]); + + + // status = H5Dread_async (dataset, H5T_NATIVE_INT, memspace, dataspace, H5P_DEFAULT, + data_out2,es_id); status = H5Dwrite_async (dataset, H5T_NATIVE_INT, memspace, dataspace, + data_transfer_propertylist, data,es_id); + + + + offset[0] = 30; + + count[0] = 10; + status = H5Sselect_hyperslab (dataspace, H5S_SELECT_SET, offset, NULL, count, NULL); + printf("MPI rank=%d Hyperslab operation on dataspace using offset %llu and count %llu + \n",mpi_rank,offset[0],count[0]); + + + offset_out[0] = 30; + + count_out[0] = 10; + status = H5Sselect_hyperslab (memspace, H5S_SELECT_SET, offset_out, NULL, count_out, NULL); + printf("----------------- on memory space using offset %llu and count %llu + \n",offset_out[0],count_out[0]); + + + // status = H5Dread_async (dataset, H5T_NATIVE_INT, memspace, dataspace, H5P_DEFAULT, + data_out2,es_id); status = H5Dwrite_async (dataset, H5T_NATIVE_INT, memspace, dataspace, + data_transfer_propertylist, data,es_id); + */ + } + + if (print_dbg_msg) + fprintf(stderr, "before H5ESwait done\n"); + + /* status = H5ESwait(es_id, H5ES_WAIT_FOREVER, &num_in_progress, &op_failed); + if (status < 0) { + fprintf(stderr, "Error with H5ESwait\n"); + + } */ + if (print_dbg_msg) + fprintf(stderr, "H5ESwait done\n"); + + MPI_Barrier(comm); + + if (mpi_rank == 0) { + + for (int p = 0; p < testcases; p++) { + drawBorder(); + fprintf(stderr, "Testcase= %d\n", p + 1); + drawBorder(); + status = H5Dread_async(dataset_array[p], H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, data_out, + es_id); + + fprintf(stderr, "\nData out from the file\n"); + + status = H5ESwait(es_id, H5ES_WAIT_FOREVER, &num_in_progress, &op_failed); + if (status < 0) { + fprintf(stderr, "Error with H5ESwait\n"); + } + for (i = 0; i < X; i++) + // if(data_out[i]!=0) + fprintf(stderr, " %4d ", data_out[i]); + fprintf(stderr, "\n"); + } + } + /* + * Close/release resources. + */ + H5Sclose(dataspace); + // fprintf(stderr,"dataspace closed\n"); + + H5Sclose(memspace); + // fprintf(stderr,"memspace closed\n"); + for (int j = 0; j < testcases; j++) { + + status = H5Dclose_async(dataset_array[j], es_id); + if (status < 0) { + fprintf(stderr, "Closing dataset failed\n"); + // ret = -1; + } + } + + status = H5Dclose_async(dataset, es_id); + if (status < 0) { + fprintf(stderr, "Closing dataset failed\n"); + // ret = -1; + } + + status = H5Fclose_async(file, es_id); + if (status < 0) { + fprintf(stderr, "Closing file failed\n"); + // ret = -1; + } + // H5Fclose(file); + status = H5ESwait(es_id, H5ES_WAIT_FOREVER, &num_in_progress, &op_failed); + if (status < 0) { + fprintf(stderr, "Error with H5ESwait\n"); + // ret = -1; + } + + status = H5ESclose(es_id); + if (status < 0) { + fprintf(stderr, "Can't close second event set\n"); + // ret = -1; + } + H5Pclose(property_list_id_MPIO); + H5Pclose(data_transfer_propertylist); + MPI_Barrier(comm); + MPI_Finalize(); + + return 0; +} From e05fcad3b1de9a0748524a57ccbe072a62506bd1 Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 10 Aug 2022 02:22:36 +0000 Subject: [PATCH 04/18] Committing clang-format changes --- test/1d_asyn_MPI_write.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/1d_asyn_MPI_write.c b/test/1d_asyn_MPI_write.c index 3673865..13a4ea9 100644 --- a/test/1d_asyn_MPI_write.c +++ b/test/1d_asyn_MPI_write.c @@ -106,7 +106,7 @@ main(int argc, char **argv) dimsm[0] = X1; memspace = H5Screate_simple(RANK_OUT, dimsm, NULL); // RANK_OUT=3 dataset = H5Dcreate_async(file, DATASETNAME, H5T_STD_I32BE, dataspace, H5P_DEFAULT, H5P_DEFAULT, - H5P_DEFAULT, es_id); + H5P_DEFAULT, es_id); int array[][4][2] = {{{0, 5}, {5, 3}, From 58014265fdf241952cc378826a93e9c2f05da717 Mon Sep 17 00:00:00 2001 From: Kamal Chowdhury <106797012+kamalchowdhurylbl@users.noreply.github.com> Date: Tue, 9 Aug 2022 21:26:01 -0500 Subject: [PATCH 05/18] Added 2d_asyn_MPI_write.c, 3d_asyn_MPI_write.c --- test/2d_asyn_MPI_write.c | 460 ++++++++++++++++++++++++++++++ test/3d_asyn_MPI_write.c | 595 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 1055 insertions(+) create mode 100644 test/2d_asyn_MPI_write.c create mode 100644 test/3d_asyn_MPI_write.c diff --git a/test/2d_asyn_MPI_write.c b/test/2d_asyn_MPI_write.c new file mode 100644 index 0000000..42988e9 --- /dev/null +++ b/test/2d_asyn_MPI_write.c @@ -0,0 +1,460 @@ +/************************************************************ + + This example shows how to write and read a hyperslab. It + is derived from the h5_read.c and h5_write.c examples in + the "Introduction to HDF5". It works on the 3 dimensional data. + ************************************************************/ + +#include "hdf5.h" + +#define FILE "sds.h5" +#define DATASETNAME "IntArray" + +#define RANK 2 // since it will work on 2 dimensional data +#define RANK_OUT 2 + +#define X 4 +#define Y 6 + +#define X1 4 +#define Y1 6 + +#define X2 4 +#define Y2 6 + +void +drawBorder() +{ + fprintf(stderr, "\n"); + for (int i = 0; i < 60; i++) { + fprintf(stderr, "#"); + } + fprintf(stderr, "\n"); +} + +int +main(int argc, char **argv) +{ + hsize_t dimsf[2]; /* dataset dimensions */ + int data[X][Y]; /* data to write */ + + /* + * Data and output buffer initialization. + */ + hid_t file, dataset; /* handles */ + hid_t dataspace; + hid_t memspace; + hsize_t dimsm[2]; /* memory space dimensions 1D*/ + hsize_t dims_out[2]; /* dataset dimensions 1D */ + herr_t status; + hid_t property_list_id_MPIO; /* property list identifier */ + hid_t data_transfer_propertylist; + + int data_out[X][Y]; // data out 3d is 6x6x3 + int data_out1[X1][Y1]; // data out1 is 2x6x3 + int data_out2[X2][Y2]; // data out1 is 3x6x3 + + hsize_t count[2]; /* size of the hyperslab in the file */ + hsize_t offset[2]; /* hyperslab offset in the file */ + hsize_t count_out[2]; /* size of the hyperslab in memory */ + hsize_t offset_out[2]; + + int i, j, k, status_n, rank; + int print_dbg_msg = 1; + hbool_t op_failed; + size_t num_in_progress; + hid_t es_id = H5EScreate(); + int provided; + /* + * MPI variables + */ + int mpi_size, mpi_rank; + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Info info = MPI_INFO_NULL; + + /* + * Initialize MPI + */ + // MPI_Init(&argc, &argv); + MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); + MPI_Comm_size(comm, &mpi_size); + MPI_Comm_rank(comm, &mpi_rank); + + property_list_id_MPIO = H5Pcreate(H5P_FILE_ACCESS); + H5Pset_fapl_mpio(property_list_id_MPIO, comm, info); + + data_transfer_propertylist = H5Pcreate(H5P_DATASET_XFER); + H5Pset_dxpl_mpio(data_transfer_propertylist, H5FD_MPIO_COLLECTIVE); + + // 3d data + int l = 0; + + for (i = 0; i < X; i++) + for (j = 0; j < Y; j++) { + data[i][j] = l; // 4x6 + l++; + } + + if (mpi_rank == 0) { + + for (i = 0; i < X; i++) { + for (j = 0; j < Y; j++) + fprintf(stderr, "%5d", data[i][j]); + fprintf(stderr, "\n"); + } + } + + // 2d operations + + file = H5Fcreate_async(FILE, H5F_ACC_TRUNC, H5P_DEFAULT, property_list_id_MPIO, es_id); + + dimsf[0] = X; + dimsf[1] = Y; + + dataspace = H5Screate_simple(RANK, dimsf, NULL); + + dataset = H5Dcreate_async(file, DATASETNAME, H5T_STD_I32BE, dataspace, H5P_DEFAULT, H5P_DEFAULT, + H5P_DEFAULT, es_id); // H5T_STD_I32BE = 32-bit big-endian signed integers + + dimsm[0] = X1; + dimsm[1] = Y1; + + memspace = H5Screate_simple(RANK_OUT, dimsm, NULL); // RANK_OUT=3 + + /* int array[][4][4]={ + { + {0,0,2,2}, + {0,2,2,1}, + {0,3,2,2}, //Testcase# 1 + {0,5,2,1} + }, + { + {0,0,2,2}, + {2,0,1,2}, + {0,3,2,2}, //Testcase# 2 + {0,5,2,1} + }, + { + {0,0,2,1}, + {0,1,2,2}, + {0,3,2,1}, //Testcase# 3 + {2,1,2,2} + }, + { + {0,1,2,3}, + {0,4,2,2}, + {2,0,2,1}, //Testcase# 4 + {2,1,2,3} + }, + { + {1,1,2,3}, + {1,4,2,1}, + {3,1,1,3}, //Testcase# 5 + {3,4,1,1} + }, + { + {1,1,2,3}, + {3,1,1,3}, + {3,4,1,1},//Testcase# 6 + {1,4,2,1}, + }, + { + {3,1,1,3}, + {1,1,2,3}, + {3,4,1,1},//Testcase# 7 + {1,4,2,1}, + }, + { + {3,1,1,3}, + {1,1,2,3}, + {1,5,2,1},//Testcase# 8 + {1,4,2,1}, + } + }; */ + int array[][4][4] = { + {{1, 1, 2, 3}, + {1, 4, 2, 1}, + {3, 1, 1, 3}, // Testcase# 5 + {3, 4, 1, 1}}, + {{1, 1, 2, 3}, + {1, 4, 2, 1}, + {3, 1, 1, 3}, // Testcase# 5 + {3, 4, 1, 1}}, + + }; + int testcases = (int)sizeof(array) / sizeof(array[0]); + hid_t dataset_array[testcases]; + char datasetname[testcases][12]; + // int arr[4][2]; + // fprintf(stderr, "%ld",sizeof(array)/sizeof(array[0])); + for (int j = 0; j < testcases; j++) { + + snprintf(datasetname[j], 12, "Testcase# %d", j + 1); + // fprintf(stderr,"dataset name= %s \n",datasetname[j]); + dataset_array[j] = H5Dcreate_async(file, datasetname[j], H5T_STD_I32BE, dataspace, H5P_DEFAULT, + H5P_DEFAULT, H5P_DEFAULT, es_id); + } + + if (mpi_rank == 0) { + + for (int j = 0; j < testcases; j++) { + drawBorder(); + fprintf(stderr, "dataset name= %s \n", datasetname[j]); + drawBorder(); + for (int i = 0; i < 4; i++) { + + offset[0] = array[j][i][0]; + offset[1] = array[j][i][1]; + count[0] = array[j][i][2]; + count[1] = array[j][i][3]; + status = H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, offset, NULL, count, NULL); + + // printf("MPI rank=%d Hyperslab operation on dataspace using offset %llux%llu and count + // %llux%llu \n",mpi_rank,offset[0],offset[1],count[0],count[1]); + offset_out[0] = array[j][i][0]; // offset=0x0 + offset_out[1] = array[j][i][1]; + count_out[0] = array[j][i][2]; // count_out=2 X 2 + count_out[1] = array[j][i][3]; + + status = H5Sselect_hyperslab(memspace, H5S_SELECT_SET, offset_out, NULL, count_out, NULL); + + // printf("----------------- on memory space using offset %llux%llu and count %llux%llu + // \n",offset_out[0],offset_out[1],count_out[0],count_out[1]); + + status = H5Dwrite_async(dataset_array[j], H5T_NATIVE_INT, memspace, dataspace, + data_transfer_propertylist, data, es_id); + } + /* status = H5ESwait(es_id, H5ES_WAIT_FOREVER, &num_in_progress, &op_failed); + if (status < 0) { + fprintf(stderr, "Error with H5ESwait\n"); + + } */ + } + + /* offset[0] = 0; + offset[1]=0; + count[0] = 2; + count[1] = 2; + status = H5Sselect_hyperslab (dataspace, H5S_SELECT_SET, offset, NULL, count, NULL); + + printf("MPI rank=%d Hyperslab operation on dataspace using offset %llux%llu and count %llux%llu + \n",mpi_rank,offset[0],offset[1],count[0],count[1]); offset_out[0] = 0; //offset=0x0 offset_out[1] = + 0; count_out[0] = 2; //count_out=2 X 2 count_out[1] = 2; + + status = H5Sselect_hyperslab (memspace, H5S_SELECT_SET, offset_out, NULL, count_out, NULL); + + printf("----------------- on memory space using offset %llux%llu and count %llux%llu + \n",offset_out[0],offset_out[1],count_out[0],count_out[1]); + + status = H5Dwrite_async (dataset, H5T_NATIVE_INT, memspace, dataspace, data_transfer_propertylist, + data,es_id); + + + offset[0] = 0; + offset[1]=2; + count[0] = 2; + count[1] = 1; + status = H5Sselect_hyperslab (dataspace, H5S_SELECT_SET, offset, NULL, count, NULL); + + printf("MPI rank=%d Hyperslab operation on dataspace using offset %llux%llu and count %llux%llu + \n",mpi_rank,offset[0],offset[1],count[0],count[1]); offset_out[0] = 0; //offset=0x0 offset_out[1] = + 2; count_out[0] = 2; //count_out=2 X 2 count_out[1] = 1; + + status = H5Sselect_hyperslab (memspace, H5S_SELECT_SET, offset_out, NULL, count_out, NULL); + + printf("----------------- on memory space using offset %llux%llu and count %llux%llu + \n",offset_out[0],offset_out[1],count_out[0],count_out[1]); + + status = H5Dwrite_async (dataset, H5T_NATIVE_INT, memspace, dataspace, data_transfer_propertylist, + data,es_id); + + + offset[0] = 0; + offset[1]=3; + count[0] = 2; + count[1] = 2; + status = H5Sselect_hyperslab (dataspace, H5S_SELECT_SET, offset, NULL, count, NULL); + + printf("MPI rank=%d Hyperslab operation on dataspace using offset %llux%llu and count %llux%llu + \n",mpi_rank,offset[0],offset[1],count[0],count[1]); offset_out[0] = 0; //offset=0x0 offset_out[1] = + 3; count_out[0] = 2; //count_out=2 X 2 count_out[1] = 2; + + status = H5Sselect_hyperslab (memspace, H5S_SELECT_SET, offset_out, NULL, count_out, NULL); + + printf("----------------- on memory space using offset %llux%llu and count %llux%llu + \n",offset_out[0],offset_out[1],count_out[0],count_out[1]); + + status = H5Dwrite_async (dataset, H5T_NATIVE_INT, memspace, dataspace, data_transfer_propertylist, + data,es_id); + + + offset[0] = 0; + offset[1]=5; + count[0] = 2; + count[1] = 1; + status = H5Sselect_hyperslab (dataspace, H5S_SELECT_SET, offset, NULL, count, NULL); + + printf("MPI rank=%d Hyperslab operation on dataspace using offset %llux%llu and count %llux%llu + \n",mpi_rank,offset[0],offset[1],count[0],count[1]); offset_out[0] = 0; //offset=0x0 offset_out[1] = + 5; count_out[0] = 2; //count_out=2 X 2 count_out[1] = 1; + + status = H5Sselect_hyperslab (memspace, H5S_SELECT_SET, offset_out, NULL, count_out, NULL); + + printf("----------------- on memory space using offset %llux%llu and count %llux%llu + \n",offset_out[0],offset_out[1],count_out[0],count_out[1]); + + status = H5Dwrite_async (dataset, H5T_NATIVE_INT, memspace, dataspace, data_transfer_propertylist, + data,es_id); + */ + } + if (mpi_rank == 1) { + + offset[0] = 2; + offset[1] = 0; + + count[0] = 2; + count[1] = 2; // count=4x2 + status = H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, offset, NULL, count, NULL); + printf("MPI rank=%d Hyperslab operation on dataspace using offset %llux%llu and count %llux%llu \n", + mpi_rank, offset[0], offset[1], count[0], count[1]); + + offset_out[0] = 2; + offset_out[1] = 0; // offset_out=2x0 + + count_out[0] = 2; + count_out[1] = 2; // count_out=4x2 + status = H5Sselect_hyperslab(memspace, H5S_SELECT_SET, offset_out, NULL, count_out, NULL); + printf("----------------- on memory space using offset %llux%llu and count %llux%llu \n", + offset_out[0], offset_out[1], count_out[0], count_out[1]); + + // status = H5Dread_async (dataset, H5T_NATIVE_INT, memspace, dataspace, H5P_DEFAULT, + // data_out2,es_id); + status = H5Dwrite_async(dataset, H5T_NATIVE_INT, memspace, dataspace, data_transfer_propertylist, + data, es_id); + + offset[0] = 2; + offset[1] = 2; + + count[0] = 2; + count[1] = 2; // count=4x2 + status = H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, offset, NULL, count, NULL); + printf("MPI rank=%d Hyperslab operation on dataspace using offset %llux%llu and count %llux%llu \n", + mpi_rank, offset[0], offset[1], count[0], count[1]); + + offset_out[0] = 2; + offset_out[1] = 2; // offset_out=2x2 + + count_out[0] = 2; + count_out[1] = 2; // count_out=4x2 + status = H5Sselect_hyperslab(memspace, H5S_SELECT_SET, offset_out, NULL, count_out, NULL); + printf("----------------- on memory space using offset %llux%llu and count %llux%llu \n", + offset_out[0], offset_out[1], count_out[0], count_out[1]); + + // status = H5Dread_async (dataset, H5T_NATIVE_INT, memspace, dataspace, H5P_DEFAULT, + // data_out2,es_id); + status = H5Dwrite_async(dataset, H5T_NATIVE_INT, memspace, dataspace, data_transfer_propertylist, + data, es_id); + + offset[0] = 2; + offset[1] = 4; + + count[0] = 2; + count[1] = 2; // count=2x2 + status = H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, offset, NULL, count, NULL); + printf("MPI rank=%d Hyperslab operation on dataspace using offset %llux%llu and count %llux%llu \n", + mpi_rank, offset[0], offset[1], count[0], count[1]); + + offset_out[0] = 2; + offset_out[1] = 4; // offset_out=2x4 + + count_out[0] = 2; + count_out[1] = 2; // count_out=2x2 + status = H5Sselect_hyperslab(memspace, H5S_SELECT_SET, offset_out, NULL, count_out, NULL); + printf("----------------- on memory space using offset %llux%llu and count %llux%llu \n", + offset_out[0], offset_out[1], count_out[0], count_out[1]); + + // status = H5Dread_async (dataset, H5T_NATIVE_INT, memspace, dataspace, H5P_DEFAULT, + // data_out2,es_id); + status = H5Dwrite_async(dataset, H5T_NATIVE_INT, memspace, dataspace, data_transfer_propertylist, + data, es_id); + } + + status = H5ESwait(es_id, H5ES_WAIT_FOREVER, &num_in_progress, &op_failed); + if (status < 0) { + fprintf(stderr, "Error with H5ESwait\n"); + } + if (print_dbg_msg) + fprintf(stderr, "H5ESwait done\n"); + + MPI_Barrier(comm); + + if (mpi_rank == 0) { + // fprintf(stderr,"%d\n",testcases); + for (int p = 0; p < testcases; p++) { + drawBorder(); + fprintf(stderr, "Testcase= %d\n", p + 1); + drawBorder(); + status = H5Dread_async(dataset_array[p], H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, data_out, + es_id); + + fprintf(stderr, "\nData out from the file\n"); + + status = H5ESwait(es_id, H5ES_WAIT_FOREVER, &num_in_progress, &op_failed); + if (status < 0) { + fprintf(stderr, "Error with H5ESwait\n"); + } + + for (i = 0; i < X; i++) { + for (j = 0; j < Y; j++) + fprintf(stderr, " %4d ", data_out[i][j]); + fprintf(stderr, "\n"); + } + } + } + + /* + * Close/release resources. + */ + H5Sclose(dataspace); + + H5Sclose(memspace); + + for (int j = 0; j < testcases; j++) { + + status = H5Dclose_async(dataset_array[j], es_id); + if (status < 0) { + fprintf(stderr, "Closing dataset failed\n"); + // ret = -1; + } + } + + status = H5Dclose_async(dataset, es_id); + if (status < 0) { + fprintf(stderr, "Closing dataset failed\n"); + // ret = -1; + } + + status = H5Fclose_async(file, es_id); + if (status < 0) { + fprintf(stderr, "Closing file failed\n"); + // ret = -1; + } + // H5Fclose(file); + status = H5ESwait(es_id, H5ES_WAIT_FOREVER, &num_in_progress, &op_failed); + if (status < 0) { + fprintf(stderr, "Error with H5ESwait\n"); + // ret = -1; + } + + status = H5ESclose(es_id); + if (status < 0) { + fprintf(stderr, "Can't close second event set\n"); + // ret = -1; + } + H5Pclose(property_list_id_MPIO); + H5Pclose(data_transfer_propertylist); + MPI_Barrier(comm); + MPI_Finalize(); + + return 0; +} \ No newline at end of file diff --git a/test/3d_asyn_MPI_write.c b/test/3d_asyn_MPI_write.c new file mode 100644 index 0000000..605a935 --- /dev/null +++ b/test/3d_asyn_MPI_write.c @@ -0,0 +1,595 @@ +/************************************************************ + + This example shows how to write and read a hyperslab. It + is derived from the h5_read.c and h5_write.c examples in + the "Introduction to HDF5". It works on the 3 dimensional data. + + ************************************************************/ + +#include "hdf5.h" + +#define FILE "sds.h5" +#define DATASETNAME "IntArray" + +#define RANK 3 // since it will work on 3 dimensional data +#define RANK_OUT 3 + +#define X 6 +#define Y 6 +#define Z 3 + +#define X1 6 +#define Y1 6 +#define Z1 3 + +#define X2 6 +#define Y2 6 +#define Z2 3 + +void +drawBorder() +{ + fprintf(stderr, "\n"); + for (int i = 0; i < 60; i++) { + fprintf(stderr, "#"); + } + fprintf(stderr, "\n"); +} +int +main(int argc, char **argv) +{ + hsize_t dimsf[3]; /* dataset dimensions */ + int data[Z][X][Y]; /* data to write */ + + /* + * Data and output buffer initialization. + */ + hid_t file, dataset; /* handles */ + hid_t dataspace; + hid_t memspace; + hsize_t dimsm[3]; /* memory space dimensions 1D*/ + hsize_t dims_out[3]; /* dataset dimensions 1D */ + herr_t status; + hid_t property_list_id_MPIO; /* property list identifier */ + hid_t data_transfer_propertylist; + + int data_out[Z][X][Y]; // data out 3d is 6x6x3 + // int data_out1[X1][Y1][Z1]; //data out1 is 2x6x3 + // int data_out2[X2][Y2][Z2]; //data out1 is 3x6x3 + + hsize_t count[3]; /* size of the hyperslab in the file */ + hsize_t offset[3]; /* hyperslab offset in the file */ + hsize_t count_out[3]; /* size of the hyperslab in memory */ + hsize_t offset_out[3]; + + int i, j, k, status_n, rank; + int print_dbg_msg = 1; + hbool_t op_failed; + size_t num_in_progress; + hid_t es_id = H5EScreate(); + int provided; + /* + * MPI variables + */ + int mpi_size, mpi_rank; + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Info info = MPI_INFO_NULL; + + /* + * Initialize MPI + */ + // MPI_Init(&argc, &argv); + MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); + MPI_Comm_size(comm, &mpi_size); + MPI_Comm_rank(comm, &mpi_rank); + + property_list_id_MPIO = H5Pcreate(H5P_FILE_ACCESS); + H5Pset_fapl_mpio(property_list_id_MPIO, comm, info); + + data_transfer_propertylist = H5Pcreate(H5P_DATASET_XFER); + H5Pset_dxpl_mpio(data_transfer_propertylist, H5FD_MPIO_COLLECTIVE); + + // 3d data + int l = 0; + for (k = 0; k < Z; k++) + for (i = 0; i < X; i++) + for (j = 0; j < Y; j++) { + data[k][i][j] = l; // 6x6x6 + l++; + } + /* data[0][0][0]=1000; + data[0][1][0]=1001; + data[0][2][0]=1002; + data[1][0][0]=1003; + data[1][1][0]=1004; + data[1][2][0]=1005; */ + // data[0][0][3]=1050; + + if (mpi_rank == 0) { + for (k = 0; k < Z; k++) { + fprintf(stderr, "\nFirst 3D Data Z=%d:\n ", k); + + for (i = 0; i < X; i++) { + for (j = 0; j < Y; j++) + fprintf(stderr, "%5d", data[k][i][j]); + fprintf(stderr, "\n "); + } + } + } + + // 3d operations + + file = H5Fcreate_async(FILE, H5F_ACC_TRUNC, H5P_DEFAULT, property_list_id_MPIO, es_id); + /* + * Describe the size of the array and create the data space for fixed + * size dataset. + */ + + dimsf[0] = Z; + dimsf[1] = X; + dimsf[2] = Y; + dataspace = H5Screate_simple(RANK, dimsf, NULL); + + /* + * Create a new dataset within the file using defined dataspace and + * default dataset creation properties. + */ + dataset = H5Dcreate_async(file, DATASETNAME, H5T_STD_I32BE, dataspace, H5P_DEFAULT, H5P_DEFAULT, + H5P_DEFAULT, es_id); // H5T_STD_I32BE = 32-bit big-endian signed integers + + dimsm[0] = Z1; + dimsm[1] = X1; + dimsm[2] = Y1; + + memspace = H5Screate_simple(RANK_OUT, dimsm, NULL); // RANK_OUT=3 + + // Test case#1 + /* int arr[4][6]={ + {0,0,0,2,2,1}, + {0,2,0,2,1,1}, + {0,3,0,2,2,1}, + {0,5,0,2,1,1} + }; */ + // Test case#2 + /* int arr[4][6]={ + {0,0,0,2,2,1}, + {2,0,0,1,2,1}, + {0,3,0,2,2,1}, + {0,5,0,2,1,1} + }; */ + // Test case#3 + /* int arr[4][6]={ + {1,1,0,3,2,1}, + {1,3,0,3,1,1}, + {1,4,0,3,2,1}, + {4,4,0,2,2,1} + }; */ + // Test case#4 + /* int arr[4][6]={ + {1,1,0,3,2,3}, + {1,3,0,3,1,3}, + {1,4,0,3,2,3}, + {4,4,0,2,2,1} + }; */ + // Test case#5 + /* int arr[4][6]={ + {0,1,1,2,2,2}, + {3,4,1,2,2,2}, + {3,3,1,2,1,2}, + {1,4,1,2,2,1} + }; */ + // Test case#6 + /* int arr[4][6]={ + {0,1,1,2,2,2}, + {3,4,1,2,2,2}, + {3,3,1,2,1,2}, + {1,4,1,2,2,2} + }; + */ + /* int array[][4][6]={ + { + {0,0,0,1,2,2}, + {0,0,2,1,2,1}, + {0,0,3,1,2,2}, //Test case#1 + {0,0,5,1,2,1} + }, + { + {0,0,0,1,2,2}, + {0,0,2,1,2,1}, + {0,0,3,1,2,2},//Test case#2 + {0,0,5,1,2,1} + }, + { + {0,1,1,1,3,2}, + {0,1,3,1,3,1}, + {0,1,4,1,3,2},//Test case#3 + {0,4,4,1,2,2} + }, + { + {0,1,1,3,3,2}, + {0,1,3,3,3,1}, + {0,1,4,3,3,2},//Test case#4 + {0,4,4,1,2,2} + }, + { + {1,0,1,2,2,2}, + {1,3,4,2,2,2}, + {1,3,3,2,2,1},//Test case#5 + {1,1,4,1,2,2} + }, + { + {1,0,1,2,2,2}, + {1,3,4,2,2,2}, + {1,3,3,2,2,1},//Test case#6 + {1,1,4,2,2,2} + }, + { + {1,1,3,2,2,1}, + {1,3,4,2,2,2}, + {1,3,3,2,2,1},//Test case#7 + {1,1,4,2,2,2} + } + + }; */ + int array[][4][6] = { + + {{0, 0, 0, 1, 2, 2}, + {0, 0, 2, 1, 2, 1}, + {0, 0, 3, 1, 2, 2}, // Test case#1 + {0, 0, 5, 1, 2, 1}}, + {{0, 0, 0, 1, 2, 2}, + {0, 0, 2, 1, 2, 1}, + {0, 0, 3, 1, 2, 2}, // Test case#2 + {0, 0, 5, 1, 2, 1}}, + {{0, 1, 1, 1, 3, 2}, + {0, 1, 3, 1, 3, 1}, + {0, 1, 4, 1, 3, 2}, // Test case#3 + {0, 4, 4, 1, 2, 2}}, + {{0, 1, 1, 3, 3, 2}, + {0, 1, 3, 3, 3, 1}, + {0, 1, 4, 3, 3, 2}, // Test case#4 + {0, 4, 4, 1, 2, 2}}, + {{1, 0, 1, 2, 2, 2}, + {1, 3, 4, 2, 2, 2}, + {1, 3, 3, 2, 2, 1}, // Test case#5 + {1, 1, 4, 1, 2, 2}}, + {{1, 0, 1, 2, 2, 2}, + {1, 3, 4, 2, 2, 2}, + {1, 3, 3, 2, 2, 1}, // Test case#6 + {1, 1, 4, 2, 2, 2}}, + {{1, 1, 3, 2, 2, 1}, + {1, 3, 4, 2, 2, 2}, + {1, 3, 3, 2, 2, 1}, // Test case#7 + {1, 1, 4, 2, 2, 2}}, + { + {0, 1, 1, 1, 2, 3}, + {0, 3, 1, 1, 1, 3}, + {0, 3, 4, 1, 1, 1}, // Testcase# 8 + {0, 1, 4, 1, 2, 1}, + }, + { + {0, 3, 1, 1, 1, 3}, + {0, 1, 1, 1, 2, 3}, + {0, 3, 4, 1, 1, 1}, // Testcase# 9 + {0, 1, 4, 1, 2, 1}, + }, + + }; + + int testcases = (int)sizeof(array) / sizeof(array[0]); + hid_t dataset_array[testcases]; + char datasetname[testcases][12]; + + /* int num_iter=sizeof(array)/sizeof(array[0][0]); + fprintf(stderr,"size_of_array=%ld size_array[0][0]=%ld + num_iter=%d\n",sizeof(array),sizeof(array[0][0]),num_iter); + */ + for (int j = 0; j < testcases; j++) { + + snprintf(datasetname[j], 12, "Testcase# %d", j + 1); + // fprintf(stderr,"dataset name= %s \n",datasetname[j]); + dataset_array[j] = H5Dcreate_async(file, datasetname[j], H5T_STD_I32BE, dataspace, H5P_DEFAULT, + H5P_DEFAULT, H5P_DEFAULT, es_id); + } + if (mpi_rank == 0) { + for (int j = 0; j < testcases; j++) { + + drawBorder(); + fprintf(stderr, "dataset name= %s \n", datasetname[j]); + drawBorder(); + + for (int i = 0; i < 4; i++) { + + offset[0] = array[j][i][0]; + offset[1] = array[j][i][1]; + offset[2] = array[j][i][2]; // select 0x0x0 + count[0] = array[j][i][3]; + count[1] = array[j][i][4]; + count[2] = array[j][i][5]; + status = H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, offset, NULL, count, NULL); + + // printf("MPI rank=%d Hyperslab operation on dataspace using offset %llux%llux%llu and count + // %llux%llux%llu \n",mpi_rank,offset[0],offset[1],offset[2],count[0],count[1],count[2]); + offset_out[0] = array[j][i][0]; // offset=0x0x0 + offset_out[1] = array[j][i][1]; + offset_out[2] = array[j][i][2]; + count_out[0] = array[j][i][3]; // count_out=2 X 2 x 3 + count_out[1] = array[j][i][4]; + count_out[2] = array[j][i][5]; + status = H5Sselect_hyperslab(memspace, H5S_SELECT_SET, offset_out, NULL, count_out, NULL); + + // printf("----------------- on memory space using offset %llux%llux%llu and count + // %llux%llux%llu + // \n",offset_out[0],offset_out[1],offset_out[2],count_out[0],count_out[1],count_out[2]); + + // status = H5Dread_async (dataset, H5T_NATIVE_INT, memspace, dataspace, H5P_DEFAULT, + // data_out1,es_id); + status = H5Dwrite_async(dataset_array[j], H5T_NATIVE_INT, memspace, dataspace, + data_transfer_propertylist, data, es_id); + } + /* status = H5ESwait(es_id, H5ES_WAIT_FOREVER, &num_in_progress, &op_failed); + if (status < 0) { + fprintf(stderr, "Error with H5ESwait\n"); + + } */ + } + + /* offset[0] = 0; + offset[1]=0; + offset[2]=0; // select 0x0x0 + count[0] = 2; + count[1] = 2; + count[2] = 1; + status = H5Sselect_hyperslab (dataspace, H5S_SELECT_SET, offset, NULL, count, NULL); + + printf("MPI rank=%d Hyperslab operation on dataspace using offset %llux%llux%llu and count + %llux%llux%llu \n",mpi_rank,offset[0],offset[1],offset[2],count[0],count[1],count[2]); offset_out[0] = 0; + //offset=0x0x0 offset_out[1] = 0; offset_out[2] = 0; count_out[0] = 2; //count_out=2 X 2 x 3 count_out[1] + = 2; count_out[2] = 1; status = H5Sselect_hyperslab (memspace, H5S_SELECT_SET, offset_out, NULL, + count_out, NULL); + + printf("----------------- on memory space using offset %llux%llux%llu and count %llux%llux%llu + \n",offset_out[0],offset_out[1],offset_out[2],count_out[0],count_out[1],count_out[2]); + + + //status = H5Dread_async (dataset, H5T_NATIVE_INT, memspace, dataspace, H5P_DEFAULT, data_out1,es_id); + status = H5Dwrite_async (dataset, H5T_NATIVE_INT, memspace, dataspace, data_transfer_propertylist, + data,es_id); + + offset[0] = 0; + offset[1]=2; + offset[2]=0; // select 0x2x0 + count[0] = 2; + count[1] = 1; + count[2] = 1; + status = H5Sselect_hyperslab (dataspace, H5S_SELECT_SET, offset, NULL, count, NULL); + + printf("MPI rank=%d Hyperslab operation on dataspace using offset %llux%llux%llu and count + %llux%llux%llu \n",mpi_rank,offset[0],offset[1],offset[2],count[0],count[1],count[2]); offset_out[0] = 0; + //offset=0x2x0 offset_out[1] = 2; offset_out[2] = 0; count_out[0] = 2; //count_out=2 X 2 x 3 count_out[1] + = 1; count_out[2] = 1; status = H5Sselect_hyperslab (memspace, H5S_SELECT_SET, offset_out, NULL, + count_out, NULL); + + printf("----------------- on memory space using offset %llux%llux%llu and count %llux%llux%llu + \n",offset_out[0],offset_out[1],offset_out[2],count_out[0],count_out[1],count_out[2]); + + + //status = H5Dread_async (dataset, H5T_NATIVE_INT, memspace, dataspace, H5P_DEFAULT, data_out1,es_id); + status = H5Dwrite_async (dataset, H5T_NATIVE_INT, memspace, dataspace, data_transfer_propertylist, + data,es_id); + + + offset[0] = 0; + offset[1]=3; + offset[2]=0; // select 0x4x0 + count[0] = 2; + count[1] = 2; + count[2] = 1; + status = H5Sselect_hyperslab (dataspace, H5S_SELECT_SET, offset, NULL, count, NULL); + + printf("MPI rank=%d Hyperslab operation on dataspace using offset %llux%llux%llu and count + %llux%llux%llu \n",mpi_rank,offset[0],offset[1],offset[2],count[0],count[1],count[2]); offset_out[0] = 0; + //offset=0x4x0 offset_out[1] = 3; offset_out[2] = 0; count_out[0] = 2; //count_out=2 X 2 x 3 count_out[1] + = 2; count_out[2] = 1; status = H5Sselect_hyperslab (memspace, H5S_SELECT_SET, offset_out, NULL, + count_out, NULL); + + printf("----------------- on memory space using offset %llux%llux%llu and count %llux%llux%llu + \n",offset_out[0],offset_out[1],offset_out[2],count_out[0],count_out[1],count_out[2]); + + + //status = H5Dread_async (dataset, H5T_NATIVE_INT, memspace, dataspace, H5P_DEFAULT, data_out1,es_id); + status = H5Dwrite_async (dataset, H5T_NATIVE_INT, memspace, dataspace, data_transfer_propertylist, + data,es_id); + + offset[0] = 0; + offset[1] = 5; + offset[2] = 0; //offset=2x0x0 + count[0] = 2; + count[1] = 1; //count=4x2x3 + count[2] = 1; + status = H5Sselect_hyperslab (dataspace, H5S_SELECT_SET, offset, NULL, count, NULL); + printf("MPI rank=%d Hyperslab operation on dataspace using offset %llux%llux%llu and count + %llux%llux%llu \n",mpi_rank,offset[0],offset[1],offset[2],count[0],count[1],count[2]); + + offset_out[0] = 0; + offset_out[1] = 5; //offset_out=2x0x0 + offset_out[2] = 0; + count_out[0] = 2; + count_out[1] = 1; //count_out=4x2x3 + count_out[2] = 1; + status = H5Sselect_hyperslab (memspace, H5S_SELECT_SET, offset_out, NULL, count_out, NULL); + printf("----------------- on memory space using offset %llux%llux%llu and count %llux%llux%llu + \n",offset_out[0],offset_out[1],offset_out[2],count_out[0],count_out[1],count_out[2]); + + + // status = H5Dread_async (dataset, H5T_NATIVE_INT, memspace, dataspace, H5P_DEFAULT, data_out2,es_id); + status = H5Dwrite_async (dataset, H5T_NATIVE_INT, memspace, dataspace, data_transfer_propertylist, + data,es_id); + */ + } + /* if(mpi_rank==1){ + + + offset[0] = 0; + offset[1] = 5; + offset[2] = 0; //offset=2x0x0 + count[0] = 2; + count[1] = 1; //count=4x2x3 + count[2] = 1; + status = H5Sselect_hyperslab (dataspace, H5S_SELECT_SET, offset, NULL, count, NULL); + printf("MPI rank=%d Hyperslab operation on dataspace using offset %llux%llux%llu and count + %llux%llux%llu \n",mpi_rank,offset[0],offset[1],offset[2],count[0],count[1],count[2]); + + offset_out[0] = 0; + offset_out[1] = 5; //offset_out=2x0x0 + offset_out[2] = 0; + count_out[0] = 2; + count_out[1] = 1; //count_out=4x2x3 + count_out[2] = 1; + status = H5Sselect_hyperslab (memspace, H5S_SELECT_SET, offset_out, NULL, count_out, NULL); + printf("----------------- on memory space using offset %llux%llux%llu and count %llux%llux%llu + \n",offset_out[0],offset_out[1],offset_out[2],count_out[0],count_out[1],count_out[2]); + + + // status = H5Dread_async (dataset, H5T_NATIVE_INT, memspace, dataspace, H5P_DEFAULT, data_out2,es_id); + status = H5Dwrite_async (dataset, H5T_NATIVE_INT, memspace, dataspace, data_transfer_propertylist, + data,es_id); + + + //offset from 3x0x0 and count 3x2x3 + offset[0] = 2; + offset[1] = 2; + offset[2] = 0; //offset=2x2x0 + count[0] = 4; + count[1] = 2; //count=4x2x3 + count[2] = 3; + status = H5Sselect_hyperslab (dataspace, H5S_SELECT_SET, offset, NULL, count, NULL); + printf("MPI rank=%d Hyperslab operation on dataspace using offset %llux%llux%llu and count + %llux%llux%llu \n",mpi_rank,offset[0],offset[1],offset[2],count[0],count[1],count[2]); + + offset_out[0] = 2; + offset_out[1] = 2; //offset_out=2x2x0 + offset_out[2] = 0; + count_out[0] = 4; + count_out[1] = 2; //count_out=4x2x3 + count_out[2] = 3; + status = H5Sselect_hyperslab (memspace, H5S_SELECT_SET, offset_out, NULL, count_out, NULL); + printf("----------------- on memory space using offset %llux%llux%llu and count %llux%llux%llu + \n",offset_out[0],offset_out[1],offset_out[2],count_out[0],count_out[1],count_out[2]); + + + // status = H5Dread_async (dataset, H5T_NATIVE_INT, memspace, dataspace, H5P_DEFAULT, data_out2,es_id); + status = H5Dwrite_async (dataset, H5T_NATIVE_INT, memspace, dataspace, data_transfer_propertylist, + data,es_id); + + + offset[0] = 2; + offset[1] = 4; + offset[2] = 0; //offset=2x4x0 + count[0] = 4; + count[1] = 2; //count=4x2x3 + count[2] = 3; + status = H5Sselect_hyperslab (dataspace, H5S_SELECT_SET, offset, NULL, count, NULL); + printf("MPI rank=%d Hyperslab operation on dataspace using offset %llux%llux%llu and count + %llux%llux%llu \n",mpi_rank,offset[0],offset[1],offset[2],count[0],count[1],count[2]); + + offset_out[0] = 2; + offset_out[1] = 4; //offset_out=2x4x0 + offset_out[2] = 0; + count_out[0] = 4; + count_out[1] = 2; //count_out=4x2x3 + count_out[2] = 3; + status = H5Sselect_hyperslab (memspace, H5S_SELECT_SET, offset_out, NULL, count_out, NULL); + printf("----------------- on memory space using offset %llux%llux%llu and count %llux%llux%llu + \n",offset_out[0],offset_out[1],offset_out[2],count_out[0],count_out[1],count_out[2]); + + + // status = H5Dread_async (dataset, H5T_NATIVE_INT, memspace, dataspace, H5P_DEFAULT, data_out2,es_id); + status = H5Dwrite_async (dataset, H5T_NATIVE_INT, memspace, dataspace, data_transfer_propertylist, + data,es_id); + + + } */ + + status = H5ESwait(es_id, H5ES_WAIT_FOREVER, &num_in_progress, &op_failed); + if (status < 0) { + fprintf(stderr, "Error with H5ESwait\n"); + } + if (print_dbg_msg) + fprintf(stderr, "H5ESwait done\n"); + + MPI_Barrier(comm); + + if (mpi_rank == 0) { + + for (int p = 0; p < testcases; p++) { + + drawBorder(); + fprintf(stderr, "Testcase= %d\n", p + 1); + drawBorder(); + status = H5Dread_async(dataset_array[p], H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, data_out, + es_id); + + status = H5ESwait(es_id, H5ES_WAIT_FOREVER, &num_in_progress, &op_failed); + if (status < 0) { + fprintf(stderr, "Error with H5ESwait\n"); + } + fprintf(stderr, "\nData out from the file\n"); + for (k = 0; k < Z; k++) { + fprintf(stderr, "\nFirst 3D Data Z=%d:\n ", k); + + for (i = 0; i < X; i++) { + for (j = 0; j < Y; j++) + fprintf(stderr, "%5d", data_out[k][i][j]); + fprintf(stderr, "\n "); + } + } + + // fprintf(stderr,"testcase=%d\n",p); + } + } + /* + * Close/release resources. + */ + H5Sclose(dataspace); + H5Sclose(memspace); + + for (int j = 0; j < testcases; j++) { + + status = H5Dclose_async(dataset_array[j], es_id); + if (status < 0) { + fprintf(stderr, "Closing dataset failed\n"); + // ret = -1; + } + } + + status = H5Dclose_async(dataset, es_id); + if (status < 0) { + fprintf(stderr, "Closing dataset failed\n"); + // ret = -1; + } + + status = H5Fclose_async(file, es_id); + if (status < 0) { + fprintf(stderr, "Closing file failed\n"); + // ret = -1; + } + // H5Fclose(file); + status = H5ESwait(es_id, H5ES_WAIT_FOREVER, &num_in_progress, &op_failed); + if (status < 0) { + fprintf(stderr, "Error with H5ESwait\n"); + // ret = -1; + } + + status = H5ESclose(es_id); + if (status < 0) { + fprintf(stderr, "Can't close second event set\n"); + // ret = -1; + } + H5Pclose(property_list_id_MPIO); + H5Pclose(data_transfer_propertylist); + MPI_Barrier(comm); + MPI_Finalize(); + + return 0; +} From 6c8e63574ea5d89fdab52957b7e1238e7b9bfaf8 Mon Sep 17 00:00:00 2001 From: Kamal Chowdhury <106797012+kamalchowdhurylbl@users.noreply.github.com> Date: Tue, 9 Aug 2022 21:27:47 -0500 Subject: [PATCH 06/18] added 1d_async_write_performance.c --- test/1d_async_write_performance.c | 314 ++++++++++++++++++++++++++++++ 1 file changed, 314 insertions(+) create mode 100644 test/1d_async_write_performance.c diff --git a/test/1d_async_write_performance.c b/test/1d_async_write_performance.c new file mode 100644 index 0000000..db4ad0a --- /dev/null +++ b/test/1d_async_write_performance.c @@ -0,0 +1,314 @@ +/************************************************************ + + This example shows how to write and read a hyperslab. It + is derived from the h5_read.c and h5_write.c examples in + the "Introduction to HDF5". It works on the 3 dimensional data. + + ************************************************************/ + +#include "hdf5.h" +#include +#include // for strtol() +#include // for time() +#include // for sleep() + + +#define FILE "sds.h5" +#define DATASETNAME "IntArray" + +#define RANK 1 //since it will work on 2 dimensional data +#define RANK_OUT 1 + +//#define X 262144 // 4 byte*256*1024=1 MB + +//#define X 256 + + + +void drawBorder(){ + fprintf(stderr,"\n"); + for(int i=0;i<60;i++) + { + fprintf(stderr,"#"); + } + fprintf(stderr,"\n"); + +} + +int +main (int argc, char **argv) +{ + hsize_t dimsf[1]; /* dataset dimensions */ + int size,counter; + int X=256; + long arg1,arg2; + fprintf(stderr,"argc=%d",argc); + //return 1; + + if( argc == 3 ) { + char* p; + + arg1 = strtol(argv[1], &p, 10); + arg2 = strtol(argv[2], &p, 10); + size=arg1; + + counter=arg2; + } + else{ + + fprintf(stderr, "Need two values for size and counter values\n"); + return 0; + } + size=size*X; + fprintf(stderr,"start=%d\n",size); // 4 byte*256*1024=1 MB + int data[size*counter]; /* data to write */ + //0-262143 262144- 2* + + // 2 parameters + //size=1KB count=50 + + //size=2KB count=50 + + //size=1024KB count=50 + + + /* + * Data and output buffer initialization. + */ + hid_t file, dataset; /* handles */ + hid_t dataspace; + hid_t memspace; + hsize_t dimsm[1]; /* memory space dimensions 1D*/ + hsize_t dims_out[1]; /* dataset dimensions 1D */ + herr_t status; + hid_t property_list_id_MPIO; /* property list identifier */ + hid_t data_transfer_propertylist; + + + + int data_out[counter*size]; //data out 1d is 60 + + + hsize_t count[1]; /* size of the hyperslab in the file */ + hsize_t offset[1]; /* hyperslab offset in the file */ + hsize_t count_out[1]; /* size of the hyperslab in memory */ + hsize_t offset_out[1]; + + int i, j, k, status_n, rank; + int print_dbg_msg=1; + hbool_t op_failed; + size_t num_in_progress; + hid_t es_id = H5EScreate(); + int provided; + /* + * MPI variables + */ + int mpi_size, mpi_rank; + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Info info = MPI_INFO_NULL; + double time1, time2,duration,global; + + /* + * Initialize MPI + */ + //MPI_Init(&argc, &argv); + MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); + MPI_Comm_size(comm, &mpi_size); + MPI_Comm_rank(comm, &mpi_rank); + + + property_list_id_MPIO = H5Pcreate(H5P_FILE_ACCESS); + H5Pset_fapl_mpio(property_list_id_MPIO, comm, info); + + + data_transfer_propertylist = H5Pcreate(H5P_DATASET_XFER); + H5Pset_dxpl_mpio(data_transfer_propertylist, H5FD_MPIO_COLLECTIVE); + + + + + //1d data + int l=0; + + for (i = 0; i < size*counter; i++) + {data[i] = l; + l++;} + + /* + if(mpi_rank==0){ + + for (i = 0; i < size*counter; i++) + fprintf(stderr,"%5d", data[i]); + fprintf(stderr,"\n"); + + } */ + + //1d operations + + file = H5Fcreate_async (FILE, H5F_ACC_TRUNC, H5P_DEFAULT, property_list_id_MPIO,es_id); + + + dimsf[0]=size*counter; + dataspace = H5Screate_simple (RANK, dimsf, NULL); + dimsm[0] = size*counter; + memspace = H5Screate_simple (RANK_OUT, dimsm, NULL); //RANK_OUT=3 + dataset = H5Dcreate_async (file, DATASETNAME, H5T_STD_I32BE, dataspace, H5P_DEFAULT,H5P_DEFAULT, H5P_DEFAULT,es_id); + + + + int p=0; + + int array[counter][size]; + for(int i=0;i Date: Wed, 10 Aug 2022 02:28:10 +0000 Subject: [PATCH 07/18] Committing clang-format changes --- test/1d_async_write_performance.c | 397 ++++++++++++++---------------- 1 file changed, 178 insertions(+), 219 deletions(-) diff --git a/test/1d_async_write_performance.c b/test/1d_async_write_performance.c index db4ad0a..a60ad6b 100644 --- a/test/1d_async_write_performance.c +++ b/test/1d_async_write_performance.c @@ -1,307 +1,268 @@ /************************************************************ - - This example shows how to write and read a hyperslab. It - is derived from the h5_read.c and h5_write.c examples in + + This example shows how to write and read a hyperslab. It + is derived from the h5_read.c and h5_write.c examples in the "Introduction to HDF5". It works on the 3 dimensional data. ************************************************************/ - + #include "hdf5.h" -#include +#include #include // for strtol() -#include // for time() -#include // for sleep() - +#include // for time() +#include // for sleep() #define FILE "sds.h5" -#define DATASETNAME "IntArray" +#define DATASETNAME "IntArray" -#define RANK 1 //since it will work on 2 dimensional data +#define RANK 1 // since it will work on 2 dimensional data #define RANK_OUT 1 //#define X 262144 // 4 byte*256*1024=1 MB //#define X 256 - - -void drawBorder(){ - fprintf(stderr,"\n"); - for(int i=0;i<60;i++) - { - fprintf(stderr,"#"); +void +drawBorder() +{ + fprintf(stderr, "\n"); + for (int i = 0; i < 60; i++) { + fprintf(stderr, "#"); } - fprintf(stderr,"\n"); - + fprintf(stderr, "\n"); } int -main (int argc, char **argv) +main(int argc, char **argv) { - hsize_t dimsf[1]; /* dataset dimensions */ - int size,counter; - int X=256; - long arg1,arg2; - fprintf(stderr,"argc=%d",argc); - //return 1; - - if( argc == 3 ) { - char* p; - - arg1 = strtol(argv[1], &p, 10); - arg2 = strtol(argv[2], &p, 10); - size=arg1; - - counter=arg2; - } - else{ - - fprintf(stderr, "Need two values for size and counter values\n"); - return 0; - } - size=size*X; - fprintf(stderr,"start=%d\n",size); // 4 byte*256*1024=1 MB - int data[size*counter]; /* data to write */ - //0-262143 262144- 2* - + hsize_t dimsf[1]; /* dataset dimensions */ + int size, counter; + int X = 256; + long arg1, arg2; + fprintf(stderr, "argc=%d", argc); + // return 1; + + if (argc == 3) { + char *p; + + arg1 = strtol(argv[1], &p, 10); + arg2 = strtol(argv[2], &p, 10); + size = arg1; + + counter = arg2; + } + else { + + fprintf(stderr, "Need two values for size and counter values\n"); + return 0; + } + size = size * X; + fprintf(stderr, "start=%d\n", size); // 4 byte*256*1024=1 MB + int data[size * counter]; /* data to write */ + // 0-262143 262144- 2* + // 2 parameters - //size=1KB count=50 - - //size=2KB count=50 + // size=1KB count=50 + + // size=2KB count=50 - //size=1024KB count=50 + // size=1024KB count=50 - - /* - * Data and output buffer initialization. + /* + * Data and output buffer initialization. */ - hid_t file, dataset; /* handles */ - hid_t dataspace; - hid_t memspace; - hsize_t dimsm[1]; /* memory space dimensions 1D*/ - hsize_t dims_out[1]; /* dataset dimensions 1D */ - herr_t status; - hid_t property_list_id_MPIO; /* property list identifier */ - hid_t data_transfer_propertylist; - - - - int data_out[counter*size]; //data out 1d is 60 - - - hsize_t count[1]; /* size of the hyperslab in the file */ - hsize_t offset[1]; /* hyperslab offset in the file */ - hsize_t count_out[1]; /* size of the hyperslab in memory */ - hsize_t offset_out[1]; - - int i, j, k, status_n, rank; - int print_dbg_msg=1; + hid_t file, dataset; /* handles */ + hid_t dataspace; + hid_t memspace; + hsize_t dimsm[1]; /* memory space dimensions 1D*/ + hsize_t dims_out[1]; /* dataset dimensions 1D */ + herr_t status; + hid_t property_list_id_MPIO; /* property list identifier */ + hid_t data_transfer_propertylist; + + int data_out[counter * size]; // data out 1d is 60 + + hsize_t count[1]; /* size of the hyperslab in the file */ + hsize_t offset[1]; /* hyperslab offset in the file */ + hsize_t count_out[1]; /* size of the hyperslab in memory */ + hsize_t offset_out[1]; + + int i, j, k, status_n, rank; + int print_dbg_msg = 1; hbool_t op_failed; size_t num_in_progress; hid_t es_id = H5EScreate(); - int provided; - /* + int provided; + /* * MPI variables */ - int mpi_size, mpi_rank; - MPI_Comm comm = MPI_COMM_WORLD; - MPI_Info info = MPI_INFO_NULL; - double time1, time2,duration,global; - + int mpi_size, mpi_rank; + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Info info = MPI_INFO_NULL; + double time1, time2, duration, global; + /* * Initialize MPI */ - //MPI_Init(&argc, &argv); + // MPI_Init(&argc, &argv); MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); MPI_Comm_size(comm, &mpi_size); - MPI_Comm_rank(comm, &mpi_rank); - + MPI_Comm_rank(comm, &mpi_rank); property_list_id_MPIO = H5Pcreate(H5P_FILE_ACCESS); H5Pset_fapl_mpio(property_list_id_MPIO, comm, info); - data_transfer_propertylist = H5Pcreate(H5P_DATASET_XFER); H5Pset_dxpl_mpio(data_transfer_propertylist, H5FD_MPIO_COLLECTIVE); - - + // 1d data + int l = 0; + + for (i = 0; i < size * counter; i++) { + data[i] = l; + l++; + } - //1d data - int l=0; - - for (i = 0; i < size*counter; i++) - {data[i] = l; - l++;} - /* - if(mpi_rank==0){ + if(mpi_rank==0){ for (i = 0; i < size*counter; i++) fprintf(stderr,"%5d", data[i]); fprintf(stderr,"\n"); - + } */ - - //1d operations - - file = H5Fcreate_async (FILE, H5F_ACC_TRUNC, H5P_DEFAULT, property_list_id_MPIO,es_id); - - - dimsf[0]=size*counter; - dataspace = H5Screate_simple (RANK, dimsf, NULL); - dimsm[0] = size*counter; - memspace = H5Screate_simple (RANK_OUT, dimsm, NULL); //RANK_OUT=3 - dataset = H5Dcreate_async (file, DATASETNAME, H5T_STD_I32BE, dataspace, H5P_DEFAULT,H5P_DEFAULT, H5P_DEFAULT,es_id); - - - - int p=0; - + + // 1d operations + + file = H5Fcreate_async(FILE, H5F_ACC_TRUNC, H5P_DEFAULT, property_list_id_MPIO, es_id); + + dimsf[0] = size * counter; + dataspace = H5Screate_simple(RANK, dimsf, NULL); + dimsm[0] = size * counter; + memspace = H5Screate_simple(RANK_OUT, dimsm, NULL); // RANK_OUT=3 + dataset = H5Dcreate_async(file, DATASETNAME, H5T_STD_I32BE, dataspace, H5P_DEFAULT, H5P_DEFAULT, + H5P_DEFAULT, es_id); + + int p = 0; + int array[counter][size]; - for(int i=0;i Date: Wed, 10 Aug 2022 16:20:23 -0500 Subject: [PATCH 08/18] added changes in the vol async, 1d,2d,3d test files --- src/h5_async_vol.c | 2 +- test/1d_asyn_MPI_write.c | 2 +- test/1d_async_write_performance.c | 2 +- test/2d_asyn_MPI_write.c | 4 ++-- test/3d_asyn_MPI_write.c | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/h5_async_vol.c b/src/h5_async_vol.c index ed968e3..3b76d93 100644 --- a/src/h5_async_vol.c +++ b/src/h5_async_vol.c @@ -1,5 +1,5 @@ /******************************************************************************* -Asynchronous I/O VOL kamal Connector (AsyncVOL) Copyright (c) 2021, The +Asynchronous I/O VOL kamal Connector (AsyncVOL) Copyright (c) 2021, The Regents of the University of California, through Lawrence Berkeley National Laboratory (subject to receipt of any required approvals from the U.S. Dept. of Energy). All rights reserved. diff --git a/test/1d_asyn_MPI_write.c b/test/1d_asyn_MPI_write.c index 13a4ea9..bffb276 100644 --- a/test/1d_asyn_MPI_write.c +++ b/test/1d_asyn_MPI_write.c @@ -1,6 +1,6 @@ /************************************************************ - This example shows how to write and read a hyperslab. It + This example shows how to write and read a hyperslab. It is derived from the h5_read.c and h5_write.c examples in the "Introduction to HDF5". It works on the 3 dimensional data. diff --git a/test/1d_async_write_performance.c b/test/1d_async_write_performance.c index a60ad6b..397f281 100644 --- a/test/1d_async_write_performance.c +++ b/test/1d_async_write_performance.c @@ -1,6 +1,6 @@ /************************************************************ - This example shows how to write and read a hyperslab. It + This example shows how to write and read a hyperslab. It is derived from the h5_read.c and h5_write.c examples in the "Introduction to HDF5". It works on the 3 dimensional data. diff --git a/test/2d_asyn_MPI_write.c b/test/2d_asyn_MPI_write.c index 42988e9..5c3ded2 100644 --- a/test/2d_asyn_MPI_write.c +++ b/test/2d_asyn_MPI_write.c @@ -1,6 +1,6 @@ /************************************************************ - This example shows how to write and read a hyperslab. It + This example shows how to write and read a hyperslab. It is derived from the h5_read.c and h5_write.c examples in the "Introduction to HDF5". It works on the 3 dimensional data. ************************************************************/ @@ -457,4 +457,4 @@ main(int argc, char **argv) MPI_Finalize(); return 0; -} \ No newline at end of file +} diff --git a/test/3d_asyn_MPI_write.c b/test/3d_asyn_MPI_write.c index 605a935..dfb869c 100644 --- a/test/3d_asyn_MPI_write.c +++ b/test/3d_asyn_MPI_write.c @@ -1,6 +1,6 @@ /************************************************************ - This example shows how to write and read a hyperslab. It + This example shows how to write and read a hyperslab. It is derived from the h5_read.c and h5_write.c examples in the "Introduction to HDF5". It works on the 3 dimensional data. From 02f6fc8f810130642b5f89827df6e7153782eec9 Mon Sep 17 00:00:00 2001 From: kamalchowdhurylbl Date: Wed, 10 Aug 2022 16:36:28 -0500 Subject: [PATCH 09/18] added vol async.c --- src/h5_async_vol.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/h5_async_vol.c b/src/h5_async_vol.c index 3b76d93..ed968e3 100644 --- a/src/h5_async_vol.c +++ b/src/h5_async_vol.c @@ -1,5 +1,5 @@ /******************************************************************************* -Asynchronous I/O VOL kamal Connector (AsyncVOL) Copyright (c) 2021, The +Asynchronous I/O VOL kamal Connector (AsyncVOL) Copyright (c) 2021, The Regents of the University of California, through Lawrence Berkeley National Laboratory (subject to receipt of any required approvals from the U.S. Dept. of Energy). All rights reserved. From 42850a405b8b08166315de45a583f0ca011e2f31 Mon Sep 17 00:00:00 2001 From: kamalchowdhurylbl Date: Wed, 10 Aug 2022 16:40:45 -0500 Subject: [PATCH 10/18] added 1d,2d, and 3d test files --- test/1d_asyn_MPI_write.c | 2 +- test/1d_async_write_performance.c | 2 +- test/2d_asyn_MPI_write.c | 2 +- test/3d_asyn_MPI_write.c | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/1d_asyn_MPI_write.c b/test/1d_asyn_MPI_write.c index bffb276..13a4ea9 100644 --- a/test/1d_asyn_MPI_write.c +++ b/test/1d_asyn_MPI_write.c @@ -1,6 +1,6 @@ /************************************************************ - This example shows how to write and read a hyperslab. It + This example shows how to write and read a hyperslab. It is derived from the h5_read.c and h5_write.c examples in the "Introduction to HDF5". It works on the 3 dimensional data. diff --git a/test/1d_async_write_performance.c b/test/1d_async_write_performance.c index 397f281..a60ad6b 100644 --- a/test/1d_async_write_performance.c +++ b/test/1d_async_write_performance.c @@ -1,6 +1,6 @@ /************************************************************ - This example shows how to write and read a hyperslab. It + This example shows how to write and read a hyperslab. It is derived from the h5_read.c and h5_write.c examples in the "Introduction to HDF5". It works on the 3 dimensional data. diff --git a/test/2d_asyn_MPI_write.c b/test/2d_asyn_MPI_write.c index 5c3ded2..fbc3231 100644 --- a/test/2d_asyn_MPI_write.c +++ b/test/2d_asyn_MPI_write.c @@ -1,6 +1,6 @@ /************************************************************ - This example shows how to write and read a hyperslab. It + This example shows how to write and read a hyperslab. It is derived from the h5_read.c and h5_write.c examples in the "Introduction to HDF5". It works on the 3 dimensional data. ************************************************************/ diff --git a/test/3d_asyn_MPI_write.c b/test/3d_asyn_MPI_write.c index dfb869c..605a935 100644 --- a/test/3d_asyn_MPI_write.c +++ b/test/3d_asyn_MPI_write.c @@ -1,6 +1,6 @@ /************************************************************ - This example shows how to write and read a hyperslab. It + This example shows how to write and read a hyperslab. It is derived from the h5_read.c and h5_write.c examples in the "Introduction to HDF5". It works on the 3 dimensional data. From 0f32b47df453f640b3df372c7e213c3da5afba51 Mon Sep 17 00:00:00 2001 From: kamalchowdhurylbl Date: Wed, 10 Aug 2022 17:29:52 -0500 Subject: [PATCH 11/18] changes in the h5_async_vol.c --- src/h5_async_vol.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/h5_async_vol.c b/src/h5_async_vol.c index ed968e3..9ef8237 100644 --- a/src/h5_async_vol.c +++ b/src/h5_async_vol.c @@ -131,7 +131,7 @@ typedef struct async_task_t { ABT_eventual eventual; int in_abt_pool; int is_done; - int is_merge; + int is_merge; ABT_thread abt_thread; hid_t err_stack; int n_dep; From bcb93b2cf6e2df12c02ae630f2c00e002ad85ea0 Mon Sep 17 00:00:00 2001 From: kamalchowdhurylbl Date: Thu, 11 Aug 2022 14:41:50 -0500 Subject: [PATCH 12/18] added modified h5_async_vol.c --- .DS_Store | Bin 6148 -> 0 bytes src/h5_async_vol.c | 662 +++++++++++++++++---------------------------- 2 files changed, 248 insertions(+), 414 deletions(-) delete mode 100644 .DS_Store diff --git a/.DS_Store b/.DS_Store deleted file mode 100644 index e68e8836983384f6deb19cb98b9c994c23fc7a5a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHK%}T>S5Z-O8O({YS3VK`cTClCP7cU{!7cim+m718M!I&*cY7V84yS|Vw;`2DO zy8(+ii`W_1{pNQ!`$6`HF~;3RIAF|Xj9JhSIVv@R?%GhpBqMShBbx`Y48Zyb<|g*n z0l&S;GM2K3p!oj%ag^o#{wHrVTU*<0t8I0yJMT#rUhe0M%=Ob7v@WHLgG%>d zW5McfGz+wNVt^Q^V*vLD0S(bHSZY*T2XuIS#&`=61$2B%APR$y!BQhcK)5ai)TP`! zF}N-Vzc6`@!BV3xXI#w;bO81@*IPuMjQqG RsvM9m0*Vmoh=E^V;0tT^Nwxq0 diff --git a/src/h5_async_vol.c b/src/h5_async_vol.c index 9ef8237..b96bcbe 100644 --- a/src/h5_async_vol.c +++ b/src/h5_async_vol.c @@ -1,5 +1,5 @@ /******************************************************************************* -Asynchronous I/O VOL kamal Connector (AsyncVOL) Copyright (c) 2021, The +Asynchronous I/O VOL Connector (AsyncVOL) Copyright (c) 2021, The Regents of the University of California, through Lawrence Berkeley National Laboratory (subject to receipt of any required approvals from the U.S. Dept. of Energy). All rights reserved. @@ -65,7 +65,7 @@ works, and perform publicly and display publicly, and to permit others to do so. /* Whether to display log messge when callback is invoked */ /* (Uncomment to enable) */ /* #define ENABLE_DBG_MSG 1 */ -#define PRINT_ERROR_STACK 1 + #define PRINT_ERROR_STACK 1 /* #define ENABLE_ASYNC_LOGGING */ #define ASYNC_DBG_MSG_RANK 0 @@ -131,7 +131,7 @@ typedef struct async_task_t { ABT_eventual eventual; int in_abt_pool; int is_done; - int is_merge; + int is_merge; ABT_thread abt_thread; hid_t err_stack; int n_dep; @@ -257,7 +257,7 @@ typedef struct async_attr_write_args_t { hid_t dxpl_id; void **req; - bool free_buf; + bool free_buf; #ifdef ENABLE_WRITE_MEMCPY hsize_t data_size; #endif @@ -333,7 +333,7 @@ typedef struct async_dataset_write_args_t { void * buf; void **req; - bool free_buf; + bool free_buf; #ifdef ENABLE_WRITE_MEMCPY hsize_t data_size; #endif @@ -8860,11 +8860,13 @@ async_dataset_write_fn(void *foo) assert(task); assert(task->async_obj); assert(task->async_obj->magic == ASYNC_MAGIC); - // fprintf(stderr,"task is_merge=%d\n",task->is_merge); + //fprintf(stderr,"task is_merge=%d\n",task->is_merge); pool_ptr = task->async_obj->pool_ptr; if (task->is_merge == 1) goto done; + + func_log(__func__, "trying to aquire global lock"); if ((attempt_count = check_app_acquire_mutex(task, &mutex_count, &acquired)) < 0) @@ -8938,7 +8940,7 @@ async_dataset_write_fn(void *foo) usleep(1000); count++; } - // fprintf(stderr," args mem_type_id=%llu\n",args->mem_type_id); + //fprintf(stderr," args mem_type_id=%llu\n",args->mem_type_id); /* Try executing operation, without default error stack handling */ H5E_BEGIN_TRY { @@ -8958,7 +8960,8 @@ async_dataset_write_fn(void *foo) } func_log(__func__, "execute success"); - + if (args->free_buf && args->buf) { + free(args->buf);} done: if (is_lib_state_restored && H5VLfinish_lib_state() < 0) fprintf(fout_g, " [ASYNC ABT ERROR] %s H5VLfinish_lib_state failed\n", __func__); @@ -8966,8 +8969,8 @@ async_dataset_write_fn(void *foo) fprintf(fout_g, " [ASYNC ABT ERROR] %s H5VLfree_lib_state failed\n", __func__); task->h5_state = NULL; - // if (args->mem_type_id > 0) - // H5Tclose(args->mem_type_id); + //if (args->mem_type_id > 0) + // H5Tclose(args->mem_type_id); if (args->mem_space_id > 0) H5Sclose(args->mem_space_id); if (args->file_space_id > 0) @@ -8992,8 +8995,9 @@ async_dataset_write_fn(void *foo) if (async_instance_g && NULL != async_instance_g->qhead.queue && async_instance_g->start_abt_push) push_task_to_abt_pool(&async_instance_g->qhead, *pool_ptr, __func__); + if (args->free_buf && args->buf) { - free(args->buf); + //free(args->buf); #ifdef ENABLE_WRITE_MEMCPY async_instance_g->used_mem -= args->data_size; #ifdef ENABLE_DBG_MSG @@ -9001,9 +9005,10 @@ async_dataset_write_fn(void *foo) (async_instance_g->mpi_rank == ASYNC_DBG_MSG_RANK || -1 == ASYNC_DBG_MSG_RANK)) fprintf(fout_g, " [ASYNC ABT DBG] %s released dset memcpy\n", __func__); #endif -#endif + #endif } + #ifdef ENABLE_TIMING task->end_time = clock(); #endif @@ -9072,30 +9077,13 @@ print_dataspace(hid_t mem_space_id) type = H5Sget_select_type(mem_space_id); - /* if(type==H5S_SEL_ERROR) - fprintf(stderr,"-----Error----\n"); - else if(type==H5S_SEL_NONE) - fprintf(stderr,"-----Empty selection-----\n"); - else if(type==H5S_SEL_POINTS) - fprintf(stderr,"-----Set of points-----\n"); - else if(type==H5S_SEL_HYPERSLABS) - fprintf(stderr,"-----Hyperslab-----\n"); - - else if(type==H5S_SEL_ALL) - fprintf(stderr,"-----Everything-----\n"); - else if(type==H5S_SEL_N) - fprintf(stderr,"-----Sentinel-----cd\n"); - */ - - // fprintf(stderr,"this is after H5Sget_select_type %d\n",type); + if (type == H5S_SEL_POINTS) { return 0; } else if (type == H5S_SEL_HYPERSLABS) { ndim = H5Sget_simple_extent_ndims(mem_space_id); - // fprintf(stderr, "ndim=%d\n",ndim); - // if (ndim != 1) - // return 0; + if (H5Sis_regular_hyperslab(mem_space_id)) { start_out = malloc(ndim * sizeof(hsize_t)); @@ -9109,31 +9097,25 @@ print_dataspace(hid_t mem_space_id) fprintf(stderr, " start = [%llu] \n", (unsigned long long)start_out[0]); // fprintf(stderr, " stride = [%llu] \n", (unsigned long long)stride_out[0]); fprintf(stderr, " count = [%llu] \n", (unsigned long long)count_out[0]); - // fprintf(stderr, " block = [%llu] \n", (unsigned long long)block_out[0]); - // fprintf(stderr, "Hyperslab operation on dataspace using start=%llu and - // count=%llu\n",start_out[0],count_out[0]); fflush(stdout); + } else if (ndim == 2) { fprintf(stderr, " start = [%llu, %llu] \n", (unsigned long long)start_out[0], (unsigned long long)start_out[1]); - // fprintf(stderr, " stride = [%llu, %llu] \n", (unsigned long long)stride_out[0], - // (unsigned long long)stride_out[1]); + fprintf(stderr, " count = [%llu, %llu] \n", (unsigned long long)count_out[0], (unsigned long long)count_out[1]); - // fprintf(stderr, " block = [%llu, %llu] \n", (unsigned long long)block_out[0], - // (unsigned long long)block_out[1]); + } else if (ndim == 3) { fprintf(stderr, " start = [%llu, %llu, %llu] \n", (unsigned long long)start_out[0], (unsigned long long)start_out[1], (unsigned long long)start_out[2]); - // fprintf(stderr, " stride = [%llu, %llu, %llu] \n", (unsigned long - // long)stride_out[0], (unsigned long long)stride_out[1], (unsigned long long)stride_out[2]); + fprintf(stderr, " count = [%llu, %llu, %llu] \n", (unsigned long long)count_out[0], (unsigned long long)count_out[1], (unsigned long long)count_out[2]); - // fprintf(stderr, " block = [%llu, %llu, %llu] \n", (unsigned long - // long)block_out[0], (unsigned long long)block_out[1], (unsigned long long)block_out[2]); + } free(start_out); @@ -9149,23 +9131,20 @@ print_dataspace(hid_t mem_space_id) slected_block = H5Sget_select_hyper_blocklist(mem_space_id, (hsize_t)0, nblocks, buffer); if (ndim == 1) { - // for(int i=0;iqhead.queue, task_list_iter) { DL_FOREACH(task_list_iter->task_list, task_iter) - { + { if (task_iter->func == async_dataset_write_fn) { // checking whether the iterator task is dataset write operation if (task_iter->parent_obj == parent_obj && task_iter->is_done != 1) { // checking whether the iterator task is operating on the same dataset of the current task - + iter_args = task_iter->args; - // fprintf(stderr,"%lld %lld - // %lld\n",task_iter->async_obj->under_object,iter_args->mem_space_id,iter_args->file_space_id); - /* fprintf(stderr,"For iterator task file space:\n"); - print_dataspace(iter_args->file_space_id); - fprintf(stderr,"For current task file space:\n"); - print_dataspace(file_space_id); */ + hid_t new_memspace; @@ -9455,108 +9375,103 @@ foreach_iteration(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t mem_typ iter_file_start = malloc(ndim * sizeof(hsize_t)); iter_file_count = malloc(ndim * sizeof(hsize_t)); + + + //time_before_contiguous_check = MPI_Wtime(); if (check_contiguous(file_space_id, iter_args->file_space_id, start, count) || check_contiguous(iter_args->file_space_id, file_space_id, start, count)) - // if(check_contiguous(file_space_id,iter_args->file_space_id,start,count)) - { + + { + time_in_contiguous1 = MPI_Wtime(); return_val = 1; - // fprintf(stderr, "\n-------#### contiguous####---------\n"); - - /* if(prev_task!=NULL){ - continue; - } */ + //fprintf(stderr, "\n-------#### contiguous####---------\n"); + + + if (prev_task != NULL) prev_task->is_merge = 1; return_task_val = task_iter; - - // fprintf(stderr, "\n-------#### 2 contiguous####---------\n"); - if (ndim == 1) { // push_contiguous(ndim,start,count); - - // fprintf(stderr, "\n start=%llu count=%llu\n", start[0], count[0]); - // H5Sclose(iter_args->file_space_id);// close the file space - - /* if(H5Smodify_select(file_space_id,H5S_SELECT_SET,iter_args->file_space_id)< 0) - {fprintf(stderr,"Error in H5Smodify_select");} */ - - // H5Sclose(iter_args->file_space_id); + + + if (ndim == 1) { + + new_memspace = H5Screate_simple(ndim, count, NULL); element_size = H5Tget_size(mem_type_id); - fprintf(stderr, "size of new_buffer= %lld \n", count[0] * element_size); + //fprintf(stderr,"size of new_buffer= %lld \n",count[0]*element_size); - new_buffer = malloc(count[0] * element_size); - - /* int *buf_temp = buf; - int *new_buffer_temp = new_buffer; */ status = H5Sget_regular_hyperslab(mem_space_id, mem_start, NULL, mem_count, NULL); status = H5Sget_regular_hyperslab(file_space_id, file_start, NULL, file_count, NULL); - // fprintf(stderr,"\nmem_start=%lld mem_count=%lld file_start=%lld file_count=%lld - // element_size=%lld\n",mem_start[0],mem_count[0],file_start[0],file_count[0],element_size); - fprintf(stderr, "\n mem_start=%lld file_start=%lld ", mem_start[0], - file_start[0]); - fprintf(stderr, "\n mem_count=%lld file_count=%lld \n", mem_count[0], - file_count[0]); + status = H5Sget_regular_hyperslab(iter_args->mem_space_id, iter_mem_start, NULL, iter_mem_count, NULL); - /* fprintf(stderr,"%d\n",status); - if (H5Sis_regular_hyperslab(iter_args->mem_space_id)) { - fprintf(stderr,"regular hyperslab\n"); - } */ + status = H5Sget_regular_hyperslab(iter_args->file_space_id, iter_file_start, NULL, iter_file_count, NULL); - // fprintf(stderr,"%d\n",status); - // fprintf(stderr, "\n iter_mem_start=%lld iter_file_start=%lld ", - // iter_mem_start[0], - // iter_file_start[0]); - // fprintf(stderr, "\n iter_mem_count=%lld iter_file_count=%lld \n", - // iter_mem_count[0], iter_file_count[0]); - if (file_start[0] >= iter_file_start[0]) { - // fprintf(stderr,"file_start[0]>=iter_file_start[0]\n"); - memcpy(new_buffer, buf + iter_file_start[0] * element_size, - iter_file_count[0] * element_size); - memcpy(new_buffer + iter_file_count[0] * element_size, - buf + file_start[0] * element_size, file_count[0] * element_size); - } - else { - // fprintf(stderr,"not file_start[0]>=iter_file_start[0]\n"); - - memcpy(new_buffer, buf + (file_start[0] * element_size), - file_count[0] * element_size); - memcpy(new_buffer + file_count[0] * element_size, - buf + (iter_file_start[0] * element_size), - iter_file_count[0] * element_size); + + + //time_memcopy1 = MPI_Wtime(); + if (iter_args->free_buf == 0) + {new_buffer = malloc(count[0] * element_size); + + + if (file_start[0] >= iter_file_start[0]) { + + memcpy(new_buffer, buf + iter_file_start[0] * element_size, + iter_file_count[0] * element_size); + memcpy(new_buffer + iter_file_count[0] * element_size, + buf + file_start[0] * element_size, file_count[0] * element_size); + } + else { + + + memcpy(new_buffer, buf + (file_start[0] * element_size), + file_count[0] * element_size); + memcpy(new_buffer + file_count[0] * element_size, + buf + (iter_file_start[0] * element_size), + iter_file_count[0] * element_size); + } + iter_args->buf = new_buffer; + iter_args->free_buf=1; } + else{ + iter_args->buf = realloc(iter_args->buf, count[0]*element_size); + if (file_start[0] >= iter_file_start[0]) { + - // memcpy(new_buffer+(file_start[0]*element_size),buf+(mem_start[0]*element_size),mem_count[0]*element_size); - // memcpy(new_buffer,buf,count[0]*element_size); + memcpy(iter_args->buf + iter_file_count[0] * element_size, + buf + file_start[0] * element_size, file_count[0] * element_size); + } + else { + + + + memcpy(iter_args->buf + file_count[0] * element_size, + buf + (iter_file_start[0] * element_size), + iter_file_count[0] * element_size); + } + iter_args->free_buf=1; + } + //if (iter_args->free_buf == 1) + // free(iter_args->buf); + time_memcopy2 = MPI_Wtime(); + duration_memcopy=time_memcopy2-time_memcopy1; - /*allocate new_buffer - get the start and count value from memspace - based on the start and count copy the data from buf to new buffer - need to be corect locations - do the same thing for iter_args->buf and iter_args->memspace. - */ + H5Sclose(iter_args->mem_space_id); - // H5Sclose(iter_args->file_space_id); + iter_args->mem_space_id = new_memspace; - // iter_args->file_space_id=new_memspace; + + - if (iter_args->free_buf == 1) - free(iter_args->buf); - - iter_args->buf = new_buffer; - iter_args->free_buf = 1; - - /* if(check_contiguous_overlap(ndim,start,count)) - {fprintf(stderr, "\ncontiguous overlap\n"); - - fprintf(stderr,"\n after contiguous overlap start=%llu - count=%llu\n",start[0],count[0]); - } */ + + + status = H5Sselect_hyperslab(file_space_id, H5S_SELECT_SET, start, NULL, count, NULL); @@ -9564,47 +9479,36 @@ foreach_iteration(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t mem_typ NULL, count, NULL); start[0] = 0; - // fprintf(stderr,"\n iter mem space mem_start=%lld mem_count=%lld file_start=%lld - // file_count=%lld - // element_size=%lld\n",start[0],count[0],start[0],count[0],element_size); + status = H5Sselect_hyperslab(iter_args->mem_space_id, H5S_SELECT_SET, start, NULL, count, NULL); - /* for (int i = 0; i < 16; i++) - fprintf(stderr, " buf[%d]=%d ", i, buf_temp[i]); - fprintf(stderr, "\n"); - - for (int i = 0; i < 16; i++) - fprintf(stderr, " new_buffer[%d]=%d ", i, new_buffer_temp[i]); - fprintf(stderr, "\n"); */ - // free(new_buffer); + } else if (ndim == 2) { - + fprintf(stderr, " start=%llux%llu count=%llux%llu\n", start[0], start[1], count[0], count[1]); - new_memspace = H5Screate_simple(ndim, count, NULL); - element_size = H5Tget_size(mem_type_id); - new_buffer = malloc(count[0] * count[1] * element_size); - int *buf_temp = buf; - int *new_buffer_temp = new_buffer; + new_memspace = H5Screate_simple(ndim, count, NULL); + element_size = H5Tget_size(mem_type_id); + new_buffer = malloc(count[0] * count[1] * element_size); + status = H5Sget_regular_hyperslab(mem_space_id, mem_start, NULL, mem_count, NULL); status = H5Sget_regular_hyperslab(file_space_id, file_start, NULL, file_count, NULL); - // fprintf(stderr,"\nmem_start=%lld mem_count=%lld file_start=%lld file_count=%lld - // element_size=%lld\n",mem_start[0],mem_count[0],file_start[0],file_count[0],element_size); + fprintf(stderr, "\n mem_start=%lldx%lld file_start=%lldx%lld ", mem_start[0], mem_start[1], file_start[0], file_start[1]); fprintf(stderr, "\n mem_count=%lldx%lld file_count=%lldx%lld \n", mem_count[0], mem_count[1], file_count[0], file_count[1]); status = H5Sget_regular_hyperslab(iter_args->mem_space_id, iter_mem_start, NULL, iter_mem_count, NULL); - + status = H5Sget_regular_hyperslab(iter_args->file_space_id, iter_file_start, NULL, iter_file_count, NULL); - // fprintf(stderr,"%d\n",status); + fprintf(stderr, "\n iter_mem_start=%lldx%lld iter_file_start=%lldx%lld ", iter_mem_start[0], iter_mem_start[1], iter_file_start[0], iter_file_start[1]); @@ -9613,11 +9517,11 @@ foreach_iteration(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t mem_typ iter_file_count[1]); H5Sget_simple_extent_dims(file_space_id, dimsm, NULL); - // fprintf(stderr, "\n dimensions=%lldx%lld \n", dimsm[0], dimsm[1]); + buffer_count = 0; feed_buf_count = 0; - if (file_start[0] == iter_file_start[0]) { + if (file_start[0] == iter_file_start[0]) { if (file_start[1] >= iter_file_start[1]) { if (file_count[0] == iter_file_count[0]) { feed_buf_count = dimsm[1] * iter_file_start[0]; @@ -9625,17 +9529,16 @@ foreach_iteration(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t mem_typ memcpy(new_buffer + (buffer_count * element_size), buf + (feed_buf_count + iter_file_start[1]) * element_size, iter_file_count[1] * element_size); - // fprintf(stderr,"element size=%lld\n",element_size); + buffer_count += iter_file_count[1]; memcpy(new_buffer + (buffer_count * element_size), buf + (feed_buf_count + file_start[1]) * element_size, file_count[1] * element_size); - // fprintf(stderr,"element size=%lld\n",element_size); + buffer_count += file_count[1]; feed_buf_count += dimsm[1]; - fprintf(stderr, "buffer_count=%d \ - feed_buf_count=%d\n", - buffer_count, feed_buf_count); + fprintf(stderr,"buffer_count=%d \ + feed_buf_count=%d\n",buffer_count,feed_buf_count); } } } @@ -9648,16 +9551,15 @@ foreach_iteration(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t mem_typ memcpy(new_buffer + (buffer_count * element_size), buf + (feed_buf_count + file_start[1]) * element_size, file_count[1] * element_size); - // fprintf(stderr,"element size=%lld\n",element_size); + buffer_count += file_count[1]; memcpy(new_buffer + (buffer_count * element_size), buf + (feed_buf_count + iter_file_start[1]) * element_size, iter_file_count[1] * element_size); - // fprintf(stderr,"element size=%lld\n",element_size); + buffer_count += iter_file_count[1]; feed_buf_count += dimsm[1]; - fprintf(stderr, "buffer_count=%d feed_buf_count=%d\n", - buffer_count, feed_buf_count); + fprintf(stderr,"buffer_count=%d feed_buf_count=%d\n",buffer_count,feed_buf_count); } } } @@ -9670,21 +9572,19 @@ foreach_iteration(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t mem_typ memcpy(new_buffer + (buffer_count * element_size), buf + (feed_buf_count + iter_file_start[1]) * element_size, iter_file_count[1] * element_size); - // fprintf(stderr,"element size=%lld\n",element_size); + buffer_count += iter_file_count[1]; feed_buf_count += dimsm[1]; - // fprintf(stderr,"buffer_count=%d - // feed_buf_count=%d\n",buffer_count,feed_buf_count); + } for (int i = 0; i < file_count[0]; i++) { memcpy(new_buffer + (buffer_count * element_size), buf + (feed_buf_count + file_start[1]) * element_size, file_count[1] * element_size); - // fprintf(stderr,"element size=%lld\n",element_size); + buffer_count += file_count[1]; feed_buf_count += dimsm[1]; - // fprintf(stderr,"buffer_count=%d - // feed_buf_count=%d\n",buffer_count,feed_buf_count); + } } } @@ -9696,58 +9596,42 @@ foreach_iteration(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t mem_typ memcpy(new_buffer + (buffer_count * element_size), buf + (feed_buf_count + file_start[1]) * element_size, file_count[1] * element_size); - // fprintf(stderr,"element size=%lld\n",element_size); + buffer_count += file_count[1]; feed_buf_count += dimsm[1]; - fprintf(stderr, "buffer_count=%d \ - // feed_buf_count=%d\n", - buffer_count, feed_buf_count); + fprintf(stderr,"buffer_count=%d \ + // feed_buf_count=%d\n",buffer_count,feed_buf_count); } for (int i = 0; i < iter_file_count[0]; i++) { memcpy(new_buffer + (buffer_count * element_size), buf + (feed_buf_count + iter_file_start[1]) * element_size, iter_file_count[1] * element_size); - // fprintf(stderr,"element size=%lld\n",element_size); + buffer_count += iter_file_count[1]; feed_buf_count += dimsm[1]; - fprintf(stderr, "buffer_count=%d \ - feed_buf_count=%d\n", - buffer_count, feed_buf_count); + fprintf(stderr,"buffer_count=%d \ + feed_buf_count=%d\n",buffer_count,feed_buf_count); } } } - } + } - // memcpy(new_buffer+(file_start[0]*element_size),buf+(mem_start[0]*element_size),mem_count[0]*element_size); - // memcpy(new_buffer,buf,count[0]*element_size); + - // for (int i = 0; i < 16; i++) - // fprintf(stderr, " buf[%d]=%d ", i, buf_temp[i]); - // fprintf(stderr, "\n"); - - // for (int i = 0; i < 16; i++) - // fprintf(stderr, " new_buffer[%d]=%d ", i, new_buffer_temp[i]); - // fprintf(stderr, "\n"); H5Sclose(iter_args->mem_space_id); - // H5Sclose(iter_args->file_space_id); - + iter_args->mem_space_id = new_memspace; - // iter_args->file_space_id=new_memspace; + - iter_args->buf = new_buffer; - iter_args->free_buf = 1; - int *iter_buf = iter_args->buf; + iter_args->buf = new_buffer; + iter_args->free_buf=1; + int *iter_buf= iter_args->buf; for (int i = 0; i < 16; i++) fprintf(stderr, " iter_args new_buffer[%d]=%d ", i, iter_buf[i]); - fprintf(stderr, "\n"); - - // if(check_contiguous_overlap(ndim,start,count)) - // {fprintf(stderr, "\ncontiguous overlap\n"); + fprintf(stderr, "\n"); - // fprintf(stderr,"\n after contiguous overlap start=%llu - // count=%llu\n",start[0],count[0]); - //} + status = H5Sselect_hyperslab(file_space_id, H5S_SELECT_SET, start, NULL, count, NULL); @@ -9756,45 +9640,32 @@ foreach_iteration(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t mem_typ start[0] = 0; start[1] = 0; - // fprintf(stderr,"\n iter mem space mem_start=%lld mem_count=%lld file_start=%lld - // file_count=%lld - // element_size=%lld\n",start[0],count[0],start[0],count[0],element_size); + status = H5Sselect_hyperslab(iter_args->mem_space_id, H5S_SELECT_SET, start, NULL, count, NULL); - // free(new_buffer); */ + } else if (ndim == 3) { - // fprintf(stderr," foreach iterator mem_type_id=%llu\n",iter_args->mem_type_id); + fprintf(stderr, " start=%llux%llux%llu count=%llux%llux%llu\n", start[0], start[1], start[2], count[0], count[1], count[2]); new_memspace = H5Screate_simple(ndim, count, NULL); element_size = H5Tget_size(mem_type_id); new_buffer = malloc(count[0] * count[1] * count[2] * element_size); - + status = H5Sget_regular_hyperslab(mem_space_id, mem_start, NULL, mem_count, NULL); status = H5Sget_regular_hyperslab(file_space_id, file_start, NULL, file_count, NULL); - /* fprintf(stderr,"\nmem_start=%lld mem_count=%lld file_start=%lld file_count=%lld - \ - element_size=%lld\n",mem_start[0],mem_count[0],file_start[0],file_count[0],element_size); - */ - /* fprintf(stderr,"\n mem_start=%lldx%lldx%lld file_start=%lldx%lldx%lld - ",mem_start[0],mem_start[1],mem_start[2],file_start[0],file_start[1],file_start[2]); - fprintf(stderr,"\n mem_count=%lldx%lldx%lld file_count=%lldx%lldx%lld - \n",mem_count[0],mem_count[1],mem_count[2],file_count[0],file_count[1],file_count[2]); - */ + status = H5Sget_regular_hyperslab(iter_args->mem_space_id, iter_mem_start, NULL, iter_mem_count, NULL); - /* fprintf(stderr,"%d\n",status); - if (H5Sis_regular_hyperslab(iter_args->mem_space_id)) { - fprintf(stderr,"regular hyperslab\n"); - } */ + status = H5Sget_regular_hyperslab(iter_args->file_space_id, iter_file_start, NULL, iter_file_count, NULL); - // fprintf(stderr,"%d\n",status); + fprintf(stderr, "\n file_start=%lldx%lldx%lld iter_file_start=%lldx%lldx%lld ", file_start[0], file_start[1], file_start[2], iter_file_start[0], iter_file_start[1], iter_file_start[2]); @@ -9802,41 +9673,38 @@ foreach_iteration(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t mem_typ file_count[0], file_count[1], file_count[2], iter_file_count[0], iter_file_count[1], iter_file_count[2]); H5Sget_simple_extent_dims(file_space_id, dimsm, NULL); - // fprintf(stderr, "\n dimensions=%lldx%lldx%lld \n", dimsm[0], dimsm[1], - // dimsm[2]); + int dim_elements = dimsm[1] * dimsm[2]; buffer_count = 0; feed_buf_count = 0; buffer_count = 0; feed_buf_count = 0; - int *buf_temp = buf; - int *new_buffer_temp = new_buffer; + if (file_start[1] == iter_file_start[1]) { if (file_start[2] >= iter_file_start[2]) { if (file_count[1] == iter_file_count[1]) { if (file_count[0] == iter_file_count[0]) { for (int d = 0; d < file_count[0]; d++) { - - feed_buf_count = file_start[0] * dim_elements + - dim_elements * d + - dimsm[2] * iter_file_start[1]; + + feed_buf_count = + file_start[0]*dim_elements+dim_elements * d + dimsm[2] * iter_file_start[1]; for (int i = 0; i < iter_file_count[1]; i++) { - memcpy(new_buffer + (buffer_count * element_size), + memcpy(new_buffer + (buffer_count * element_size), buf + (feed_buf_count + iter_file_start[2]) * element_size, - iter_file_count[2] * element_size); - // fprintf(stderr,"element size=%lld\n",element_size); + iter_file_count[2] * element_size); + buffer_count += iter_file_count[2]; - memcpy(new_buffer + (buffer_count * element_size), + memcpy(new_buffer + (buffer_count * element_size), buf + (feed_buf_count + file_start[2]) * element_size, - file_count[2] * element_size); - // fprintf(stderr,"element size=%lld\n",element_size); + file_count[2] * element_size); + buffer_count += file_count[2]; feed_buf_count += dimsm[2]; - // fprintf(stderr,"buffer_count=%d \ + // fprintf(stderr,"buffer_count=%d \ //feed_buf_count=%d\n",buffer_count,feed_buf_count); } } @@ -9849,22 +9717,21 @@ foreach_iteration(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t mem_typ if (file_count[0] == iter_file_count[0]) { for (int d = 0; d < file_count[0]; d++) { - feed_buf_count = file_start[0] * dim_elements + - dim_elements * d + dimsm[2] * file_start[1]; - // feed_buf_count=dimsm[2]*file_start[1]; + feed_buf_count = file_start[0]*dim_elements+dim_elements * d + dimsm[2] * file_start[1]; + for (int i = 0; i < file_count[1]; i++) { - memcpy(new_buffer + (buffer_count * element_size), + memcpy(new_buffer + (buffer_count * element_size), buf + (feed_buf_count + file_start[2]) * element_size, - file_count[2] * element_size); - // fprintf(stderr,"element size=%lld\n",element_size); + file_count[2] * element_size); + buffer_count += file_count[2]; - memcpy(new_buffer + (buffer_count * element_size), + memcpy(new_buffer + (buffer_count * element_size), buf + (feed_buf_count + iter_file_start[2]) * element_size, - iter_file_count[2] * element_size); - // fprintf(stderr,"element size=%lld\n",element_size); + iter_file_count[2] * element_size); + buffer_count += iter_file_count[2]; feed_buf_count += dimsm[2]; //fprintf(stderr,"buffer_count=%d \ @@ -9881,28 +9748,27 @@ foreach_iteration(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t mem_typ if (file_count[0] == iter_file_count[0]) { for (int d = 0; d < file_count[0]; d++) { - feed_buf_count = file_start[0] * dim_elements + - dim_elements * d + - dimsm[2] * iter_file_start[1]; + feed_buf_count = + file_start[0]*dim_elements+dim_elements * d + dimsm[2] * iter_file_start[1]; - // feed_buf_count=dimsm[2]*iter_file_start[1]; + for (int i = 0; i < iter_file_count[1]; i++) { - memcpy(new_buffer + (buffer_count * element_size), + memcpy(new_buffer + (buffer_count * element_size), buf + (feed_buf_count + iter_file_start[2]) * element_size, - iter_file_count[2] * element_size); - // fprintf(stderr,"element size=%lld\n",element_size); + iter_file_count[2] * element_size); + buffer_count += iter_file_count[2]; feed_buf_count += dimsm[2]; - // fprintf(stderr, "buffer_count=%d feed_buf_count=%d\n", - // buffer_count, feed_buf_count); + //fprintf(stderr, "buffer_count=%d feed_buf_count=%d\n", + // buffer_count, feed_buf_count); } for (int i = 0; i < file_count[1]; i++) { - memcpy(new_buffer + (buffer_count * element_size), + memcpy(new_buffer + (buffer_count * element_size), buf + (feed_buf_count + file_start[2]) * element_size, - file_count[2] * element_size); - // fprintf(stderr,"element size=%lld\n",element_size); + file_count[2] * element_size); + buffer_count += file_count[2]; feed_buf_count += dimsm[2]; //fprintf(stderr,"buffer_count=%d \ @@ -9917,28 +9783,26 @@ foreach_iteration(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t mem_typ if (file_count[0] == iter_file_count[0]) { for (int d = 0; d < file_count[0]; d++) { - feed_buf_count = file_start[0] * dim_elements + - dim_elements * d + dimsm[2] * file_start[1]; - - // feed_buf_count=dimsm[2]*file_start[1]; + feed_buf_count = file_start[0]*dim_elements+dim_elements * d + dimsm[2] * file_start[1]; + for (int i = 0; i < file_count[1]; i++) { - memcpy(new_buffer + (buffer_count * element_size), + memcpy(new_buffer + (buffer_count * element_size), buf + (feed_buf_count + file_start[2]) * element_size, - file_count[2] * element_size); - // fprintf(stderr,"element size=%lld\n",element_size); + file_count[2] * element_size); + buffer_count += file_count[2]; feed_buf_count += dimsm[2]; - // fprintf(stderr,"buffer_count=%d \ + // fprintf(stderr,"buffer_count=%d \ // feed_buf_count=%d\n",buffer_count,feed_buf_count); } for (int i = 0; i < iter_file_count[1]; i++) { - memcpy(new_buffer + (buffer_count * element_size), + memcpy(new_buffer + (buffer_count * element_size), buf + (feed_buf_count + iter_file_start[2]) * element_size, iter_file_count[2] * element_size); - // fprintf(stderr,"element size=%lld\n",element_size); + buffer_count += iter_file_count[2]; feed_buf_count += dimsm[2]; // fprintf(stderr,"buffer_count=%d \ @@ -9950,44 +9814,13 @@ foreach_iteration(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t mem_typ } } - // memcpy(new_buffer+(file_start[0]*element_size),buf+(mem_start[0]*element_size),mem_count[0]*element_size); - // memcpy(new_buffer,buf,count[0]*element_size); - - /*allocate new_buffer - get the start and count value from memspace - based on the start and count copy the data from buf to new buffer - need to be corect locations - do the same thing for iter_args->buf and iter_args->memspace. - */ - - /* for (int i = 0; i < 16; i++) - fprintf(stderr, " buf[%d]=%d ", i, buf_temp[i]); - fprintf(stderr, "\n"); - - for (int i = 0; i < 16; i++) - fprintf(stderr, " new_buffer[%d]=%d ", i, new_buffer_temp[i]); - fprintf(stderr, "\n"); */ - H5Sclose(iter_args->mem_space_id); - // H5Sclose(iter_args->file_space_id); - iter_args->mem_space_id = new_memspace; - // iter_args->file_space_id=new_memspace; - - iter_args->buf = new_buffer; - iter_args->free_buf = 1; - - /* int *iter_buf= iter_args->buf; - for (int i = 0; i < 16; i++) - fprintf(stderr, " iter_args new_buffer[%d]=%d ", i, iter_buf[i]); - fprintf(stderr, "\n"); */ - - /* if(check_contiguous_overlap(ndim,start,count)) - {fprintf(stderr, "\ncontiguous overlap\n"); - - fprintf(stderr,"\n after contiguous overlap start=%llu - count=%llu\n",start[0],count[0]); - } */ + + iter_args->buf = new_buffer; + iter_args->free_buf=1; + + status = H5Sselect_hyperslab(file_space_id, H5S_SELECT_SET, start, NULL, count, NULL); @@ -10003,17 +9836,25 @@ foreach_iteration(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t mem_typ status = H5Sselect_hyperslab(iter_args->mem_space_id, H5S_SELECT_SET, start, NULL, count, NULL); - // fprintf(stderr," end of foreach iterator - // mem_type_id=%llu\n",iter_args->mem_type_id); + //fprintf(stderr," end of foreach iterator mem_type_id=%llu\n",iter_args->mem_type_id); + + } - // return 1; + + + //time_in_contiguous2= MPI_Wtime(); + //duration_in_contiguous_check= time_in_contiguous2 - time_in_contiguous1; + //fprintf(stderr," Duration in contiguous Runtime is %f \n", duration_contiguous_check); } else { - // free(new_buffer); - // iter_args->buf=buf; - // fprintf(stderr, "-------- Not contiguous---------\n"); + + //fprintf(stderr, "-------- Not contiguous---------\n"); } + //time_after_contiguous_check = MPI_Wtime(); + //duration_contiguous_check= time_after_contiguous_check - time_before_contiguous_check; + // fprintf(stderr," Duration contiguous Runtime is %f \n", duration_contiguous_check); + free(dimsm); free(start); free(count); @@ -10025,13 +9866,16 @@ foreach_iteration(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t mem_typ free(iter_mem_count); free(iter_file_start); free(iter_file_count); + } } } } - // free(new_buffer); - // free(buf); - // return return_val; + //time_foreach2 = MPI_Wtime(); + + //duration_foreach = time_foreach2 - time_foreach1; + //fprintf(stderr,"Foreach Duration Runtime is %f \n", duration_foreach); + return return_task_val; } static herr_t @@ -10061,59 +9905,49 @@ async_dataset_write_merge(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t async_task_t * return_task_val; async_task_list_t *task_list_iter; int buffer_count, feed_buf_count; - double time1, time2, duration; - double time11, time12, duration1; - double time21, time22, duration2; + double time1, time2,duration; + double time11,time12,duration1; + double time21,time22,duration2; assert(aid); assert(parent_obj); assert(parent_obj->magic == ASYNC_MAGIC); assert(mem_space_id); - // fprintf(stderr,"this is before H5Sget_select_type\n"); - - // print_dataspace(mem_space_id); - // print_dataspace(file_space_id); - /*if(type==H5S_SEL_HYPERSLABS) - fprintf(stderr,"A hyperslab or compound hyperslab is selected\n"); - else if(type==H5S_SEL_POINTS) - fprintf(stderr,"A sequence of points is selected\n"); - else if(type==H5S_SEL_ALL) - fprintf(stderr,"The entire dataset is selected\n"); - else if(type==H5S_SEL_NONE) - fprintf(stderr,"No selection is defined\n"); - */ - // fprintf(stderr," merged func mem_type_id=%llu\n",mem_type_id); - time1 = MPI_Wtime(); - time11 = MPI_Wtime(); + + //fprintf(stderr,"################# \n"); + //time1 = MPI_Wtime(); + //time11=MPI_Wtime(); return_task_val = foreach_iteration(aid, parent_obj, mem_type_id, mem_space_id, file_space_id, plist_id, buf, NULL); // return_val=foreach_iteration(aid,parent_obj,mem_type_id,mem_space_id,file_space_id,plist_id,buf,NULL); - time12 = MPI_Wtime(); - duration1 = time12 - time11; - fprintf(stderr, "1st Foreach iter Runtime is %f \n", duration1); + //time12 = MPI_Wtime(); + //duration1 = time12 - time11; + //fprintf(stderr,"\t1st Foreach iter Runtime is %f \n", duration1); if (return_task_val == NULL) return_val = 0; else return_val = 1; - - for (int i = 0; return_task_val != NULL; i++) { - time21 = MPI_Wtime(); - - return_task_val = foreach_iteration(aid, parent_obj, mem_type_id, mem_space_id, file_space_id, - plist_id, buf, return_task_val); - time22 = MPI_Wtime(); - - duration2 = time22 - time21; - fprintf(stderr, "in the for loop Runtime is i=%d %f \n", i, duration1); - } - - time2 = MPI_Wtime(); - duration = time2 - time1; - - fprintf(stderr, "Foreach Runtime is %f \n", duration); - + + for (int i = 0; return_task_val != NULL; i++) + { //time21=MPI_Wtime(); + + return_task_val = foreach_iteration(aid, parent_obj, mem_type_id, mem_space_id, file_space_id, + plist_id, buf, return_task_val); + //time22=MPI_Wtime(); + + //duration2 = time22 - time21; + //fprintf(stderr,"\t\tin the for loop Runtime is i=%d %f \n", i, duration1); + } + + //time2 = MPI_Wtime(); + //duration = time2 - time1; + + + + //fprintf(stderr,"Foreach Runtime is %f \n", duration); + return return_val; } @@ -10134,13 +9968,13 @@ async_dataset_write(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t mem_t assert(aid); assert(parent_obj); assert(parent_obj->magic == ASYNC_MAGIC); - // fprintf(stderr,"mem_type_id=%llu\n",mem_type_id); + return_val = async_dataset_write_merge(aid, parent_obj, mem_type_id, mem_space_id, file_space_id, plist_id, buf); if (return_val == 1) - goto done; + goto done; async_instance_g->prev_push_state = async_instance_g->start_abt_push; if ((args = (async_dataset_write_args_t *)calloc(1, sizeof(async_dataset_write_args_t))) == NULL) { From 8b77076a1c783ce70133c73f7c7d1bcdd9a04a9a Mon Sep 17 00:00:00 2001 From: kamalchowdhurylbl Date: Thu, 11 Aug 2022 16:43:23 -0500 Subject: [PATCH 13/18] changes in the makefile --- test/Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/Makefile b/test/Makefile index f930d59..5749335 100644 --- a/test/Makefile +++ b/test/Makefile @@ -37,7 +37,8 @@ SRC = async_test_serial.c async_test_serial2.c \ async_test_parallel.c async_test_parallel2.c async_test_parallel3.c \ async_test_parallel4.c async_test_parallel5.c \ async_test_serial_event_set.c async_test_serial_error_stack.c \ - async_test_serial_event_set_error_stack.c + async_test_serial_event_set_error_stack.c 1d_asyn_MPI_write.c 2d_asyn_MPI_write.c \ + 3d_asyn_MPI_write.c 1d_async_write_performance #async_test_no_fclose.c OBJ = $(SRC:.c=.o) From 2243c1405ef6f01e69a1cca2244d29020c4a02f6 Mon Sep 17 00:00:00 2001 From: kamalchowdhurylbl Date: Fri, 12 Aug 2022 12:49:44 -0500 Subject: [PATCH 14/18] changes after review async vol.c --- src/h5_async_vol.c | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/h5_async_vol.c b/src/h5_async_vol.c index b96bcbe..0090791 100644 --- a/src/h5_async_vol.c +++ b/src/h5_async_vol.c @@ -1,5 +1,5 @@ /******************************************************************************* -Asynchronous I/O VOL Connector (AsyncVOL) Copyright (c) 2021, The +Asynchronous I/O VOL Connector (AsyncVOL) Copyright (c) 2021, The Regents of the University of California, through Lawrence Berkeley National Laboratory (subject to receipt of any required approvals from the U.S. Dept. of Energy). All rights reserved. @@ -65,7 +65,7 @@ works, and perform publicly and display publicly, and to permit others to do so. /* Whether to display log messge when callback is invoked */ /* (Uncomment to enable) */ /* #define ENABLE_DBG_MSG 1 */ - #define PRINT_ERROR_STACK 1 + #define PRINT_ERROR_STACK 1 /* #define ENABLE_ASYNC_LOGGING */ #define ASYNC_DBG_MSG_RANK 0 @@ -206,6 +206,7 @@ typedef struct async_instance_t { bool pause; /* Pause background thread execution */ bool disable_implicit_file; /* Disable implicit async execution globally */ bool disable_implicit; /* Disable implicit async execution for dxpl */ + bool disable_async_dset_get; /* Disable async execution for dataset get */ bool delay_time_env; /* Flag that indicates the delay time is set by env variable */ uint64_t delay_time; /* Sleep time before background thread trying to acquire global mutex */ int sleep_time; /* Sleep time between checking the global mutex attemp count */ @@ -256,7 +257,6 @@ typedef struct async_attr_write_args_t { void * buf; hid_t dxpl_id; void **req; - bool free_buf; #ifdef ENABLE_WRITE_MEMCPY hsize_t data_size; @@ -332,7 +332,6 @@ typedef struct async_dataset_write_args_t { hid_t plist_id; void * buf; void **req; - bool free_buf; #ifdef ENABLE_WRITE_MEMCPY hsize_t data_size; @@ -1300,7 +1299,7 @@ async_instance_init(int backing_thread_count) aid->disable_implicit = false; aid->disable_implicit_file = false; aid->delay_time_env = false; - + aid->disable_async_dset_get = true; // Check for delaying operations to file / group / dataset close operations env_var = getenv("HDF5_ASYNC_EXE_FCLOSE"); if (env_var && *env_var && atoi(env_var) > 0) @@ -1321,6 +1320,10 @@ async_instance_init(int backing_thread_count) /* Set "delay execution" convenience flag, if any of the others are set */ if (aid->ex_fclose || aid->ex_gclose || aid->ex_dclose) aid->ex_delay = true; + + env_var = getenv("HDF5_ASYNC_DISABLE_DSET_GET"); + if (env_var && *env_var && atoi(env_var) <= 0) + aid->disable_async_dset_get = false; #ifdef ENABLE_WRITE_MEMCPY // Get max memory allowed for async memcpy @@ -8860,7 +8863,7 @@ async_dataset_write_fn(void *foo) assert(task); assert(task->async_obj); assert(task->async_obj->magic == ASYNC_MAGIC); - //fprintf(stderr,"task is_merge=%d\n",task->is_merge); + pool_ptr = task->async_obj->pool_ptr; if (task->is_merge == 1) goto done; @@ -8969,8 +8972,8 @@ async_dataset_write_fn(void *foo) fprintf(fout_g, " [ASYNC ABT ERROR] %s H5VLfree_lib_state failed\n", __func__); task->h5_state = NULL; - //if (args->mem_type_id > 0) - // H5Tclose(args->mem_type_id); + if (args->mem_type_id > 0) + H5Tclose(args->mem_type_id); if (args->mem_space_id > 0) H5Sclose(args->mem_space_id); if (args->file_space_id > 0) From 1aca59536d622d41137480961300d2545d0da061 Mon Sep 17 00:00:00 2001 From: kamalchowdhurylbl Date: Fri, 12 Aug 2022 14:58:35 -0500 Subject: [PATCH 15/18] changes after review async vol.c, 1d,2d,and 3d test write code --- src/h5_async_vol.c | 157 +++++++---------------- test/1d_asyn_MPI_write.c | 123 +----------------- test/2d_asyn_MPI_write.c | 76 +---------- test/3d_asyn_MPI_write.c | 267 +-------------------------------------- 4 files changed, 54 insertions(+), 569 deletions(-) diff --git a/src/h5_async_vol.c b/src/h5_async_vol.c index 0090791..855577d 100644 --- a/src/h5_async_vol.c +++ b/src/h5_async_vol.c @@ -8943,7 +8943,7 @@ async_dataset_write_fn(void *foo) usleep(1000); count++; } - //fprintf(stderr," args mem_type_id=%llu\n",args->mem_type_id); + /* Try executing operation, without default error stack handling */ H5E_BEGIN_TRY { @@ -9056,20 +9056,15 @@ print_dataspace(hid_t mem_space_id) int ndim; int num_elements; hsize_t nblocks; - int N1 = 20; - hsize_t dimsm[1]; hsize_t *buffer; H5S_sel_type type; herr_t slected_block; hssize_t numblocks; async_dataset_write_args_t *args = NULL; - hsize_t *start_out, *stride_out, *count_out, *block_out; - herr_t status; - async_task_t * task_iter; - async_task_list_t *task_list_iter; + if (mem_space_id == H5S_SEL_ALL) fprintf(stderr, "-----H5S_SEL_ALL----\n"); @@ -9167,23 +9162,13 @@ check_contiguous(hid_t current_task_space_id, hid_t task_iterator_file_space_id, { int ndim; - int num_elements; - hsize_t nblocks; - int N1 = 20; - hsize_t dimsm[1]; - hsize_t * buffer; - hid_t memspace, dataspace; - H5S_sel_type type; - herr_t slected_block; - hssize_t numblocks; - async_dataset_write_args_t *args = NULL; - async_dataset_write_args_t *iter_args = NULL; + hsize_t *current_task_start_out, *current_task_stride_out, *current_task_count_out, *current_task_block_out; hsize_t *iterator_task_start_out, *iterator_task_stride_out, *iterator_task_count_out, *iterator_task_block_out; - herr_t status; + herr_t status; H5S_sel_type current_task_type, iterator_task_type; @@ -9324,30 +9309,35 @@ foreach_iteration(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t mem_typ hid_t file_space_id, hid_t plist_id, const void *buf, async_task_t *prev_task) { int ndim; - int num_elements; - hsize_t nblocks; - int N1 = 20; hsize_t * dimsm; void * buffer, *new_buffer; - hid_t memspace, dataspace; - H5S_sel_type type; - herr_t slected_block; - hssize_t numblocks; - async_dataset_write_args_t *args = NULL; - async_dataset_write_args_t *iter_args = NULL; + async_dataset_write_args_t *iter_args = NULL; hsize_t *start_out, *stride_out, *count_out, *block_out; hsize_t *start, *count, *mem_start, *mem_count, *file_start, *file_count, *iter_mem_start, *iter_mem_count, *iter_file_start, *iter_file_count, element_size; + hid_t new_memspace; herr_t status, return_val = 0; async_task_t * task_iter; async_task_t * return_task_val = NULL; async_task_list_t *task_list_iter; int buffer_count, feed_buf_count; - double time_foreach1,time_foreach2,duration_foreach,time_before_contiguous_check, time_after_contiguous_check,duration_contiguous_check; - double duration_in_contiguous_check,time_in_contiguous1,time_in_contiguous2,duration_memcopy,time_memcopy1,time_memcopy2; - + //These variables are for testing the run time + //double time_foreach1,time_foreach2,duration_foreach,time_before_contiguous_check, time_after_contiguous_check,duration_contiguous_check; + //double duration_in_contiguous_check,time_in_contiguous1,time_in_contiguous2,duration_memcopy,time_memcopy1,time_memcopy2; + ndim = H5Sget_simple_extent_ndims(file_space_id); + dimsm = malloc(ndim * sizeof(hsize_t)); + start = malloc(ndim * sizeof(hsize_t)); + count = malloc(ndim * sizeof(hsize_t)); + mem_start = malloc(ndim * sizeof(hsize_t)); + mem_count = malloc(ndim * sizeof(hsize_t)); + file_start = malloc(ndim * sizeof(hsize_t)); + file_count = malloc(ndim * sizeof(hsize_t)); + iter_mem_start = malloc(ndim * sizeof(hsize_t)); + iter_mem_count = malloc(ndim * sizeof(hsize_t)); + iter_file_start = malloc(ndim * sizeof(hsize_t)); + iter_file_count = malloc(ndim * sizeof(hsize_t)); //time_foreach1 = MPI_Wtime(); DL_FOREACH(async_instance_g->qhead.queue, task_list_iter) @@ -9360,50 +9350,21 @@ foreach_iteration(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t mem_typ // checking whether the iterator task is operating on the same dataset of the current task iter_args = task_iter->args; - - - - hid_t new_memspace; - - ndim = H5Sget_simple_extent_ndims(file_space_id); - dimsm = malloc(ndim * sizeof(hsize_t)); - start = malloc(ndim * sizeof(hsize_t)); - count = malloc(ndim * sizeof(hsize_t)); - mem_start = malloc(ndim * sizeof(hsize_t)); - mem_count = malloc(ndim * sizeof(hsize_t)); - file_start = malloc(ndim * sizeof(hsize_t)); - file_count = malloc(ndim * sizeof(hsize_t)); - iter_mem_start = malloc(ndim * sizeof(hsize_t)); - iter_mem_count = malloc(ndim * sizeof(hsize_t)); - iter_file_start = malloc(ndim * sizeof(hsize_t)); - iter_file_count = malloc(ndim * sizeof(hsize_t)); - - - //time_before_contiguous_check = MPI_Wtime(); if (check_contiguous(file_space_id, iter_args->file_space_id, start, count) || check_contiguous(iter_args->file_space_id, file_space_id, start, count)) { - time_in_contiguous1 = MPI_Wtime(); + //time_in_contiguous1 = MPI_Wtime(); return_val = 1; - //fprintf(stderr, "\n-------#### contiguous####---------\n"); - - - + //fprintf(stderr, "\n-------#### contiguous####---------\n"); if (prev_task != NULL) prev_task->is_merge = 1; return_task_val = task_iter; - - if (ndim == 1) { - - new_memspace = H5Screate_simple(ndim, count, NULL); element_size = H5Tget_size(mem_type_id); //fprintf(stderr,"size of new_buffer= %lld \n",count[0]*element_size); - - status = H5Sget_regular_hyperslab(mem_space_id, mem_start, NULL, mem_count, NULL); status = H5Sget_regular_hyperslab(file_space_id, file_start, NULL, file_count, NULL); @@ -9458,23 +9419,15 @@ foreach_iteration(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t mem_typ } iter_args->free_buf=1; } - //if (iter_args->free_buf == 1) - // free(iter_args->buf); - time_memcopy2 = MPI_Wtime(); - duration_memcopy=time_memcopy2-time_memcopy1; + + //time_memcopy2 = MPI_Wtime(); + //duration_memcopy=time_memcopy2-time_memcopy1; H5Sclose(iter_args->mem_space_id); - iter_args->mem_space_id = new_memspace; - - - - - - status = H5Sselect_hyperslab(file_space_id, H5S_SELECT_SET, start, NULL, count, NULL); @@ -9858,22 +9811,24 @@ foreach_iteration(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t mem_typ //duration_contiguous_check= time_after_contiguous_check - time_before_contiguous_check; // fprintf(stderr," Duration contiguous Runtime is %f \n", duration_contiguous_check); - free(dimsm); - free(start); - free(count); - free(mem_start); - free(mem_count); - free(file_start); - free(file_count); - free(iter_mem_start); - free(iter_mem_count); - free(iter_file_start); - free(iter_file_count); + } } } } + + free(dimsm); + free(start); + free(count); + free(mem_start); + free(mem_count); + free(file_start); + free(file_count); + free(iter_mem_start); + free(iter_mem_count); + free(iter_file_start); + free(iter_file_count); //time_foreach2 = MPI_Wtime(); //duration_foreach = time_foreach2 - time_foreach1; @@ -9886,31 +9841,14 @@ async_dataset_write_merge(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t hid_t mem_space_id, hid_t file_space_id, hid_t plist_id, const void *buf) { - int ndim; - int num_elements; - hsize_t nblocks; - int N1 = 20; - hsize_t * dimsm; - void * buffer, *new_buffer; - hid_t memspace, dataspace; - H5S_sel_type type; - herr_t slected_block; - hssize_t numblocks; - async_dataset_write_args_t *args = NULL; - async_dataset_write_args_t *iter_args = NULL; - - hsize_t *start_out, *stride_out, *count_out, *block_out; - hsize_t *start, *count, *mem_start, *mem_count, *file_start, *file_count, *iter_mem_start, - *iter_mem_count, *iter_file_start, *iter_file_count, element_size; - herr_t status, return_val = 0; async_task_t * task_iter; async_task_t * return_task_val; async_task_list_t *task_list_iter; int buffer_count, feed_buf_count; - double time1, time2,duration; - double time11,time12,duration1; - double time21,time22,duration2; + //double time1, time2,duration; + //double time11,time12,duration1; + //double time21,time22,duration2; assert(aid); assert(parent_obj); @@ -9971,13 +9909,14 @@ async_dataset_write(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t mem_t assert(aid); assert(parent_obj); assert(parent_obj->magic == ASYNC_MAGIC); - + #ifdef ENABLE_WRITE_MERGE + return_val = + async_dataset_write_merge(aid, parent_obj, mem_type_id, mem_space_id, file_space_id, plist_id, buf); - return_val = - async_dataset_write_merge(aid, parent_obj, mem_type_id, mem_space_id, file_space_id, plist_id, buf); + if (return_val == 1) + goto done; + #endif - if (return_val == 1) - goto done; async_instance_g->prev_push_state = async_instance_g->start_abt_push; if ((args = (async_dataset_write_args_t *)calloc(1, sizeof(async_dataset_write_args_t))) == NULL) { diff --git a/test/1d_asyn_MPI_write.c b/test/1d_asyn_MPI_write.c index 13a4ea9..07ccaa4 100644 --- a/test/1d_asyn_MPI_write.c +++ b/test/1d_asyn_MPI_write.c @@ -189,83 +189,7 @@ main(int argc, char **argv) } } - /* - offset[0] = 0; - count[0] = 5; - - status = H5Sselect_hyperslab (dataspace, H5S_SELECT_SET, offset, NULL, count, NULL); - - printf("MPI rank=%d Hyperslab operation on dataspace using offset %llu and count %llu - \n",mpi_rank,offset[0],count[0]); offset_out[0] = 0; //offset=0x0 count_out[0] = 5; //count_out=2 X - 2 - - status = H5Sselect_hyperslab (memspace, H5S_SELECT_SET, offset_out, NULL, count_out, NULL); - - fprintf(stderr,"----------------- on memory space using offset %llu and count %llu - \n",offset_out[0],count_out[0]); - - status = H5Dwrite_async (dataset, H5T_NATIVE_INT, memspace, dataspace, data_transfer_propertylist, - data,es_id); - - - - offset[0] = 5; - count[0] = 3; - - status = H5Sselect_hyperslab (dataspace, H5S_SELECT_SET, offset, NULL, count, NULL); - - printf("MPI rank=%d Hyperslab operation on dataspace using offset %llu and count %llu - \n",mpi_rank,offset[0],count[0]); offset_out[0] = 5; //offset=0x0 count_out[0] = 3; //count_out=2 X - 2 - - status = H5Sselect_hyperslab (memspace, H5S_SELECT_SET, offset_out, NULL, count_out, NULL); - - - printf("----------------- on memory space using offset %llu and count %llu - \n",offset_out[0],count_out[0]); - - status = H5Dwrite_async (dataset, H5T_NATIVE_INT, memspace, dataspace, data_transfer_propertylist, - data,es_id); - - - offset[0] = 8; - count[0] = 2; - - status = H5Sselect_hyperslab (dataspace, H5S_SELECT_SET, offset, NULL, count, NULL); - - printf("MPI rank=%d Hyperslab operation on dataspace using offset %llu and count %llu - \n",mpi_rank,offset[0],count[0]); offset_out[0] = 8; //offset=0x0 count_out[0] = 2; //count_out=2 X - 2 - - status = H5Sselect_hyperslab (memspace, H5S_SELECT_SET, offset_out, NULL, count_out, NULL); - - printf("----------------- on memory space using offset %llu and count %llu - \n",offset_out[0],count_out[0]); - - status = H5Dwrite_async (dataset, H5T_NATIVE_INT, memspace, dataspace, data_transfer_propertylist, - data,es_id); - - - - offset[0] = 10; - count[0] = 10; - - status = H5Sselect_hyperslab (dataspace, H5S_SELECT_SET, offset, NULL, count, NULL); - - printf("MPI rank=%d Hyperslab operation on dataspace using offset %llu and count %llu - \n",mpi_rank,offset[0],count[0]); offset_out[0] = 10; //offset=0x0 count_out[0] = 10; //count_out=2 - X 2 - - status = H5Sselect_hyperslab (memspace, H5S_SELECT_SET, offset_out, NULL, count_out, NULL); - - printf("----------------- on memory space using offset %llu and count %llu - \n",offset_out[0],count_out[0]); - - status = H5Dwrite_async (dataset, H5T_NATIVE_INT, memspace, dataspace, data_transfer_propertylist, - data,es_id); - - - */ + } if (mpi_rank == 1) { @@ -307,50 +231,7 @@ main(int argc, char **argv) status = H5Dwrite_async(dataset, H5T_NATIVE_INT, memspace, dataspace, data_transfer_propertylist, data, es_id); - /* - - offset[0] = 28; - - count[0] = 2; - status = H5Sselect_hyperslab (dataspace, H5S_SELECT_SET, offset, NULL, count, NULL); - printf("MPI rank=%d Hyperslab operation on dataspace using offset %llu and count %llu - \n",mpi_rank,offset[0],count[0]); - - - offset_out[0] = 28; - - count_out[0] = 2; - status = H5Sselect_hyperslab (memspace, H5S_SELECT_SET, offset_out, NULL, count_out, NULL); - printf("----------------- on memory space using offset %llu and count %llu - \n",offset_out[0],count_out[0]); - - - // status = H5Dread_async (dataset, H5T_NATIVE_INT, memspace, dataspace, H5P_DEFAULT, - data_out2,es_id); status = H5Dwrite_async (dataset, H5T_NATIVE_INT, memspace, dataspace, - data_transfer_propertylist, data,es_id); - - - - offset[0] = 30; - - count[0] = 10; - status = H5Sselect_hyperslab (dataspace, H5S_SELECT_SET, offset, NULL, count, NULL); - printf("MPI rank=%d Hyperslab operation on dataspace using offset %llu and count %llu - \n",mpi_rank,offset[0],count[0]); - - - offset_out[0] = 30; - - count_out[0] = 10; - status = H5Sselect_hyperslab (memspace, H5S_SELECT_SET, offset_out, NULL, count_out, NULL); - printf("----------------- on memory space using offset %llu and count %llu - \n",offset_out[0],count_out[0]); - - - // status = H5Dread_async (dataset, H5T_NATIVE_INT, memspace, dataspace, H5P_DEFAULT, - data_out2,es_id); status = H5Dwrite_async (dataset, H5T_NATIVE_INT, memspace, dataspace, - data_transfer_propertylist, data,es_id); - */ + } if (print_dbg_msg) diff --git a/test/2d_asyn_MPI_write.c b/test/2d_asyn_MPI_write.c index fbc3231..fcd7eb1 100644 --- a/test/2d_asyn_MPI_write.c +++ b/test/2d_asyn_MPI_write.c @@ -231,81 +231,7 @@ main(int argc, char **argv) } */ } - /* offset[0] = 0; - offset[1]=0; - count[0] = 2; - count[1] = 2; - status = H5Sselect_hyperslab (dataspace, H5S_SELECT_SET, offset, NULL, count, NULL); - - printf("MPI rank=%d Hyperslab operation on dataspace using offset %llux%llu and count %llux%llu - \n",mpi_rank,offset[0],offset[1],count[0],count[1]); offset_out[0] = 0; //offset=0x0 offset_out[1] = - 0; count_out[0] = 2; //count_out=2 X 2 count_out[1] = 2; - - status = H5Sselect_hyperslab (memspace, H5S_SELECT_SET, offset_out, NULL, count_out, NULL); - - printf("----------------- on memory space using offset %llux%llu and count %llux%llu - \n",offset_out[0],offset_out[1],count_out[0],count_out[1]); - - status = H5Dwrite_async (dataset, H5T_NATIVE_INT, memspace, dataspace, data_transfer_propertylist, - data,es_id); - - - offset[0] = 0; - offset[1]=2; - count[0] = 2; - count[1] = 1; - status = H5Sselect_hyperslab (dataspace, H5S_SELECT_SET, offset, NULL, count, NULL); - - printf("MPI rank=%d Hyperslab operation on dataspace using offset %llux%llu and count %llux%llu - \n",mpi_rank,offset[0],offset[1],count[0],count[1]); offset_out[0] = 0; //offset=0x0 offset_out[1] = - 2; count_out[0] = 2; //count_out=2 X 2 count_out[1] = 1; - - status = H5Sselect_hyperslab (memspace, H5S_SELECT_SET, offset_out, NULL, count_out, NULL); - - printf("----------------- on memory space using offset %llux%llu and count %llux%llu - \n",offset_out[0],offset_out[1],count_out[0],count_out[1]); - - status = H5Dwrite_async (dataset, H5T_NATIVE_INT, memspace, dataspace, data_transfer_propertylist, - data,es_id); - - - offset[0] = 0; - offset[1]=3; - count[0] = 2; - count[1] = 2; - status = H5Sselect_hyperslab (dataspace, H5S_SELECT_SET, offset, NULL, count, NULL); - - printf("MPI rank=%d Hyperslab operation on dataspace using offset %llux%llu and count %llux%llu - \n",mpi_rank,offset[0],offset[1],count[0],count[1]); offset_out[0] = 0; //offset=0x0 offset_out[1] = - 3; count_out[0] = 2; //count_out=2 X 2 count_out[1] = 2; - - status = H5Sselect_hyperslab (memspace, H5S_SELECT_SET, offset_out, NULL, count_out, NULL); - - printf("----------------- on memory space using offset %llux%llu and count %llux%llu - \n",offset_out[0],offset_out[1],count_out[0],count_out[1]); - - status = H5Dwrite_async (dataset, H5T_NATIVE_INT, memspace, dataspace, data_transfer_propertylist, - data,es_id); - - - offset[0] = 0; - offset[1]=5; - count[0] = 2; - count[1] = 1; - status = H5Sselect_hyperslab (dataspace, H5S_SELECT_SET, offset, NULL, count, NULL); - - printf("MPI rank=%d Hyperslab operation on dataspace using offset %llux%llu and count %llux%llu - \n",mpi_rank,offset[0],offset[1],count[0],count[1]); offset_out[0] = 0; //offset=0x0 offset_out[1] = - 5; count_out[0] = 2; //count_out=2 X 2 count_out[1] = 1; - - status = H5Sselect_hyperslab (memspace, H5S_SELECT_SET, offset_out, NULL, count_out, NULL); - - printf("----------------- on memory space using offset %llux%llu and count %llux%llu - \n",offset_out[0],offset_out[1],count_out[0],count_out[1]); - - status = H5Dwrite_async (dataset, H5T_NATIVE_INT, memspace, dataspace, data_transfer_propertylist, - data,es_id); - */ + } if (mpi_rank == 1) { diff --git a/test/3d_asyn_MPI_write.c b/test/3d_asyn_MPI_write.c index 605a935..a21812b 100644 --- a/test/3d_asyn_MPI_write.c +++ b/test/3d_asyn_MPI_write.c @@ -143,94 +143,7 @@ main(int argc, char **argv) memspace = H5Screate_simple(RANK_OUT, dimsm, NULL); // RANK_OUT=3 - // Test case#1 - /* int arr[4][6]={ - {0,0,0,2,2,1}, - {0,2,0,2,1,1}, - {0,3,0,2,2,1}, - {0,5,0,2,1,1} - }; */ - // Test case#2 - /* int arr[4][6]={ - {0,0,0,2,2,1}, - {2,0,0,1,2,1}, - {0,3,0,2,2,1}, - {0,5,0,2,1,1} - }; */ - // Test case#3 - /* int arr[4][6]={ - {1,1,0,3,2,1}, - {1,3,0,3,1,1}, - {1,4,0,3,2,1}, - {4,4,0,2,2,1} - }; */ - // Test case#4 - /* int arr[4][6]={ - {1,1,0,3,2,3}, - {1,3,0,3,1,3}, - {1,4,0,3,2,3}, - {4,4,0,2,2,1} - }; */ - // Test case#5 - /* int arr[4][6]={ - {0,1,1,2,2,2}, - {3,4,1,2,2,2}, - {3,3,1,2,1,2}, - {1,4,1,2,2,1} - }; */ - // Test case#6 - /* int arr[4][6]={ - {0,1,1,2,2,2}, - {3,4,1,2,2,2}, - {3,3,1,2,1,2}, - {1,4,1,2,2,2} - }; - */ - /* int array[][4][6]={ - { - {0,0,0,1,2,2}, - {0,0,2,1,2,1}, - {0,0,3,1,2,2}, //Test case#1 - {0,0,5,1,2,1} - }, - { - {0,0,0,1,2,2}, - {0,0,2,1,2,1}, - {0,0,3,1,2,2},//Test case#2 - {0,0,5,1,2,1} - }, - { - {0,1,1,1,3,2}, - {0,1,3,1,3,1}, - {0,1,4,1,3,2},//Test case#3 - {0,4,4,1,2,2} - }, - { - {0,1,1,3,3,2}, - {0,1,3,3,3,1}, - {0,1,4,3,3,2},//Test case#4 - {0,4,4,1,2,2} - }, - { - {1,0,1,2,2,2}, - {1,3,4,2,2,2}, - {1,3,3,2,2,1},//Test case#5 - {1,1,4,1,2,2} - }, - { - {1,0,1,2,2,2}, - {1,3,4,2,2,2}, - {1,3,3,2,2,1},//Test case#6 - {1,1,4,2,2,2} - }, - { - {1,1,3,2,2,1}, - {1,3,4,2,2,2}, - {1,3,3,2,2,1},//Test case#7 - {1,1,4,2,2,2} - } - - }; */ + int array[][4][6] = { {{0, 0, 0, 1, 2, 2}, @@ -334,182 +247,7 @@ main(int argc, char **argv) } */ } - /* offset[0] = 0; - offset[1]=0; - offset[2]=0; // select 0x0x0 - count[0] = 2; - count[1] = 2; - count[2] = 1; - status = H5Sselect_hyperslab (dataspace, H5S_SELECT_SET, offset, NULL, count, NULL); - - printf("MPI rank=%d Hyperslab operation on dataspace using offset %llux%llux%llu and count - %llux%llux%llu \n",mpi_rank,offset[0],offset[1],offset[2],count[0],count[1],count[2]); offset_out[0] = 0; - //offset=0x0x0 offset_out[1] = 0; offset_out[2] = 0; count_out[0] = 2; //count_out=2 X 2 x 3 count_out[1] - = 2; count_out[2] = 1; status = H5Sselect_hyperslab (memspace, H5S_SELECT_SET, offset_out, NULL, - count_out, NULL); - - printf("----------------- on memory space using offset %llux%llux%llu and count %llux%llux%llu - \n",offset_out[0],offset_out[1],offset_out[2],count_out[0],count_out[1],count_out[2]); - - - //status = H5Dread_async (dataset, H5T_NATIVE_INT, memspace, dataspace, H5P_DEFAULT, data_out1,es_id); - status = H5Dwrite_async (dataset, H5T_NATIVE_INT, memspace, dataspace, data_transfer_propertylist, - data,es_id); - - offset[0] = 0; - offset[1]=2; - offset[2]=0; // select 0x2x0 - count[0] = 2; - count[1] = 1; - count[2] = 1; - status = H5Sselect_hyperslab (dataspace, H5S_SELECT_SET, offset, NULL, count, NULL); - - printf("MPI rank=%d Hyperslab operation on dataspace using offset %llux%llux%llu and count - %llux%llux%llu \n",mpi_rank,offset[0],offset[1],offset[2],count[0],count[1],count[2]); offset_out[0] = 0; - //offset=0x2x0 offset_out[1] = 2; offset_out[2] = 0; count_out[0] = 2; //count_out=2 X 2 x 3 count_out[1] - = 1; count_out[2] = 1; status = H5Sselect_hyperslab (memspace, H5S_SELECT_SET, offset_out, NULL, - count_out, NULL); - - printf("----------------- on memory space using offset %llux%llux%llu and count %llux%llux%llu - \n",offset_out[0],offset_out[1],offset_out[2],count_out[0],count_out[1],count_out[2]); - - - //status = H5Dread_async (dataset, H5T_NATIVE_INT, memspace, dataspace, H5P_DEFAULT, data_out1,es_id); - status = H5Dwrite_async (dataset, H5T_NATIVE_INT, memspace, dataspace, data_transfer_propertylist, - data,es_id); - - - offset[0] = 0; - offset[1]=3; - offset[2]=0; // select 0x4x0 - count[0] = 2; - count[1] = 2; - count[2] = 1; - status = H5Sselect_hyperslab (dataspace, H5S_SELECT_SET, offset, NULL, count, NULL); - - printf("MPI rank=%d Hyperslab operation on dataspace using offset %llux%llux%llu and count - %llux%llux%llu \n",mpi_rank,offset[0],offset[1],offset[2],count[0],count[1],count[2]); offset_out[0] = 0; - //offset=0x4x0 offset_out[1] = 3; offset_out[2] = 0; count_out[0] = 2; //count_out=2 X 2 x 3 count_out[1] - = 2; count_out[2] = 1; status = H5Sselect_hyperslab (memspace, H5S_SELECT_SET, offset_out, NULL, - count_out, NULL); - - printf("----------------- on memory space using offset %llux%llux%llu and count %llux%llux%llu - \n",offset_out[0],offset_out[1],offset_out[2],count_out[0],count_out[1],count_out[2]); - - - //status = H5Dread_async (dataset, H5T_NATIVE_INT, memspace, dataspace, H5P_DEFAULT, data_out1,es_id); - status = H5Dwrite_async (dataset, H5T_NATIVE_INT, memspace, dataspace, data_transfer_propertylist, - data,es_id); - - offset[0] = 0; - offset[1] = 5; - offset[2] = 0; //offset=2x0x0 - count[0] = 2; - count[1] = 1; //count=4x2x3 - count[2] = 1; - status = H5Sselect_hyperslab (dataspace, H5S_SELECT_SET, offset, NULL, count, NULL); - printf("MPI rank=%d Hyperslab operation on dataspace using offset %llux%llux%llu and count - %llux%llux%llu \n",mpi_rank,offset[0],offset[1],offset[2],count[0],count[1],count[2]); - - offset_out[0] = 0; - offset_out[1] = 5; //offset_out=2x0x0 - offset_out[2] = 0; - count_out[0] = 2; - count_out[1] = 1; //count_out=4x2x3 - count_out[2] = 1; - status = H5Sselect_hyperslab (memspace, H5S_SELECT_SET, offset_out, NULL, count_out, NULL); - printf("----------------- on memory space using offset %llux%llux%llu and count %llux%llux%llu - \n",offset_out[0],offset_out[1],offset_out[2],count_out[0],count_out[1],count_out[2]); - - - // status = H5Dread_async (dataset, H5T_NATIVE_INT, memspace, dataspace, H5P_DEFAULT, data_out2,es_id); - status = H5Dwrite_async (dataset, H5T_NATIVE_INT, memspace, dataspace, data_transfer_propertylist, - data,es_id); - */ - } - /* if(mpi_rank==1){ - - - offset[0] = 0; - offset[1] = 5; - offset[2] = 0; //offset=2x0x0 - count[0] = 2; - count[1] = 1; //count=4x2x3 - count[2] = 1; - status = H5Sselect_hyperslab (dataspace, H5S_SELECT_SET, offset, NULL, count, NULL); - printf("MPI rank=%d Hyperslab operation on dataspace using offset %llux%llux%llu and count - %llux%llux%llu \n",mpi_rank,offset[0],offset[1],offset[2],count[0],count[1],count[2]); - - offset_out[0] = 0; - offset_out[1] = 5; //offset_out=2x0x0 - offset_out[2] = 0; - count_out[0] = 2; - count_out[1] = 1; //count_out=4x2x3 - count_out[2] = 1; - status = H5Sselect_hyperslab (memspace, H5S_SELECT_SET, offset_out, NULL, count_out, NULL); - printf("----------------- on memory space using offset %llux%llux%llu and count %llux%llux%llu - \n",offset_out[0],offset_out[1],offset_out[2],count_out[0],count_out[1],count_out[2]); - - - // status = H5Dread_async (dataset, H5T_NATIVE_INT, memspace, dataspace, H5P_DEFAULT, data_out2,es_id); - status = H5Dwrite_async (dataset, H5T_NATIVE_INT, memspace, dataspace, data_transfer_propertylist, - data,es_id); - - - //offset from 3x0x0 and count 3x2x3 - offset[0] = 2; - offset[1] = 2; - offset[2] = 0; //offset=2x2x0 - count[0] = 4; - count[1] = 2; //count=4x2x3 - count[2] = 3; - status = H5Sselect_hyperslab (dataspace, H5S_SELECT_SET, offset, NULL, count, NULL); - printf("MPI rank=%d Hyperslab operation on dataspace using offset %llux%llux%llu and count - %llux%llux%llu \n",mpi_rank,offset[0],offset[1],offset[2],count[0],count[1],count[2]); - - offset_out[0] = 2; - offset_out[1] = 2; //offset_out=2x2x0 - offset_out[2] = 0; - count_out[0] = 4; - count_out[1] = 2; //count_out=4x2x3 - count_out[2] = 3; - status = H5Sselect_hyperslab (memspace, H5S_SELECT_SET, offset_out, NULL, count_out, NULL); - printf("----------------- on memory space using offset %llux%llux%llu and count %llux%llux%llu - \n",offset_out[0],offset_out[1],offset_out[2],count_out[0],count_out[1],count_out[2]); - - - // status = H5Dread_async (dataset, H5T_NATIVE_INT, memspace, dataspace, H5P_DEFAULT, data_out2,es_id); - status = H5Dwrite_async (dataset, H5T_NATIVE_INT, memspace, dataspace, data_transfer_propertylist, - data,es_id); - - - offset[0] = 2; - offset[1] = 4; - offset[2] = 0; //offset=2x4x0 - count[0] = 4; - count[1] = 2; //count=4x2x3 - count[2] = 3; - status = H5Sselect_hyperslab (dataspace, H5S_SELECT_SET, offset, NULL, count, NULL); - printf("MPI rank=%d Hyperslab operation on dataspace using offset %llux%llux%llu and count - %llux%llux%llu \n",mpi_rank,offset[0],offset[1],offset[2],count[0],count[1],count[2]); - - offset_out[0] = 2; - offset_out[1] = 4; //offset_out=2x4x0 - offset_out[2] = 0; - count_out[0] = 4; - count_out[1] = 2; //count_out=4x2x3 - count_out[2] = 3; - status = H5Sselect_hyperslab (memspace, H5S_SELECT_SET, offset_out, NULL, count_out, NULL); - printf("----------------- on memory space using offset %llux%llux%llu and count %llux%llux%llu - \n",offset_out[0],offset_out[1],offset_out[2],count_out[0],count_out[1],count_out[2]); - - - // status = H5Dread_async (dataset, H5T_NATIVE_INT, memspace, dataspace, H5P_DEFAULT, data_out2,es_id); - status = H5Dwrite_async (dataset, H5T_NATIVE_INT, memspace, dataspace, data_transfer_propertylist, - data,es_id); - - - } */ + status = H5ESwait(es_id, H5ES_WAIT_FOREVER, &num_in_progress, &op_failed); if (status < 0) { @@ -593,3 +331,4 @@ main(int argc, char **argv) return 0; } + \ No newline at end of file From 32872ac92a61a65ee597726c95f22155b54d5e18 Mon Sep 17 00:00:00 2001 From: kamalchowdhurylbl Date: Fri, 12 Aug 2022 15:23:34 -0500 Subject: [PATCH 16/18] changes after review async vol.c --- src/h5_async_vol.c | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/h5_async_vol.c b/src/h5_async_vol.c index 855577d..9d1e6bd 100644 --- a/src/h5_async_vol.c +++ b/src/h5_async_vol.c @@ -10326,8 +10326,9 @@ async_dataset_get(task_list_qtype qtype, async_instance_t *aid, H5VL_async_t *pa if (dxpl_id > 0) args->dxpl_id = H5Pcopy(dxpl_id); args->req = req; - - if (req) { + + // Temporary fix for data space get that could cause a pthread deadlock + if (req && !aid->disable_async_dset_get) { H5VL_async_t *new_req; if ((new_req = H5VL_async_new_obj(NULL, parent_obj->under_vol_id)) == NULL) { fprintf(fout_g, " [ASYNC VOL ERROR] %s with request object calloc\n", __func__); @@ -21371,9 +21372,15 @@ H5VL_async_str_to_info(const char *str, void **_info) /* Retrieve the underlying VOL connector value and info */ sscanf(str, "under_vol=%u;", &under_vol_value); - under_vol_id = H5VLregister_connector_by_value((H5VL_class_value_t)under_vol_value, H5P_DEFAULT); - under_vol_info_start = strchr(str, '{'); - under_vol_info_end = strrchr(str, '}'); + under_vol_id = H5VLregister_connector_by_value((H5VL_class_value_t)under_vol_value, H5P_DEFAULT); + if (strstr(str, "[") && strstr(str, "]")) { + under_vol_info_start = strchr(str, '['); + under_vol_info_end = strrchr(str, ']'); + } + else { + under_vol_info_start = strchr(str, '{'); + under_vol_info_end = strrchr(str, '}'); + } assert(under_vol_info_end > under_vol_info_start); if (under_vol_info_end != (under_vol_info_start + 1)) { char *under_vol_info_str; From f48d330757dc489776f7e37d9be2f3cef6a5035c Mon Sep 17 00:00:00 2001 From: kamalchowdhurylbl Date: Fri, 12 Aug 2022 16:37:31 -0500 Subject: [PATCH 17/18] changes after review async vol.c,test 1d,2d and 3d, makefile --- src/h5_async_vol.c | 1218 ++++++++++++++++++-------------------- test/1d_asyn_MPI_write.c | 15 +- test/2d_asyn_MPI_write.c | 102 ++-- test/3d_asyn_MPI_write.c | 5 +- test/Makefile | 2 +- 5 files changed, 633 insertions(+), 709 deletions(-) diff --git a/src/h5_async_vol.c b/src/h5_async_vol.c index 9d1e6bd..6680aa7 100644 --- a/src/h5_async_vol.c +++ b/src/h5_async_vol.c @@ -65,7 +65,7 @@ works, and perform publicly and display publicly, and to permit others to do so. /* Whether to display log messge when callback is invoked */ /* (Uncomment to enable) */ /* #define ENABLE_DBG_MSG 1 */ - #define PRINT_ERROR_STACK 1 +/* #define PRINT_ERROR_STACK 1 */ /* #define ENABLE_ASYNC_LOGGING */ #define ASYNC_DBG_MSG_RANK 0 @@ -105,7 +105,7 @@ FILE *fout_g; /* The async VOL wrapper context */ typedef struct H5VL_async_wrap_ctx_t { hid_t under_vol_id; /* VOL ID for under VOL */ - void * under_wrap_ctx; /* Object wrapping context for under VOL */ + void *under_wrap_ctx; /* Object wrapping context for under VOL */ struct H5VL_async_t *file_async_obj; } H5VL_async_wrap_ctx_t; @@ -123,11 +123,11 @@ typedef struct async_task_t { hid_t under_vol_id; int magic; ABT_mutex task_mutex; - void * h5_state; + void *h5_state; void (*func)(void *); - void * args; + void *args; obj_op_type op; - struct H5VL_async_t * async_obj; + struct H5VL_async_t *async_obj; ABT_eventual eventual; int in_abt_pool; int is_done; @@ -155,19 +155,19 @@ typedef struct async_task_t { typedef struct H5VL_async_t { hid_t under_vol_id; - void * under_object; + void *under_object; int magic; int is_obj_valid; - async_task_t * create_task; /* task that creates the object */ - async_task_t * close_task; - async_task_t * my_task; /* for request */ - async_task_t * file_task_list_head; + async_task_t *create_task; /* task that creates the object */ + async_task_t *close_task; + async_task_t *my_task; /* for request */ + async_task_t *file_task_list_head; ABT_mutex file_task_list_mutex; struct H5VL_async_t *file_async_obj; int task_cnt; int attempt_check_cnt; ABT_mutex obj_mutex; - ABT_pool * pool_ptr; + ABT_pool *pool_ptr; hbool_t is_col_meta; #ifdef ENABLE_WRITE_MEMCPY hsize_t data_size; @@ -176,7 +176,7 @@ typedef struct H5VL_async_t { typedef struct async_task_list_t { task_list_qtype type; - async_task_t * task_list; + async_task_t *task_list; struct async_task_list_t *prev; struct async_task_list_t *next; @@ -191,23 +191,23 @@ typedef struct async_instance_t { async_qhead_t qhead; ABT_pool pool; int num_xstreams; - ABT_xstream * xstreams; - ABT_xstream * scheds; - ABT_sched * progress_scheds; + ABT_xstream *xstreams; + ABT_xstream *scheds; + ABT_sched *progress_scheds; int nfopen; int mpi_size; int mpi_rank; - bool ex_delay; /* Delay background thread execution */ - bool ex_fclose; /* Delay background thread execution until file close */ - bool ex_gclose; /* Delay background thread execution until group close */ - bool ex_dclose; /* Delay background thread execution until dset close */ - bool start_abt_push; /* Start pushing tasks to Argobots pool */ - bool prev_push_state; /* Previous state of start_abt_push before a change*/ - bool pause; /* Pause background thread execution */ - bool disable_implicit_file; /* Disable implicit async execution globally */ - bool disable_implicit; /* Disable implicit async execution for dxpl */ + bool ex_delay; /* Delay background thread execution */ + bool ex_fclose; /* Delay background thread execution until file close */ + bool ex_gclose; /* Delay background thread execution until group close */ + bool ex_dclose; /* Delay background thread execution until dset close */ + bool start_abt_push; /* Start pushing tasks to Argobots pool */ + bool prev_push_state; /* Previous state of start_abt_push before a change*/ + bool pause; /* Pause background thread execution */ + bool disable_implicit_file; /* Disable implicit async execution globally */ + bool disable_implicit; /* Disable implicit async execution for dxpl */ bool disable_async_dset_get; /* Disable async execution for dataset get */ - bool delay_time_env; /* Flag that indicates the delay time is set by env variable */ + bool delay_time_env; /* Flag that indicates the delay time is set by env variable */ uint64_t delay_time; /* Sleep time before background thread trying to acquire global mutex */ int sleep_time; /* Sleep time between checking the global mutex attemp count */ #ifdef ENABLE_WRITE_MEMCPY @@ -223,186 +223,186 @@ typedef struct async_future_obj_t { } async_future_obj_t; typedef struct async_attr_create_args_t { - void * obj; + void *obj; H5VL_loc_params_t *loc_params; - char * name; + char *name; hid_t type_id; hid_t space_id; hid_t acpl_id; hid_t aapl_id; hid_t dxpl_id; - void ** req; + void **req; } async_attr_create_args_t; typedef struct async_attr_open_args_t { - void * obj; + void *obj; H5VL_loc_params_t *loc_params; - char * name; + char *name; hid_t aapl_id; hid_t dxpl_id; - void ** req; + void **req; } async_attr_open_args_t; typedef struct async_attr_read_args_t { - void * attr; + void *attr; hid_t mem_type_id; - void * buf; + void *buf; hid_t dxpl_id; void **req; } async_attr_read_args_t; typedef struct async_attr_write_args_t { - void * attr; + void *attr; hid_t mem_type_id; - void * buf; + void *buf; hid_t dxpl_id; void **req; - bool free_buf; + bool free_buf; #ifdef ENABLE_WRITE_MEMCPY hsize_t data_size; #endif } async_attr_write_args_t; typedef struct async_attr_get_args_t { - void * obj; + void *obj; H5VL_attr_get_args_t args; hid_t dxpl_id; - void ** req; + void **req; } async_attr_get_args_t; typedef struct async_attr_specific_args_t { - void * obj; - H5VL_loc_params_t * loc_params; + void *obj; + H5VL_loc_params_t *loc_params; H5VL_attr_specific_args_t args; hid_t dxpl_id; - void ** req; + void **req; } async_attr_specific_args_t; typedef struct async_attr_optional_args_t { - void * obj; + void *obj; H5VL_optional_args_t args; H5VL_native_attr_optional_args_t opt_args; hid_t dxpl_id; - void ** req; + void **req; } async_attr_optional_args_t; typedef struct async_attr_close_args_t { - void * attr; + void *attr; hid_t dxpl_id; void **req; } async_attr_close_args_t; typedef struct async_dataset_create_args_t { - void * obj; + void *obj; H5VL_loc_params_t *loc_params; - char * name; + char *name; hid_t lcpl_id; hid_t type_id; hid_t space_id; hid_t dcpl_id; hid_t dapl_id; hid_t dxpl_id; - void ** req; + void **req; } async_dataset_create_args_t; typedef struct async_dataset_open_args_t { - void * obj; + void *obj; H5VL_loc_params_t *loc_params; - char * name; + char *name; hid_t dapl_id; hid_t dxpl_id; - void ** req; + void **req; } async_dataset_open_args_t; typedef struct async_dataset_read_args_t { - void * dset; + void *dset; hid_t mem_type_id; hid_t mem_space_id; hid_t file_space_id; hid_t plist_id; - void * buf; + void *buf; void **req; } async_dataset_read_args_t; typedef struct async_dataset_write_args_t { - void * dset; + void *dset; hid_t mem_type_id; hid_t mem_space_id; hid_t file_space_id; hid_t plist_id; - void * buf; + void *buf; void **req; - bool free_buf; + bool free_buf; #ifdef ENABLE_WRITE_MEMCPY hsize_t data_size; #endif } async_dataset_write_args_t; typedef struct async_dataset_get_args_t { - void * dset; + void *dset; H5VL_dataset_get_args_t args; hid_t dxpl_id; - void ** req; + void **req; } async_dataset_get_args_t; typedef struct async_dataset_specific_args_t { - void * obj; + void *obj; H5VL_dataset_specific_args_t args; hid_t dxpl_id; - void ** req; + void **req; } async_dataset_specific_args_t; typedef struct async_dataset_optional_args_t { - void * obj; + void *obj; H5VL_optional_args_t args; H5VL_native_dataset_optional_args_t opt_args; hid_t dxpl_id; - void ** req; + void **req; } async_dataset_optional_args_t; typedef struct async_dataset_close_args_t { - void * dset; + void *dset; hid_t dxpl_id; void **req; } async_dataset_close_args_t; typedef struct async_datatype_commit_args_t { - void * obj; + void *obj; H5VL_loc_params_t *loc_params; - char * name; + char *name; hid_t type_id; hid_t lcpl_id; hid_t tcpl_id; hid_t tapl_id; hid_t dxpl_id; - void ** req; + void **req; } async_datatype_commit_args_t; typedef struct async_datatype_open_args_t { - void * obj; + void *obj; H5VL_loc_params_t *loc_params; - char * name; + char *name; hid_t tapl_id; hid_t dxpl_id; - void ** req; + void **req; } async_datatype_open_args_t; typedef struct async_datatype_get_args_t { - void * dt; + void *dt; H5VL_datatype_get_args_t args; hid_t dxpl_id; - void ** req; + void **req; } async_datatype_get_args_t; typedef struct async_datatype_specific_args_t { - void * obj; + void *obj; H5VL_datatype_specific_args_t args; hid_t dxpl_id; - void ** req; + void **req; } async_datatype_specific_args_t; typedef struct async_datatype_optional_args_t { - void * obj; + void *obj; H5VL_optional_args_t args; #ifdef NOT_YET H5VL_native_datatype_optional_args_t opt_args; @@ -412,157 +412,157 @@ typedef struct async_datatype_optional_args_t { } async_datatype_optional_args_t; typedef struct async_datatype_close_args_t { - void * dt; + void *dt; hid_t dxpl_id; void **req; } async_datatype_close_args_t; typedef struct async_file_create_args_t { - char * name; + char *name; unsigned flags; hid_t fcpl_id; hid_t fapl_id; hid_t dxpl_id; - void ** req; + void **req; } async_file_create_args_t; typedef struct async_file_open_args_t { - char * name; + char *name; unsigned flags; hid_t fapl_id; hid_t dxpl_id; - void ** req; + void **req; } async_file_open_args_t; typedef struct async_file_get_args_t { - void * file; + void *file; H5VL_file_get_args_t args; hid_t dxpl_id; - void ** req; + void **req; } async_file_get_args_t; typedef struct async_file_specific_args_t { - void * file; + void *file; H5VL_file_specific_args_t args; hid_t dxpl_id; - void ** req; + void **req; } async_file_specific_args_t; typedef struct async_file_optional_args_t { - void * file; + void *file; H5VL_optional_args_t args; H5VL_native_file_optional_args_t opt_args; hid_t dxpl_id; - void ** req; + void **req; } async_file_optional_args_t; typedef struct async_file_close_args_t { - void * file; + void *file; hid_t dxpl_id; void **req; bool is_reopen; } async_file_close_args_t; typedef struct async_group_create_args_t { - void * obj; + void *obj; H5VL_loc_params_t *loc_params; - char * name; + char *name; hid_t lcpl_id; hid_t gcpl_id; hid_t gapl_id; hid_t dxpl_id; - void ** req; + void **req; } async_group_create_args_t; typedef struct async_group_open_args_t { - void * obj; + void *obj; H5VL_loc_params_t *loc_params; - char * name; + char *name; hid_t gapl_id; hid_t dxpl_id; - void ** req; + void **req; } async_group_open_args_t; typedef struct async_group_get_args_t { - void * obj; + void *obj; H5VL_group_get_args_t args; hid_t dxpl_id; - void ** req; + void **req; } async_group_get_args_t; typedef struct async_group_specific_args_t { - void * obj; + void *obj; H5VL_group_specific_args_t args; hid_t dxpl_id; - void ** req; + void **req; } async_group_specific_args_t; typedef struct async_group_optional_args_t { - void * obj; + void *obj; H5VL_optional_args_t args; H5VL_native_group_optional_args_t opt_args; hid_t dxpl_id; - void ** req; + void **req; } async_group_optional_args_t; typedef struct async_group_close_args_t { - void * grp; + void *grp; hid_t dxpl_id; void **req; } async_group_close_args_t; typedef struct async_link_create_args_t { H5VL_link_create_args_t args; - void * obj; - H5VL_loc_params_t * loc_params; + void *obj; + H5VL_loc_params_t *loc_params; hid_t lcpl_id; hid_t lapl_id; hid_t dxpl_id; - void ** req; + void **req; va_list arguments; } async_link_create_args_t; typedef struct async_link_copy_args_t { - void * src_obj; + void *src_obj; H5VL_loc_params_t *loc_params1; - void * dst_obj; + void *dst_obj; H5VL_loc_params_t *loc_params2; hid_t lcpl_id; hid_t lapl_id; hid_t dxpl_id; - void ** req; + void **req; } async_link_copy_args_t; typedef struct async_link_move_args_t { - void * src_obj; + void *src_obj; H5VL_loc_params_t *loc_params1; - void * dst_obj; + void *dst_obj; H5VL_loc_params_t *loc_params2; hid_t lcpl_id; hid_t lapl_id; hid_t dxpl_id; - void ** req; + void **req; } async_link_move_args_t; typedef struct async_link_get_args_t { - void * obj; - H5VL_loc_params_t * loc_params; + void *obj; + H5VL_loc_params_t *loc_params; H5VL_link_get_args_t args; hid_t dxpl_id; - void ** req; + void **req; } async_link_get_args_t; typedef struct async_link_specific_args_t { - void * obj; - H5VL_loc_params_t * loc_params; + void *obj; + H5VL_loc_params_t *loc_params; H5VL_link_specific_args_t args; hid_t dxpl_id; - void ** req; + void **req; } async_link_specific_args_t; typedef struct async_link_optional_args_t { - void * obj; - H5VL_loc_params_t * loc_params; + void *obj; + H5VL_loc_params_t *loc_params; H5VL_optional_args_t args; #ifdef NOT_YET H5VL_native_link_optional_args_t opt_args; @@ -572,49 +572,49 @@ typedef struct async_link_optional_args_t { } async_link_optional_args_t; typedef struct async_object_open_args_t { - void * obj; + void *obj; H5VL_loc_params_t *loc_params; - H5I_type_t * opened_type; + H5I_type_t *opened_type; hid_t dxpl_id; - void ** req; + void **req; } async_object_open_args_t; typedef struct async_object_copy_args_t { - void * src_obj; + void *src_obj; H5VL_loc_params_t *src_loc_params; - char * src_name; - void * dst_obj; + char *src_name; + void *dst_obj; H5VL_loc_params_t *dst_loc_params; - char * dst_name; + char *dst_name; hid_t ocpypl_id; hid_t lcpl_id; hid_t dxpl_id; - void ** req; + void **req; } async_object_copy_args_t; typedef struct async_object_get_args_t { - void * obj; - H5VL_loc_params_t * loc_params; + void *obj; + H5VL_loc_params_t *loc_params; H5VL_object_get_args_t args; hid_t dxpl_id; - void ** req; + void **req; } async_object_get_args_t; typedef struct async_object_specific_args_t { - void * obj; - H5VL_loc_params_t * loc_params; + void *obj; + H5VL_loc_params_t *loc_params; H5VL_object_specific_args_t args; hid_t dxpl_id; - void ** req; + void **req; } async_object_specific_args_t; typedef struct async_object_optional_args_t { - void * obj; - H5VL_loc_params_t * loc_params; + void *obj; + H5VL_loc_params_t *loc_params; H5VL_optional_args_t args; H5VL_native_object_optional_args_t opt_args; hid_t dxpl_id; - void ** req; + void **req; } async_object_optional_args_t; /*******************/ @@ -638,24 +638,24 @@ static herr_t H5VL_async_init(hid_t vipl_id); static herr_t H5VL_async_term(void); /* VOL info callbacks */ -static void * H5VL_async_info_copy(const void *info); +static void *H5VL_async_info_copy(const void *info); static herr_t H5VL_async_info_cmp(int *cmp_value, const void *info1, const void *info2); static herr_t H5VL_async_info_free(void *info); static herr_t H5VL_async_info_to_str(const void *info, char **str); static herr_t H5VL_async_str_to_info(const char *str, void **info); /* VOL object wrap / retrieval callbacks */ -static void * H5VL_async_get_object(const void *obj); +static void *H5VL_async_get_object(const void *obj); static herr_t H5VL_async_get_wrap_ctx(const void *obj, void **wrap_ctx); -static void * H5VL_async_wrap_object(void *obj, H5I_type_t obj_type, void *wrap_ctx); -static void * H5VL_async_unwrap_object(void *obj); +static void *H5VL_async_wrap_object(void *obj, H5I_type_t obj_type, void *wrap_ctx); +static void *H5VL_async_unwrap_object(void *obj); static herr_t H5VL_async_free_wrap_ctx(void *obj); /* Attribute callbacks */ -static void * H5VL_async_attr_create(void *obj, const H5VL_loc_params_t *loc_params, const char *name, +static void *H5VL_async_attr_create(void *obj, const H5VL_loc_params_t *loc_params, const char *name, hid_t type_id, hid_t space_id, hid_t acpl_id, hid_t aapl_id, hid_t dxpl_id, void **req); -static void * H5VL_async_attr_open(void *obj, const H5VL_loc_params_t *loc_params, const char *name, +static void *H5VL_async_attr_open(void *obj, const H5VL_loc_params_t *loc_params, const char *name, hid_t aapl_id, hid_t dxpl_id, void **req); static herr_t H5VL_async_attr_read(void *attr, hid_t mem_type_id, void *buf, hid_t dxpl_id, void **req); static herr_t H5VL_async_attr_write(void *attr, hid_t mem_type_id, const void *buf, hid_t dxpl_id, @@ -667,10 +667,10 @@ static herr_t H5VL_async_attr_optional(void *obj, H5VL_optional_args_t *args, hi static herr_t H5VL_async_attr_close(void *attr, hid_t dxpl_id, void **req); /* Dataset callbacks */ -static void * H5VL_async_dataset_create(void *obj, const H5VL_loc_params_t *loc_params, const char *name, +static void *H5VL_async_dataset_create(void *obj, const H5VL_loc_params_t *loc_params, const char *name, hid_t lcpl_id, hid_t type_id, hid_t space_id, hid_t dcpl_id, hid_t dapl_id, hid_t dxpl_id, void **req); -static void * H5VL_async_dataset_open(void *obj, const H5VL_loc_params_t *loc_params, const char *name, +static void *H5VL_async_dataset_open(void *obj, const H5VL_loc_params_t *loc_params, const char *name, hid_t dapl_id, hid_t dxpl_id, void **req); static herr_t H5VL_async_dataset_read(void *dset, hid_t mem_type_id, hid_t mem_space_id, hid_t file_space_id, hid_t plist_id, void *buf, void **req); @@ -683,10 +683,10 @@ static herr_t H5VL_async_dataset_optional(void *obj, H5VL_optional_args_t *args, static herr_t H5VL_async_dataset_close(void *dset, hid_t dxpl_id, void **req); /* Datatype callbacks */ -static void * H5VL_async_datatype_commit(void *obj, const H5VL_loc_params_t *loc_params, const char *name, +static void *H5VL_async_datatype_commit(void *obj, const H5VL_loc_params_t *loc_params, const char *name, hid_t type_id, hid_t lcpl_id, hid_t tcpl_id, hid_t tapl_id, hid_t dxpl_id, void **req); -static void * H5VL_async_datatype_open(void *obj, const H5VL_loc_params_t *loc_params, const char *name, +static void *H5VL_async_datatype_open(void *obj, const H5VL_loc_params_t *loc_params, const char *name, hid_t tapl_id, hid_t dxpl_id, void **req); static herr_t H5VL_async_datatype_get(void *dt, H5VL_datatype_get_args_t *args, hid_t dxpl_id, void **req); static herr_t H5VL_async_datatype_specific(void *obj, H5VL_datatype_specific_args_t *args, hid_t dxpl_id, @@ -705,9 +705,9 @@ static herr_t H5VL_async_file_optional(void *file, H5VL_optional_args_t *args, h static herr_t H5VL_async_file_close(void *file, hid_t dxpl_id, void **req); /* Group callbacks */ -static void * H5VL_async_group_create(void *obj, const H5VL_loc_params_t *loc_params, const char *name, +static void *H5VL_async_group_create(void *obj, const H5VL_loc_params_t *loc_params, const char *name, hid_t lcpl_id, hid_t gcpl_id, hid_t gapl_id, hid_t dxpl_id, void **req); -static void * H5VL_async_group_open(void *obj, const H5VL_loc_params_t *loc_params, const char *name, +static void *H5VL_async_group_open(void *obj, const H5VL_loc_params_t *loc_params, const char *name, hid_t gapl_id, hid_t dxpl_id, void **req); static herr_t H5VL_async_group_get(void *obj, H5VL_group_get_args_t *args, hid_t dxpl_id, void **req); static herr_t H5VL_async_group_specific(void *obj, H5VL_group_specific_args_t *args, hid_t dxpl_id, @@ -733,7 +733,7 @@ static herr_t H5VL_async_link_optional(void *obj, const H5VL_loc_params_t *loc_p H5VL_optional_args_t *args, hid_t dxpl_id, void **req); /* Object callbacks */ -static void * H5VL_async_object_open(void *obj, const H5VL_loc_params_t *loc_params, H5I_type_t *opened_type, +static void *H5VL_async_object_open(void *obj, const H5VL_loc_params_t *loc_params, H5I_type_t *opened_type, hid_t dxpl_id, void **req); static herr_t H5VL_async_object_copy(void *src_obj, const H5VL_loc_params_t *src_loc_params, const char *src_name, void *dst_obj, @@ -1184,10 +1184,10 @@ async_instance_init(int backing_thread_count) async_instance_t *aid; ABT_pool pool; ABT_xstream self_xstream; - ABT_xstream * progress_xstreams = NULL; - ABT_sched * progress_scheds = NULL; + ABT_xstream *progress_xstreams = NULL; + ABT_sched *progress_scheds = NULL; int abt_ret, i; - const char * env_var; + const char *env_var; if (backing_thread_count < 0) return -1; @@ -1285,21 +1285,22 @@ async_instance_init(int backing_thread_count) } // end for } // end else - aid->pool = pool; - aid->xstreams = progress_xstreams; - aid->num_xstreams = backing_thread_count; - aid->progress_scheds = progress_scheds; - aid->nfopen = 0; - aid->ex_delay = false; - aid->ex_fclose = false; - aid->ex_gclose = false; - aid->ex_dclose = false; - aid->pause = false; - aid->start_abt_push = false; - aid->disable_implicit = false; - aid->disable_implicit_file = false; - aid->delay_time_env = false; + aid->pool = pool; + aid->xstreams = progress_xstreams; + aid->num_xstreams = backing_thread_count; + aid->progress_scheds = progress_scheds; + aid->nfopen = 0; + aid->ex_delay = false; + aid->ex_fclose = false; + aid->ex_gclose = false; + aid->ex_dclose = false; + aid->pause = false; + aid->start_abt_push = false; + aid->disable_implicit = false; + aid->disable_implicit_file = false; + aid->delay_time_env = false; aid->disable_async_dset_get = true; + // Check for delaying operations to file / group / dataset close operations env_var = getenv("HDF5_ASYNC_EXE_FCLOSE"); if (env_var && *env_var && atoi(env_var) > 0) @@ -1320,7 +1321,7 @@ async_instance_init(int backing_thread_count) /* Set "delay execution" convenience flag, if any of the others are set */ if (aid->ex_fclose || aid->ex_gclose || aid->ex_dclose) aid->ex_delay = true; - + env_var = getenv("HDF5_ASYNC_DISABLE_DSET_GET"); if (env_var && *env_var && atoi(env_var) <= 0) aid->disable_async_dset_get = false; @@ -1629,7 +1630,8 @@ H5VL_async_dxpl_set_pause(hid_t dxpl) * VOL documentation. * */ -static herr_t H5VL_async_init(hid_t __attribute__((unused)) vipl_id) +static herr_t +H5VL_async_init(hid_t __attribute__((unused)) vipl_id) { /* Initialize the Argobots I/O instance */ if (NULL == async_instance_g) { @@ -2104,7 +2106,7 @@ get_n_running_task_in_queue(async_task_t *task, const char *call_func) { size_t pool_size = 0; ABT_thread_state thread_state; - async_task_t * task_elt; + async_task_t *task_elt; ABT_thread self_thread; ABT_bool is_equal; @@ -2156,7 +2158,7 @@ get_n_running_task_in_queue_obj(H5VL_async_t *async_obj, const char *call_func) { size_t pool_size = 0; ABT_thread_state thread_state; - async_task_t * task_elt; + async_task_t *task_elt; ABT_thread self_thread; ABT_bool is_equal; @@ -2259,7 +2261,7 @@ push_task_to_abt_pool(async_qhead_t *qhead, ABT_pool pool, const char *call_func { int i, is_dep_done = 1, ntask; ABT_thread_state thread_state; - async_task_t * task_elt, *task_tmp; + async_task_t *task_elt, *task_tmp; async_task_list_t *task_list_tmp, *task_list_elt; assert(qhead); @@ -2467,7 +2469,7 @@ add_task_to_queue(async_qhead_t *qhead, async_task_t *task, task_list_qtype task int is_end, is_end2; /* int is_dep = 0; */ async_task_list_t *tail_list, *task_list_elt; - async_task_t * task_elt, *tail_task; + async_task_t *task_elt, *tail_task; assert(qhead); assert(task); @@ -3519,7 +3521,7 @@ free_loc_param(H5VL_loc_params_t *loc_params) static int dup_attr_get_args(H5VL_attr_get_args_t *dst_args, H5VL_attr_get_args_t *src_args, async_task_t *task) { - hid_t * future_id_ptr = NULL; /* Pointer to ID for future ID */ + hid_t *future_id_ptr = NULL; /* Pointer to ID for future ID */ H5I_type_t future_id_type = H5I_BADID; /* Type ("class") of future ID */ hbool_t need_future_id = false; /* Whether an operation needs a future ID */ @@ -3778,7 +3780,7 @@ free_native_attr_optional_args(async_attr_optional_args_t *args) static int dup_dataset_get_args(H5VL_dataset_get_args_t *dst_args, H5VL_dataset_get_args_t *src_args, async_task_t *task) { - hid_t * future_id_ptr = NULL; /* Pointer to ID for future ID */ + hid_t *future_id_ptr = NULL; /* Pointer to ID for future ID */ H5I_type_t future_id_type = H5I_BADID; /* Type ("class") of future ID */ hbool_t need_future_id = false; /* Whether an operation needs a future ID */ @@ -3965,7 +3967,7 @@ free_dataset_spec_args(H5VL_dataset_specific_args_t *args) static void dup_native_dataset_optional_args(async_dataset_optional_args_t *dst_args, - const H5VL_optional_args_t * src_args) + const H5VL_optional_args_t *src_args) { assert(dst_args); assert(src_args); @@ -4057,7 +4059,7 @@ static int dup_datatype_get_args(H5VL_datatype_get_args_t *dst_args, H5VL_datatype_get_args_t *src_args, async_task_t *task) { - hid_t * future_id_ptr = NULL; /* Pointer to ID for future ID */ + hid_t *future_id_ptr = NULL; /* Pointer to ID for future ID */ H5I_type_t future_id_type = H5I_BADID; /* Type ("class") of future ID */ hbool_t need_future_id = false; /* Whether an operation needs a future ID */ @@ -4194,7 +4196,7 @@ free_datatype_spec_args(H5VL_datatype_specific_args_t *args) static void dup_native_datatype_optional_args(async_datatype_optional_args_t *dst_args, - const H5VL_optional_args_t * src_args) + const H5VL_optional_args_t *src_args) { assert(dst_args); assert(src_args); @@ -4239,7 +4241,7 @@ free_native_datatype_optional_args(async_datatype_optional_args_t *args) static int dup_file_get_args(H5VL_file_get_args_t *dst_args, H5VL_file_get_args_t *src_args, async_task_t *task) { - hid_t * future_id_ptr = NULL; /* Pointer to ID for future ID */ + hid_t *future_id_ptr = NULL; /* Pointer to ID for future ID */ H5I_type_t future_id_type = H5I_BADID; /* Type ("class") of future ID */ hbool_t need_future_id = false; /* Whether an operation needs a future ID */ @@ -4551,7 +4553,7 @@ free_native_file_optional_args(async_file_optional_args_t *args) static int dup_group_get_args(H5VL_group_get_args_t *dst_args, H5VL_group_get_args_t *src_args, async_task_t *task) { - hid_t * future_id_ptr = NULL; /* Pointer to ID for future ID */ + hid_t *future_id_ptr = NULL; /* Pointer to ID for future ID */ H5I_type_t future_id_type = H5I_BADID; /* Type ("class") of future ID */ hbool_t need_future_id = false; /* Whether an operation needs a future ID */ @@ -5115,14 +5117,14 @@ free_native_object_optional_args(async_object_optional_args_t *args) static void async_attr_create_fn(void *foo) { - void * obj; + void *obj; hbool_t acquired = false; unsigned int mutex_count = 1; int attempt_count = 0; int is_lock = 0; hbool_t is_lib_state_restored = false; - ABT_pool * pool_ptr; - async_task_t * task = (async_task_t *)foo; + ABT_pool *pool_ptr; + async_task_t *task = (async_task_t *)foo; async_attr_create_args_t *args = (async_attr_create_args_t *)(task->args); #ifdef ENABLE_TIMING @@ -5205,7 +5207,6 @@ async_attr_create_fn(void *foo) usleep(1000); } is_lock = 1; - /* Try executing operation, without default error stack handling */ H5E_BEGIN_TRY { @@ -5277,8 +5278,8 @@ async_attr_create(async_instance_t *aid, H5VL_async_t *parent_obj, const H5VL_lo const char *name, hid_t type_id, hid_t space_id, hid_t acpl_id, hid_t aapl_id, hid_t dxpl_id, void **req) { - H5VL_async_t * async_obj = NULL; - async_task_t * async_task = NULL; + H5VL_async_t *async_obj = NULL; + async_task_t *async_task = NULL; async_attr_create_args_t *args = NULL; bool lock_parent = false; bool is_blocking = false; @@ -5490,14 +5491,14 @@ async_attr_create(async_instance_t *aid, H5VL_async_t *parent_obj, const H5VL_lo static void async_attr_open_fn(void *foo) { - void * obj; + void *obj; hbool_t acquired = false; unsigned int mutex_count = 1; int attempt_count = 0; int is_lock = 0; hbool_t is_lib_state_restored = false; - ABT_pool * pool_ptr; - async_task_t * task = (async_task_t *)foo; + ABT_pool *pool_ptr; + async_task_t *task = (async_task_t *)foo; async_attr_open_args_t *args = (async_attr_open_args_t *)(task->args); #ifdef ENABLE_TIMING @@ -5644,8 +5645,8 @@ static H5VL_async_t * async_attr_open(async_instance_t *aid, H5VL_async_t *parent_obj, const H5VL_loc_params_t *loc_params, const char *name, hid_t aapl_id, hid_t dxpl_id, void **req) { - H5VL_async_t * async_obj = NULL; - async_task_t * async_task = NULL; + H5VL_async_t *async_obj = NULL; + async_task_t *async_task = NULL; async_attr_open_args_t *args = NULL; bool lock_parent = false; bool is_blocking = false; @@ -5842,8 +5843,8 @@ async_attr_read_fn(void *foo) int attempt_count = 0; int is_lock = 0; hbool_t is_lib_state_restored = false; - ABT_pool * pool_ptr; - async_task_t * task = (async_task_t *)foo; + ABT_pool *pool_ptr; + async_task_t *task = (async_task_t *)foo; async_attr_read_args_t *args = (async_attr_read_args_t *)(task->args); herr_t status; @@ -5986,7 +5987,7 @@ async_attr_read(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t mem_type_ { // For implicit mode (env var), make all read to be blocking assert(async_instance_g); - async_task_t * async_task = NULL; + async_task_t *async_task = NULL; async_attr_read_args_t *args = NULL; bool lock_parent = false; bool is_blocking = false; @@ -6160,8 +6161,8 @@ async_attr_write_fn(void *foo) int attempt_count = 0; int is_lock = 0; hbool_t is_lib_state_restored = false; - ABT_pool * pool_ptr; - async_task_t * task = (async_task_t *)foo; + ABT_pool *pool_ptr; + async_task_t *task = (async_task_t *)foo; async_attr_write_args_t *args = (async_attr_write_args_t *)(task->args); herr_t status; @@ -6310,7 +6311,7 @@ static herr_t async_attr_write(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t mem_type_id, const void *buf, hid_t dxpl_id, void **req) { - async_task_t * async_task = NULL; + async_task_t *async_task = NULL; async_attr_write_args_t *args = NULL; bool lock_parent = false; bool is_blocking = false; @@ -6504,8 +6505,8 @@ async_attr_get_fn(void *foo) int attempt_count = 0; int is_lock = 0; hbool_t is_lib_state_restored = false; - ABT_pool * pool_ptr; - async_task_t * task = (async_task_t *)foo; + ABT_pool *pool_ptr; + async_task_t *task = (async_task_t *)foo; async_attr_get_args_t *args = (async_attr_get_args_t *)(task->args); herr_t status; @@ -6644,7 +6645,7 @@ static herr_t async_attr_get(task_list_qtype qtype, async_instance_t *aid, H5VL_async_t *parent_obj, H5VL_attr_get_args_t *get_args, hid_t dxpl_id, void **req) { - async_task_t * async_task = NULL; + async_task_t *async_task = NULL; async_attr_get_args_t *args = NULL; bool lock_parent = false; bool is_blocking = false; @@ -6819,8 +6820,8 @@ async_attr_specific_fn(void *foo) int attempt_count = 0; int is_lock = 0; hbool_t is_lib_state_restored = false; - ABT_pool * pool_ptr; - async_task_t * task = (async_task_t *)foo; + ABT_pool *pool_ptr; + async_task_t *task = (async_task_t *)foo; async_attr_specific_args_t *args = (async_attr_specific_args_t *)(task->args); herr_t status; @@ -6964,7 +6965,7 @@ async_attr_specific(task_list_qtype qtype, async_instance_t *aid, H5VL_async_t * const H5VL_loc_params_t *loc_params, H5VL_attr_specific_args_t *spec_args, hid_t dxpl_id, void **req) { - async_task_t * async_task = NULL; + async_task_t *async_task = NULL; async_attr_specific_args_t *args = NULL; bool lock_parent = false; bool is_blocking = false; @@ -7152,8 +7153,8 @@ async_attr_optional_fn(void *foo) int attempt_count = 0; int is_lock = 0; hbool_t is_lib_state_restored = false; - ABT_pool * pool_ptr; - async_task_t * task = (async_task_t *)foo; + ABT_pool *pool_ptr; + async_task_t *task = (async_task_t *)foo; async_attr_optional_args_t *args = (async_attr_optional_args_t *)(task->args); herr_t status; @@ -7292,7 +7293,7 @@ static herr_t async_attr_optional(task_list_qtype qtype, async_instance_t *aid, H5VL_async_t *parent_obj, H5VL_optional_args_t *opt_args, hid_t dxpl_id, void **req) { - async_task_t * async_task = NULL; + async_task_t *async_task = NULL; async_attr_optional_args_t *args = NULL; bool lock_parent = false; bool is_blocking = false; @@ -7464,8 +7465,8 @@ async_attr_close_fn(void *foo) int attempt_count = 0; int is_lock = 0; hbool_t is_lib_state_restored = false; - ABT_pool * pool_ptr; - async_task_t * task = (async_task_t *)foo; + ABT_pool *pool_ptr; + async_task_t *task = (async_task_t *)foo; async_attr_close_args_t *args = (async_attr_close_args_t *)(task->args); herr_t status; @@ -7603,7 +7604,7 @@ static herr_t async_attr_close(task_list_qtype qtype, async_instance_t *aid, H5VL_async_t *parent_obj, hid_t dxpl_id, void **req) { - async_task_t * async_task = NULL; + async_task_t *async_task = NULL; async_attr_close_args_t *args = NULL; bool lock_parent = false; bool is_blocking = false; @@ -7779,14 +7780,14 @@ async_attr_close(task_list_qtype qtype, async_instance_t *aid, H5VL_async_t *par static void async_dataset_create_fn(void *foo) { - void * obj; + void *obj; hbool_t acquired = false; unsigned int mutex_count = 1; int attempt_count = 0; int is_lock = 0; hbool_t is_lib_state_restored = false; - ABT_pool * pool_ptr; - async_task_t * task = (async_task_t *)foo; + ABT_pool *pool_ptr; + async_task_t *task = (async_task_t *)foo; async_dataset_create_args_t *args = (async_dataset_create_args_t *)(task->args); #ifdef ENABLE_TIMING @@ -7943,8 +7944,8 @@ async_dataset_create(async_instance_t *aid, H5VL_async_t *parent_obj, const H5VL const char *name, hid_t lcpl_id, hid_t type_id, hid_t space_id, hid_t dcpl_id, hid_t dapl_id, hid_t dxpl_id, void **req) { - H5VL_async_t * async_obj = NULL; - async_task_t * async_task = NULL; + H5VL_async_t *async_obj = NULL; + async_task_t *async_task = NULL; async_dataset_create_args_t *args = NULL; bool lock_parent = false; bool is_blocking = false; @@ -8161,14 +8162,14 @@ async_dataset_create(async_instance_t *aid, H5VL_async_t *parent_obj, const H5VL static void async_dataset_open_fn(void *foo) { - void * obj; + void *obj; hbool_t acquired = false; unsigned int mutex_count = 1; int attempt_count = 0; int is_lock = 0; hbool_t is_lib_state_restored = false; - ABT_pool * pool_ptr; - async_task_t * task = (async_task_t *)foo; + ABT_pool *pool_ptr; + async_task_t *task = (async_task_t *)foo; async_dataset_open_args_t *args = (async_dataset_open_args_t *)(task->args); #ifdef ENABLE_TIMING @@ -8321,8 +8322,8 @@ async_dataset_open(task_list_qtype qtype, async_instance_t *aid, H5VL_async_t *p const H5VL_loc_params_t *loc_params, const char *name, hid_t dapl_id, hid_t dxpl_id, void **req) { - H5VL_async_t * async_obj = NULL; - async_task_t * async_task = NULL; + H5VL_async_t *async_obj = NULL; + async_task_t *async_task = NULL; async_dataset_open_args_t *args = NULL; bool lock_parent = false; bool is_blocking = false; @@ -8521,8 +8522,8 @@ async_dataset_read_fn(void *foo) int attempt_count = 0; int is_lock = 0; hbool_t is_lib_state_restored = false; - ABT_pool * pool_ptr; - async_task_t * task = (async_task_t *)foo; + ABT_pool *pool_ptr; + async_task_t *task = (async_task_t *)foo; async_dataset_read_args_t *args = (async_dataset_read_args_t *)(task->args); herr_t status; @@ -8669,7 +8670,7 @@ async_dataset_read(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t mem_ty { // For implicit mode (env var), make all read to be blocking assert(async_instance_g); - async_task_t * async_task = NULL; + async_task_t *async_task = NULL; async_dataset_read_args_t *args = NULL; bool lock_parent = false; bool is_blocking = false; @@ -8848,8 +8849,8 @@ async_dataset_write_fn(void *foo) int attempt_count = 0; int is_lock = 0, count = 0; hbool_t is_lib_state_restored = false; - ABT_pool * pool_ptr; - async_task_t * task = (async_task_t *)foo; + ABT_pool *pool_ptr; + async_task_t *task = (async_task_t *)foo; async_dataset_write_args_t *args = (async_dataset_write_args_t *)(task->args); herr_t status; @@ -8863,13 +8864,11 @@ async_dataset_write_fn(void *foo) assert(task); assert(task->async_obj); assert(task->async_obj->magic == ASYNC_MAGIC); - + pool_ptr = task->async_obj->pool_ptr; if (task->is_merge == 1) goto done; - - func_log(__func__, "trying to aquire global lock"); if ((attempt_count = check_app_acquire_mutex(task, &mutex_count, &acquired)) < 0) @@ -8943,7 +8942,7 @@ async_dataset_write_fn(void *foo) usleep(1000); count++; } - + /* Try executing operation, without default error stack handling */ H5E_BEGIN_TRY { @@ -8963,8 +8962,10 @@ async_dataset_write_fn(void *foo) } func_log(__func__, "execute success"); + if (args->free_buf && args->buf) { - free(args->buf);} + free(args->buf); + } done: if (is_lib_state_restored && H5VLfinish_lib_state() < 0) fprintf(fout_g, " [ASYNC ABT ERROR] %s H5VLfinish_lib_state failed\n", __func__); @@ -8998,9 +8999,8 @@ async_dataset_write_fn(void *foo) if (async_instance_g && NULL != async_instance_g->qhead.queue && async_instance_g->start_abt_push) push_task_to_abt_pool(&async_instance_g->qhead, *pool_ptr, __func__); - if (args->free_buf && args->buf) { - //free(args->buf); + // free(args->buf); #ifdef ENABLE_WRITE_MEMCPY async_instance_g->used_mem -= args->data_size; #ifdef ENABLE_DBG_MSG @@ -9008,10 +9008,9 @@ async_dataset_write_fn(void *foo) (async_instance_g->mpi_rank == ASYNC_DBG_MSG_RANK || -1 == ASYNC_DBG_MSG_RANK)) fprintf(fout_g, " [ASYNC ABT DBG] %s released dset memcpy\n", __func__); #endif - #endif +#endif } - #ifdef ENABLE_TIMING task->end_time = clock(); #endif @@ -9062,9 +9061,8 @@ print_dataspace(hid_t mem_space_id) herr_t slected_block; hssize_t numblocks; async_dataset_write_args_t *args = NULL; - hsize_t *start_out, *stride_out, *count_out, *block_out; - herr_t status; - + hsize_t *start_out, *stride_out, *count_out, *block_out; + herr_t status; if (mem_space_id == H5S_SEL_ALL) fprintf(stderr, "-----H5S_SEL_ALL----\n"); @@ -9075,13 +9073,11 @@ print_dataspace(hid_t mem_space_id) type = H5Sget_select_type(mem_space_id); - if (type == H5S_SEL_POINTS) { return 0; } else if (type == H5S_SEL_HYPERSLABS) { ndim = H5Sget_simple_extent_ndims(mem_space_id); - if (H5Sis_regular_hyperslab(mem_space_id)) { start_out = malloc(ndim * sizeof(hsize_t)); @@ -9095,25 +9091,22 @@ print_dataspace(hid_t mem_space_id) fprintf(stderr, " start = [%llu] \n", (unsigned long long)start_out[0]); // fprintf(stderr, " stride = [%llu] \n", (unsigned long long)stride_out[0]); fprintf(stderr, " count = [%llu] \n", (unsigned long long)count_out[0]); - } else if (ndim == 2) { fprintf(stderr, " start = [%llu, %llu] \n", (unsigned long long)start_out[0], (unsigned long long)start_out[1]); - + fprintf(stderr, " count = [%llu, %llu] \n", (unsigned long long)count_out[0], (unsigned long long)count_out[1]); - } else if (ndim == 3) { fprintf(stderr, " start = [%llu, %llu, %llu] \n", (unsigned long long)start_out[0], (unsigned long long)start_out[1], (unsigned long long)start_out[2]); - + fprintf(stderr, " count = [%llu, %llu, %llu] \n", (unsigned long long)count_out[0], (unsigned long long)count_out[1], (unsigned long long)count_out[2]); - } free(start_out); @@ -9129,20 +9122,20 @@ print_dataspace(hid_t mem_space_id) slected_block = H5Sget_select_hyper_blocklist(mem_space_id, (hsize_t)0, nblocks, buffer); if (ndim == 1) { - + fprintf(stderr, "Hyperslab operation on dataspace using offset=%llu and count=%llu\n", buffer[0], buffer[num_elements - 1] - buffer[0] + 1); } else if (ndim == 2) { - + fprintf(stderr, "Hyperslab operation on dataspace using offset=%llux%llu and count=%llux%llu\n", buffer[0], buffer[1], buffer[num_elements - 2] - buffer[0] + 1, buffer[num_elements - 1] - buffer[1] + 1); } else if (ndim == 3) { - + fprintf( stderr, "Hyperslab operation on dataspace using offset=%llux%llux%llu and count=%llux%llux%llu\n", @@ -9161,14 +9154,13 @@ check_contiguous(hid_t current_task_space_id, hid_t task_iterator_file_space_id, hsize_t *count) { - int ndim; - + int ndim; hsize_t *current_task_start_out, *current_task_stride_out, *current_task_count_out, *current_task_block_out; hsize_t *iterator_task_start_out, *iterator_task_stride_out, *iterator_task_count_out, *iterator_task_block_out; - herr_t status; + herr_t status; H5S_sel_type current_task_type, iterator_task_type; @@ -9197,8 +9189,7 @@ check_contiguous(hid_t current_task_space_id, hid_t task_iterator_file_space_id, iterator_task_block_out); if (ndim == 1) { - if (current_task_start_out[0] == iterator_task_start_out[0] + iterator_task_count_out[0]) - { + if (current_task_start_out[0] == iterator_task_start_out[0] + iterator_task_count_out[0]) { start[0] = iterator_task_start_out[0]; count[0] = current_task_count_out[0] + iterator_task_count_out[0]; // fprintf(stderr,"start=%llu and @@ -9211,7 +9202,6 @@ check_contiguous(hid_t current_task_space_id, hid_t task_iterator_file_space_id, } else if (ndim == 2) { - if (current_task_start_out[0] == iterator_task_start_out[0]) { if ((current_task_start_out[1] == iterator_task_start_out[1] + iterator_task_count_out[1]) && @@ -9257,8 +9247,7 @@ check_contiguous(hid_t current_task_space_id, hid_t task_iterator_file_space_id, iterator_task_start_out[2] + iterator_task_count_out[2]) && (current_task_count_out[1] == iterator_task_count_out[1]) && (current_task_count_out[0] == iterator_task_count_out[0]) && - (current_task_start_out[0] == iterator_task_start_out[0])) - { + (current_task_start_out[0] == iterator_task_start_out[0])) { start[0] = iterator_task_start_out[0]; start[1] = iterator_task_start_out[1]; start[2] = iterator_task_start_out[2]; @@ -9279,7 +9268,7 @@ check_contiguous(hid_t current_task_space_id, hid_t task_iterator_file_space_id, (current_task_count_out[2] == iterator_task_count_out[2]) && (current_task_count_out[0] == iterator_task_count_out[0]) && (current_task_start_out[0] == iterator_task_start_out[0])) - + { start[0] = iterator_task_start_out[0]; start[1] = iterator_task_start_out[1]; @@ -9309,9 +9298,9 @@ foreach_iteration(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t mem_typ hid_t file_space_id, hid_t plist_id, const void *buf, async_task_t *prev_task) { int ndim; - hsize_t * dimsm; - void * buffer, *new_buffer; - async_dataset_write_args_t *iter_args = NULL; + hsize_t *dimsm; + void *buffer, *new_buffer; + async_dataset_write_args_t *iter_args = NULL; hsize_t *start_out, *stride_out, *count_out, *block_out; hsize_t *start, *count, *mem_start, *mem_count, *file_start, *file_count, *iter_mem_start, @@ -9319,13 +9308,14 @@ foreach_iteration(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t mem_typ hid_t new_memspace; herr_t status, return_val = 0; - async_task_t * task_iter; - async_task_t * return_task_val = NULL; + async_task_t *task_iter; + async_task_t *return_task_val = NULL; async_task_list_t *task_list_iter; int buffer_count, feed_buf_count; - //These variables are for testing the run time - //double time_foreach1,time_foreach2,duration_foreach,time_before_contiguous_check, time_after_contiguous_check,duration_contiguous_check; - //double duration_in_contiguous_check,time_in_contiguous1,time_in_contiguous2,duration_memcopy,time_memcopy1,time_memcopy2; + // These variables are for testing the run time + // double time_foreach1,time_foreach2,duration_foreach,time_before_contiguous_check, + // time_after_contiguous_check,duration_contiguous_check; double + // duration_in_contiguous_check,time_in_contiguous1,time_in_contiguous2,duration_memcopy,time_memcopy1,time_memcopy2; ndim = H5Sget_simple_extent_ndims(file_space_id); dimsm = malloc(ndim * sizeof(hsize_t)); start = malloc(ndim * sizeof(hsize_t)); @@ -9338,92 +9328,83 @@ foreach_iteration(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t mem_typ iter_mem_count = malloc(ndim * sizeof(hsize_t)); iter_file_start = malloc(ndim * sizeof(hsize_t)); iter_file_count = malloc(ndim * sizeof(hsize_t)); - - //time_foreach1 = MPI_Wtime(); + + // time_foreach1 = MPI_Wtime(); DL_FOREACH(async_instance_g->qhead.queue, task_list_iter) { DL_FOREACH(task_list_iter->task_list, task_iter) - { + { if (task_iter->func == async_dataset_write_fn) { // checking whether the iterator task is dataset write operation if (task_iter->parent_obj == parent_obj && task_iter->is_done != 1) { // checking whether the iterator task is operating on the same dataset of the current task - + iter_args = task_iter->args; - //time_before_contiguous_check = MPI_Wtime(); + // time_before_contiguous_check = MPI_Wtime(); if (check_contiguous(file_space_id, iter_args->file_space_id, start, count) || check_contiguous(iter_args->file_space_id, file_space_id, start, count)) - - { - //time_in_contiguous1 = MPI_Wtime(); + + { + // time_in_contiguous1 = MPI_Wtime(); return_val = 1; - //fprintf(stderr, "\n-------#### contiguous####---------\n"); + // fprintf(stderr, "\n-------#### contiguous####---------\n"); if (prev_task != NULL) prev_task->is_merge = 1; return_task_val = task_iter; - if (ndim == 1) { + if (ndim == 1) { new_memspace = H5Screate_simple(ndim, count, NULL); element_size = H5Tget_size(mem_type_id); - //fprintf(stderr,"size of new_buffer= %lld \n",count[0]*element_size); + // fprintf(stderr,"size of new_buffer= %lld \n",count[0]*element_size); status = H5Sget_regular_hyperslab(mem_space_id, mem_start, NULL, mem_count, NULL); status = H5Sget_regular_hyperslab(file_space_id, file_start, NULL, file_count, NULL); - status = H5Sget_regular_hyperslab(iter_args->mem_space_id, iter_mem_start, NULL, iter_mem_count, NULL); - + status = H5Sget_regular_hyperslab(iter_args->file_space_id, iter_file_start, NULL, iter_file_count, NULL); - - - //time_memcopy1 = MPI_Wtime(); - if (iter_args->free_buf == 0) - {new_buffer = malloc(count[0] * element_size); - - + + // time_memcopy1 = MPI_Wtime(); + if (iter_args->free_buf == 0) { + new_buffer = malloc(count[0] * element_size); + if (file_start[0] >= iter_file_start[0]) { - + memcpy(new_buffer, buf + iter_file_start[0] * element_size, - iter_file_count[0] * element_size); + iter_file_count[0] * element_size); memcpy(new_buffer + iter_file_count[0] * element_size, - buf + file_start[0] * element_size, file_count[0] * element_size); + buf + file_start[0] * element_size, file_count[0] * element_size); } else { - memcpy(new_buffer, buf + (file_start[0] * element_size), - file_count[0] * element_size); + file_count[0] * element_size); memcpy(new_buffer + file_count[0] * element_size, - buf + (iter_file_start[0] * element_size), - iter_file_count[0] * element_size); + buf + (iter_file_start[0] * element_size), + iter_file_count[0] * element_size); } - iter_args->buf = new_buffer; - iter_args->free_buf=1; + iter_args->buf = new_buffer; + iter_args->free_buf = 1; } - else{ - iter_args->buf = realloc(iter_args->buf, count[0]*element_size); + else { + iter_args->buf = realloc(iter_args->buf, count[0] * element_size); if (file_start[0] >= iter_file_start[0]) { - memcpy(iter_args->buf + iter_file_count[0] * element_size, - buf + file_start[0] * element_size, file_count[0] * element_size); + buf + file_start[0] * element_size, file_count[0] * element_size); } else { - - memcpy(iter_args->buf + file_count[0] * element_size, - buf + (iter_file_start[0] * element_size), - iter_file_count[0] * element_size); + buf + (iter_file_start[0] * element_size), + iter_file_count[0] * element_size); } - iter_args->free_buf=1; + iter_args->free_buf = 1; } - - //time_memcopy2 = MPI_Wtime(); - //duration_memcopy=time_memcopy2-time_memcopy1; - + // time_memcopy2 = MPI_Wtime(); + // duration_memcopy=time_memcopy2-time_memcopy1; H5Sclose(iter_args->mem_space_id); @@ -9435,36 +9416,32 @@ foreach_iteration(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t mem_typ NULL, count, NULL); start[0] = 0; - status = H5Sselect_hyperslab(iter_args->mem_space_id, H5S_SELECT_SET, start, NULL, count, NULL); - } else if (ndim == 2) { - + fprintf(stderr, " start=%llux%llu count=%llux%llu\n", start[0], start[1], count[0], count[1]); new_memspace = H5Screate_simple(ndim, count, NULL); element_size = H5Tget_size(mem_type_id); new_buffer = malloc(count[0] * count[1] * element_size); - status = H5Sget_regular_hyperslab(mem_space_id, mem_start, NULL, mem_count, NULL); status = H5Sget_regular_hyperslab(file_space_id, file_start, NULL, file_count, NULL); - fprintf(stderr, "\n mem_start=%lldx%lld file_start=%lldx%lld ", mem_start[0], mem_start[1], file_start[0], file_start[1]); fprintf(stderr, "\n mem_count=%lldx%lld file_count=%lldx%lld \n", mem_count[0], mem_count[1], file_count[0], file_count[1]); status = H5Sget_regular_hyperslab(iter_args->mem_space_id, iter_mem_start, NULL, iter_mem_count, NULL); - + status = H5Sget_regular_hyperslab(iter_args->file_space_id, iter_file_start, NULL, iter_file_count, NULL); - + fprintf(stderr, "\n iter_mem_start=%lldx%lld iter_file_start=%lldx%lld ", iter_mem_start[0], iter_mem_start[1], iter_file_start[0], iter_file_start[1]); @@ -9473,11 +9450,10 @@ foreach_iteration(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t mem_typ iter_file_count[1]); H5Sget_simple_extent_dims(file_space_id, dimsm, NULL); - buffer_count = 0; feed_buf_count = 0; - if (file_start[0] == iter_file_start[0]) { + if (file_start[0] == iter_file_start[0]) { if (file_start[1] >= iter_file_start[1]) { if (file_count[0] == iter_file_count[0]) { feed_buf_count = dimsm[1] * iter_file_start[0]; @@ -9485,16 +9461,17 @@ foreach_iteration(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t mem_typ memcpy(new_buffer + (buffer_count * element_size), buf + (feed_buf_count + iter_file_start[1]) * element_size, iter_file_count[1] * element_size); - + buffer_count += iter_file_count[1]; memcpy(new_buffer + (buffer_count * element_size), buf + (feed_buf_count + file_start[1]) * element_size, file_count[1] * element_size); - + buffer_count += file_count[1]; feed_buf_count += dimsm[1]; - fprintf(stderr,"buffer_count=%d \ - feed_buf_count=%d\n",buffer_count,feed_buf_count); + fprintf(stderr, "buffer_count=%d \ + feed_buf_count=%d\n", + buffer_count, feed_buf_count); } } } @@ -9507,15 +9484,16 @@ foreach_iteration(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t mem_typ memcpy(new_buffer + (buffer_count * element_size), buf + (feed_buf_count + file_start[1]) * element_size, file_count[1] * element_size); - + buffer_count += file_count[1]; memcpy(new_buffer + (buffer_count * element_size), buf + (feed_buf_count + iter_file_start[1]) * element_size, iter_file_count[1] * element_size); - + buffer_count += iter_file_count[1]; feed_buf_count += dimsm[1]; - fprintf(stderr,"buffer_count=%d feed_buf_count=%d\n",buffer_count,feed_buf_count); + fprintf(stderr, "buffer_count=%d feed_buf_count=%d\n", + buffer_count, feed_buf_count); } } } @@ -9528,19 +9506,17 @@ foreach_iteration(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t mem_typ memcpy(new_buffer + (buffer_count * element_size), buf + (feed_buf_count + iter_file_start[1]) * element_size, iter_file_count[1] * element_size); - + buffer_count += iter_file_count[1]; feed_buf_count += dimsm[1]; - } for (int i = 0; i < file_count[0]; i++) { memcpy(new_buffer + (buffer_count * element_size), buf + (feed_buf_count + file_start[1]) * element_size, file_count[1] * element_size); - + buffer_count += file_count[1]; feed_buf_count += dimsm[1]; - } } } @@ -9552,42 +9528,39 @@ foreach_iteration(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t mem_typ memcpy(new_buffer + (buffer_count * element_size), buf + (feed_buf_count + file_start[1]) * element_size, file_count[1] * element_size); - + buffer_count += file_count[1]; feed_buf_count += dimsm[1]; - fprintf(stderr,"buffer_count=%d \ - // feed_buf_count=%d\n",buffer_count,feed_buf_count); + fprintf(stderr, "buffer_count=%d \ + // feed_buf_count=%d\n", + buffer_count, feed_buf_count); } for (int i = 0; i < iter_file_count[0]; i++) { memcpy(new_buffer + (buffer_count * element_size), buf + (feed_buf_count + iter_file_start[1]) * element_size, iter_file_count[1] * element_size); - + buffer_count += iter_file_count[1]; feed_buf_count += dimsm[1]; - fprintf(stderr,"buffer_count=%d \ - feed_buf_count=%d\n",buffer_count,feed_buf_count); + fprintf(stderr, "buffer_count=%d \ + feed_buf_count=%d\n", + buffer_count, feed_buf_count); } } } - } - - - + } H5Sclose(iter_args->mem_space_id); - + iter_args->mem_space_id = new_memspace; - - iter_args->buf = new_buffer; - iter_args->free_buf=1; - int *iter_buf= iter_args->buf; + iter_args->buf = new_buffer; + iter_args->free_buf = 1; + int *iter_buf = iter_args->buf; for (int i = 0; i < 16; i++) fprintf(stderr, " iter_args new_buffer[%d]=%d ", i, iter_buf[i]); - fprintf(stderr, "\n"); + fprintf(stderr, "\n"); - status = H5Sselect_hyperslab(file_space_id, H5S_SELECT_SET, start, NULL, count, NULL); @@ -9596,32 +9569,29 @@ foreach_iteration(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t mem_typ start[0] = 0; start[1] = 0; - status = H5Sselect_hyperslab(iter_args->mem_space_id, H5S_SELECT_SET, start, NULL, count, NULL); - } else if (ndim == 3) { - + fprintf(stderr, " start=%llux%llux%llu count=%llux%llux%llu\n", start[0], start[1], start[2], count[0], count[1], count[2]); new_memspace = H5Screate_simple(ndim, count, NULL); element_size = H5Tget_size(mem_type_id); new_buffer = malloc(count[0] * count[1] * count[2] * element_size); - + status = H5Sget_regular_hyperslab(mem_space_id, mem_start, NULL, mem_count, NULL); status = H5Sget_regular_hyperslab(file_space_id, file_start, NULL, file_count, NULL); - status = H5Sget_regular_hyperslab(iter_args->mem_space_id, iter_mem_start, NULL, iter_mem_count, NULL); - + status = H5Sget_regular_hyperslab(iter_args->file_space_id, iter_file_start, NULL, iter_file_count, NULL); - + fprintf(stderr, "\n file_start=%lldx%lldx%lld iter_file_start=%lldx%lldx%lld ", file_start[0], file_start[1], file_start[2], iter_file_start[0], iter_file_start[1], iter_file_start[2]); @@ -9629,38 +9599,38 @@ foreach_iteration(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t mem_typ file_count[0], file_count[1], file_count[2], iter_file_count[0], iter_file_count[1], iter_file_count[2]); H5Sget_simple_extent_dims(file_space_id, dimsm, NULL); - - int dim_elements = dimsm[1] * dimsm[2]; - buffer_count = 0; - feed_buf_count = 0; - buffer_count = 0; - feed_buf_count = 0; - + + int dim_elements = dimsm[1] * dimsm[2]; + buffer_count = 0; + feed_buf_count = 0; + buffer_count = 0; + feed_buf_count = 0; if (file_start[1] == iter_file_start[1]) { if (file_start[2] >= iter_file_start[2]) { if (file_count[1] == iter_file_count[1]) { if (file_count[0] == iter_file_count[0]) { for (int d = 0; d < file_count[0]; d++) { - - feed_buf_count = - file_start[0]*dim_elements+dim_elements * d + dimsm[2] * iter_file_start[1]; + + feed_buf_count = file_start[0] * dim_elements + + dim_elements * d + + dimsm[2] * iter_file_start[1]; for (int i = 0; i < iter_file_count[1]; i++) { - memcpy(new_buffer + (buffer_count * element_size), + memcpy(new_buffer + (buffer_count * element_size), buf + (feed_buf_count + iter_file_start[2]) * element_size, - iter_file_count[2] * element_size); - + iter_file_count[2] * element_size); + buffer_count += iter_file_count[2]; - memcpy(new_buffer + (buffer_count * element_size), + memcpy(new_buffer + (buffer_count * element_size), buf + (feed_buf_count + file_start[2]) * element_size, - file_count[2] * element_size); - + file_count[2] * element_size); + buffer_count += file_count[2]; feed_buf_count += dimsm[2]; - // fprintf(stderr,"buffer_count=%d \ + // fprintf(stderr,"buffer_count=%d \ //feed_buf_count=%d\n",buffer_count,feed_buf_count); } } @@ -9673,21 +9643,22 @@ foreach_iteration(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t mem_typ if (file_count[0] == iter_file_count[0]) { for (int d = 0; d < file_count[0]; d++) { - feed_buf_count = file_start[0]*dim_elements+dim_elements * d + dimsm[2] * file_start[1]; - + feed_buf_count = file_start[0] * dim_elements + + dim_elements * d + dimsm[2] * file_start[1]; + for (int i = 0; i < file_count[1]; i++) { - memcpy(new_buffer + (buffer_count * element_size), + memcpy(new_buffer + (buffer_count * element_size), buf + (feed_buf_count + file_start[2]) * element_size, - file_count[2] * element_size); - + file_count[2] * element_size); + buffer_count += file_count[2]; - memcpy(new_buffer + (buffer_count * element_size), + memcpy(new_buffer + (buffer_count * element_size), buf + (feed_buf_count + iter_file_start[2]) * element_size, - iter_file_count[2] * element_size); - + iter_file_count[2] * element_size); + buffer_count += iter_file_count[2]; feed_buf_count += dimsm[2]; //fprintf(stderr,"buffer_count=%d \ @@ -9704,27 +9675,27 @@ foreach_iteration(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t mem_typ if (file_count[0] == iter_file_count[0]) { for (int d = 0; d < file_count[0]; d++) { - feed_buf_count = - file_start[0]*dim_elements+dim_elements * d + dimsm[2] * iter_file_start[1]; + feed_buf_count = file_start[0] * dim_elements + + dim_elements * d + + dimsm[2] * iter_file_start[1]; - for (int i = 0; i < iter_file_count[1]; i++) { - memcpy(new_buffer + (buffer_count * element_size), + memcpy(new_buffer + (buffer_count * element_size), buf + (feed_buf_count + iter_file_start[2]) * element_size, - iter_file_count[2] * element_size); - + iter_file_count[2] * element_size); + buffer_count += iter_file_count[2]; feed_buf_count += dimsm[2]; - //fprintf(stderr, "buffer_count=%d feed_buf_count=%d\n", - // buffer_count, feed_buf_count); + // fprintf(stderr, "buffer_count=%d feed_buf_count=%d\n", + // buffer_count, feed_buf_count); } for (int i = 0; i < file_count[1]; i++) { - memcpy(new_buffer + (buffer_count * element_size), + memcpy(new_buffer + (buffer_count * element_size), buf + (feed_buf_count + file_start[2]) * element_size, - file_count[2] * element_size); - + file_count[2] * element_size); + buffer_count += file_count[2]; feed_buf_count += dimsm[2]; //fprintf(stderr,"buffer_count=%d \ @@ -9739,26 +9710,26 @@ foreach_iteration(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t mem_typ if (file_count[0] == iter_file_count[0]) { for (int d = 0; d < file_count[0]; d++) { - feed_buf_count = file_start[0]*dim_elements+dim_elements * d + dimsm[2] * file_start[1]; + feed_buf_count = file_start[0] * dim_elements + + dim_elements * d + dimsm[2] * file_start[1]; - for (int i = 0; i < file_count[1]; i++) { - memcpy(new_buffer + (buffer_count * element_size), + memcpy(new_buffer + (buffer_count * element_size), buf + (feed_buf_count + file_start[2]) * element_size, - file_count[2] * element_size); - + file_count[2] * element_size); + buffer_count += file_count[2]; feed_buf_count += dimsm[2]; - // fprintf(stderr,"buffer_count=%d \ + // fprintf(stderr,"buffer_count=%d \ // feed_buf_count=%d\n",buffer_count,feed_buf_count); } for (int i = 0; i < iter_file_count[1]; i++) { - memcpy(new_buffer + (buffer_count * element_size), + memcpy(new_buffer + (buffer_count * element_size), buf + (feed_buf_count + iter_file_start[2]) * element_size, iter_file_count[2] * element_size); - + buffer_count += iter_file_count[2]; feed_buf_count += dimsm[2]; // fprintf(stderr,"buffer_count=%d \ @@ -9772,11 +9743,10 @@ foreach_iteration(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t mem_typ H5Sclose(iter_args->mem_space_id); iter_args->mem_space_id = new_memspace; - - iter_args->buf = new_buffer; - iter_args->free_buf=1; - - + + iter_args->buf = new_buffer; + iter_args->free_buf = 1; + status = H5Sselect_hyperslab(file_space_id, H5S_SELECT_SET, start, NULL, count, NULL); @@ -9792,27 +9762,23 @@ foreach_iteration(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t mem_typ status = H5Sselect_hyperslab(iter_args->mem_space_id, H5S_SELECT_SET, start, NULL, count, NULL); - //fprintf(stderr," end of foreach iterator mem_type_id=%llu\n",iter_args->mem_type_id); - - + // fprintf(stderr," end of foreach iterator + // mem_type_id=%llu\n",iter_args->mem_type_id); } - - //time_in_contiguous2= MPI_Wtime(); - //duration_in_contiguous_check= time_in_contiguous2 - time_in_contiguous1; - //fprintf(stderr," Duration in contiguous Runtime is %f \n", duration_contiguous_check); + // time_in_contiguous2= MPI_Wtime(); + // duration_in_contiguous_check= time_in_contiguous2 - time_in_contiguous1; + // fprintf(stderr," Duration in contiguous Runtime is %f \n", + // duration_contiguous_check); } else { - - //fprintf(stderr, "-------- Not contiguous---------\n"); - } - //time_after_contiguous_check = MPI_Wtime(); - //duration_contiguous_check= time_after_contiguous_check - time_before_contiguous_check; - // fprintf(stderr," Duration contiguous Runtime is %f \n", duration_contiguous_check); - - + // fprintf(stderr, "-------- Not contiguous---------\n"); + } + // time_after_contiguous_check = MPI_Wtime(); + // duration_contiguous_check= time_after_contiguous_check - time_before_contiguous_check; + // fprintf(stderr," Duration contiguous Runtime is %f \n", duration_contiguous_check); } } } @@ -9829,11 +9795,11 @@ foreach_iteration(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t mem_typ free(iter_mem_count); free(iter_file_start); free(iter_file_count); - //time_foreach2 = MPI_Wtime(); + // time_foreach2 = MPI_Wtime(); + + // duration_foreach = time_foreach2 - time_foreach1; + // fprintf(stderr,"Foreach Duration Runtime is %f \n", duration_foreach); - //duration_foreach = time_foreach2 - time_foreach1; - //fprintf(stderr,"Foreach Duration Runtime is %f \n", duration_foreach); - return return_task_val; } static herr_t @@ -9842,53 +9808,49 @@ async_dataset_write_merge(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t { herr_t status, return_val = 0; - async_task_t * task_iter; - async_task_t * return_task_val; + async_task_t *task_iter; + async_task_t *return_task_val; async_task_list_t *task_list_iter; int buffer_count, feed_buf_count; - //double time1, time2,duration; - //double time11,time12,duration1; - //double time21,time22,duration2; + // double time1, time2,duration; + // double time11,time12,duration1; + // double time21,time22,duration2; assert(aid); assert(parent_obj); assert(parent_obj->magic == ASYNC_MAGIC); assert(mem_space_id); - - //fprintf(stderr,"################# \n"); - //time1 = MPI_Wtime(); - //time11=MPI_Wtime(); + // fprintf(stderr,"################# \n"); + // time1 = MPI_Wtime(); + // time11=MPI_Wtime(); return_task_val = foreach_iteration(aid, parent_obj, mem_type_id, mem_space_id, file_space_id, plist_id, buf, NULL); // return_val=foreach_iteration(aid,parent_obj,mem_type_id,mem_space_id,file_space_id,plist_id,buf,NULL); - //time12 = MPI_Wtime(); - //duration1 = time12 - time11; - //fprintf(stderr,"\t1st Foreach iter Runtime is %f \n", duration1); + // time12 = MPI_Wtime(); + // duration1 = time12 - time11; + // fprintf(stderr,"\t1st Foreach iter Runtime is %f \n", duration1); if (return_task_val == NULL) return_val = 0; else return_val = 1; - - for (int i = 0; return_task_val != NULL; i++) - { //time21=MPI_Wtime(); - - return_task_val = foreach_iteration(aid, parent_obj, mem_type_id, mem_space_id, file_space_id, - plist_id, buf, return_task_val); - //time22=MPI_Wtime(); - - //duration2 = time22 - time21; - //fprintf(stderr,"\t\tin the for loop Runtime is i=%d %f \n", i, duration1); - } - - //time2 = MPI_Wtime(); - //duration = time2 - time1; - - - - //fprintf(stderr,"Foreach Runtime is %f \n", duration); - + + for (int i = 0; return_task_val != NULL; i++) { // time21=MPI_Wtime(); + + return_task_val = foreach_iteration(aid, parent_obj, mem_type_id, mem_space_id, file_space_id, + plist_id, buf, return_task_val); + // time22=MPI_Wtime(); + + // duration2 = time22 - time21; + // fprintf(stderr,"\t\tin the for loop Runtime is i=%d %f \n", i, duration1); + } + + // time2 = MPI_Wtime(); + // duration = time2 - time1; + + // fprintf(stderr,"Foreach Runtime is %f \n", duration); + return return_val; } @@ -9896,7 +9858,7 @@ static herr_t async_dataset_write(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t mem_type_id, hid_t mem_space_id, hid_t file_space_id, hid_t plist_id, const void *buf, void **req) { - async_task_t * async_task = NULL; + async_task_t *async_task = NULL; async_dataset_write_args_t *args = NULL; bool lock_parent = false; bool is_blocking = false; @@ -9909,13 +9871,13 @@ async_dataset_write(async_instance_t *aid, H5VL_async_t *parent_obj, hid_t mem_t assert(aid); assert(parent_obj); assert(parent_obj->magic == ASYNC_MAGIC); - #ifdef ENABLE_WRITE_MERGE - return_val = - async_dataset_write_merge(aid, parent_obj, mem_type_id, mem_space_id, file_space_id, plist_id, buf); +#ifdef ENABLE_WRITE_MERGE + return_val = + async_dataset_write_merge(aid, parent_obj, mem_type_id, mem_space_id, file_space_id, plist_id, buf); - if (return_val == 1) - goto done; - #endif + if (return_val == 1) + goto done; +#endif async_instance_g->prev_push_state = async_instance_g->start_abt_push; @@ -10145,8 +10107,8 @@ async_dataset_get_fn(void *foo) int attempt_count = 0; int is_lock = 0; hbool_t is_lib_state_restored = false; - ABT_pool * pool_ptr; - async_task_t * task = (async_task_t *)foo; + ABT_pool *pool_ptr; + async_task_t *task = (async_task_t *)foo; async_dataset_get_args_t *args = (async_dataset_get_args_t *)(task->args); herr_t status; @@ -10285,7 +10247,7 @@ static herr_t async_dataset_get(task_list_qtype qtype, async_instance_t *aid, H5VL_async_t *parent_obj, H5VL_dataset_get_args_t *get_args, hid_t dxpl_id, void **req) { - async_task_t * async_task = NULL; + async_task_t *async_task = NULL; async_dataset_get_args_t *args = NULL; bool lock_parent = false; bool is_blocking = false; @@ -10326,7 +10288,7 @@ async_dataset_get(task_list_qtype qtype, async_instance_t *aid, H5VL_async_t *pa if (dxpl_id > 0) args->dxpl_id = H5Pcopy(dxpl_id); args->req = req; - + // Temporary fix for data space get that could cause a pthread deadlock if (req && !aid->disable_async_dset_get) { H5VL_async_t *new_req; @@ -10466,8 +10428,8 @@ async_dataset_specific_fn(void *foo) int attempt_count = 0; int is_lock = 0; hbool_t is_lib_state_restored = false; - ABT_pool * pool_ptr; - async_task_t * task = (async_task_t *)foo; + ABT_pool *pool_ptr; + async_task_t *task = (async_task_t *)foo; async_dataset_specific_args_t *args = (async_dataset_specific_args_t *)(task->args); herr_t status; @@ -10606,7 +10568,7 @@ static herr_t async_dataset_specific(task_list_qtype qtype, async_instance_t *aid, H5VL_async_t *parent_obj, H5VL_dataset_specific_args_t *spec_args, hid_t dxpl_id, void **req) { - async_task_t * async_task = NULL; + async_task_t *async_task = NULL; async_dataset_specific_args_t *args = NULL; bool lock_parent = false; bool is_blocking = false; @@ -10783,8 +10745,8 @@ async_dataset_optional_fn(void *foo) int attempt_count = 0; int is_lock = 0; hbool_t is_lib_state_restored = false; - ABT_pool * pool_ptr; - async_task_t * task = (async_task_t *)foo; + ABT_pool *pool_ptr; + async_task_t *task = (async_task_t *)foo; async_dataset_optional_args_t *args = (async_dataset_optional_args_t *)(task->args); herr_t status; @@ -10923,7 +10885,7 @@ static herr_t async_dataset_optional(task_list_qtype qtype, async_instance_t *aid, H5VL_async_t *parent_obj, H5VL_optional_args_t *opt_args, hid_t dxpl_id, void **req) { - async_task_t * async_task = NULL; + async_task_t *async_task = NULL; async_dataset_optional_args_t *args = NULL; bool lock_parent = false; bool is_blocking = false; @@ -11095,8 +11057,8 @@ async_dataset_close_fn(void *foo) int attempt_count = 0; int is_lock = 0; hbool_t is_lib_state_restored = false; - ABT_pool * pool_ptr; - async_task_t * task = (async_task_t *)foo; + ABT_pool *pool_ptr; + async_task_t *task = (async_task_t *)foo; async_dataset_close_args_t *args = (async_dataset_close_args_t *)(task->args); herr_t status; @@ -11235,7 +11197,7 @@ static herr_t async_dataset_close(task_list_qtype qtype, async_instance_t *aid, H5VL_async_t *parent_obj, hid_t dxpl_id, void **req) { - async_task_t * async_task = NULL; + async_task_t *async_task = NULL; async_dataset_close_args_t *args = NULL; bool lock_parent = false; bool is_blocking = false; @@ -11422,10 +11384,10 @@ async_datatype_commit_fn(void *foo) int attempt_count = 0; int is_lock = 0; hbool_t is_lib_state_restored = false; - ABT_pool * pool_ptr; - async_task_t * task = (async_task_t *)foo; + ABT_pool *pool_ptr; + async_task_t *task = (async_task_t *)foo; async_datatype_commit_args_t *args = (async_datatype_commit_args_t *)(task->args); - void * under_obj; + void *under_obj; #ifdef ENABLE_TIMING task->start_time = clock(); @@ -11576,8 +11538,8 @@ async_datatype_commit(async_instance_t *aid, H5VL_async_t *parent_obj, const H5V const char *name, hid_t type_id, hid_t lcpl_id, hid_t tcpl_id, hid_t tapl_id, hid_t dxpl_id, void **req) { - H5VL_async_t * async_obj = NULL; - async_task_t * async_task = NULL; + H5VL_async_t *async_obj = NULL; + async_task_t *async_task = NULL; async_datatype_commit_args_t *args = NULL; bool lock_parent = false; bool is_blocking = false; @@ -11777,14 +11739,14 @@ async_datatype_commit(async_instance_t *aid, H5VL_async_t *parent_obj, const H5V static void async_datatype_open_fn(void *foo) { - void * obj; + void *obj; hbool_t acquired = false; unsigned int mutex_count = 1; int attempt_count = 0; int is_lock = 0; hbool_t is_lib_state_restored = false; - ABT_pool * pool_ptr; - async_task_t * task = (async_task_t *)foo; + ABT_pool *pool_ptr; + async_task_t *task = (async_task_t *)foo; async_datatype_open_args_t *args = (async_datatype_open_args_t *)(task->args); #ifdef ENABLE_TIMING @@ -11931,8 +11893,8 @@ static H5VL_async_t * async_datatype_open(async_instance_t *aid, H5VL_async_t *parent_obj, const H5VL_loc_params_t *loc_params, const char *name, hid_t tapl_id, hid_t dxpl_id, void **req) { - H5VL_async_t * async_obj = NULL; - async_task_t * async_task = NULL; + H5VL_async_t *async_obj = NULL; + async_task_t *async_task = NULL; async_datatype_open_args_t *args = NULL; bool lock_parent = false; bool is_blocking = false; @@ -12129,8 +12091,8 @@ async_datatype_get_fn(void *foo) int attempt_count = 0; int is_lock = 0; hbool_t is_lib_state_restored = false; - ABT_pool * pool_ptr; - async_task_t * task = (async_task_t *)foo; + ABT_pool *pool_ptr; + async_task_t *task = (async_task_t *)foo; async_datatype_get_args_t *args = (async_datatype_get_args_t *)(task->args); herr_t status; @@ -12269,7 +12231,7 @@ static herr_t async_datatype_get(task_list_qtype qtype, async_instance_t *aid, H5VL_async_t *parent_obj, H5VL_datatype_get_args_t *get_args, hid_t dxpl_id, void **req) { - async_task_t * async_task = NULL; + async_task_t *async_task = NULL; async_datatype_get_args_t *args = NULL; bool lock_parent = false; bool is_blocking = false; @@ -12444,8 +12406,8 @@ async_datatype_specific_fn(void *foo) int attempt_count = 0; int is_lock = 0; hbool_t is_lib_state_restored = false; - ABT_pool * pool_ptr; - async_task_t * task = (async_task_t *)foo; + ABT_pool *pool_ptr; + async_task_t *task = (async_task_t *)foo; async_datatype_specific_args_t *args = (async_datatype_specific_args_t *)(task->args); herr_t status; @@ -12584,7 +12546,7 @@ static herr_t async_datatype_specific(task_list_qtype qtype, async_instance_t *aid, H5VL_async_t *parent_obj, H5VL_datatype_specific_args_t *spec_args, hid_t dxpl_id, void **req) { - async_task_t * async_task = NULL; + async_task_t *async_task = NULL; async_datatype_specific_args_t *args = NULL; bool lock_parent = false; bool is_blocking = false; @@ -12757,8 +12719,8 @@ async_datatype_optional_fn(void *foo) int attempt_count = 0; int is_lock = 0; hbool_t is_lib_state_restored = false; - ABT_pool * pool_ptr; - async_task_t * task = (async_task_t *)foo; + ABT_pool *pool_ptr; + async_task_t *task = (async_task_t *)foo; async_datatype_optional_args_t *args = (async_datatype_optional_args_t *)(task->args); herr_t status; @@ -12897,7 +12859,7 @@ static herr_t async_datatype_optional(task_list_qtype qtype, async_instance_t *aid, H5VL_async_t *parent_obj, H5VL_optional_args_t *opt_args, hid_t dxpl_id, void **req) { - async_task_t * async_task = NULL; + async_task_t *async_task = NULL; async_datatype_optional_args_t *args = NULL; bool lock_parent = false; bool is_blocking = false; @@ -13070,8 +13032,8 @@ async_datatype_close_fn(void *foo) int attempt_count = 0; int is_lock = 0; hbool_t is_lib_state_restored = false; - ABT_pool * pool_ptr; - async_task_t * task = (async_task_t *)foo; + ABT_pool *pool_ptr; + async_task_t *task = (async_task_t *)foo; async_datatype_close_args_t *args = (async_datatype_close_args_t *)(task->args); herr_t status; @@ -13209,7 +13171,7 @@ static herr_t async_datatype_close(task_list_qtype qtype, async_instance_t *aid, H5VL_async_t *parent_obj, hid_t dxpl_id, void **req) { - async_task_t * async_task = NULL; + async_task_t *async_task = NULL; async_datatype_close_args_t *args = NULL; bool lock_parent = false; bool is_blocking = false; @@ -13381,15 +13343,15 @@ async_datatype_close(task_list_qtype qtype, async_instance_t *aid, H5VL_async_t static void async_file_create_fn(void *foo) { - void * obj; + void *obj; hbool_t acquired = false; unsigned int mutex_count = 1; int attempt_count = 0; int is_lock = 0; hbool_t is_lib_state_restored = false; - ABT_pool * pool_ptr; - H5VL_async_info_t * info = NULL; - async_task_t * task = (async_task_t *)foo; + ABT_pool *pool_ptr; + H5VL_async_info_t *info = NULL; + async_task_t *task = (async_task_t *)foo; async_file_create_args_t *args = (async_file_create_args_t *)(task->args); /* herr_t status; */ hid_t under_vol_id; @@ -13549,8 +13511,8 @@ async_file_create(async_instance_t *aid, const char *name, unsigned flags, hid_t hid_t dxpl_id, void **req) { hid_t under_vol_id; - H5VL_async_t * async_obj = NULL; - async_task_t * async_task = NULL; + H5VL_async_t *async_obj = NULL; + async_task_t *async_task = NULL; async_file_create_args_t *args = NULL; bool lock_self = false; bool is_blocking = false; @@ -13730,15 +13692,15 @@ async_file_create(async_instance_t *aid, const char *name, unsigned flags, hid_t static void async_file_open_fn(void *foo) { - void * obj; + void *obj; hbool_t acquired = false; unsigned int mutex_count = 1; int attempt_count = 0; int is_lock = 0; hbool_t is_lib_state_restored = false; - ABT_pool * pool_ptr; - H5VL_async_info_t * info = NULL; - async_task_t * task = (async_task_t *)foo; + ABT_pool *pool_ptr; + H5VL_async_info_t *info = NULL; + async_task_t *task = (async_task_t *)foo; async_file_open_args_t *args = (async_file_open_args_t *)(task->args); /* herr_t status; */ hid_t under_vol_id; @@ -13900,8 +13862,8 @@ async_file_open(task_list_qtype qtype, async_instance_t *aid, const char *name, hid_t dxpl_id, void **req) { hid_t under_vol_id; - H5VL_async_t * async_obj = NULL; - async_task_t * async_task = NULL; + H5VL_async_t *async_obj = NULL; + async_task_t *async_task = NULL; async_file_open_args_t *args = NULL; bool lock_self = false; bool is_blocking = false; @@ -14084,8 +14046,8 @@ async_file_get_fn(void *foo) int attempt_count = 0; int is_lock = 0; hbool_t is_lib_state_restored = false; - ABT_pool * pool_ptr; - async_task_t * task = (async_task_t *)foo; + ABT_pool *pool_ptr; + async_task_t *task = (async_task_t *)foo; async_file_get_args_t *args = (async_file_get_args_t *)(task->args); herr_t status; @@ -14224,7 +14186,7 @@ static herr_t async_file_get(task_list_qtype qtype, async_instance_t *aid, H5VL_async_t *parent_obj, H5VL_file_get_args_t *get_args, hid_t dxpl_id, void **req) { - async_task_t * async_task = NULL; + async_task_t *async_task = NULL; async_file_get_args_t *args = NULL; bool lock_parent = false; bool is_blocking = false; @@ -14399,8 +14361,8 @@ async_file_specific_fn(void *foo) int attempt_count = 0; int is_lock = 0; hbool_t is_lib_state_restored = false; - ABT_pool * pool_ptr; - async_task_t * task = (async_task_t *)foo; + ABT_pool *pool_ptr; + async_task_t *task = (async_task_t *)foo; async_file_specific_args_t *args = (async_file_specific_args_t *)(task->args); herr_t status; @@ -14539,7 +14501,7 @@ static herr_t async_file_specific(task_list_qtype qtype, async_instance_t *aid, H5VL_async_t *parent_obj, H5VL_file_specific_args_t *spec_args, hid_t dxpl_id, void **req) { - async_task_t * async_task = NULL; + async_task_t *async_task = NULL; async_file_specific_args_t *args = NULL; bool lock_parent = false; bool is_blocking = false; @@ -14720,8 +14682,8 @@ async_file_optional_fn(void *foo) int attempt_count = 0; int is_lock = 0; hbool_t is_lib_state_restored = false; - ABT_pool * pool_ptr; - async_task_t * task = (async_task_t *)foo; + ABT_pool *pool_ptr; + async_task_t *task = (async_task_t *)foo; async_file_optional_args_t *args = (async_file_optional_args_t *)(task->args); herr_t status; @@ -14879,7 +14841,7 @@ static herr_t async_file_optional(task_list_qtype qtype, async_instance_t *aid, H5VL_async_t *parent_obj, H5VL_optional_args_t *opt_args, hid_t dxpl_id, void **req) { - async_task_t * async_task = NULL; + async_task_t *async_task = NULL; async_file_optional_args_t *args = NULL; bool lock_parent = false; bool is_blocking = false; @@ -15054,8 +15016,8 @@ async_file_close_fn(void *foo) int attempt_count = 0; int is_lock = 0; hbool_t is_lib_state_restored = false; - ABT_pool * pool_ptr; - async_task_t * task = (async_task_t *)foo; + ABT_pool *pool_ptr; + async_task_t *task = (async_task_t *)foo; async_file_close_args_t *args = (async_file_close_args_t *)(task->args); herr_t status; @@ -15213,7 +15175,7 @@ static herr_t async_file_close(task_list_qtype qtype, async_instance_t *aid, H5VL_async_t *parent_obj, hid_t dxpl_id, void **req) { - async_task_t * async_task = NULL; + async_task_t *async_task = NULL; async_file_close_args_t *args = NULL; bool lock_parent = false; bool is_blocking = false; @@ -15412,14 +15374,14 @@ async_file_close(task_list_qtype qtype, async_instance_t *aid, H5VL_async_t *par static void async_group_create_fn(void *foo) { - void * obj; + void *obj; hbool_t acquired = false; unsigned int mutex_count = 1; int attempt_count = 0; int is_lock = 0; hbool_t is_lib_state_restored = false; - ABT_pool * pool_ptr; - async_task_t * task = (async_task_t *)foo; + ABT_pool *pool_ptr; + async_task_t *task = (async_task_t *)foo; async_group_create_args_t *args = (async_group_create_args_t *)(task->args); #ifdef ENABLE_TIMING @@ -15572,8 +15534,8 @@ static H5VL_async_t * async_group_create(async_instance_t *aid, H5VL_async_t *parent_obj, const H5VL_loc_params_t *loc_params, const char *name, hid_t lcpl_id, hid_t gcpl_id, hid_t gapl_id, hid_t dxpl_id, void **req) { - H5VL_async_t * async_obj = NULL; - async_task_t * async_task = NULL; + H5VL_async_t *async_obj = NULL; + async_task_t *async_task = NULL; async_group_create_args_t *args = NULL; bool lock_parent = false; bool is_blocking = false; @@ -15779,14 +15741,14 @@ async_group_create(async_instance_t *aid, H5VL_async_t *parent_obj, const H5VL_l static void async_group_open_fn(void *foo) { - void * obj; + void *obj; hbool_t acquired = false; unsigned int mutex_count = 1; int attempt_count = 0; int is_lock = 0; hbool_t is_lib_state_restored = false; - ABT_pool * pool_ptr; - async_task_t * task = (async_task_t *)foo; + ABT_pool *pool_ptr; + async_task_t *task = (async_task_t *)foo; async_group_open_args_t *args = (async_group_open_args_t *)(task->args); #ifdef ENABLE_TIMING @@ -15933,8 +15895,8 @@ static H5VL_async_t * async_group_open(async_instance_t *aid, H5VL_async_t *parent_obj, const H5VL_loc_params_t *loc_params, const char *name, hid_t gapl_id, hid_t dxpl_id, void **req) { - H5VL_async_t * async_obj = NULL; - async_task_t * async_task = NULL; + H5VL_async_t *async_obj = NULL; + async_task_t *async_task = NULL; async_group_open_args_t *args = NULL; bool lock_parent = false; bool is_blocking = false; @@ -16131,8 +16093,8 @@ async_group_get_fn(void *foo) int attempt_count = 0; int is_lock = 0; hbool_t is_lib_state_restored = false; - ABT_pool * pool_ptr; - async_task_t * task = (async_task_t *)foo; + ABT_pool *pool_ptr; + async_task_t *task = (async_task_t *)foo; async_group_get_args_t *args = (async_group_get_args_t *)(task->args); herr_t status; @@ -16271,7 +16233,7 @@ static herr_t async_group_get(task_list_qtype qtype, async_instance_t *aid, H5VL_async_t *parent_obj, H5VL_group_get_args_t *get_args, hid_t dxpl_id, void **req) { - async_task_t * async_task = NULL; + async_task_t *async_task = NULL; async_group_get_args_t *args = NULL; bool lock_parent = false; bool is_blocking = false; @@ -16451,8 +16413,8 @@ async_group_specific_fn(void *foo) int attempt_count = 0; int is_lock = 0; hbool_t is_lib_state_restored = false; - ABT_pool * pool_ptr; - async_task_t * task = (async_task_t *)foo; + ABT_pool *pool_ptr; + async_task_t *task = (async_task_t *)foo; async_group_specific_args_t *args = (async_group_specific_args_t *)(task->args); herr_t status; @@ -16591,7 +16553,7 @@ static herr_t async_group_specific(task_list_qtype qtype, async_instance_t *aid, H5VL_async_t *parent_obj, H5VL_group_specific_args_t *spec_args, hid_t dxpl_id, void **req) { - async_task_t * async_task = NULL; + async_task_t *async_task = NULL; async_group_specific_args_t *args = NULL; bool lock_parent = false; bool is_blocking = false; @@ -16763,8 +16725,8 @@ async_group_optional_fn(void *foo) int attempt_count = 0; int is_lock = 0; hbool_t is_lib_state_restored = false; - ABT_pool * pool_ptr; - async_task_t * task = (async_task_t *)foo; + ABT_pool *pool_ptr; + async_task_t *task = (async_task_t *)foo; async_group_optional_args_t *args = (async_group_optional_args_t *)(task->args); herr_t status; @@ -16903,7 +16865,7 @@ static herr_t async_group_optional(task_list_qtype qtype, async_instance_t *aid, H5VL_async_t *parent_obj, H5VL_optional_args_t *opt_args, hid_t dxpl_id, void **req) { - async_task_t * async_task = NULL; + async_task_t *async_task = NULL; async_group_optional_args_t *args = NULL; bool lock_parent = false; bool is_blocking = false; @@ -17075,8 +17037,8 @@ async_group_close_fn(void *foo) int attempt_count = 0; int is_lock = 0; hbool_t is_lib_state_restored = false; - ABT_pool * pool_ptr; - async_task_t * task = (async_task_t *)foo; + ABT_pool *pool_ptr; + async_task_t *task = (async_task_t *)foo; async_group_close_args_t *args = (async_group_close_args_t *)(task->args); herr_t status; @@ -17218,7 +17180,7 @@ static herr_t async_group_close(task_list_qtype qtype, async_instance_t *aid, H5VL_async_t *parent_obj, hid_t dxpl_id, void **req) { - async_task_t * async_task = NULL; + async_task_t *async_task = NULL; async_group_close_args_t *args = NULL; bool lock_parent = false; bool is_blocking = false; @@ -17409,8 +17371,8 @@ async_link_create_fn(void *foo) int attempt_count = 0; int is_lock = 0; hbool_t is_lib_state_restored = false; - ABT_pool * pool_ptr; - async_task_t * task = (async_task_t *)foo; + ABT_pool *pool_ptr; + async_task_t *task = (async_task_t *)foo; async_link_create_args_t *args = (async_link_create_args_t *)(task->args); herr_t status; @@ -17559,8 +17521,8 @@ async_link_create(task_list_qtype qtype, async_instance_t *aid, H5VL_link_create H5VL_async_t *parent_obj, const H5VL_loc_params_t *loc_params, hid_t lcpl_id, hid_t lapl_id, hid_t dxpl_id, void **req) { - H5VL_async_t * async_obj = NULL; - async_task_t * async_task = NULL; + H5VL_async_t *async_obj = NULL; + async_task_t *async_task = NULL; async_link_create_args_t *args = NULL; bool lock_parent = false; bool is_blocking = false; @@ -17759,8 +17721,8 @@ async_link_copy_fn(void *foo) int attempt_count = 0; int is_lock = 0; hbool_t is_lib_state_restored = false; - ABT_pool * pool_ptr; - async_task_t * task = (async_task_t *)foo; + ABT_pool *pool_ptr; + async_task_t *task = (async_task_t *)foo; async_link_copy_args_t *args = (async_link_copy_args_t *)(task->args); herr_t status; @@ -17907,13 +17869,13 @@ async_link_copy(async_instance_t *aid, H5VL_async_t *parent_obj1, const H5VL_loc H5VL_async_t *parent_obj2, const H5VL_loc_params_t *loc_params2, hid_t lcpl_id, hid_t lapl_id, hid_t dxpl_id, void **req) { - async_task_t * async_task = NULL; + async_task_t *async_task = NULL; async_link_copy_args_t *args = NULL; bool lock_parent = false; bool is_blocking = false; hbool_t acquired = false; unsigned int mutex_count = 1; - H5VL_async_t * parent_obj = parent_obj1 ? parent_obj1 : parent_obj2; + H5VL_async_t *parent_obj = parent_obj1 ? parent_obj1 : parent_obj2; func_enter(__func__, NULL); @@ -18107,8 +18069,8 @@ async_link_move_fn(void *foo) int attempt_count = 0; int is_lock = 0; hbool_t is_lib_state_restored = false; - ABT_pool * pool_ptr; - async_task_t * task = (async_task_t *)foo; + ABT_pool *pool_ptr; + async_task_t *task = (async_task_t *)foo; async_link_move_args_t *args = (async_link_move_args_t *)(task->args); herr_t status; @@ -18255,13 +18217,13 @@ async_link_move(async_instance_t *aid, H5VL_async_t *parent_obj1, const H5VL_loc H5VL_async_t *parent_obj2, const H5VL_loc_params_t *loc_params2, hid_t lcpl_id, hid_t lapl_id, hid_t dxpl_id, void **req) { - async_task_t * async_task = NULL; + async_task_t *async_task = NULL; async_link_move_args_t *args = NULL; bool lock_parent = false; bool is_blocking = false; hbool_t acquired = false; unsigned int mutex_count = 1; - H5VL_async_t * parent_obj = parent_obj1 ? parent_obj1 : parent_obj2; + H5VL_async_t *parent_obj = parent_obj1 ? parent_obj1 : parent_obj2; func_enter(__func__, NULL); @@ -18456,8 +18418,8 @@ async_link_get_fn(void *foo) int attempt_count = 0; int is_lock = 0; hbool_t is_lib_state_restored = false; - ABT_pool * pool_ptr; - async_task_t * task = (async_task_t *)foo; + ABT_pool *pool_ptr; + async_task_t *task = (async_task_t *)foo; async_link_get_args_t *args = (async_link_get_args_t *)(task->args); herr_t status; @@ -18598,7 +18560,7 @@ static herr_t async_link_get(task_list_qtype qtype, async_instance_t *aid, H5VL_async_t *parent_obj, const H5VL_loc_params_t *loc_params, H5VL_link_get_args_t *get_args, hid_t dxpl_id, void **req) { - async_task_t * async_task = NULL; + async_task_t *async_task = NULL; async_link_get_args_t *args = NULL; bool lock_parent = false; bool is_blocking = false; @@ -18780,8 +18742,8 @@ async_link_specific_fn(void *foo) int attempt_count = 0; int is_lock = 0; hbool_t is_lib_state_restored = false; - ABT_pool * pool_ptr; - async_task_t * task = (async_task_t *)foo; + ABT_pool *pool_ptr; + async_task_t *task = (async_task_t *)foo; async_link_specific_args_t *args = (async_link_specific_args_t *)(task->args); herr_t status; @@ -18926,7 +18888,7 @@ async_link_specific(task_list_qtype qtype, async_instance_t *aid, H5VL_async_t * const H5VL_loc_params_t *loc_params, H5VL_link_specific_args_t *spec_args, hid_t dxpl_id, void **req) { - async_task_t * async_task = NULL; + async_task_t *async_task = NULL; async_link_specific_args_t *args = NULL; bool lock_parent = false; bool is_blocking = false; @@ -19108,8 +19070,8 @@ async_link_optional_fn(void *foo) int attempt_count = 0; int is_lock = 0; hbool_t is_lib_state_restored = false; - ABT_pool * pool_ptr; - async_task_t * task = (async_task_t *)foo; + ABT_pool *pool_ptr; + async_task_t *task = (async_task_t *)foo; async_link_optional_args_t *args = (async_link_optional_args_t *)(task->args); herr_t status; @@ -19251,7 +19213,7 @@ async_link_optional(task_list_qtype qtype, async_instance_t *aid, H5VL_async_t * const H5VL_loc_params_t *loc_params, H5VL_optional_args_t *opt_args, hid_t dxpl_id, void **req) { - async_task_t * async_task = NULL; + async_task_t *async_task = NULL; async_link_optional_args_t *args = NULL; bool lock_parent = false; bool is_blocking = false; @@ -19433,10 +19395,10 @@ async_object_open_fn(void *foo) int attempt_count = 0; int is_lock = 0; hbool_t is_lib_state_restored = false; - ABT_pool * pool_ptr; - async_task_t * task = (async_task_t *)foo; + ABT_pool *pool_ptr; + async_task_t *task = (async_task_t *)foo; async_object_open_args_t *args = (async_object_open_args_t *)(task->args); - void * obj; + void *obj; #ifdef ENABLE_TIMING task->start_time = clock(); @@ -19578,8 +19540,8 @@ static H5VL_async_t * async_object_open(task_list_qtype qtype, async_instance_t *aid, H5VL_async_t *parent_obj, const H5VL_loc_params_t *loc_params, H5I_type_t *opened_type, hid_t dxpl_id, void **req) { - H5VL_async_t * async_obj = NULL; - async_task_t * async_task = NULL; + H5VL_async_t *async_obj = NULL; + async_task_t *async_task = NULL; async_object_open_args_t *args = NULL; bool lock_parent = false; bool is_blocking = false; @@ -19778,8 +19740,8 @@ async_object_copy_fn(void *foo) int attempt_count = 0; int is_lock = 0; hbool_t is_lib_state_restored = false; - ABT_pool * pool_ptr; - async_task_t * task = (async_task_t *)foo; + ABT_pool *pool_ptr; + async_task_t *task = (async_task_t *)foo; async_object_copy_args_t *args = (async_object_copy_args_t *)(task->args); herr_t status; @@ -19937,13 +19899,13 @@ async_object_copy(task_list_qtype qtype, async_instance_t *aid, H5VL_async_t *pa const H5VL_loc_params_t *dst_loc_params, const char *dst_name, hid_t ocpypl_id, hid_t lcpl_id, hid_t dxpl_id, void **req) { - async_task_t * async_task = NULL; + async_task_t *async_task = NULL; async_object_copy_args_t *args = NULL; bool lock_parent = false; bool is_blocking = false; hbool_t acquired = false; unsigned int mutex_count = 1; - H5VL_async_t * parent_obj = parent_obj1 ? parent_obj1 : parent_obj2; + H5VL_async_t *parent_obj = parent_obj1 ? parent_obj1 : parent_obj2; func_enter(__func__, NULL); @@ -20132,8 +20094,8 @@ async_object_get_fn(void *foo) int attempt_count = 0; int is_lock = 0; hbool_t is_lib_state_restored = false; - ABT_pool * pool_ptr; - async_task_t * task = (async_task_t *)foo; + ABT_pool *pool_ptr; + async_task_t *task = (async_task_t *)foo; async_object_get_args_t *args = (async_object_get_args_t *)(task->args); herr_t status; @@ -20278,7 +20240,7 @@ async_object_get(task_list_qtype qtype, async_instance_t *aid, H5VL_async_t *par const H5VL_loc_params_t *loc_params, H5VL_object_get_args_t *get_args, hid_t dxpl_id, void **req) { - async_task_t * async_task = NULL; + async_task_t *async_task = NULL; async_object_get_args_t *args = NULL; bool lock_parent = false; bool is_blocking = false; @@ -20465,8 +20427,8 @@ async_object_specific_fn(void *foo) int attempt_count = 0; int is_lock = 0; hbool_t is_lib_state_restored = false; - ABT_pool * pool_ptr; - async_task_t * task = (async_task_t *)foo; + ABT_pool *pool_ptr; + async_task_t *task = (async_task_t *)foo; async_object_specific_args_t *args = (async_object_specific_args_t *)(task->args); herr_t status; @@ -20610,7 +20572,7 @@ async_object_specific(task_list_qtype qtype, async_instance_t *aid, H5VL_async_t const H5VL_loc_params_t *loc_params, H5VL_object_specific_args_t *spec_args, hid_t dxpl_id, void **req) { - async_task_t * async_task = NULL; + async_task_t *async_task = NULL; async_object_specific_args_t *args = NULL; bool lock_parent = false; bool is_blocking = false; @@ -20798,8 +20760,8 @@ async_object_optional_fn(void *foo) int attempt_count = 0; int is_lock = 0; hbool_t is_lib_state_restored = false; - ABT_pool * pool_ptr; - async_task_t * task = (async_task_t *)foo; + ABT_pool *pool_ptr; + async_task_t *task = (async_task_t *)foo; async_object_optional_args_t *args = (async_object_optional_args_t *)(task->args); herr_t status; @@ -20941,7 +20903,7 @@ async_object_optional(task_list_qtype qtype, async_instance_t *aid, H5VL_async_t const H5VL_loc_params_t *loc_params, H5VL_optional_args_t *opt_args, hid_t dxpl_id, void **req) { - async_task_t * async_task = NULL; + async_task_t *async_task = NULL; async_object_optional_args_t *args = NULL; bool lock_parent = false; bool is_blocking = false; @@ -21205,7 +21167,7 @@ static void * H5VL_async_info_copy(const void *_info) { const H5VL_async_info_t *info = (const H5VL_async_info_t *)_info; - H5VL_async_info_t * new_info; + H5VL_async_info_t *new_info; #ifdef ENABLE_ASYNC_LOGGING printf("------- ASYNC VOL INFO Copy\n"); @@ -21317,7 +21279,7 @@ H5VL_async_info_to_str(const void *_info, char **str) { const H5VL_async_info_t *info = (const H5VL_async_info_t *)_info; H5VL_class_value_t under_value = (H5VL_class_value_t)-1; - char * under_vol_string = NULL; + char *under_vol_string = NULL; size_t under_vol_str_len = 0; #ifdef ENABLE_ASYNC_LOGGING @@ -21362,9 +21324,9 @@ H5VL_async_str_to_info(const char *str, void **_info) { H5VL_async_info_t *info; unsigned under_vol_value; - const char * under_vol_info_start, *under_vol_info_end; + const char *under_vol_info_start, *under_vol_info_end; hid_t under_vol_id; - void * under_vol_info = NULL; + void *under_vol_info = NULL; #ifdef ENABLE_ASYNC_LOGGING printf("------- ASYNC VOL INFO String To Info\n"); @@ -21441,9 +21403,9 @@ H5VL_async_get_object(const void *obj) static herr_t H5VL_async_get_wrap_ctx(const void *obj, void **wrap_ctx) { - const H5VL_async_t * o_async = (const H5VL_async_t *)obj; + const H5VL_async_t *o_async = (const H5VL_async_t *)obj; hid_t under_vol_id = 0; - void * under_object = NULL; + void *under_object = NULL; H5VL_async_wrap_ctx_t *new_wrap_ctx; #ifdef ENABLE_ASYNC_LOGGING @@ -21496,8 +21458,8 @@ static void * H5VL_async_wrap_object(void *obj, H5I_type_t obj_type, void *_wrap_ctx) { H5VL_async_wrap_ctx_t *wrap_ctx = (H5VL_async_wrap_ctx_t *)_wrap_ctx; - H5VL_async_t * new_obj; - void * under; + H5VL_async_t *new_obj; + void *under; #ifdef ENABLE_ASYNC_LOGGING printf("------- ASYNC VOL WRAP Object\n"); @@ -21530,7 +21492,7 @@ static void * H5VL_async_unwrap_object(void *obj) { H5VL_async_t *o = (H5VL_async_t *)obj; - void * under; + void *under; #ifdef ENABLE_ASYNC_LOGGING printf("------- ASYNC VOL UNWRAP Object\n"); @@ -21599,7 +21561,7 @@ H5VL_async_attr_create(void *obj, const H5VL_loc_params_t *loc_params, const cha { H5VL_async_t *attr; H5VL_async_t *o = (H5VL_async_t *)obj; - void * under; + void *under; #ifdef ENABLE_ASYNC_LOGGING printf("------- ASYNC VOL ATTRIBUTE Create\n"); @@ -21643,7 +21605,7 @@ H5VL_async_attr_open(void *obj, const H5VL_loc_params_t *loc_params, const char { H5VL_async_t *attr; H5VL_async_t *o = (H5VL_async_t *)obj; - void * under; + void *under; #ifdef ENABLE_ASYNC_LOGGING printf("------- ASYNC VOL ATTRIBUTE Open\n"); @@ -21758,7 +21720,7 @@ H5VL_async_attr_write(void *attr, hid_t mem_type_id, const void *buf, hid_t dxpl static herr_t H5VL_async_attr_get(void *obj, H5VL_attr_get_args_t *args, hid_t dxpl_id, void **req) { - H5VL_async_t * o = (H5VL_async_t *)obj; + H5VL_async_t *o = (H5VL_async_t *)obj; herr_t ret_value; task_list_qtype qtype = ISOLATED; @@ -21797,7 +21759,7 @@ static herr_t H5VL_async_attr_specific(void *obj, const H5VL_loc_params_t *loc_params, H5VL_attr_specific_args_t *args, hid_t dxpl_id, void **req) { - H5VL_async_t * o = (H5VL_async_t *)obj; + H5VL_async_t *o = (H5VL_async_t *)obj; herr_t ret_value; task_list_qtype qtype = ISOLATED; @@ -21838,7 +21800,7 @@ H5VL_async_attr_specific(void *obj, const H5VL_loc_params_t *loc_params, H5VL_at static herr_t H5VL_async_attr_optional(void *obj, H5VL_optional_args_t *args, hid_t dxpl_id, void **req) { - H5VL_async_t * o = (H5VL_async_t *)obj; + H5VL_async_t *o = (H5VL_async_t *)obj; herr_t ret_value; task_list_qtype qtype = ISOLATED; @@ -21882,7 +21844,7 @@ H5VL_async_attr_optional(void *obj, H5VL_optional_args_t *args, hid_t dxpl_id, v static herr_t H5VL_async_attr_close(void *attr, hid_t dxpl_id, void **req) { - H5VL_async_t * o = (H5VL_async_t *)attr; + H5VL_async_t *o = (H5VL_async_t *)attr; herr_t ret_value; hbool_t is_term; task_list_qtype qtype = DEPENDENT; @@ -21936,7 +21898,7 @@ H5VL_async_dataset_create(void *obj, const H5VL_loc_params_t *loc_params, const { H5VL_async_t *dset; H5VL_async_t *o = (H5VL_async_t *)obj; - void * under; + void *under; #ifdef ENABLE_ASYNC_LOGGING printf("------- ASYNC VOL DATASET Create\n"); @@ -21984,10 +21946,10 @@ static void * H5VL_async_dataset_open(void *obj, const H5VL_loc_params_t *loc_params, const char *name, hid_t dapl_id, hid_t dxpl_id, void **req) { - H5VL_async_t * dset; - H5VL_async_t * o = (H5VL_async_t *)obj; + H5VL_async_t *dset; + H5VL_async_t *o = (H5VL_async_t *)obj; task_list_qtype qtype = REGULAR; - void * under; + void *under; #ifdef ENABLE_ASYNC_LOGGING printf("------- ASYNC VOL DATASET Open\n"); @@ -22111,7 +22073,7 @@ H5VL_async_dataset_write(void *dset, hid_t mem_type_id, hid_t mem_space_id, hid_ static herr_t H5VL_async_dataset_get(void *dset, H5VL_dataset_get_args_t *args, hid_t dxpl_id, void **req) { - H5VL_async_t * o = (H5VL_async_t *)dset; + H5VL_async_t *o = (H5VL_async_t *)dset; herr_t ret_value; task_list_qtype qtype = ISOLATED; @@ -22149,7 +22111,7 @@ H5VL_async_dataset_get(void *dset, H5VL_dataset_get_args_t *args, hid_t dxpl_id, static herr_t H5VL_async_dataset_specific(void *obj, H5VL_dataset_specific_args_t *args, hid_t dxpl_id, void **req) { - H5VL_async_t * o = (H5VL_async_t *)obj; + H5VL_async_t *o = (H5VL_async_t *)obj; hid_t under_vol_id; herr_t ret_value; task_list_qtype qtype = ISOLATED; @@ -22195,7 +22157,7 @@ H5VL_async_dataset_specific(void *obj, H5VL_dataset_specific_args_t *args, hid_t static herr_t H5VL_async_dataset_optional(void *obj, H5VL_optional_args_t *args, hid_t dxpl_id, void **req) { - H5VL_async_t * o = (H5VL_async_t *)obj; + H5VL_async_t *o = (H5VL_async_t *)obj; herr_t ret_value; task_list_qtype qtype = ISOLATED; @@ -22253,7 +22215,7 @@ H5VL_async_dataset_optional(void *obj, H5VL_optional_args_t *args, hid_t dxpl_id static herr_t H5VL_async_dataset_close(void *dset, hid_t dxpl_id, void **req) { - H5VL_async_t * o = (H5VL_async_t *)dset; + H5VL_async_t *o = (H5VL_async_t *)dset; herr_t ret_value; hbool_t is_term; task_list_qtype qtype = REGULAR; @@ -22310,7 +22272,7 @@ H5VL_async_datatype_commit(void *obj, const H5VL_loc_params_t *loc_params, const { H5VL_async_t *dt; H5VL_async_t *o = (H5VL_async_t *)obj; - void * under; + void *under; #ifdef ENABLE_ASYNC_LOGGING printf("------- ASYNC VOL DATATYPE Commit\n"); @@ -22355,7 +22317,7 @@ H5VL_async_datatype_open(void *obj, const H5VL_loc_params_t *loc_params, const c { H5VL_async_t *dt; H5VL_async_t *o = (H5VL_async_t *)obj; - void * under; + void *under; #ifdef ENABLE_ASYNC_LOGGING printf("------- ASYNC VOL DATATYPE Open\n"); @@ -22395,7 +22357,7 @@ H5VL_async_datatype_open(void *obj, const H5VL_loc_params_t *loc_params, const c static herr_t H5VL_async_datatype_get(void *dt, H5VL_datatype_get_args_t *args, hid_t dxpl_id, void **req) { - H5VL_async_t * o = (H5VL_async_t *)dt; + H5VL_async_t *o = (H5VL_async_t *)dt; herr_t ret_value; task_list_qtype qtype = ISOLATED; @@ -22433,7 +22395,7 @@ H5VL_async_datatype_get(void *dt, H5VL_datatype_get_args_t *args, hid_t dxpl_id, static herr_t H5VL_async_datatype_specific(void *obj, H5VL_datatype_specific_args_t *args, hid_t dxpl_id, void **req) { - H5VL_async_t * o = (H5VL_async_t *)obj; + H5VL_async_t *o = (H5VL_async_t *)obj; herr_t ret_value; task_list_qtype qtype = ISOLATED; hid_t under_vol_id; @@ -22476,7 +22438,7 @@ H5VL_async_datatype_specific(void *obj, H5VL_datatype_specific_args_t *args, hid static herr_t H5VL_async_datatype_optional(void *obj, H5VL_optional_args_t *args, hid_t dxpl_id, void **req) { - H5VL_async_t * o = (H5VL_async_t *)obj; + H5VL_async_t *o = (H5VL_async_t *)obj; herr_t ret_value; task_list_qtype qtype = ISOLATED; @@ -22520,7 +22482,7 @@ H5VL_async_datatype_optional(void *obj, H5VL_optional_args_t *args, hid_t dxpl_i static herr_t H5VL_async_datatype_close(void *dt, hid_t dxpl_id, void **req) { - H5VL_async_t * o = (H5VL_async_t *)dt; + H5VL_async_t *o = (H5VL_async_t *)dt; herr_t ret_value; hbool_t is_term; task_list_qtype qtype = DEPENDENT; @@ -22574,9 +22536,9 @@ H5VL_async_file_create(const char *name, unsigned flags, hid_t fcpl_id, hid_t fa void **req) { H5VL_async_info_t *info; - H5VL_async_t * file; + H5VL_async_t *file; hid_t under_fapl_id; - void * under; + void *under; #ifdef ENABLE_ASYNC_LOGGING printf("------- ASYNC VOL FILE Create\n"); @@ -22672,10 +22634,10 @@ static void * H5VL_async_file_open(const char *name, unsigned flags, hid_t fapl_id, hid_t dxpl_id, void **req) { H5VL_async_info_t *info; - H5VL_async_t * file; + H5VL_async_t *file; hid_t under_fapl_id; task_list_qtype qtype = REGULAR; - void * under; + void *under; #ifdef ENABLE_ASYNC_LOGGING printf("------- ASYNC VOL FILE Open\n"); @@ -22771,7 +22733,7 @@ H5VL_async_file_open(const char *name, unsigned flags, hid_t fapl_id, hid_t dxpl static herr_t H5VL_async_file_get(void *file, H5VL_file_get_args_t *args, hid_t dxpl_id, void **req) { - H5VL_async_t * o = (H5VL_async_t *)file; + H5VL_async_t *o = (H5VL_async_t *)file; herr_t ret_value; task_list_qtype qtype = ISOLATED; @@ -22809,7 +22771,7 @@ H5VL_async_file_get(void *file, H5VL_file_get_args_t *args, hid_t dxpl_id, void static herr_t H5VL_async_file_specific(void *file, H5VL_file_specific_args_t *args, hid_t dxpl_id, void **req) { - H5VL_async_t * o = (H5VL_async_t *)file; + H5VL_async_t *o = (H5VL_async_t *)file; herr_t ret_value; task_list_qtype qtype = ISOLATED; @@ -22828,7 +22790,7 @@ H5VL_async_file_specific(void *file, H5VL_file_specific_args_t *args, hid_t dxpl /* Check for 'is accessible' operation */ if (args->op_type == H5VL_FILE_IS_ACCESSIBLE || args->op_type == H5VL_FILE_DELETE) { H5VL_file_specific_args_t my_args; - H5VL_async_info_t * info; + H5VL_async_info_t *info; hid_t new_fapl_id; /* Don't currently support asynchronous execution of these operations */ @@ -22912,7 +22874,7 @@ H5VL_async_file_specific(void *file, H5VL_file_specific_args_t *args, hid_t dxpl static herr_t H5VL_async_file_optional(void *file, H5VL_optional_args_t *args, hid_t dxpl_id, void **req) { - H5VL_async_t * o = (H5VL_async_t *)file; + H5VL_async_t *o = (H5VL_async_t *)file; herr_t ret_value; task_list_qtype qtype = REGULAR; @@ -22970,7 +22932,7 @@ H5VL_async_file_optional(void *file, H5VL_optional_args_t *args, hid_t dxpl_id, static herr_t H5VL_async_file_close(void *file, hid_t dxpl_id, void **req) { - H5VL_async_t * o = (H5VL_async_t *)file; + H5VL_async_t *o = (H5VL_async_t *)file; herr_t ret_value; hbool_t is_term; task_list_qtype qtype = REGULAR; @@ -23023,7 +22985,7 @@ H5VL_async_group_create(void *obj, const H5VL_loc_params_t *loc_params, const ch { H5VL_async_t *group; H5VL_async_t *o = (H5VL_async_t *)obj; - void * under; + void *under; #ifdef ENABLE_ASYNC_LOGGING printf("------- ASYNC VOL GROUP Create\n"); @@ -23068,7 +23030,7 @@ H5VL_async_group_open(void *obj, const H5VL_loc_params_t *loc_params, const char { H5VL_async_t *group; H5VL_async_t *o = (H5VL_async_t *)obj; - void * under; + void *under; #ifdef ENABLE_ASYNC_LOGGING printf("------- ASYNC VOL GROUP Open\n"); @@ -23108,7 +23070,7 @@ H5VL_async_group_open(void *obj, const H5VL_loc_params_t *loc_params, const char static herr_t H5VL_async_group_get(void *obj, H5VL_group_get_args_t *args, hid_t dxpl_id, void **req) { - H5VL_async_t * o = (H5VL_async_t *)obj; + H5VL_async_t *o = (H5VL_async_t *)obj; herr_t ret_value; task_list_qtype qtype = ISOLATED; @@ -23149,7 +23111,7 @@ H5VL_async_group_get(void *obj, H5VL_group_get_args_t *args, hid_t dxpl_id, void static herr_t H5VL_async_group_specific(void *obj, H5VL_group_specific_args_t *args, hid_t dxpl_id, void **req) { - H5VL_async_t * o = (H5VL_async_t *)obj; + H5VL_async_t *o = (H5VL_async_t *)obj; H5VL_group_specific_args_t my_args; H5VL_group_specific_args_t *new_args; herr_t ret_value; @@ -23209,7 +23171,7 @@ H5VL_async_group_specific(void *obj, H5VL_group_specific_args_t *args, hid_t dxp static herr_t H5VL_async_group_optional(void *obj, H5VL_optional_args_t *args, hid_t dxpl_id, void **req) { - H5VL_async_t * o = (H5VL_async_t *)obj; + H5VL_async_t *o = (H5VL_async_t *)obj; herr_t ret_value; task_list_qtype qtype = ISOLATED; @@ -23253,7 +23215,7 @@ H5VL_async_group_optional(void *obj, H5VL_optional_args_t *args, hid_t dxpl_id, static herr_t H5VL_async_group_close(void *grp, hid_t dxpl_id, void **req) { - H5VL_async_t * o = (H5VL_async_t *)grp; + H5VL_async_t *o = (H5VL_async_t *)grp; herr_t ret_value; hbool_t is_term; task_list_qtype qtype = REGULAR; @@ -23304,7 +23266,7 @@ static herr_t H5VL_async_link_create(H5VL_link_create_args_t *args, void *obj, const H5VL_loc_params_t *loc_params, hid_t lcpl_id, hid_t lapl_id, hid_t dxpl_id, void **req) { - H5VL_async_t * o = (H5VL_async_t *)obj; + H5VL_async_t *o = (H5VL_async_t *)obj; herr_t ret_value; task_list_qtype qtype = ISOLATED; @@ -23518,7 +23480,7 @@ static herr_t H5VL_async_link_get(void *obj, const H5VL_loc_params_t *loc_params, H5VL_link_get_args_t *args, hid_t dxpl_id, void **req) { - H5VL_async_t * o = (H5VL_async_t *)obj; + H5VL_async_t *o = (H5VL_async_t *)obj; herr_t ret_value; task_list_qtype qtype = ISOLATED; @@ -23557,7 +23519,7 @@ static herr_t H5VL_async_link_specific(void *obj, const H5VL_loc_params_t *loc_params, H5VL_link_specific_args_t *args, hid_t dxpl_id, void **req) { - H5VL_async_t * o = (H5VL_async_t *)obj; + H5VL_async_t *o = (H5VL_async_t *)obj; herr_t ret_value; task_list_qtype qtype = ISOLATED; @@ -23596,7 +23558,7 @@ static herr_t H5VL_async_link_optional(void *obj, const H5VL_loc_params_t *loc_params, H5VL_optional_args_t *args, hid_t dxpl_id, void **req) { - H5VL_async_t * o = (H5VL_async_t *)obj; + H5VL_async_t *o = (H5VL_async_t *)obj; herr_t ret_value; task_list_qtype qtype = ISOLATED; @@ -23641,10 +23603,10 @@ static void * H5VL_async_object_open(void *obj, const H5VL_loc_params_t *loc_params, H5I_type_t *opened_type, hid_t dxpl_id, void **req) { - H5VL_async_t * new_obj; - H5VL_async_t * o = (H5VL_async_t *)obj; + H5VL_async_t *new_obj; + H5VL_async_t *o = (H5VL_async_t *)obj; task_list_qtype qtype = BLOCKING; - void * under; + void *under; #ifdef ENABLE_ASYNC_LOGGING printf("------- ASYNC VOL OBJECT Open\n"); @@ -23688,8 +23650,8 @@ H5VL_async_object_copy(void *src_obj, const H5VL_loc_params_t *src_loc_params, c void *dst_obj, const H5VL_loc_params_t *dst_loc_params, const char *dst_name, hid_t ocpypl_id, hid_t lcpl_id, hid_t dxpl_id, void **req) { - H5VL_async_t * o_src = (H5VL_async_t *)src_obj; - H5VL_async_t * o_dst = (H5VL_async_t *)dst_obj; + H5VL_async_t *o_src = (H5VL_async_t *)src_obj; + H5VL_async_t *o_dst = (H5VL_async_t *)dst_obj; herr_t ret_value; task_list_qtype qtype = ISOLATED; @@ -23731,7 +23693,7 @@ static herr_t H5VL_async_object_get(void *obj, const H5VL_loc_params_t *loc_params, H5VL_object_get_args_t *args, hid_t dxpl_id, void **req) { - H5VL_async_t * o = (H5VL_async_t *)obj; + H5VL_async_t *o = (H5VL_async_t *)obj; herr_t ret_value; task_list_qtype qtype = ISOLATED; @@ -23770,7 +23732,7 @@ static herr_t H5VL_async_object_specific(void *obj, const H5VL_loc_params_t *loc_params, H5VL_object_specific_args_t *args, hid_t dxpl_id, void **req) { - H5VL_async_t * o = (H5VL_async_t *)obj; + H5VL_async_t *o = (H5VL_async_t *)obj; herr_t ret_value; task_list_qtype qtype = ISOLATED; hid_t under_vol_id; @@ -23816,7 +23778,7 @@ static herr_t H5VL_async_object_optional(void *obj, const H5VL_loc_params_t *loc_params, H5VL_optional_args_t *args, hid_t dxpl_id, void **req) { - H5VL_async_t * o = (H5VL_async_t *)obj; + H5VL_async_t *o = (H5VL_async_t *)obj; herr_t ret_value; task_list_qtype qtype = ISOLATED; @@ -23960,8 +23922,8 @@ H5VL_async_request_wait(void *obj, uint64_t timeout, H5VL_request_status_t *stat herr_t ret_value = 0; clock_t start_time, now_time; double elapsed, trigger; - H5VL_async_t * request; - async_task_t * task; + H5VL_async_t *request; + async_task_t *task; ABT_thread_state state; hbool_t acquired = false; unsigned int mutex_count = 1; diff --git a/test/1d_asyn_MPI_write.c b/test/1d_asyn_MPI_write.c index 07ccaa4..29d064a 100644 --- a/test/1d_asyn_MPI_write.c +++ b/test/1d_asyn_MPI_write.c @@ -106,7 +106,7 @@ main(int argc, char **argv) dimsm[0] = X1; memspace = H5Screate_simple(RANK_OUT, dimsm, NULL); // RANK_OUT=3 dataset = H5Dcreate_async(file, DATASETNAME, H5T_STD_I32BE, dataspace, H5P_DEFAULT, H5P_DEFAULT, - H5P_DEFAULT, es_id); + H5P_DEFAULT, es_id); int array[][4][2] = {{{0, 5}, {5, 3}, @@ -136,20 +136,13 @@ main(int argc, char **argv) {11, 10}, // testcase 7 {5, 3}, {8, 3}}}; - /* int array[][4][2]={ - {{0,5}, - {5,3}, - {8,2}, //testcase 1 - {10,10} - } - }; */ int testcases = (int)sizeof(array) / sizeof(array[0]); // fprintf(stderr, "%d",testcases); // testcases=7; hid_t dataset_array[testcases]; char datasetname[testcases][12]; - // int arr[4][2]; + // int arr[4][2]; for (int j = 0; j < testcases; j++) { @@ -188,8 +181,6 @@ main(int argc, char **argv) data_transfer_propertylist, data, es_id); } } - - } if (mpi_rank == 1) { @@ -230,8 +221,6 @@ main(int argc, char **argv) // data_out2,es_id); status = H5Dwrite_async(dataset, H5T_NATIVE_INT, memspace, dataspace, data_transfer_propertylist, data, es_id); - - } if (print_dbg_msg) diff --git a/test/2d_asyn_MPI_write.c b/test/2d_asyn_MPI_write.c index fcd7eb1..b0927f7 100644 --- a/test/2d_asyn_MPI_write.c +++ b/test/2d_asyn_MPI_write.c @@ -121,67 +121,45 @@ main(int argc, char **argv) memspace = H5Screate_simple(RANK_OUT, dimsm, NULL); // RANK_OUT=3 - /* int array[][4][4]={ - { - {0,0,2,2}, - {0,2,2,1}, - {0,3,2,2}, //Testcase# 1 - {0,5,2,1} - }, - { - {0,0,2,2}, - {2,0,1,2}, - {0,3,2,2}, //Testcase# 2 - {0,5,2,1} - }, - { - {0,0,2,1}, - {0,1,2,2}, - {0,3,2,1}, //Testcase# 3 - {2,1,2,2} - }, - { - {0,1,2,3}, - {0,4,2,2}, - {2,0,2,1}, //Testcase# 4 - {2,1,2,3} - }, - { - {1,1,2,3}, - {1,4,2,1}, - {3,1,1,3}, //Testcase# 5 - {3,4,1,1} - }, - { - {1,1,2,3}, - {3,1,1,3}, - {3,4,1,1},//Testcase# 6 - {1,4,2,1}, - }, - { - {3,1,1,3}, - {1,1,2,3}, - {3,4,1,1},//Testcase# 7 - {1,4,2,1}, - }, - { - {3,1,1,3}, - {1,1,2,3}, - {1,5,2,1},//Testcase# 8 - {1,4,2,1}, - } - }; */ - int array[][4][4] = { - {{1, 1, 2, 3}, - {1, 4, 2, 1}, - {3, 1, 1, 3}, // Testcase# 5 - {3, 4, 1, 1}}, - {{1, 1, 2, 3}, - {1, 4, 2, 1}, - {3, 1, 1, 3}, // Testcase# 5 - {3, 4, 1, 1}}, - - }; + int array[][4][4] = {{{0, 0, 2, 2}, + {0, 2, 2, 1}, + {0, 3, 2, 2}, // Testcase# 1 + {0, 5, 2, 1}}, + {{0, 0, 2, 2}, + {2, 0, 1, 2}, + {0, 3, 2, 2}, // Testcase# 2 + {0, 5, 2, 1}}, + {{0, 0, 2, 1}, + {0, 1, 2, 2}, + {0, 3, 2, 1}, // Testcase# 3 + {2, 1, 2, 2}}, + {{0, 1, 2, 3}, + {0, 4, 2, 2}, + {2, 0, 2, 1}, // Testcase# 4 + {2, 1, 2, 3}}, + {{1, 1, 2, 3}, + {1, 4, 2, 1}, + {3, 1, 1, 3}, // Testcase# 5 + {3, 4, 1, 1}}, + { + {1, 1, 2, 3}, + {3, 1, 1, 3}, + {3, 4, 1, 1}, // Testcase# 6 + {1, 4, 2, 1}, + }, + { + {3, 1, 1, 3}, + {1, 1, 2, 3}, + {3, 4, 1, 1}, // Testcase# 7 + {1, 4, 2, 1}, + }, + { + {3, 1, 1, 3}, + {1, 1, 2, 3}, + {1, 5, 2, 1}, // Testcase# 8 + {1, 4, 2, 1}, + }}; + int testcases = (int)sizeof(array) / sizeof(array[0]); hid_t dataset_array[testcases]; char datasetname[testcases][12]; @@ -230,8 +208,6 @@ main(int argc, char **argv) } */ } - - } if (mpi_rank == 1) { diff --git a/test/3d_asyn_MPI_write.c b/test/3d_asyn_MPI_write.c index a21812b..e84f46d 100644 --- a/test/3d_asyn_MPI_write.c +++ b/test/3d_asyn_MPI_write.c @@ -143,7 +143,6 @@ main(int argc, char **argv) memspace = H5Screate_simple(RANK_OUT, dimsm, NULL); // RANK_OUT=3 - int array[][4][6] = { {{0, 0, 0, 1, 2, 2}, @@ -246,8 +245,7 @@ main(int argc, char **argv) } */ } - - + } status = H5ESwait(es_id, H5ES_WAIT_FOREVER, &num_in_progress, &op_failed); if (status < 0) { @@ -331,4 +329,3 @@ main(int argc, char **argv) return 0; } - \ No newline at end of file diff --git a/test/Makefile b/test/Makefile index 5749335..6e2dae0 100644 --- a/test/Makefile +++ b/test/Makefile @@ -38,7 +38,7 @@ SRC = async_test_serial.c async_test_serial2.c \ async_test_parallel4.c async_test_parallel5.c \ async_test_serial_event_set.c async_test_serial_error_stack.c \ async_test_serial_event_set_error_stack.c 1d_asyn_MPI_write.c 2d_asyn_MPI_write.c \ - 3d_asyn_MPI_write.c 1d_async_write_performance + 3d_asyn_MPI_write.c 1d_async_write_performance.c #async_test_no_fclose.c OBJ = $(SRC:.c=.o) From e524cb7647c3acfc8f762befc072e47d72495406 Mon Sep 17 00:00:00 2001 From: kamalchowdhurylbl Date: Fri, 12 Aug 2022 16:53:18 -0500 Subject: [PATCH 18/18] added async vol.c with merge, and makefile --- src/Makefile | 2 +- src/h5_async_vol.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Makefile b/src/Makefile index b0d0ddf..358bba3 100644 --- a/src/Makefile +++ b/src/Makefile @@ -15,7 +15,7 @@ ABT_LIBDIR = $(shell if [ -d $$ABT_DIR/lib ]; then echo "lib"; else echo "lib64 # These lines may need to be adjusted for your compiler environment: DEBUG = -g -O0 -CFLAGS = $(DEBUG) -fPIC -I$(ABT_DIR)/include -I$(HDF5_DIR)/include -Wall +CFLAGS = $(DEBUG) -fPIC -I$(ABT_DIR)/include -I$(HDF5_DIR)/include -Wall -DENABLE_WRITE_MERGE LIBS = -L$(HDF5_DIR)/$(HDF5_LIBDIR) -L$(ABT_DIR)/$(ABT_LIBDIR) -labt -lhdf5 -lz ARFLAGS = rs diff --git a/src/h5_async_vol.c b/src/h5_async_vol.c index 6680aa7..1d7129b 100644 --- a/src/h5_async_vol.c +++ b/src/h5_async_vol.c @@ -206,8 +206,8 @@ typedef struct async_instance_t { bool pause; /* Pause background thread execution */ bool disable_implicit_file; /* Disable implicit async execution globally */ bool disable_implicit; /* Disable implicit async execution for dxpl */ - bool disable_async_dset_get; /* Disable async execution for dataset get */ bool delay_time_env; /* Flag that indicates the delay time is set by env variable */ + bool disable_async_dset_get; /* Disable async execution for dataset get */ uint64_t delay_time; /* Sleep time before background thread trying to acquire global mutex */ int sleep_time; /* Sleep time between checking the global mutex attemp count */ #ifdef ENABLE_WRITE_MEMCPY