diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index f1c4c53..a030ab5 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -51,6 +51,8 @@ Bug fix: + Fix compression threshold calculation. See a9ea841 Other changes + + Remove the HDF5 metadata memory footprint profiling, as H5get_alloc_stats + has been removed from HDF5 since 1.14.0. See PR #5. + Remove the requirement of group `/spill`. See d4e0852 + Change chunk setting for small datasets. Set the chunk size to 256K element, no matter if the dataset dim[0] is smaller than 256K element or diff --git a/configure.ac b/configure.ac index cbc22a0..47b632c 100644 --- a/configure.ac +++ b/configure.ac @@ -354,9 +354,6 @@ AC_CHECK_DECL(H5Ovisit3, AC_CHECK_DECL(H5Treclaim, [AC_DEFINE(HAS_H5TRECLAIM, 1, [Define if API H5Treclaim is defined])], [], [[#include ]]) -AC_CHECK_DECL(H5get_alloc_stats, - [AC_DEFINE(HAS_H5GET_ALLOC_STATS, 1, [Define if API H5get_alloc_stats is defined])], - [], [[#include ]]) AC_ARG_VAR(TESTMPIRUN, [MPI run command for "make check", @<:@default: mpiexec@:>@]) if test "x${TESTMPIRUN}" = x ; then diff --git a/main.cpp b/main.cpp index 49bce5e..30ad138 100644 --- a/main.cpp +++ b/main.cpp @@ -361,10 +361,6 @@ int main(int argc, char **argv) GET_MEM(step_vmrss[0], step_vmsize[0]) -#if defined HAS_H5GET_ALLOC_STATS && HAS_H5GET_ALLOC_STATS - size_t cur_bytes, md_malloc; - H5get_alloc_stats(NULL, &cur_bytes, NULL, NULL, NULL, NULL, NULL); -#endif SET_TIMER(ts) /* Each process reads assigned input files to collect all group and dataset * metadata and stores them in the object concat. The input datasets are @@ -379,10 +375,6 @@ int main(int argc, char **argv) GET_TIMER(ts, step_time[0]) PRN_TIMER(step_time[0], "Read metadata from input files") GET_MEM(step_vmrss[1], step_vmsize[1]) -#if defined HAS_H5GET_ALLOC_STATS && HAS_H5GET_ALLOC_STATS - H5get_alloc_stats(NULL, &md_malloc, NULL, NULL, NULL, NULL, NULL); - md_malloc -= cur_bytes; -#endif SET_TIMER(ts) if (opt.append_mode) @@ -641,60 +633,6 @@ int main(int argc, char **argv) printf(" VmSize: %9s %9s %9ld\n", "","", total_vmsize[8]); printf("\n"); } - -#if defined HAS_H5GET_ALLOC_STATS && HAS_H5GET_ALLOC_STATS - unsigned long long total_alloc_bytes, max_ull, min_ull, avg_ull; - size_t curr_alloc_bytes, peak_alloc_bytes, max_block_size; - size_t total_alloc_blocks_count, curr_alloc_blocks_count; - size_t peak_alloc_blocks_count, zd[7], max_zd[7], min_zd[7], avg_zd[7]; - - err = H5get_alloc_stats(&total_alloc_bytes, &curr_alloc_bytes, - &peak_alloc_bytes, &max_block_size, - &total_alloc_blocks_count, - &curr_alloc_blocks_count, - &peak_alloc_blocks_count); - - MPI_Reduce(&total_alloc_bytes, &min_ull, 1, MPI_UNSIGNED_LONG_LONG, MPI_MIN, 0, MPI_COMM_WORLD); - MPI_Reduce(&total_alloc_bytes, &max_ull, 1, MPI_UNSIGNED_LONG_LONG, MPI_MAX, 0, MPI_COMM_WORLD); - MPI_Reduce(&total_alloc_bytes, &avg_ull, 1, MPI_UNSIGNED_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD); - min_ull /= 1048576; - max_ull /= 1048576; - avg_ull /= nprocs * 1048576; - zd[0] = md_malloc; - zd[1] = curr_alloc_bytes; - zd[2] = peak_alloc_bytes; - zd[3] = max_block_size; - zd[4] = total_alloc_blocks_count; - zd[5] = curr_alloc_blocks_count; - zd[6] = peak_alloc_blocks_count; - MPI_Reduce(zd, &min_zd, 7, MPI_UNSIGNED_LONG, MPI_MIN, 0, MPI_COMM_WORLD); - MPI_Reduce(zd, &max_zd, 7, MPI_UNSIGNED_LONG, MPI_MAX, 0, MPI_COMM_WORLD); - MPI_Reduce(zd, &avg_zd, 7, MPI_UNSIGNED_LONG, MPI_SUM, 0, MPI_COMM_WORLD); - for (int i=0; i<4; i++) { - min_zd[i] /= 1048576; - max_zd[i] /= 1048576; - avg_zd[i] /= nprocs * 1048576; - } - for (int i=4; i<7; i++) { - min_zd[i] /= 1024; - max_zd[i] /= 1024; - avg_zd[i] /= nprocs * 1024; - } - - if (!opt.quiet && rank == 0) { /* only rank 0 reports timings */ - printf("-------------------------------------------------------------\n"); - printf("Memory footprints (min, max, avg among all processes):\n"); - printf("construct metadata (MiB) min=%8zd max=%8zd avg=%8zd\n",min_zd[0],max_zd[0],avg_zd[0]); - printf("total_alloc_bytes (MiB) min=%8llu max=%8llu avg=%8llu\n",min_ull,max_ull,avg_ull); - // printf("curr_alloc_bytes (MiB) min=%8zd max=%8zd avg=%8zd\n",min_zd[1],max_zd[1],avg_zd[1]); - printf("peak_alloc_bytes (MiB) min=%8zd max=%8zd avg=%8zd\n",min_zd[2],max_zd[2],avg_zd[2]); - printf("max_block_size (MiB) min=%8zd max=%8zd avg=%8zd\n",min_zd[3],max_zd[3],avg_zd[3]); - printf("total_alloc_blocks_count (K) min=%8zd max=%8zd avg=%8zd\n",min_zd[4],max_zd[4],avg_zd[4]); - // printf("curr_alloc_blocks_count (K) min=%8zd max=%8zd avg=%8zd\n",min_zd[5],max_zd[5],avg_zd[5]); - printf("peak_alloc_blocks_count (K) min=%8zd max=%8zd avg=%8zd\n",min_zd[6],max_zd[6],avg_zd[6]); - printf("-------------------------------------------------------------\n"); - } -#endif #endif prog_exit: