Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Remove use of H5get_alloc_stats #5

Merged
merged 2 commits into from
Aug 16, 2023
Merged

Remove use of H5get_alloc_stats #5

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e09e1dc
Remove use of H5get_alloc_stats
wkliao Aug 16, 2023
3a75971
add PR 5 to release note
wkliao Aug 16, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions RELEASE_NOTES.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,8 @@ Bug fix:
+ Fix compression threshold calculation. See a9ea841

Other changes
+ Remove the HDF5 metadata memory footprint profiling, as H5get_alloc_stats
has been removed from HDF5 since 1.14.0. See PR #5.
+ Remove the requirement of group `/spill`. See d4e0852
+ Change chunk setting for small datasets. Set the chunk size to 256K
element, no matter if the dataset dim[0] is smaller than 256K element or
Expand Down
3 changes: 0 additions & 3 deletions configure.ac
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -354,9 +354,6 @@ AC_CHECK_DECL(H5Ovisit3,
AC_CHECK_DECL(H5Treclaim,
[AC_DEFINE(HAS_H5TRECLAIM, 1, [Define if API H5Treclaim is defined])],
[], [[#include <hdf5.h>]])
AC_CHECK_DECL(H5get_alloc_stats,
[AC_DEFINE(HAS_H5GET_ALLOC_STATS, 1, [Define if API H5get_alloc_stats is defined])],
[], [[#include <hdf5.h>]])

AC_ARG_VAR(TESTMPIRUN, [MPI run command for "make check", @<:@default: mpiexec@:>@])
if test "x${TESTMPIRUN}" = x ; then
Expand Down
62 changes: 0 additions & 62 deletions main.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -361,10 +361,6 @@ int main(int argc, char **argv)

GET_MEM(step_vmrss[0], step_vmsize[0])

#if defined HAS_H5GET_ALLOC_STATS && HAS_H5GET_ALLOC_STATS
size_t cur_bytes, md_malloc;
H5get_alloc_stats(NULL, &cur_bytes, NULL, NULL, NULL, NULL, NULL);
#endif
SET_TIMER(ts)
/* Each process reads assigned input files to collect all group and dataset
* metadata and stores them in the object concat. The input datasets are
Expand All @@ -379,10 +375,6 @@ int main(int argc, char **argv)
GET_TIMER(ts, step_time[0])
PRN_TIMER(step_time[0], "Read metadata from input files")
GET_MEM(step_vmrss[1], step_vmsize[1])
#if defined HAS_H5GET_ALLOC_STATS && HAS_H5GET_ALLOC_STATS
H5get_alloc_stats(NULL, &md_malloc, NULL, NULL, NULL, NULL, NULL);
md_malloc -= cur_bytes;
#endif

SET_TIMER(ts)
if (opt.append_mode)
Expand Down Expand Up @@ -641,60 +633,6 @@ int main(int argc, char **argv)
printf(" VmSize: %9s %9s %9ld\n", "","", total_vmsize[8]);
printf("\n");
}

#if defined HAS_H5GET_ALLOC_STATS && HAS_H5GET_ALLOC_STATS
unsigned long long total_alloc_bytes, max_ull, min_ull, avg_ull;
size_t curr_alloc_bytes, peak_alloc_bytes, max_block_size;
size_t total_alloc_blocks_count, curr_alloc_blocks_count;
size_t peak_alloc_blocks_count, zd[7], max_zd[7], min_zd[7], avg_zd[7];

err = H5get_alloc_stats(&total_alloc_bytes, &curr_alloc_bytes,
&peak_alloc_bytes, &max_block_size,
&total_alloc_blocks_count,
&curr_alloc_blocks_count,
&peak_alloc_blocks_count);

MPI_Reduce(&total_alloc_bytes, &min_ull, 1, MPI_UNSIGNED_LONG_LONG, MPI_MIN, 0, MPI_COMM_WORLD);
MPI_Reduce(&total_alloc_bytes, &max_ull, 1, MPI_UNSIGNED_LONG_LONG, MPI_MAX, 0, MPI_COMM_WORLD);
MPI_Reduce(&total_alloc_bytes, &avg_ull, 1, MPI_UNSIGNED_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
min_ull /= 1048576;
max_ull /= 1048576;
avg_ull /= nprocs * 1048576;
zd[0] = md_malloc;
zd[1] = curr_alloc_bytes;
zd[2] = peak_alloc_bytes;
zd[3] = max_block_size;
zd[4] = total_alloc_blocks_count;
zd[5] = curr_alloc_blocks_count;
zd[6] = peak_alloc_blocks_count;
MPI_Reduce(zd, &min_zd, 7, MPI_UNSIGNED_LONG, MPI_MIN, 0, MPI_COMM_WORLD);
MPI_Reduce(zd, &max_zd, 7, MPI_UNSIGNED_LONG, MPI_MAX, 0, MPI_COMM_WORLD);
MPI_Reduce(zd, &avg_zd, 7, MPI_UNSIGNED_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
for (int i=0; i<4; i++) {
min_zd[i] /= 1048576;
max_zd[i] /= 1048576;
avg_zd[i] /= nprocs * 1048576;
}
for (int i=4; i<7; i++) {
min_zd[i] /= 1024;
max_zd[i] /= 1024;
avg_zd[i] /= nprocs * 1024;
}

if (!opt.quiet && rank == 0) { /* only rank 0 reports timings */
printf("-------------------------------------------------------------\n");
printf("Memory footprints (min, max, avg among all processes):\n");
printf("construct metadata (MiB) min=%8zd max=%8zd avg=%8zd\n",min_zd[0],max_zd[0],avg_zd[0]);
printf("total_alloc_bytes (MiB) min=%8llu max=%8llu avg=%8llu\n",min_ull,max_ull,avg_ull);
// printf("curr_alloc_bytes (MiB) min=%8zd max=%8zd avg=%8zd\n",min_zd[1],max_zd[1],avg_zd[1]);
printf("peak_alloc_bytes (MiB) min=%8zd max=%8zd avg=%8zd\n",min_zd[2],max_zd[2],avg_zd[2]);
printf("max_block_size (MiB) min=%8zd max=%8zd avg=%8zd\n",min_zd[3],max_zd[3],avg_zd[3]);
printf("total_alloc_blocks_count (K) min=%8zd max=%8zd avg=%8zd\n",min_zd[4],max_zd[4],avg_zd[4]);
// printf("curr_alloc_blocks_count (K) min=%8zd max=%8zd avg=%8zd\n",min_zd[5],max_zd[5],avg_zd[5]);
printf("peak_alloc_blocks_count (K) min=%8zd max=%8zd avg=%8zd\n",min_zd[6],max_zd[6],avg_zd[6]);
printf("-------------------------------------------------------------\n");
}
#endif
#endif

prog_exit:
Expand Down