From 540b7622a407510e99c639d6a438ddb584f9b2e5 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Wed, 3 Jan 2024 08:50:13 -0500 Subject: [PATCH 001/124] Fix the C example stepping and reading data. Simplify code for shape/start/count --- examples/basics/globalArray1D/decomp.c | 24 ++++----- examples/basics/globalArray1D/decomp.h | 8 +-- .../basics/globalArray1D/globalArray1DRead.c | 54 +++++++++++-------- .../globalArray1D/globalArray1DWrite.F90 | 2 +- .../basics/globalArray1D/globalArray1DWrite.c | 34 ++++-------- 5 files changed, 61 insertions(+), 61 deletions(-) diff --git a/examples/basics/globalArray1D/decomp.c b/examples/basics/globalArray1D/decomp.c index bb7d3300a5..421f76c2ba 100644 --- a/examples/basics/globalArray1D/decomp.c +++ b/examples/basics/globalArray1D/decomp.c @@ -15,23 +15,23 @@ /* random integer from {minv, minv+1, ..., maxv} including minv and maxv */ -long long int get_random(int minv, int maxv) +size_t get_random(int minv, int maxv, int rank) { - long long int n; + size_t n; time_t t; /* Intializes random number generator */ - srand((unsigned)time(&t)); - n = (rand() % (maxv - minv + 1)) + minv; + srand((unsigned)time(&t) + rank); + n = (size_t)((rand() % (maxv - minv + 1)) + minv); return n; } /* gather the local sizes of arrays and sum them up so that each process knows the global shape and its own offset in the global space */ -void gather_decomp_1d(long long int *mysize, long long int *myshape, long long int *myoffset) +void gather_decomp_1d(size_t *mysize, size_t *myshape, size_t *myoffset) { - long long int *sizes; + size_t *sizes; int i; - sizes = malloc(sizeof(long long int) * (size_t)nproc); + sizes = malloc(sizeof(size_t) * (size_t)nproc); MPI_Allgather(mysize, 1, MPI_LONG_LONG, sizes, 1, MPI_LONG_LONG, app_comm); *myshape = 0; @@ -49,14 +49,14 @@ void gather_decomp_1d(long long int *mysize, long long int *myshape, long long i return; } -void decomp_1d(long long int globalsize, long long int *myoffset, long long int *mysize) +void decomp_1d(size_t *globalsize, size_t *myoffset, size_t *mysize) { - long long int rem; - *mysize = globalsize / nproc; - rem = globalsize - (nproc * *mysize); + size_t rem; + *mysize = *globalsize / nproc; + rem = *globalsize - (nproc * *mysize); if (rank < rem) { - mysize = mysize + 1; + *mysize = *mysize + 1; *myoffset = rank * *mysize; } else diff --git a/examples/basics/globalArray1D/decomp.h b/examples/basics/globalArray1D/decomp.h index e972714844..e3fd49985a 100644 --- a/examples/basics/globalArray1D/decomp.h +++ b/examples/basics/globalArray1D/decomp.h @@ -8,7 +8,9 @@ #ifndef ADIOS2EXAMPLES_DECOMP_H #define ADIOS2EXAMPLES_DECOMP_H -extern long long int get_random(int, int); -extern void gather_decomp_1d(long long int *, long long int *, long long int *); -extern void decomp_1d(long long int, long long int *, long long int *); +#include + +extern size_t get_random(int minv, int maxv, int rank); +extern void gather_decomp_1d(size_t *, size_t *, size_t *); +extern void decomp_1d(size_t *, size_t *, size_t *); #endif // ADIOS2EXAMPLES_DECOMP_H diff --git a/examples/basics/globalArray1D/globalArray1DRead.c b/examples/basics/globalArray1D/globalArray1DRead.c index 288ce9f11b..d01ccdb3de 100644 --- a/examples/basics/globalArray1D/globalArray1DRead.c +++ b/examples/basics/globalArray1D/globalArray1DRead.c @@ -7,57 +7,67 @@ #include "decomp.h" #include "mpivars.h" #include +#include #include #include void reader(adios2_adios *adios) { int step; - float *g; + float *g = NULL; const char *streamname = "adios2-global-array-1d-c.bp"; adios2_step_status err; - long long int fixed_shape = 0, fixed_start = 0, fixed_count = 0; + size_t shape[1], start[1], count[1]; adios2_io *io = adios2_declare_io(adios, "input"); - size_t shape[1]; - shape[0] = (size_t)fixed_shape; adios2_engine *engine = adios2_open(io, streamname, adios2_mode_read); step = 0; - do + while (1) { adios2_begin_step(engine, adios2_step_mode_read, 10.0, &err); + if (err == adios2_step_status_end_of_stream) + { + break; + } + if (err != adios2_step_status_ok) + { + printf("Unexpected status when calling adios2_begin_step() for step %d", step); + break; + } adios2_variable *var_g = adios2_inquire_variable(io, "GlobalArray"); if (step == 0) { /* fixed_shape is allocated in the next call*/ adios2_variable_shape(shape, var_g); - fixed_shape = (long long int)shape[0]; - decomp_1d(fixed_shape, &fixed_start, &fixed_count); - g = malloc((size_t)fixed_count * sizeof(float)); + decomp_1d(shape, start, count); + g = malloc(count[0] * sizeof(float)); + + printf("Read plan rank = %d global shape = %zu local count = %zu offset = %zu\n", rank, + shape[0], count[0], start[0]); + } - printf("Read plan rank = %d global shape = %lld local count = %lld " - "offset = %lld\n", - rank, fixed_shape, fixed_count, fixed_start); + adios2_variable *var = adios2_inquire_variable(io, "GlobalArray"); + if (!var) + { + printf("ERROR: Variable 'GlobalArray' was not found in step %d", step); + break; } + + adios2_set_selection(var, 1, start, count); + // Initiate reading data in default/deferred mode: data is available after end_step + adios2_get(engine, var, g, adios2_mode_deferred); + adios2_end_step(engine); step++; - } while (err != adios2_step_status_end_of_stream); + } // Close the output adios2_close(engine); - free(g); - if (rank == 0) + if (g != NULL) { - printf("Try the following: \n"); - printf(" bpls -la adios2-global-array-1d-c.bp GlobalArray -d -n " - "%lld \n", - fixed_shape); - printf(" bpls -la adios2-global-array-1d-c.bp GlobalArray -d -t -n " - "%lld \n ", - fixed_shape); - printf(" mpirun -n 2 ./adios2-global-array-1d-read-c \n"); + free(g); } } diff --git a/examples/basics/globalArray1D/globalArray1DWrite.F90 b/examples/basics/globalArray1D/globalArray1DWrite.F90 index 90d0778909..51c31d1223 100644 --- a/examples/basics/globalArray1D/globalArray1DWrite.F90 +++ b/examples/basics/globalArray1D/globalArray1DWrite.F90 @@ -85,7 +85,7 @@ subroutine writer " bpls -la adios2-global-array-1d-f.bp ", & "GlobalArray -d -t -n ", fixed_shape(1) write (*,'(a)') & - " mpirun -n 2 ./adios2-global-array-1d-read-f " + " mpirun -n 2 ./adios2_basics_globalArray1DRead_f " endif end subroutine writer diff --git a/examples/basics/globalArray1D/globalArray1DWrite.c b/examples/basics/globalArray1D/globalArray1DWrite.c index 4ae38ed3da..a1744de399 100644 --- a/examples/basics/globalArray1D/globalArray1DWrite.c +++ b/examples/basics/globalArray1D/globalArray1DWrite.c @@ -7,6 +7,7 @@ #include "decomp.h" #include "mpivars.h" #include +#include #include #include @@ -19,35 +20,26 @@ void writer(adios2_adios *adios) const int numsteps = 5; adios2_step_status err; - long long int fixed_shape = 0, fixed_start = 0, fixed_count = 0; + size_t shape[1], start[1], count[1]; /* Application variables g = 1D distributed array, global shape and per-process size is fixed */ - fixed_count = get_random(mincount, maxcount); - g = malloc((size_t)fixed_count * sizeof(float)); - gather_decomp_1d(&fixed_count, &fixed_shape, &fixed_start); + count[0] = get_random(mincount, maxcount, rank); + g = malloc((size_t)count[0] * sizeof(float)); + gather_decomp_1d(count, shape, start); adios2_io *io = adios2_declare_io(adios, "output"); - size_t shape[1]; - shape[0] = (size_t)fixed_shape; - - size_t start[1]; - start[0] = (size_t)fixed_start; - - size_t count[1]; - count[0] = (size_t)fixed_count; adios2_variable *var_g = adios2_define_variable(io, "GlobalArray", adios2_type_float, 1, shape, start, count, adios2_constant_dims_true); adios2_engine *engine = adios2_open(io, "adios2-global-array-1d-c.bp", adios2_mode_write); - printf("Decmp rank = %d global shape = %lld local count = %lld offset = " - "%lld\n", - rank, fixed_shape, fixed_count, fixed_start); + printf("Decomp rank = %d global shape = %zu local count = %zu offset = %zu\n", rank, shape[0], + count[0], start[0]); for (step = 0; step < numsteps; step++) { - for (i = 0; i < fixed_count; i++) + for (i = 0; i < count[0]; i++) { g[i] = (float)(rank + (step + 1) / 100.0); } @@ -63,13 +55,9 @@ void writer(adios2_adios *adios) if (rank == 0) { printf("Try the following: \n"); - printf(" bpls -la adios2-global-array-1d-c.bp GlobalArray -d -n " - "%lld \n", - fixed_shape); - printf(" bpls -la adios2-global-array-1d-c.bp GlobalArray -d -t -n " - "%lld \n ", - fixed_shape); - printf(" mpirun -n 2 ./adios2-global-array-1d-read-c \n"); + printf(" bpls -la adios2-global-array-1d-c.bp GlobalArray -d -n %zu \n", shape[0]); + printf(" bpls -la adios2-global-array-1d-c.bp GlobalArray -d -t -n %zu \n ", shape[0]); + printf(" mpirun -n 2 ./adios2_basics_globalArray1DRead_c \n"); } } From c1bd7399abafd6d42bd88ba22b18a5520bcd2a76 Mon Sep 17 00:00:00 2001 From: Greg Eisenhauer Date: Mon, 8 Jan 2024 16:26:21 -0500 Subject: [PATCH 002/124] Fix read request length on BP5 Local arrays with operators --- .../toolkit/format/bp5/BP5Deserializer.cpp | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/source/adios2/toolkit/format/bp5/BP5Deserializer.cpp b/source/adios2/toolkit/format/bp5/BP5Deserializer.cpp index 0803ad88cf..9df8d55239 100644 --- a/source/adios2/toolkit/format/bp5/BP5Deserializer.cpp +++ b/source/adios2/toolkit/format/bp5/BP5Deserializer.cpp @@ -1577,13 +1577,23 @@ BP5Deserializer::GenerateReadRequests(const bool doAllocTempBuffers, size_t *max throw std::runtime_error("No data exists for this variable"); if (Req->MemSpace != MemorySpace::Host) RR.DirectToAppMemory = false; + else if (VarRec->Operator != NULL) + RR.DirectToAppMemory = false; else RR.DirectToAppMemory = IsContiguousTransfer(Req, &writer_meta_base->Offsets[StartDim], &writer_meta_base->Count[StartDim]); - RR.ReadLength = - helper::GetDataTypeSize(VarRec->Type) * - CalcBlockLength(VarRec->DimCount, &writer_meta_base->Count[StartDim]); + if (VarRec->Operator) + { + // have to have the whole thing + RR.ReadLength = writer_meta_base->DataBlockSize[NeededBlock]; + } + else + { + RR.ReadLength = + helper::GetDataTypeSize(VarRec->Type) * + CalcBlockLength(VarRec->DimCount, &writer_meta_base->Count[StartDim]); + } RR.OffsetInBlock = 0; if (RR.DirectToAppMemory) { From 732255d2cf8a2d93a53b0279cd4f28cff95c2580 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Tue, 9 Jan 2024 12:03:57 -0500 Subject: [PATCH 003/124] Fix 1: bpls handle when BP5 throws logic error on minmax for unsupported type Fix 2: examples/basics/values.F90 need to use adios2_mode_readRandomAccess --- examples/basics/values/values.F90 | 2 +- examples/basics/values/valuesWrite.cpp | 8 ++++++++ source/utils/bpls/bpls.cpp | 28 ++++++++++++++++---------- 3 files changed, 26 insertions(+), 12 deletions(-) diff --git a/examples/basics/values/values.F90 b/examples/basics/values/values.F90 index 608ecf37b6..a1f0d9038b 100644 --- a/examples/basics/values/values.F90 +++ b/examples/basics/values/values.F90 @@ -111,7 +111,7 @@ subroutine reader ! Note, every process reads everything in this example call adios2_declare_io(io, adios, "ValuesInput", ierr) - call adios2_open(engine, io, "adios2-values-f.bp", adios2_mode_read, MPI_COMM_SELF, ierr) + call adios2_open(engine, io, "adios2-values-f.bp", adios2_mode_readRandomAccess, MPI_COMM_SELF, ierr) call adios2_inquire_variable(var_gc, io, "GlobalConstant", ierr) call adios2_get(engine, var_gc, gc , ierr) diff --git a/examples/basics/values/valuesWrite.cpp b/examples/basics/values/valuesWrite.cpp index 4f9f556798..8300e4e682 100644 --- a/examples/basics/values/valuesWrite.cpp +++ b/examples/basics/values/valuesWrite.cpp @@ -19,6 +19,7 @@ */ #include +#include #include #include @@ -80,6 +81,8 @@ int main(int argc, char *argv[]) // 2. Global value, same value across processes, varying value over time adios2::Variable varStep = io.DefineVariable("Step"); + adios2::Variable varGlobalString = + io.DefineVariable("GlobalString"); // 3. Local value, varying across processes, constant over time adios2::Variable varProcessID = @@ -110,6 +113,11 @@ int main(int argc, char *argv[]) writer.Put(varNproc, nproc); } writer.Put(varStep, step); + + std::string str = "This is step " + std::to_string(step); + // str will go out of scope before EndStep(), so we must use + // Sync mode in Put() + writer.Put(varGlobalString, str, adios2::Mode::Sync); } // 3. and 4. Writing a local value on every process. Will be shown diff --git a/source/utils/bpls/bpls.cpp b/source/utils/bpls/bpls.cpp index fd263b5748..0d3ce8e73c 100644 --- a/source/utils/bpls/bpls.cpp +++ b/source/utils/bpls/bpls.cpp @@ -1144,21 +1144,27 @@ int printVariableInfo(core::Engine *fp, core::IO *io, core::Variable *variabl if (timestep == false) { MinMaxStruct MinMax; - if (fp->VariableMinMax(*variable, DefaultSizeT, MinMax)) + try { - fprintf(outf, " = "); - print_data(&MinMax.MinUnion, 0, adiosvartype, false); - fprintf(outf, " / "); - print_data(&MinMax.MaxUnion, 0, adiosvartype, false); + if (fp->VariableMinMax(*variable, DefaultSizeT, MinMax)) + { + fprintf(outf, " = "); + print_data(&MinMax.MinUnion, 0, adiosvartype, false); + fprintf(outf, " / "); + print_data(&MinMax.MaxUnion, 0, adiosvartype, false); + } + else + { + fprintf(outf, " = "); + print_data(&variable->m_Min, 0, adiosvartype, false); + fprintf(outf, " / "); + print_data(&variable->m_Max, 0, adiosvartype, false); + } + // fprintf(outf," {MIN / MAX} "); } - else + catch (std::logic_error &) { - fprintf(outf, " = "); - print_data(&variable->m_Min, 0, adiosvartype, false); - fprintf(outf, " / "); - print_data(&variable->m_Max, 0, adiosvartype, false); } - // fprintf(outf," {MIN / MAX} "); } #if 0 else From 924670cbd1ed2afee329f395d97b74ab02a41aa1 Mon Sep 17 00:00:00 2001 From: Vicente Bolea Date: Fri, 12 Jan 2024 06:36:40 -0500 Subject: [PATCH 004/124] python,cmake: update version script to support rc (#3992) (cherry picked from commit 19a7c35bb72b31bc0fd848c0e0f8f08608e3396d) --- scripts/ci/gh-actions/config/adios-version.cmake | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/scripts/ci/gh-actions/config/adios-version.cmake b/scripts/ci/gh-actions/config/adios-version.cmake index d3f4e76852..adac361c59 100644 --- a/scripts/ci/gh-actions/config/adios-version.cmake +++ b/scripts/ci/gh-actions/config/adios-version.cmake @@ -37,7 +37,17 @@ else() elseif (out MATCHES "^v([^-]*)-([^-]*)-g[a-f0-9]*") set(ver_tag ${CMAKE_MATCH_1}) set(ver_ncommits ${CMAKE_MATCH_2}) - set(adios2_pip_package_version "${ver_tag}.${ver_ncommits}") + math(EXPR ver_tweak "100000+${ver_ncommits}") + set(adios2_pip_package_version "${ver_tag}.${ver_tweak}") + elseif (out MATCHES "^v([^-]*)-rc([0-9]*)(-([^-]*)-g[a-f0-9]*)?") + set(ver_tag ${CMAKE_MATCH_1}) + set(ver_rc ${CMAKE_MATCH_2}) + set(ver_ncommits 0) + if (CMAKE_MATCH_COUNT EQUAL 4) + set(ver_ncommits ${CMAKE_MATCH_4}) + endif() + math(EXPR ver_tweak "(1000*${ver_rc})+${ver_ncommits}") + set(adios2_pip_package_version "${ver_tag}.${ver_tweak}") endif() endif() if (NOT res EQUAL 0 OR out MATCHES "fatal: not a git repository") @@ -46,4 +56,5 @@ else() endif() endif() +message(${adios2_pip_package_version}) file(WRITE "VERSION.TXT" ${adios2_pip_package_version}) From 5110209dfcea43fb60e3d1ed937122e9641bb7d7 Mon Sep 17 00:00:00 2001 From: pnorbert Date: Fri, 12 Jan 2024 08:20:46 -0500 Subject: [PATCH 005/124] =?UTF-8?q?Python=20API:=20Adios()=20new=20argumen?= =?UTF-8?q?t=20'config=5Ffile',=20Stream()=20new=20argument=E2=80=A6=20(#3?= =?UTF-8?q?984)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Python API: Adios() new argument 'config_file', Stream() new argument 'io_name' that must go together with 'config_file' Gray-Scott example's python scripts fixed to use the new Stream() object * Make 'comm' a named optional argument for Stream. --------- Co-authored-by: Greg Eisenhauer --- .../simulations/gray-scott/plot/decomp.py | 3 +- .../simulations/gray-scott/plot/gsplot.py | 37 +++++++++---------- .../simulations/gray-scott/plot/pdfplot.py | 23 +++++------- python/adios2/adios.py | 14 +++++-- python/adios2/file_reader.py | 11 +++++- python/adios2/stream.py | 22 ++++++++++- .../python/TestBPChangingShapeHighLevelAPI.py | 4 +- .../adios2/python/TestBPPNGHighLevelAPI.py | 4 +- .../adios2/python/TestBPWriteReadString.py | 8 ++-- .../python/TestBPWriteTypesHighLevelAPI.py | 4 +- .../TestBPWriteTypesHighLevelAPILocal.py | 4 +- .../adios2/python/TestBPZfpHighLevelAPI.py | 4 +- 12 files changed, 82 insertions(+), 56 deletions(-) diff --git a/examples/simulations/gray-scott/plot/decomp.py b/examples/simulations/gray-scott/plot/decomp.py index 6edbfbd9e2..7e068693d8 100644 --- a/examples/simulations/gray-scott/plot/decomp.py +++ b/examples/simulations/gray-scott/plot/decomp.py @@ -14,7 +14,7 @@ def Locate(rank, nproc, datasize): class MPISetup(object): readargs = [] size = 1 - rank = {"app": 0, "x": 0, "y": 0} + rank = {"app": 0, "x": 0, "y": 0, "z": 0} def __init__(self, args, appID): @@ -56,6 +56,7 @@ def __init__(self, args, appID): raise ValueError("nx must = 1 without MPI") if self.ny != 1: raise ValueError("ny must = 1 without MPI") + self.comm_app = None # self.readargs.extend([args.xmlfile, "heat"]) diff --git a/examples/simulations/gray-scott/plot/gsplot.py b/examples/simulations/gray-scott/plot/gsplot.py index 5ab88340bd..5573ffd996 100644 --- a/examples/simulations/gray-scott/plot/gsplot.py +++ b/examples/simulations/gray-scott/plot/gsplot.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -import adios2 # pylint: disable=import-error +from adios2 import Stream # pylint: disable=import-error import argparse import numpy as np # pylint: disable=import-error import matplotlib.pyplot as plt # pylint: disable=import-error @@ -98,21 +98,14 @@ def Plot2D(plane_direction, data, args, fullshape, step, fontsize): plt.show() plt.pause(displaysec) elif args.outfile.endswith(".bp"): - if step == 0: - global adios - global ioWriter - global var - global writer - adios = adios2.ADIOS(mpi.comm_app) - ioWriter = adios.DeclareIO("VizOutput") - var = ioWriter.DefineVariable( - args.varname, data.shape, [0, 0], data.shape, adios2.ConstantDims, data - ) - writer = ioWriter.Open(args.outfile, adios2.Mode.Write) - - writer.BeginStep() - writer.Put(var, data, adios2.Mode.Sync) - writer.EndStep() + global writer +# print("plot to file, step = ", step) +# if step == 0: +# writer = Stream(args.outfile, "w") +# + writer.begin_step() + writer.write(args.varname, data, data.shape, [0, 0], data.shape) + writer.end_step() else: imgfile = args.outfile + "{0:0>5}".format(step) + "_" + plane_direction + ".png" fig.savefig(imgfile) @@ -139,14 +132,18 @@ def read_data(args, fr, start_coord, size_dims): myrank = mpi.rank["app"] # Read the data from this object - fr = adios2.open(args.instream, "r", mpi.comm_app, "adios2.xml", "SimulationOutput") - # vars_info = fr.availablevariables() + fr = Stream(args.instream, "r", comm=mpi.comm_app, config_file="adios2.xml", + io_name="SimulationOutput") + + if args.outfile.endswith(".bp"): + global writer + writer = Stream(args.outfile, "w") # Get the ADIOS selections -- equally partition the data if parallelization is requested # Read through the steps, one at a time plot_step = 0 - for fr_step in fr: + for fr_step in fr.steps(): # if fr_step.current_step() start, size, fullshape = mpi.Partition_3D_3D(fr, args) cur_step = fr_step.current_step() @@ -185,3 +182,5 @@ def read_data(args, fr, start_coord, size_dims): plot_step = plot_step + 1 fr.close() + if args.outfile.endswith(".bp"): + writer.close() diff --git a/examples/simulations/gray-scott/plot/pdfplot.py b/examples/simulations/gray-scott/plot/pdfplot.py index 77ad246bc7..4984da5ed2 100644 --- a/examples/simulations/gray-scott/plot/pdfplot.py +++ b/examples/simulations/gray-scott/plot/pdfplot.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -import adios2 # pylint: disable=import-error +from adios2 import Stream # pylint: disable=import-error import argparse import numpy as np # pylint: disable=import-error import matplotlib.pyplot as plt # pylint: disable=import-error @@ -85,16 +85,12 @@ def read_data(args, fr, start_coord, size_dims): myrank = mpi.rank["app"] # Read the data from this object - if not args.nompi: - fr = adios2.open( - args.instream, "r", mpi.comm_app, "adios2.xml", "PDFAnalysisOutput" - ) - else: - fr = adios2.open(args.instream, "r", "adios2.xml", "PDFAnalysisOutput") + fr = Stream(args.instream, "r", comm=mpi.comm_app, config_file="adios2.xml", + io_name="PDFAnalysisOutput") # Read through the steps, one at a time plot_step = 0 - for fr_step in fr: + for fr_step in fr.steps(): cur_step = fr_step.current_step() vars_info = fr_step.available_variables() # print (vars_info) @@ -103,13 +99,12 @@ def read_data(args, fr, start_coord, size_dims): shape2_str = vars_info[pdfvar]["Shape"].split(",") shape2 = list(map(int, shape2_str)) - start = np.zeros(2, dtype=np.int64) - count = np.zeros(2, dtype=np.int64) # Equally partition the PDF arrays among readers - start[0], count[0] = decomp.Locate(myrank, mpi.size, shape2[0]) - start[1], count[1] = (0, shape2[1]) - start_bins = np.array([0], dtype=np.int64) - count_bins = np.array([shape2[1]], dtype=np.int64) + s0, c0 = decomp.Locate(myrank, mpi.size, shape2[0]) + start = [s0, 0] + count = [c0, shape2[1]] + start_bins = [0] + count_bins = [shape2[1]] # print("Rank {0} reads {1} slices from offset {2}".format(myrank, count[0], start[0])) diff --git a/python/adios2/adios.py b/python/adios2/adios.py index c16108956b..5deacad5a3 100644 --- a/python/adios2/adios.py +++ b/python/adios2/adios.py @@ -11,14 +11,20 @@ class Adios: """High level representation of the ADIOS class in the adios2.bindings""" - def __init__(self, comm=None): + def __init__(self, config_file=None, comm=None): if comm and not bindings.is_built_with_mpi: raise RuntimeError("Cannot use MPI since ADIOS2 was built without MPI support") - if comm: - self.impl = bindings.ADIOS(comm) + if config_file: + if comm: + self.impl = bindings.ADIOS(config_file, comm) + else: + self.impl = bindings.ADIOS(config_file) else: - self.impl = bindings.ADIOS() + if comm: + self.impl = bindings.ADIOS(comm) + else: + self.impl = bindings.ADIOS() @property def impl(self): diff --git a/python/adios2/file_reader.py b/python/adios2/file_reader.py index 731356e24d..7b80c11b91 100644 --- a/python/adios2/file_reader.py +++ b/python/adios2/file_reader.py @@ -12,8 +12,15 @@ class FileReader(Stream): def __repr__(self): return f"" - def __init__(self, path, comm=None, engine_type="BPStream", config_file=None): - super().__init__(path, "rra", comm, engine_type, config_file) + def __init__(self, path, *, comm=None, engine_type=None, config_file=None, io_name=None): + super().__init__( + path, + "rra", + comm=comm, + engine_type=engine_type, + config_file=config_file, + io_name=io_name, + ) def variables(self): """Returns the list of variables contained in the opened file""" diff --git a/python/adios2/stream.py b/python/adios2/stream.py index 8718a226cb..4ba245d545 100644 --- a/python/adios2/stream.py +++ b/python/adios2/stream.py @@ -29,24 +29,42 @@ def type_adios_to_numpy(name): class Stream: """High level implementation of the Stream class from the core API""" - def __init__(self, path, mode="r", comm=None, engine_type="BPStream", config_file=None): + def __init__( + self, path, mode="r", *, comm=None, engine_type=None, config_file=None, io_name=None + ): + + # pylint: disable=R0912 # Too many branches if comm and not bindings.is_built_with_mpi: raise RuntimeError("Cannot use MPI since ADIOS2 was built without MPI support") + if config_file and engine_type: + raise RuntimeError("Arguments 'engine_type' and 'config_file' cannot be used together") + + if config_file and not io_name: + raise RuntimeError("Argument 'io_name' is required when using 'config_file'") + + if not engine_type: + engine_type = "File" + # pylint: disable=E1121 if config_file: if comm: self._adios = Adios(config_file, comm) else: self._adios = Adios(config_file) + self._io_name = io_name + else: if comm: self._adios = Adios(comm) else: self._adios = Adios() + self._io_name = f"stream:{path}:engine_type:{engine_type}:mode:{mode}" + # pylint: enable=E1121 - self._io_name = f"stream:{path}:engine_type:{engine_type}:mode:{mode}" self._io = self._adios.declare_io(self._io_name) + if not config_file: + self._io.set_engine(engine_type) if mode == "r": self._mode = bindings.Mode.Read diff --git a/testing/adios2/python/TestBPChangingShapeHighLevelAPI.py b/testing/adios2/python/TestBPChangingShapeHighLevelAPI.py index 6497650341..ac2aa58388 100644 --- a/testing/adios2/python/TestBPChangingShapeHighLevelAPI.py +++ b/testing/adios2/python/TestBPChangingShapeHighLevelAPI.py @@ -25,7 +25,7 @@ count = [[nx[0]], [nx[1]]] # Write different sized arrays as separate steps -with Stream("out.bp", "w", comm) as s: +with Stream("out.bp", "w", comm=comm) as s: s.begin_step() s.write("z", data[0], shape[0], start[0], count[0]) s.end_step() @@ -34,7 +34,7 @@ s.end_step() # Read back arrays -with Stream("out.bp", "r", comm) as s: +with Stream("out.bp", "r", comm=comm) as s: for step in s.steps(): shape_z = int(step.available_variables()["z"]["Shape"]) print(shape_z) diff --git a/testing/adios2/python/TestBPPNGHighLevelAPI.py b/testing/adios2/python/TestBPPNGHighLevelAPI.py index cc8ac10fa3..fc73821179 100644 --- a/testing/adios2/python/TestBPPNGHighLevelAPI.py +++ b/testing/adios2/python/TestBPPNGHighLevelAPI.py @@ -51,7 +51,7 @@ def compress_png(compression_level): count2D = [Nx, Ny] # writer - with Stream(fname, "w", comm) as s: + with Stream(fname, "w", comm=comm) as s: for step in s.steps(NSteps): s.write( "u8", @@ -89,7 +89,7 @@ def compress_png(compression_level): ) # reader - with Stream(fname, "r", comm) as s: + with Stream(fname, "r", comm=comm) as s: for step in s.steps(): in_u8s = step.read("u8", start3D, count3D) in_u32s = step.read("u32", start2D, count2D) diff --git a/testing/adios2/python/TestBPWriteReadString.py b/testing/adios2/python/TestBPWriteReadString.py index 52979560a8..72eda69f94 100644 --- a/testing/adios2/python/TestBPWriteReadString.py +++ b/testing/adios2/python/TestBPWriteReadString.py @@ -21,11 +21,11 @@ def test_write_read_string_high_api(self): bpFilename = "string_test_highAPI.bp" varname = "mystringvar" - with Stream(bpFilename, "w", comm) as s: + with Stream(bpFilename, "w", comm=comm) as s: for step in s.steps(N_STEPS): s.write(varname, theString + str(step.current_step())) - with Stream(bpFilename, "r", comm) as s: + with Stream(bpFilename, "r", comm=comm) as s: for _ in s.steps(): step = s.current_step() result = s.read(varname) @@ -34,13 +34,13 @@ def test_write_read_string_high_api(self): def test_read_strings_all_steps(self): comm = MPI.COMM_WORLD fileName = "string_test_all.bp" - with Stream(fileName, "w", comm) as s: + with Stream(fileName, "w", comm=comm) as s: i = 0 for _ in s.steps(N_STEPS): s.write("string_variable", "written {}".format(i)) i += 1 - # with Stream(fileName, "rra", comm) as s: + # with Stream(fileName, "rra", comm = comm) as s: # n = s.num_steps() # name = "string_variable" # result = s.read_string(name, 0, n) diff --git a/testing/adios2/python/TestBPWriteTypesHighLevelAPI.py b/testing/adios2/python/TestBPWriteTypesHighLevelAPI.py index cd900ba706..e6da7abb6f 100644 --- a/testing/adios2/python/TestBPWriteTypesHighLevelAPI.py +++ b/testing/adios2/python/TestBPWriteTypesHighLevelAPI.py @@ -27,7 +27,7 @@ count = [nx] # Writer -with Stream("types_np.bp", "w", comm) as s: +with Stream("types_np.bp", "w", comm=comm) as s: for step in s.steps(5): data.update(rank, step.current_step(), size) s.write("rank", np.array(rank), shape=[LocalValueDim]) @@ -100,7 +100,7 @@ # Reader data = SmallTestData() -with Stream("types_np.bp", "r", comm) as fr: +with Stream("types_np.bp", "r", comm=comm) as fr: # file only assert fr.num_steps() == 5 diff --git a/testing/adios2/python/TestBPWriteTypesHighLevelAPILocal.py b/testing/adios2/python/TestBPWriteTypesHighLevelAPILocal.py index eb824761bb..6a719b3ba9 100644 --- a/testing/adios2/python/TestBPWriteTypesHighLevelAPILocal.py +++ b/testing/adios2/python/TestBPWriteTypesHighLevelAPILocal.py @@ -33,7 +33,7 @@ def check_array(np1, np2, hint): count = [nx] # Writer -with Stream("types_np_local.bp", "w", comm) as s: +with Stream("types_np_local.bp", "w", comm=comm) as s: for step in s.steps(5): data.update(rank, step.current_step(), size) s.write("varI8", data.i8, shape, start, count) @@ -50,7 +50,7 @@ def check_array(np1, np2, hint): # Reader data = SmallTestData() -with Stream("types_np_local.bp", "r", comm) as s: +with Stream("types_np_local.bp", "r", comm=comm) as s: for fr_step in s.steps(): step = fr_step.current_step() diff --git a/testing/adios2/python/TestBPZfpHighLevelAPI.py b/testing/adios2/python/TestBPZfpHighLevelAPI.py index 0ca1600101..bd68bff52b 100644 --- a/testing/adios2/python/TestBPZfpHighLevelAPI.py +++ b/testing/adios2/python/TestBPZfpHighLevelAPI.py @@ -42,13 +42,13 @@ def CompressZfp2D(rate): count = [Ny, Nx] # writer - with Stream(fname, "w", comm) as s: + with Stream(fname, "w", comm=comm) as s: for _ in s.steps(NSteps): s.write("r32", r32s, shape, start, count, [("zfp", {"accuracy": str(rate)})]) s.write("r64", r64s, shape, start, count, [("zfp", {"accuracy": str(rate)})]) # reader - with Stream(fname, "r", comm) as s: + with Stream(fname, "r", comm=comm) as s: for _ in s.steps(): in_r32s = s.read("r32", start, count) in_r64s = s.read("r64", start, count) From 11560ecb2317faa22e44b96a259fa9ef7301e8c7 Mon Sep 17 00:00:00 2001 From: Greg Eisenhauer Date: Fri, 12 Jan 2024 19:48:35 -0500 Subject: [PATCH 006/124] Backport recent ASAN fixes to 2.10 release branch (#3985) * BP5internal - Switch to name based record lookup * Fix Var reference * ASAN fixes * kill comment --- CTestConfig.cmake | 12 --------- scripts/ci/cmake/adios-asan.supp | 4 ++- .../toolkit/format/bp5/BP5Deserializer.cpp | 2 ++ .../toolkit/format/bp5/BP5Serializer.cpp | 27 ++++++++++++------- .../adios2/toolkit/format/bp5/BP5Serializer.h | 2 +- source/adios2/toolkit/remote/Remote.cpp | 13 +++++++-- source/adios2/toolkit/remote/Remote.h | 3 ++- source/adios2/toolkit/remote/remote_common.h | 1 + source/adios2/toolkit/sst/cp/cp_reader.c | 1 + 9 files changed, 39 insertions(+), 26 deletions(-) diff --git a/CTestConfig.cmake b/CTestConfig.cmake index dae4b567bc..be9c516a6a 100644 --- a/CTestConfig.cmake +++ b/CTestConfig.cmake @@ -14,16 +14,4 @@ set(MEMORYCHECK_SUPPRESSIONS_FILE ${CMAKE_SOURCE_DIR}/scripts/dashboard/nightly/ # Ignore tests that are currently failing, remove tests here as they are fixed list(APPEND CTEST_CUSTOM_MEMCHECK_IGNORE -Engine.BP.BPWriteReadAsStreamTestADIOS2.ReaderWriterDefineVariable.BP5.Serial -Remote.BPWriteReadADIOS2stdio.GetRemote -Remote.BPWriteMemorySelectionRead.GetRemote -Remote.BPWriteMemorySelectionRead.FileRemote -remoteServerCleanup -Engine.SST.SstWriteFails.InvalidBeginStep.Serial -Staging.1x1.Local2.CommMin.BP5.SST -Staging.1x1Struct.CommMin.BP5.SST -Staging.WriteMemorySelectionRead.1x1.CommMin.BP5.SST -Staging.1x1.Local2.CommMin.BP.SST -Staging.WriteMemorySelectionRead.1x1.CommMin.BP.SST -Staging.1x1Struct.BP5 ) diff --git a/scripts/ci/cmake/adios-asan.supp b/scripts/ci/cmake/adios-asan.supp index d36b448f8d..9f9184380e 100644 --- a/scripts/ci/cmake/adios-asan.supp +++ b/scripts/ci/cmake/adios-asan.supp @@ -1,2 +1,4 @@ leak:ps_make_timer_name_ -leak:ibv_get_device_list \ No newline at end of file +leak:ibv_get_device_list +leak:add_transport_to_cm +leak:INT_CMadd_delayed_task diff --git a/source/adios2/toolkit/format/bp5/BP5Deserializer.cpp b/source/adios2/toolkit/format/bp5/BP5Deserializer.cpp index 9df8d55239..0e7574a17c 100644 --- a/source/adios2/toolkit/format/bp5/BP5Deserializer.cpp +++ b/source/adios2/toolkit/format/bp5/BP5Deserializer.cpp @@ -1996,6 +1996,8 @@ BP5Deserializer::~BP5Deserializer() free(VarRec.second->VarName); if (VarRec.second->Operator) free(VarRec.second->Operator); + if (VarRec.second->Def) + delete VarRec.second->Def; delete VarRec.second; } if (m_FreeableMBA) diff --git a/source/adios2/toolkit/format/bp5/BP5Serializer.cpp b/source/adios2/toolkit/format/bp5/BP5Serializer.cpp index 2ff1e5e802..2f618089dc 100644 --- a/source/adios2/toolkit/format/bp5/BP5Serializer.cpp +++ b/source/adios2/toolkit/format/bp5/BP5Serializer.cpp @@ -37,14 +37,16 @@ namespace format BP5Serializer::BP5Serializer() { Init(); } BP5Serializer::~BP5Serializer() { - if (!Info.RecMap.empty()) + if (CurDataBuffer) + delete CurDataBuffer; + if (!Info.RecNameMap.empty()) { - for (auto &rec : Info.RecMap) + for (auto &rec : Info.RecNameMap) { if (rec.second.OperatorType) free(rec.second.OperatorType); } - Info.RecMap.clear(); + Info.RecNameMap.clear(); } if (Info.MetaFieldCount) free_FMfield_list(Info.MetaFields); @@ -82,12 +84,13 @@ void BP5Serializer::Init() ((BP5MetadataInfoStruct *)MetadataBuf)->BitField = (std::size_t *)malloc(sizeof(size_t)); ((BP5MetadataInfoStruct *)MetadataBuf)->DataBlockSize = 0; } -BP5Serializer::BP5WriterRec BP5Serializer::LookupWriterRec(void *Key) const +BP5Serializer::BP5WriterRec BP5Serializer::LookupWriterRec(void *Variable) const { - auto it = Info.RecMap.find(Key); - if (it != Info.RecMap.end()) + core::VariableBase *VB = static_cast(Variable); + auto it2 = Info.RecNameMap.find(VB->m_Name); + if (it2 != Info.RecNameMap.end()) { - return const_cast(&(it->second)); + return const_cast(&(it2->second)); } return NULL; } @@ -467,6 +470,8 @@ void BP5Serializer::AddDoubleArrayField(FMFieldList *FieldP, int *CountP, const void BP5Serializer::ValidateWriterRec(BP5Serializer::BP5WriterRec Rec, void *Variable) { core::VariableBase *VB = static_cast(Variable); + + Rec->Key = Variable; // reset this, because Variable might have been destroyed and recreated if ((VB->m_Operations.size() == 0) && Rec->OperatorType) { // removed operator case @@ -505,7 +510,7 @@ BP5Serializer::BP5WriterRec BP5Serializer::CreateWriterRec(void *Variable, const #ifdef ADIOS2_HAVE_DERIVED_VARIABLE core::VariableDerived *VD = dynamic_cast(VB); #endif - auto obj = Info.RecMap.insert(std::make_pair(Variable, _BP5WriterRec())); + auto obj = Info.RecNameMap.insert(std::make_pair(VB->m_Name, _BP5WriterRec())); BP5WriterRec Rec = &obj.first->second; if (Type == DataType::String) ElemSize = sizeof(char *); @@ -550,6 +555,8 @@ BP5Serializer::BP5WriterRec BP5Serializer::CreateWriterRec(void *Variable, const struct_list[0].struct_size = (int)SD->StructSize(); FMFormat Format = register_data_format(Info.LocalFMContext, &struct_list[0]); + free_FMfield_list(List); + free((void *)struct_list[0].format_name); int IDLength; char *ServerID = get_server_ID_FMformat(Format, &IDLength); @@ -636,6 +643,8 @@ BP5Serializer::BP5WriterRec BP5Serializer::CreateWriterRec(void *Variable, const // Changing the formats renders these invalid Info.MetaFormat = NULL; } + if (TextStructID) + free((void *)TextStructID); Info.RecCount++; return Rec; } @@ -1250,7 +1259,7 @@ BufferV *BP5Serializer::ReinitStepData(BufferV *DataBuffer, bool forceCopyDeferr void BP5Serializer::CollectFinalShapeValues() { - for (auto it : Info.RecMap) + for (auto it : Info.RecNameMap) { BP5WriterRec Rec = &it.second; if (Rec->Shape == ShapeID::GlobalArray) diff --git a/source/adios2/toolkit/format/bp5/BP5Serializer.h b/source/adios2/toolkit/format/bp5/BP5Serializer.h index d07aa727fa..bd5830cfc1 100644 --- a/source/adios2/toolkit/format/bp5/BP5Serializer.h +++ b/source/adios2/toolkit/format/bp5/BP5Serializer.h @@ -167,7 +167,7 @@ class BP5Serializer : virtual public BP5Base FMFormat AttributeFormat = NULL; void *AttributeData = NULL; int AttributeSize = 0; - std::unordered_map RecMap; + std::unordered_map RecNameMap; }; FMFormat GenericAttributeFormat = NULL; diff --git a/source/adios2/toolkit/remote/Remote.cpp b/source/adios2/toolkit/remote/Remote.cpp index 3a397b05a9..29c0da3184 100644 --- a/source/adios2/toolkit/remote/Remote.cpp +++ b/source/adios2/toolkit/remote/Remote.cpp @@ -15,6 +15,12 @@ namespace adios2 Remote::Remote() {} #ifdef ADIOS2_HAVE_SST +Remote::~Remote() +{ + if (m_conn) + CMConnection_close(m_conn); +} + void OpenResponseHandler(CManager cm, CMConnection conn, void *vevent, void *client_data, attr_list attrs) { @@ -86,9 +92,10 @@ void Remote::Open(const std::string hostname, const int32_t port, const std::str atom_t CM_IP_HOSTNAME = -1; CM_IP_HOSTNAME = attr_atom_from_string("IP_HOST"); CM_IP_PORT = attr_atom_from_string("IP_PORT"); - add_attr(contact_list, CM_IP_HOSTNAME, Attr_String, (attr_value)hostname.c_str()); + add_attr(contact_list, CM_IP_HOSTNAME, Attr_String, (attr_value)strdup(hostname.c_str())); add_attr(contact_list, CM_IP_PORT, Attr_Int4, (attr_value)port); m_conn = CMinitiate_conn(ev_state.cm, contact_list); + free_attr_list(contact_list); if (!m_conn) return; @@ -124,9 +131,10 @@ void Remote::OpenSimpleFile(const std::string hostname, const int32_t port, atom_t CM_IP_HOSTNAME = -1; CM_IP_HOSTNAME = attr_atom_from_string("IP_HOST"); CM_IP_PORT = attr_atom_from_string("IP_PORT"); - add_attr(contact_list, CM_IP_HOSTNAME, Attr_String, (attr_value)hostname.c_str()); + add_attr(contact_list, CM_IP_HOSTNAME, Attr_String, (attr_value)strdup(hostname.c_str())); add_attr(contact_list, CM_IP_PORT, Attr_Int4, (attr_value)port); m_conn = CMinitiate_conn(ev_state.cm, contact_list); + free_attr_list(contact_list); if (!m_conn) return; @@ -193,5 +201,6 @@ Remote::GetHandle Remote::Read(size_t Start, size_t Size, void *Dest) { return static_cast(0); }; +Remote::~Remote() {} #endif } // end namespace adios2 diff --git a/source/adios2/toolkit/remote/Remote.h b/source/adios2/toolkit/remote/Remote.h index 4e4f24f579..9e432516fa 100644 --- a/source/adios2/toolkit/remote/Remote.h +++ b/source/adios2/toolkit/remote/Remote.h @@ -32,6 +32,7 @@ class Remote * @param comm passed to m_Comm */ Remote(); + ~Remote(); explicit operator bool() const { return m_Active; } @@ -55,7 +56,7 @@ class Remote #ifdef ADIOS2_HAVE_SST void InitCMData(); RemoteCommon::Remote_evpath_state ev_state; - CMConnection m_conn; + CMConnection m_conn = NULL; std::mutex m_CMInitMutex; #endif bool m_Active = false; diff --git a/source/adios2/toolkit/remote/remote_common.h b/source/adios2/toolkit/remote/remote_common.h index fc75b4a1f0..0d78bd290a 100644 --- a/source/adios2/toolkit/remote/remote_common.h +++ b/source/adios2/toolkit/remote/remote_common.h @@ -95,6 +95,7 @@ typedef struct _CloseFileMsg typedef struct _KillServerMsg { int KillResponseCondition; + size_t unused; // small messages call stack addressing issues? } *KillServerMsg; typedef struct _KillResponseMsg diff --git a/source/adios2/toolkit/sst/cp/cp_reader.c b/source/adios2/toolkit/sst/cp/cp_reader.c index ed58f0da42..13a504f07b 100644 --- a/source/adios2/toolkit/sst/cp/cp_reader.c +++ b/source/adios2/toolkit/sst/cp/cp_reader.c @@ -1221,6 +1221,7 @@ static void waitForMetadataWithTimeout(SstStream Stream, float timeout_secs) if (timercmp(&now, &end, >)) { CP_verbose(Stream, PerRankVerbose, "Returning from wait after timing out\n"); + free(TimeoutTask); return; } /* wait until we get the timestep metadata or something else changes */ From c9f76bf45e103ca79b74e08e02ef937cd03307f4 Mon Sep 17 00:00:00 2001 From: Greg Eisenhauer Date: Tue, 30 Jan 2024 15:13:07 -0500 Subject: [PATCH 007/124] dill 2023-11-20 (7583783f) (#4005) Code extracted from: https://github.com/GTkorvo/dill.git at commit 7583783fbcb6e6401f8986c8bf8dd281cc5f2897 (master). Upstream Shortlog ----------------- Co-authored-by: dill Upstream --- thirdparty/dill/dill/arm8.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/thirdparty/dill/dill/arm8.c b/thirdparty/dill/dill/arm8.c index d189463581..372899d8a9 100644 --- a/thirdparty/dill/dill/arm8.c +++ b/thirdparty/dill/dill/arm8.c @@ -1517,6 +1517,8 @@ arm8_PLT_emit(dill_stream s, int package) } } +extern void arm8_rt_call_link(char *code, call_t *t); + static void arm8_call_link(dill_stream s) { From 3e0d9201d8fbcd10a9f7b72f1188c6dee889d7a3 Mon Sep 17 00:00:00 2001 From: Greg Eisenhauer Date: Thu, 4 Jan 2024 14:16:35 -0500 Subject: [PATCH 008/124] Fix compilation errors on strict compiler (#3983) --- examples/campaign/campaign_write.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/campaign/campaign_write.cpp b/examples/campaign/campaign_write.cpp index aed73cc22a..3805480817 100644 --- a/examples/campaign/campaign_write.cpp +++ b/examples/campaign/campaign_write.cpp @@ -292,7 +292,7 @@ int main(int argc, char *argv[]) writerAll.Put(varYAll, y.data()); writerAll.Put(varZAll, z.data()); writerAll.Put(varStepAll, step); - writerAll.Put(varPhysStepAll, physicalStep); + writerAll.Put(varPhysStepAll, (size_t)physicalStep); writerAll.Put(varPhysTimeAll, physicalTime); writerAll.EndStep(); @@ -301,7 +301,7 @@ int main(int argc, char *argv[]) ioStep.Open("dataStep" + std::to_string(step) + ".bp", adios2::Mode::Write); writerStep.Put(varZStep, z.data()); writerStep.Put(varStepStep, step); - writerStep.Put(varPhysStepStep, physicalStep); + writerStep.Put(varPhysStepStep, (size_t)physicalStep); writerStep.Put(varPhysTimeStep, physicalTime); writerStep.Close(); @@ -334,7 +334,7 @@ int main(int argc, char *argv[]) ioNew.Open("dataNew" + std::to_string(step) + ".bp", adios2::Mode::Write); writerNew.Put(varYNew, y.data()); writerNew.Put(varStepNew, step); - writerNew.Put(varPhysStepNew, physicalStep); + writerNew.Put(varPhysStepNew, (size_t)physicalStep); writerNew.Put(varPhysTimeNew, physicalTime); writerNew.Close(); } @@ -379,7 +379,7 @@ int main(int argc, char *argv[]) adios2::Engine writerFinal = ioFinal.Open("dataFinal.bp", mode); writerFinal.Put(varXFinal, x.data()); writerFinal.Put(varStepFinal, startStep + nSteps - 1); - writerFinal.Put(varPhysStepFinal, physicalStep); + writerFinal.Put(varPhysStepFinal, (size_t)physicalStep); writerFinal.Put(varPhysTimeFinal, physicalTime); writerFinal.Close(); } From 44026e4452707993c12af135989d0030a01162de Mon Sep 17 00:00:00 2001 From: Ana Gainaru Date: Fri, 5 Jan 2024 10:32:03 -0500 Subject: [PATCH 009/124] Update the tutorial with links to external tutorials (#3978) --- docs/user_guide/source/tutorials/overview.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/user_guide/source/tutorials/overview.rst b/docs/user_guide/source/tutorials/overview.rst index b9cb47a573..4130a8bfff 100644 --- a/docs/user_guide/source/tutorials/overview.rst +++ b/docs/user_guide/source/tutorials/overview.rst @@ -13,3 +13,8 @@ More specifically, we will go through the following examples: 3. :ref:`Attributes ` 4. :ref:`Operators ` 5. :ref:`Steps ` + +More advance tutorials that cover more information related to: + +- Running ADIOS2 at scale (through files or streaming) with hands-on excercises: `Exascale I/O tutorial ` +- Using ADIOS2 with Paraview, TAU, Catalyst, FIDES, VTK-M: `ADIOS2 tutorial at SC23 ` From d7add2a855a36b0df0e24ef60c91c4feb0806f41 Mon Sep 17 00:00:00 2001 From: Vicente Adolfo Bolea Sanchez Date: Tue, 16 Jan 2024 18:06:37 +0900 Subject: [PATCH 010/124] update CODEOWNERS --- CODEOWNERS | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/CODEOWNERS b/CODEOWNERS index 2d2c4cd680..02365836a9 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1,6 +1,3 @@ -# KW's stuff -CMakeLists.txt @vicentebolea @caitlinross - # Caitlin's stuff plugins/ @caitlinross @@ -10,11 +7,11 @@ plugins/ @caitlinross *.in @vicentebolea *.json @vicentebolea *.sh @vicentebolea -*.txt @vicentebolea *.yaml @vicentebolea *.yml @vicentebolea cmake/ @vicentebolea scripts/ @vicentebolea +python/ @vicentebolea .github/ @vicentebolea .circleci/ @vicentebolea source/adios2/toolkit/sst/dp/mpi_dp.c @vicentebolea From 616c7e4c6bcc8489e2e6bd3ee059af2a0eb4cf03 Mon Sep 17 00:00:00 2001 From: Vicente Adolfo Bolea Sanchez Date: Tue, 16 Jan 2024 17:18:07 +0900 Subject: [PATCH 011/124] update readthedocs deps --- docs/environment.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/environment.yml b/docs/environment.yml index 20b0939ac2..87792f5d94 100644 --- a/docs/environment.yml +++ b/docs/environment.yml @@ -4,14 +4,14 @@ channels: - conda-forge dependencies: - - breathe==4.35.0 + - breathe - cmake - - docutils==0.17 + - docutils - libpython-static - numpy - pip - - sphinx=5.3 + - sphinx - pip: - - blockdiag==3.0.0 - - sphinxcontrib-blockdiag==3.0.0 - - sphinx_rtd_theme==1.1.1 + - blockdiag + - sphinxcontrib-blockdiag + - sphinx_rtd_theme From c52b4d3ca42b475b657e0fdb4085599689d260c5 Mon Sep 17 00:00:00 2001 From: dmitry-ganyushin Date: Wed, 10 Jan 2024 14:28:16 -0500 Subject: [PATCH 012/124] WIP. Implementing sqlite database instead of json file. Writing part. --- .../engine/campaign/CampaignManager.cpp | 69 ++++++++++++++----- 1 file changed, 53 insertions(+), 16 deletions(-) diff --git a/source/adios2/engine/campaign/CampaignManager.cpp b/source/adios2/engine/campaign/CampaignManager.cpp index ddc6a6bc0b..cb5e3bc4ef 100644 --- a/source/adios2/engine/campaign/CampaignManager.cpp +++ b/source/adios2/engine/campaign/CampaignManager.cpp @@ -17,6 +17,7 @@ #include #include +#include namespace adios2 { @@ -25,20 +26,55 @@ namespace core namespace engine { -static std::string CMapToJson(const CampaignRecordMap &cmap, const int rank, const std::string name) +int CMapToSqlite(const CampaignRecordMap &cmap, const int rank, std::string name) { - nlohmann::json j = nlohmann::json::array(); + sqlite3 *db; + int rc; + char *zErrMsg = 0; + std::string sqlcmd; + std::string db_name = name + ".db"; + rc = sqlite3_open(db_name.c_str(), &db); + + if (rc != SQLITE_OK) + { + std::cout << "SQL error: " << zErrMsg << std::endl; + std::string m(zErrMsg); + helper::Throw("Engine", "CampaignReader", "WriteCampaignData", + "SQL error on writing records:"); + sqlite3_free(zErrMsg); + } + sqlcmd = + "CREATE TABLE table1 (name TEXT,varying_deltas INT,delta_step INT, delta_time REAL);\n"; + sqlcmd += "INSERT INTO table1 VALUES\n"; + for (auto &r : cmap) { - nlohmann::json c = nlohmann::json{{"name", r.first}, - {"varying_deltas", r.second.varying_deltas}, - {"delta_step", r.second.delta_step}, - {"delta_time", r.second.delta_time}, - {"steps", r.second.steps}, - {"times", r.second.times}}; - j.push_back(c); + // vectors r.second.steps and std::to_string(r.second.times) should go to another table + // check for nans + double delta_time = 0.0; + if (!std::isnan(r.second.delta_time)) + delta_time = r.second.delta_time; + size_t delta_step = 0; + if (!std::isnan(r.second.delta_step)) + delta_time = r.second.delta_step; + sqlcmd += " ('" + r.first + "'" + "," + std::to_string(r.second.varying_deltas) + "," + + std::to_string(delta_step) + "," + std::to_string(delta_time) + "),\n"; } - return nlohmann::to_string(j); + sqlcmd.replace(sqlcmd.size() - 2, 1, ";"); + std::cout << sqlcmd << std::endl; + rc = sqlite3_exec(db, sqlcmd.c_str(), 0, 0, &zErrMsg); + if (rc != SQLITE_OK) + { + std::cout << "SQL error: " << zErrMsg << std::endl; + std::string m(zErrMsg); + helper::Throw("Engine", "CampaignReader", "WriteCampaignData", + "SQL error on writing records:"); + sqlite3_free(zErrMsg); + } + + sqlite3_close(db); + + return 0; } CampaignManager::CampaignManager(adios2::helper::Comm &comm) @@ -65,7 +101,7 @@ CampaignManager::~CampaignManager() void CampaignManager::Open(const std::string &name) { - m_Name = m_CampaignDir + "/" + name + "_" + std::to_string(m_WriterRank) + ".json"; + m_Name = m_CampaignDir + "/" + name + "_" + std::to_string(m_WriterRank); if (m_Verbosity == 5) { std::cout << "Campaign Manager " << m_WriterRank << " Open(" << m_Name << ")\n"; @@ -118,11 +154,12 @@ void CampaignManager::Close() { if (!cmap.empty()) { - m_Output.open(m_Name, std::ofstream::out); - m_Opened = true; - m_Output << std::setw(4) << CMapToJson(cmap, m_WriterRank, m_Name) << std::endl; - m_Output.close(); - m_Opened = false; + CMapToSqlite(cmap, m_WriterRank, m_Name); +// m_Output.open(m_Name, std::ofstream::out); +// m_Opened = true; +// m_Output << std::setw(4) << CMapToJson(cmap, m_WriterRank, m_Name) << std::endl; +// m_Output.close(); +// m_Opened = false; } } From cfa72c6edbf87ca8231c2a4a17eb9d090273de23 Mon Sep 17 00:00:00 2001 From: Dmitry Ganyushin Date: Fri, 12 Jan 2024 17:42:17 -0500 Subject: [PATCH 013/124] added rowid --- .../engine/campaign/CampaignManager.cpp | 49 ++++++++++--------- 1 file changed, 27 insertions(+), 22 deletions(-) diff --git a/source/adios2/engine/campaign/CampaignManager.cpp b/source/adios2/engine/campaign/CampaignManager.cpp index cb5e3bc4ef..0118366bca 100644 --- a/source/adios2/engine/campaign/CampaignManager.cpp +++ b/source/adios2/engine/campaign/CampaignManager.cpp @@ -43,33 +43,43 @@ int CMapToSqlite(const CampaignRecordMap &cmap, const int rank, std::string name "SQL error on writing records:"); sqlite3_free(zErrMsg); } - sqlcmd = - "CREATE TABLE table1 (name TEXT,varying_deltas INT,delta_step INT, delta_time REAL);\n"; - sqlcmd += "INSERT INTO table1 VALUES\n"; + sqlcmd = "CREATE TABLE files (name, varying_deltas, delta_step, delta_time);"; + rc = sqlite3_exec(db, sqlcmd.c_str(), 0, 0, &zErrMsg); + if (rc != SQLITE_OK) + { + std::cout << "SQL error: " << zErrMsg << std::endl; + std::string m(zErrMsg); + helper::Throw("Engine", "CampaignReader", "WriteCampaignData", + "SQL error on writing records:"); + sqlite3_free(zErrMsg); + } + size_t rowid = 1; for (auto &r : cmap) { // vectors r.second.steps and std::to_string(r.second.times) should go to another table - // check for nans + // check for NaNs double delta_time = 0.0; if (!std::isnan(r.second.delta_time)) delta_time = r.second.delta_time; size_t delta_step = 0; if (!std::isnan(r.second.delta_step)) delta_time = r.second.delta_step; - sqlcmd += " ('" + r.first + "'" + "," + std::to_string(r.second.varying_deltas) + "," + - std::to_string(delta_step) + "," + std::to_string(delta_time) + "),\n"; - } - sqlcmd.replace(sqlcmd.size() - 2, 1, ";"); - std::cout << sqlcmd << std::endl; - rc = sqlite3_exec(db, sqlcmd.c_str(), 0, 0, &zErrMsg); - if (rc != SQLITE_OK) - { - std::cout << "SQL error: " << zErrMsg << std::endl; - std::string m(zErrMsg); - helper::Throw("Engine", "CampaignReader", "WriteCampaignData", - "SQL error on writing records:"); - sqlite3_free(zErrMsg); + sqlcmd = "INSERT INTO files (rowid, name, varying_deltas, delta_step, delta_time)\n"; + sqlcmd += "VALUES(" + std::to_string(rowid) + "," + "'" + r.first + "'" + "," + + std::to_string(r.second.varying_deltas) + "," + std::to_string(delta_step) + "," + + std::to_string(delta_time) + ");"; + rowid++; + std::cout << sqlcmd << std::endl; + rc = sqlite3_exec(db, sqlcmd.c_str(), 0, 0, &zErrMsg); + if (rc != SQLITE_OK) + { + std::cout << "SQL error: " << zErrMsg << std::endl; + std::string m(zErrMsg); + helper::Throw("Engine", "CampaignReader", "WriteCampaignData", + "SQL error on writing records:"); + sqlite3_free(zErrMsg); + } } sqlite3_close(db); @@ -155,11 +165,6 @@ void CampaignManager::Close() if (!cmap.empty()) { CMapToSqlite(cmap, m_WriterRank, m_Name); -// m_Output.open(m_Name, std::ofstream::out); -// m_Opened = true; -// m_Output << std::setw(4) << CMapToJson(cmap, m_WriterRank, m_Name) << std::endl; -// m_Output.close(); -// m_Opened = false; } } From 16627b31059413c1641956f62ef251aa31e1a875 Mon Sep 17 00:00:00 2001 From: Dmitry Ganyushin Date: Mon, 15 Jan 2024 16:12:48 -0500 Subject: [PATCH 014/124] WIP. Read part --- .../engine/campaign/CampaignManager.cpp | 25 ++++++++++++---- .../adios2_campaign_manager.py | 29 ++++++++++++++----- 2 files changed, 40 insertions(+), 14 deletions(-) diff --git a/source/adios2/engine/campaign/CampaignManager.cpp b/source/adios2/engine/campaign/CampaignManager.cpp index 0118366bca..a821d52a38 100644 --- a/source/adios2/engine/campaign/CampaignManager.cpp +++ b/source/adios2/engine/campaign/CampaignManager.cpp @@ -43,7 +43,7 @@ int CMapToSqlite(const CampaignRecordMap &cmap, const int rank, std::string name "SQL error on writing records:"); sqlite3_free(zErrMsg); } - sqlcmd = "CREATE TABLE files (name, varying_deltas, delta_step, delta_time);"; + sqlcmd = "CREATE TABLE bpfiles (name);"; rc = sqlite3_exec(db, sqlcmd.c_str(), 0, 0, &zErrMsg); if (rc != SQLITE_OK) { @@ -54,7 +54,18 @@ int CMapToSqlite(const CampaignRecordMap &cmap, const int rank, std::string name sqlite3_free(zErrMsg); } - size_t rowid = 1; +// sqlcmd = "CREATE TABLE data (name);"; +// rc = sqlite3_exec(db, sqlcmd.c_str(), 0, 0, &zErrMsg); +// if (rc != SQLITE_OK) +// { +// std::cout << "SQL error: " << zErrMsg << std::endl; +// std::string m(zErrMsg); +// helper::Throw("Engine", "CampaignReader", "WriteCampaignData", +// "SQL error on writing records:"); +// sqlite3_free(zErrMsg); +// } + + size_t rowid = 1000; for (auto &r : cmap) { // vectors r.second.steps and std::to_string(r.second.times) should go to another table @@ -65,10 +76,12 @@ int CMapToSqlite(const CampaignRecordMap &cmap, const int rank, std::string name size_t delta_step = 0; if (!std::isnan(r.second.delta_step)) delta_time = r.second.delta_step; - sqlcmd = "INSERT INTO files (rowid, name, varying_deltas, delta_step, delta_time)\n"; - sqlcmd += "VALUES(" + std::to_string(rowid) + "," + "'" + r.first + "'" + "," + - std::to_string(r.second.varying_deltas) + "," + std::to_string(delta_step) + "," + - std::to_string(delta_time) + ");"; +// sqlcmd = "INSERT INTO files (rowid, name, varying_deltas, delta_step, delta_time)\n"; +// sqlcmd += "VALUES(" + std::to_string(rowid) + "," + "'" + r.first + "'" + "," + +// std::to_string(r.second.varying_deltas) + "," + std::to_string(delta_step) + "," + +// std::to_string(delta_time) + ");"; + sqlcmd = "INSERT INTO bpfiles (rowid, name)\n"; + sqlcmd += "VALUES(" + std::to_string(rowid) + "," + "'" + r.first + "'" + ");"; rowid++; std::cout << sqlcmd << std::endl; rc = sqlite3_exec(db, sqlcmd.c_str(), 0, 0, &zErrMsg); diff --git a/source/utils/adios_campaign_manager/adios2_campaign_manager.py b/source/utils/adios_campaign_manager/adios2_campaign_manager.py index 5b4b096517..7122287767 100755 --- a/source/utils/adios_campaign_manager/adios2_campaign_manager.py +++ b/source/utils/adios_campaign_manager/adios2_campaign_manager.py @@ -222,10 +222,10 @@ def Update(args: dict, cur: sqlite3.Cursor): dirID = curDir.lastrowid con.commit() - jsonlist = MergeJsonFiles(jsonFileList) + db_list = MergeDBFiles(dbFileList) # print(f"Merged json = {jsonlist}") - ProcessJsonFile(args, jsonlist, cur, hostID, dirID) + ProcessJsonFile(args, db_list, cur, hostID, dirID) con.commit() @@ -250,11 +250,24 @@ def Create(args: dict, cur: sqlite3.Cursor): Update(args, cur) -def MergeJsonFiles(jsonfiles: list): +def MergeDBFiles(dbfiles: list): + # read db files here result = list() - for f1 in jsonfiles: - with open(f1, 'r') as infile: - result.extend(json.load(infile)) + for f1 in dbfiles: + try: + con = sqlite3.connect(f1) + except sqlite3.Error as e: + print(e) + + cur = con.cursor() + try: + cur.execute("select * from bpfiles") + except sqlite3.Error as e: + print(e) + record = cur.fetchall() + for item in record: + result.append({"name": item[0]}) + cur.close() return result @@ -291,8 +304,8 @@ def MergeJsonFiles(jsonfiles: list): else: CheckLocalCampaignDir(args) # List the local campaign directory - jsonFileList = glob.glob(args.LocalCampaignDir + '/*.json') - if len(jsonFileList) == 0: + dbFileList = glob.glob(args.LocalCampaignDir + '/*.db') + if len(dbFileList) == 0: print("There are no campaign data files in " + args.LocalCampaignDir) exit(2) From efa961deb8a571ea4142767f61e1354c544dabfa Mon Sep 17 00:00:00 2001 From: Dmitry Ganyushin Date: Tue, 16 Jan 2024 13:29:36 -0500 Subject: [PATCH 015/124] WIP. Read part. Formatting --- .../engine/campaign/CampaignManager.cpp | 29 ++++++++++--------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/source/adios2/engine/campaign/CampaignManager.cpp b/source/adios2/engine/campaign/CampaignManager.cpp index a821d52a38..e2834f6aa7 100644 --- a/source/adios2/engine/campaign/CampaignManager.cpp +++ b/source/adios2/engine/campaign/CampaignManager.cpp @@ -54,16 +54,16 @@ int CMapToSqlite(const CampaignRecordMap &cmap, const int rank, std::string name sqlite3_free(zErrMsg); } -// sqlcmd = "CREATE TABLE data (name);"; -// rc = sqlite3_exec(db, sqlcmd.c_str(), 0, 0, &zErrMsg); -// if (rc != SQLITE_OK) -// { -// std::cout << "SQL error: " << zErrMsg << std::endl; -// std::string m(zErrMsg); -// helper::Throw("Engine", "CampaignReader", "WriteCampaignData", -// "SQL error on writing records:"); -// sqlite3_free(zErrMsg); -// } + // sqlcmd = "CREATE TABLE data (name);"; + // rc = sqlite3_exec(db, sqlcmd.c_str(), 0, 0, &zErrMsg); + // if (rc != SQLITE_OK) + // { + // std::cout << "SQL error: " << zErrMsg << std::endl; + // std::string m(zErrMsg); + // helper::Throw("Engine", "CampaignReader", "WriteCampaignData", + // "SQL error on writing records:"); + // sqlite3_free(zErrMsg); + // } size_t rowid = 1000; for (auto &r : cmap) @@ -76,10 +76,11 @@ int CMapToSqlite(const CampaignRecordMap &cmap, const int rank, std::string name size_t delta_step = 0; if (!std::isnan(r.second.delta_step)) delta_time = r.second.delta_step; -// sqlcmd = "INSERT INTO files (rowid, name, varying_deltas, delta_step, delta_time)\n"; -// sqlcmd += "VALUES(" + std::to_string(rowid) + "," + "'" + r.first + "'" + "," + -// std::to_string(r.second.varying_deltas) + "," + std::to_string(delta_step) + "," + -// std::to_string(delta_time) + ");"; + // sqlcmd = "INSERT INTO files (rowid, name, varying_deltas, delta_step, + // delta_time)\n"; sqlcmd += "VALUES(" + std::to_string(rowid) + "," + "'" + r.first + // + "'" + "," + + // std::to_string(r.second.varying_deltas) + "," + + // std::to_string(delta_step) + "," + std::to_string(delta_time) + ");"; sqlcmd = "INSERT INTO bpfiles (rowid, name)\n"; sqlcmd += "VALUES(" + std::to_string(rowid) + "," + "'" + r.first + "'" + ");"; rowid++; From bda474189b535fad1ad50b91ab1f661ee88ebffb Mon Sep 17 00:00:00 2001 From: Dmitry Ganyushin Date: Tue, 16 Jan 2024 14:54:40 -0500 Subject: [PATCH 016/124] Cleaned-up --- .../engine/campaign/CampaignManager.cpp | 25 ------------------- 1 file changed, 25 deletions(-) diff --git a/source/adios2/engine/campaign/CampaignManager.cpp b/source/adios2/engine/campaign/CampaignManager.cpp index e2834f6aa7..5a218b68b8 100644 --- a/source/adios2/engine/campaign/CampaignManager.cpp +++ b/source/adios2/engine/campaign/CampaignManager.cpp @@ -54,37 +54,12 @@ int CMapToSqlite(const CampaignRecordMap &cmap, const int rank, std::string name sqlite3_free(zErrMsg); } - // sqlcmd = "CREATE TABLE data (name);"; - // rc = sqlite3_exec(db, sqlcmd.c_str(), 0, 0, &zErrMsg); - // if (rc != SQLITE_OK) - // { - // std::cout << "SQL error: " << zErrMsg << std::endl; - // std::string m(zErrMsg); - // helper::Throw("Engine", "CampaignReader", "WriteCampaignData", - // "SQL error on writing records:"); - // sqlite3_free(zErrMsg); - // } - size_t rowid = 1000; for (auto &r : cmap) { - // vectors r.second.steps and std::to_string(r.second.times) should go to another table - // check for NaNs - double delta_time = 0.0; - if (!std::isnan(r.second.delta_time)) - delta_time = r.second.delta_time; - size_t delta_step = 0; - if (!std::isnan(r.second.delta_step)) - delta_time = r.second.delta_step; - // sqlcmd = "INSERT INTO files (rowid, name, varying_deltas, delta_step, - // delta_time)\n"; sqlcmd += "VALUES(" + std::to_string(rowid) + "," + "'" + r.first - // + "'" + "," + - // std::to_string(r.second.varying_deltas) + "," + - // std::to_string(delta_step) + "," + std::to_string(delta_time) + ");"; sqlcmd = "INSERT INTO bpfiles (rowid, name)\n"; sqlcmd += "VALUES(" + std::to_string(rowid) + "," + "'" + r.first + "'" + ");"; rowid++; - std::cout << sqlcmd << std::endl; rc = sqlite3_exec(db, sqlcmd.c_str(), 0, 0, &zErrMsg); if (rc != SQLITE_OK) { From 7996076c9d8ae8b6ade4766bba00dd35c8f5e79c Mon Sep 17 00:00:00 2001 From: anagainaru Date: Sun, 10 Dec 2023 14:03:41 -0500 Subject: [PATCH 017/124] Update the BP Kokkos example to be able to use any combination of memory spaces --- .../bpStepsWriteReadKokkos.cpp | 187 ++++++++++-------- 1 file changed, 102 insertions(+), 85 deletions(-) diff --git a/examples/hello/bpStepsWriteReadKokkos/bpStepsWriteReadKokkos.cpp b/examples/hello/bpStepsWriteReadKokkos/bpStepsWriteReadKokkos.cpp index 268d75adf3..9fd98b4027 100644 --- a/examples/hello/bpStepsWriteReadKokkos/bpStepsWriteReadKokkos.cpp +++ b/examples/hello/bpStepsWriteReadKokkos/bpStepsWriteReadKokkos.cpp @@ -3,144 +3,161 @@ * accompanying file Copyright.txt for details. * * bpStepsWriteReadKokkos.cpp Simple example of writing and reading bpFloats through ADIOS2 BP - * engine with multiple simulations steps for every IO step using Kokkos + * engine using Kokkos on any memory space */ #include #include -#include //std::invalid_argument std::exception #include #include #include -void writer(adios2::ADIOS &adios, const std::string &engine, const std::string &fname, - const size_t Nx, unsigned int nSteps) +template +int BPWrite(const std::string fname, const size_t Nx, const size_t Ny, const size_t nSteps, + const std::string engine) { - // Initialize the simulation bpFloats with the default memory space - using mem_space = Kokkos::DefaultExecutionSpace::memory_space; - Kokkos::View gpuSimData("simBuffer", Nx); - Kokkos::parallel_for( - "initBuffer", Kokkos::RangePolicy(0, Nx), - KOKKOS_LAMBDA(int i) { gpuSimData(i) = static_cast(i); }); + // Initialize the simulation data + Kokkos::View gpuSimData("simBuffer", Nx, Ny); + static_assert(Kokkos::SpaceAccessibility::accessible, ""); + Kokkos::parallel_for("initBuffer", Kokkos::RangePolicy(0, Nx), KOKKOS_LAMBDA(int i) { + for (int j = 0; j < Ny; j++) + gpuSimData(i, j) = static_cast(i); + }); Kokkos::fence(); - // Set up the ADIOS structures - adios2::IO bpIO = adios.DeclareIO("WriteIO"); - bpIO.SetEngine(engine); + adios2::ADIOS adios; + adios2::IO io = adios.DeclareIO("WriteIO"); + io.SetEngine(engine); - const adios2::Dims shape{static_cast(Nx)}; - const adios2::Dims start{static_cast(0)}; - const adios2::Dims count{Nx}; - auto bpFloats = bpIO.DefineVariable("bpFloats", shape, start, count); - auto bpStep = bpIO.DefineVariable("bpStep"); + const adios2::Dims shape{Nx, Ny}; + const adios2::Dims start{0, 0}; + const adios2::Dims count{Nx, Ny}; + auto data = io.DefineVariable("bpFloats", shape, start, count); - adios2::Engine bpWriter = bpIO.Open(fname, adios2::Mode::Write); + adios2::Engine bpWriter = io.Open(fname, adios2::Mode::Write); // Simulation steps for (unsigned int step = 0; step < nSteps; ++step) { - // Make a 1D selection to describe the local dimensions of the - // variable we write and its offsets in the global spaces - adios2::Box sel({0}, {Nx}); - bpFloats.SetSelection(sel); + adios2::Box sel({0, 0}, {Nx, Ny}); + data.SetSelection(sel); - // Start IO step every write step bpWriter.BeginStep(); - bpWriter.Put(bpFloats, gpuSimData.data()); - bpWriter.Put(bpStep, step); + bpWriter.Put(data, gpuSimData.data()); bpWriter.EndStep(); - // Update values in the simulation bpFloats using the default - // execution space - Kokkos::parallel_for( - "updateBuffer", Kokkos::RangePolicy(0, Nx), - KOKKOS_LAMBDA(int i) { gpuSimData(i) += 10; }); + // Update values in the simulation data + Kokkos::parallel_for("updateBuffer", Kokkos::RangePolicy(0, Nx), + KOKKOS_LAMBDA(int i) { + for (int j = 0; j < Ny; j++) + gpuSimData(i, j) += 10; + }); Kokkos::fence(); } bpWriter.Close(); - Kokkos::DefaultExecutionSpace exe_space; + ExecSpace exe_space; std::cout << "Done writing on memory space: " << exe_space.name() << std::endl; + return 0; +} + +template +std::array GetDimenstions(adios2::Variable data) +{ + return {data.Shape()[1], data.Shape()[0]}; } -void reader(adios2::ADIOS &adios, const std::string &engine, const std::string &fname, - const size_t Nx, unsigned int /*nSteps*/) +template <> +std::array GetDimenstions(adios2::Variable data) { - // Create ADIOS structures - adios2::IO bpIO = adios.DeclareIO("ReadIO"); - bpIO.SetEngine(engine); + return {data.Shape()[0], data.Shape()[1]}; +} + +template +int BPRead(const std::string fname, const std::string engine) +{ + adios2::ADIOS adios; + adios2::IO io = adios.DeclareIO("ReadIO"); + io.SetEngine(engine); - Kokkos::DefaultExecutionSpace exe_space; + ExecSpace exe_space; std::cout << "Read on memory space: " << exe_space.name() << std::endl; - adios2::Engine bpReader = bpIO.Open(fname, adios2::Mode::Read); + adios2::Engine bpReader = io.Open(fname, adios2::Mode::Read); - using mem_space = Kokkos::DefaultExecutionSpace::memory_space; - Kokkos::View gpuSimData("simBuffer", Nx); - unsigned int inStep = 0; - for (unsigned int step = 0; bpReader.BeginStep() == adios2::StepStatus::OK; ++step) + unsigned int step = 0; + for (; bpReader.BeginStep() == adios2::StepStatus::OK; ++step) { - auto bpFloats = bpIO.InquireVariable("bpFloats"); - if (bpFloats) - { - const adios2::Dims start{0}; - const adios2::Dims count{Nx}; - const adios2::Box sel(start, count); - bpFloats.SetSelection(sel); - - bpReader.Get(bpFloats, gpuSimData.data()); - } - auto bpStep = bpIO.InquireVariable("bpStep"); - if (bpStep) + auto data = io.InquireVariable("bpFloats"); + if (data.Shape().size() != 2) { - bpReader.Get(bpStep, &inStep); + std::cout << "Error, the bpFloats variable in the BP file " + " on step " + << step << "needs to have two dimensions" << std::endl; + break; } + + const adios2::Dims start{0, 0}; + const adios2::Box sel(start, data.Shape()); + data.SetSelection(sel); + + auto dims = GetDimenstions(data); + size_t Nx = dims[0], Ny = dims[1]; + Kokkos::View gpuSimData("simBuffer", Nx, Ny); + bpReader.Get(data, gpuSimData.data()); bpReader.EndStep(); - if (inStep != step) + + auto cpuData = Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace{}, gpuSimData); + for (size_t i = 0; i < Nx; i++) { - std::cout << "ERROR: step mismatch\n"; - return; + for (size_t j = 0; j < Ny; j++) + std::cout << cpuData(i, j) << " "; + std::cout << std::endl; } } bpReader.Close(); + return 0; } int main(int argc, char **argv) { - Kokkos::initialize(argc, argv); - const std::string engine = argv[1] ? argv[1] : "BPFile"; std::cout << "Using engine " << engine << std::endl; + const size_t Nx = 6, Ny = 10, nSteps = 1; + const std::string fromMemSpace = "Default"; + const std::string toMemSpace = "Default"; - const std::string filename = engine + "StepsWriteReadCuda.bp"; - const unsigned int nSteps = 10; - const unsigned int Nx = 6000; - try + const std::string filename = engine + "StepsWriteReadKokkos"; + Kokkos::initialize(argc, argv); { - /** ADIOS class factory of IO class objects */ - adios2::ADIOS adios; + std::cout << "Using engine " << engine << std::endl; - writer(adios, engine, filename, Nx, nSteps); - reader(adios, engine, filename, Nx, nSteps); - } - catch (std::invalid_argument &e) - { - std::cout << "Invalid argument exception, STOPPING PROGRAM\n"; - std::cout << e.what() << "\n"; - } - catch (std::ios_base::failure &e) - { - std::cout << "IO System base failure exception, STOPPING PROGRAM\n"; - std::cout << e.what() << "\n"; - } - catch (std::exception &e) - { - std::cout << "Exception, STOPPING PROGRAM\n"; - std::cout << e.what() << "\n"; + if (fromMemSpace == "Default") + { + using mem_space = Kokkos::DefaultExecutionSpace::memory_space; + std::cout << "Writing on memory space: DefaultMemorySpace" << std::endl; + BPWrite(filename + ".bp", Nx, Ny, nSteps, + engine); + } + else + { + std::cout << "Writing on memory space: HostSpace" << std::endl; + BPWrite(filename + ".bp", nSteps, Nx, Ny, engine); + } + if (toMemSpace == "Default") + { + using mem_space = Kokkos::DefaultExecutionSpace::memory_space; + std::cout << "Reading on memory space: DefaultMemorySpace" << std::endl; + BPRead(filename + ".bp", engine); + } + else + { + std::cout << "Reading on memory space: HostSpace" << std::endl; + BPRead(filename + ".bp", engine); + } } Kokkos::finalize(); - return 0; } From a7edaaada309f0d1b652999aa83a68ab62d32d05 Mon Sep 17 00:00:00 2001 From: anagainaru Date: Sun, 10 Dec 2023 14:00:40 -0500 Subject: [PATCH 018/124] Fix the memory space to one value for all Put/Get calls to the same variable --- bindings/CXX11/adios2/cxx11/Engine.cpp | 6 ++++-- bindings/CXX11/adios2/cxx11/Engine.h | 14 ++++++++++---- bindings/CXX11/adios2/cxx11/Variable.cpp | 2 +- bindings/CXX11/adios2/cxx11/Variable.h | 2 +- source/adios2/core/VariableBase.cpp | 14 ++++++-------- source/adios2/engine/bp5/BP5Writer.cpp | 6 +++--- source/adios2/engine/daos/DaosWriter.cpp | 6 +++--- source/adios2/engine/sst/SstWriter.tcc | 3 ++- source/adios2/toolkit/interop/hdf5/HDF5Common.tcc | 3 ++- 9 files changed, 32 insertions(+), 24 deletions(-) diff --git a/bindings/CXX11/adios2/cxx11/Engine.cpp b/bindings/CXX11/adios2/cxx11/Engine.cpp index c37d8e1624..cd9af86b6a 100644 --- a/bindings/CXX11/adios2/cxx11/Engine.cpp +++ b/bindings/CXX11/adios2/cxx11/Engine.cpp @@ -20,8 +20,10 @@ namespace adios2 #ifdef ADIOS2_HAVE_GPU_SUPPORT void Engine::CheckMemorySpace(MemorySpace variableMem, MemorySpace bufferMem) { - if (variableMem != MemorySpace::Detect && variableMem != bufferMem) - helper::Throw("CXX-Bindings", "Engine", "Put", "Memory space mismatch"); + if (variableMem != bufferMem) + helper::Throw( + "CXX-Bindings", "Engine", "Put", + "Memory space mismatch between the pre-set value and the Kokkos::View"); } #endif diff --git a/bindings/CXX11/adios2/cxx11/Engine.h b/bindings/CXX11/adios2/cxx11/Engine.h index 2d5a1de68a..0ed357bd8c 100644 --- a/bindings/CXX11/adios2/cxx11/Engine.h +++ b/bindings/CXX11/adios2/cxx11/Engine.h @@ -214,8 +214,9 @@ class Engine auto bufferView = static_cast>(data); auto bufferMem = bufferView.memory_space(); #ifdef ADIOS2_HAVE_GPU_SUPPORT - auto variableMem = variable.GetMemorySpace(); - CheckMemorySpace(variableMem, bufferMem); + auto varMemSpace = variable.GetDefaultMemorySpace(); + if (varMemSpace != MemorySpace::Detect) + CheckMemorySpace(varMemSpace, bufferMem); #endif variable.SetMemorySpace(bufferMem); Put(variable, bufferView.data(), launch); @@ -418,8 +419,13 @@ class Engine void Get(Variable variable, U const &data, const Mode launch = Mode::Deferred) { auto adios_data = static_cast>(data); - auto mem_space = adios_data.memory_space(); - variable.SetMemorySpace(mem_space); + auto bufferMem = adios_data.memory_space(); +#ifdef ADIOS2_HAVE_GPU_SUPPORT + auto varMemSpace = variable.GetDefaultMemorySpace(); + if (varMemSpace != MemorySpace::Detect) + CheckMemorySpace(varMemSpace, bufferMem); +#endif + variable.SetMemorySpace(bufferMem); Get(variable, adios_data.data(), launch); } diff --git a/bindings/CXX11/adios2/cxx11/Variable.cpp b/bindings/CXX11/adios2/cxx11/Variable.cpp index d7d47c3428..a6cb79330a 100644 --- a/bindings/CXX11/adios2/cxx11/Variable.cpp +++ b/bindings/CXX11/adios2/cxx11/Variable.cpp @@ -37,7 +37,7 @@ namespace adios2 } \ \ template <> \ - MemorySpace Variable::GetMemorySpace() \ + MemorySpace Variable::GetDefaultMemorySpace() \ { \ return m_Variable->m_MemSpace; \ } \ diff --git a/bindings/CXX11/adios2/cxx11/Variable.h b/bindings/CXX11/adios2/cxx11/Variable.h index 4f8ea8f071..59f1061673 100644 --- a/bindings/CXX11/adios2/cxx11/Variable.h +++ b/bindings/CXX11/adios2/cxx11/Variable.h @@ -158,7 +158,7 @@ class Variable * Get the memory space that was set by the application * @return the memory space stored in the Variable object */ - MemorySpace GetMemorySpace(); + MemorySpace GetDefaultMemorySpace(); /** * Set new shape, care must be taken when reading back the variable for diff --git a/source/adios2/core/VariableBase.cpp b/source/adios2/core/VariableBase.cpp index 3f76bb1f48..0481bf67d8 100644 --- a/source/adios2/core/VariableBase.cpp +++ b/source/adios2/core/VariableBase.cpp @@ -45,17 +45,15 @@ size_t VariableBase::TotalSize() const noexcept { return helper::GetTotalSize(m_ MemorySpace VariableBase::GetMemorySpace(const void *ptr) { #ifdef ADIOS2_HAVE_GPU_SUPPORT - if (m_MemSpace != MemorySpace::Detect) + if (m_MemSpace == MemorySpace::Detect) { - return m_MemSpace; - } - - if (helper::IsGPUbuffer(ptr)) - { - return MemorySpace::GPU; + if (helper::IsGPUbuffer(ptr)) + m_MemSpace = MemorySpace::GPU; + else + m_MemSpace = MemorySpace::Host; } #endif - return MemorySpace::Host; + return m_MemSpace; } void VariableBase::SetMemorySpace(const MemorySpace mem) { m_MemSpace = mem; } diff --git a/source/adios2/engine/bp5/BP5Writer.cpp b/source/adios2/engine/bp5/BP5Writer.cpp index 69c0c4ec86..8afeae2643 100644 --- a/source/adios2/engine/bp5/BP5Writer.cpp +++ b/source/adios2/engine/bp5/BP5Writer.cpp @@ -1729,7 +1729,8 @@ void BP5Writer::PutCommon(VariableBase &variable, const void *values, bool sync) } // if the user buffer is allocated on the GPU always use sync mode - if (variable.GetMemorySpace(values) != MemorySpace::Host) + auto memSpace = variable.GetMemorySpace(values); + if (memSpace != MemorySpace::Host) sync = true; size_t *Shape = NULL; @@ -1800,8 +1801,7 @@ void BP5Writer::PutCommon(VariableBase &variable, const void *values, bool sync) helper::NdCopy((const char *)values, helper::CoreDims(ZeroDims), MemoryCount, sourceRowMajor, false, (char *)ptr, MemoryStart, varCount, sourceRowMajor, false, (int)ObjSize, helper::CoreDims(), helper::CoreDims(), - helper::CoreDims(), helper::CoreDims(), false /* safemode */, - variable.m_MemSpace); + helper::CoreDims(), helper::CoreDims(), false /* safemode */, memSpace); } else { diff --git a/source/adios2/engine/daos/DaosWriter.cpp b/source/adios2/engine/daos/DaosWriter.cpp index 7c9e2100bf..6567d427fc 100644 --- a/source/adios2/engine/daos/DaosWriter.cpp +++ b/source/adios2/engine/daos/DaosWriter.cpp @@ -1708,7 +1708,8 @@ void DaosWriter::PutCommon(VariableBase &variable, const void *values, bool sync } // if the user buffer is allocated on the GPU always use sync mode - if (variable.GetMemorySpace(values) != MemorySpace::Host) + auto memSpace = variable.GetMemorySpace(values); + if (memSpace != MemorySpace::Host) sync = true; size_t *Shape = NULL; @@ -1769,8 +1770,7 @@ void DaosWriter::PutCommon(VariableBase &variable, const void *values, bool sync helper::NdCopy((const char *)values, helper::CoreDims(ZeroDims), variable.m_MemoryCount, sourceRowMajor, false, (char *)ptr, variable.m_MemoryStart, variable.m_Count, sourceRowMajor, false, ObjSize, helper::CoreDims(), helper::CoreDims(), - helper::CoreDims(), helper::CoreDims(), false /* safemode */, - variable.m_MemSpace); + helper::CoreDims(), helper::CoreDims(), false /* safemode */, memSpace); } else { diff --git a/source/adios2/engine/sst/SstWriter.tcc b/source/adios2/engine/sst/SstWriter.tcc index 237c902d8e..2bcd0c4b49 100644 --- a/source/adios2/engine/sst/SstWriter.tcc +++ b/source/adios2/engine/sst/SstWriter.tcc @@ -37,6 +37,7 @@ void SstWriter::PutSyncCommon(Variable &variable, const T *values) "BeginStep/EndStep pairs"); } + auto memSpace = variable.GetMemorySpace(values); if ((Params.MarshalMethod == SstMarshalFFS) || (Params.MarshalMethod == SstMarshalBP5)) { size_t *Shape = NULL; @@ -107,7 +108,7 @@ void SstWriter::PutSyncCommon(Variable &variable, const T *values) sourceRowMajor, false, (char *)ptr, MemoryStart, varCount, sourceRowMajor, false, (int)ObjSize, helper::CoreDims(), helper::CoreDims(), helper::CoreDims(), helper::CoreDims(), - false /* safemode */, variable.m_MemSpace); + false /* safemode */, memSpace); } else { diff --git a/source/adios2/toolkit/interop/hdf5/HDF5Common.tcc b/source/adios2/toolkit/interop/hdf5/HDF5Common.tcc index 9ef93b3498..ba898f9db6 100644 --- a/source/adios2/toolkit/interop/hdf5/HDF5Common.tcc +++ b/source/adios2/toolkit/interop/hdf5/HDF5Common.tcc @@ -241,7 +241,8 @@ void HDF5Common::Write(core::Variable &variable, const T *values) #ifdef NO_STAT size_t valuesSize = adios2::helper::GetTotalSize(variable.m_Count); T min, max; - adios2::helper::GetMinMaxThreads(values, valuesSize, min, max, 1, variable.m_MemSpace); + auto memSpace = variable.GetMemorySpace(values); + adios2::helper::GetMinMaxThreads(values, valuesSize, min, max, 1, memSpace); int chainSize = chain.size(); hid_t parentId = m_GroupId; From 6191bd74e6afd5826c31b6905bb12a4613e3d9f4 Mon Sep 17 00:00:00 2001 From: anagainaru Date: Mon, 22 Jan 2024 11:41:31 -0500 Subject: [PATCH 019/124] Separate the concept of layout from the one of memory space and use it to adjust the variable dimensions --- bindings/CXX11/adios2/cxx11/Engine.cpp | 10 --- bindings/CXX11/adios2/cxx11/Engine.h | 28 +++----- bindings/CXX11/adios2/cxx11/KokkosView.h | 21 ++++++ bindings/CXX11/adios2/cxx11/Variable.cpp | 10 ++- bindings/CXX11/adios2/cxx11/Variable.h | 16 ++++- bindings/CXX11/adios2/cxx11/Variable.tcc | 14 ++++ source/adios2/core/IO.tcc | 3 + source/adios2/core/VariableBase.cpp | 85 +++++++++++++++++++++++- source/adios2/core/VariableBase.h | 22 ++++++ 9 files changed, 179 insertions(+), 30 deletions(-) diff --git a/bindings/CXX11/adios2/cxx11/Engine.cpp b/bindings/CXX11/adios2/cxx11/Engine.cpp index cd9af86b6a..93a6bf2b3c 100644 --- a/bindings/CXX11/adios2/cxx11/Engine.cpp +++ b/bindings/CXX11/adios2/cxx11/Engine.cpp @@ -17,16 +17,6 @@ namespace adios2 { -#ifdef ADIOS2_HAVE_GPU_SUPPORT -void Engine::CheckMemorySpace(MemorySpace variableMem, MemorySpace bufferMem) -{ - if (variableMem != bufferMem) - helper::Throw( - "CXX-Bindings", "Engine", "Put", - "Memory space mismatch between the pre-set value and the Kokkos::View"); -} -#endif - Engine::operator bool() const noexcept { if (m_Engine == nullptr) diff --git a/bindings/CXX11/adios2/cxx11/Engine.h b/bindings/CXX11/adios2/cxx11/Engine.h index 0ed357bd8c..100ab495b6 100644 --- a/bindings/CXX11/adios2/cxx11/Engine.h +++ b/bindings/CXX11/adios2/cxx11/Engine.h @@ -38,10 +38,6 @@ class Engine friend class IO; friend class QueryWorker; -#ifdef ADIOS2_HAVE_GPU_SUPPORT - void CheckMemorySpace(MemorySpace variableMem, MemorySpace bufferMem); -#endif - public: /** * Empty (default) constructor, use it as a placeholder for future @@ -212,13 +208,12 @@ class Engine void Put(Variable variable, U const &data, const Mode launch = Mode::Deferred) { auto bufferView = static_cast>(data); +#if defined(ADIOS2_HAVE_KOKKOS) || defined(ADIOS2_HAVE_GPU_SUPPORT) auto bufferMem = bufferView.memory_space(); -#ifdef ADIOS2_HAVE_GPU_SUPPORT - auto varMemSpace = variable.GetDefaultMemorySpace(); - if (varMemSpace != MemorySpace::Detect) - CheckMemorySpace(varMemSpace, bufferMem); -#endif + auto bufferLayout = bufferView.layout(); variable.SetMemorySpace(bufferMem); + variable.SetArrayLayout(bufferLayout); +#endif Put(variable, bufferView.data(), launch); } @@ -418,15 +413,14 @@ class Engine class = typename std::enable_if>::value>::type> void Get(Variable variable, U const &data, const Mode launch = Mode::Deferred) { - auto adios_data = static_cast>(data); - auto bufferMem = adios_data.memory_space(); -#ifdef ADIOS2_HAVE_GPU_SUPPORT - auto varMemSpace = variable.GetDefaultMemorySpace(); - if (varMemSpace != MemorySpace::Detect) - CheckMemorySpace(varMemSpace, bufferMem); -#endif + auto bufferView = static_cast>(data); +#if defined(ADIOS2_HAVE_KOKKOS) || defined(ADIOS2_HAVE_GPU_SUPPORT) + auto bufferMem = bufferView.memory_space(); + auto bufferLayout = bufferView.layout(); variable.SetMemorySpace(bufferMem); - Get(variable, adios_data.data(), launch); + variable.SetArrayLayout(bufferLayout); +#endif + Get(variable, bufferView.data(), launch); } /** Perform all Get calls in Deferred mode up to this point */ diff --git a/bindings/CXX11/adios2/cxx11/KokkosView.h b/bindings/CXX11/adios2/cxx11/KokkosView.h index 20fa32ae03..75200623bd 100644 --- a/bindings/CXX11/adios2/cxx11/KokkosView.h +++ b/bindings/CXX11/adios2/cxx11/KokkosView.h @@ -28,6 +28,23 @@ struct memspace_kokkos_to_adios2 }; #endif +template +struct layout_kokkos_to_adios2 +{ + static constexpr adios2::ArrayOrdering value = adios2::ArrayOrdering::RowMajor; +}; + +template <> +struct layout_kokkos_to_adios2 +{ + static constexpr adios2::ArrayOrdering value = adios2::ArrayOrdering::ColumnMajor; +}; + +template <> +struct layout_kokkos_to_adios2 +{ + static constexpr adios2::ArrayOrdering value = adios2::ArrayOrdering::RowMajor; +}; } // namespace detail template @@ -36,6 +53,7 @@ class AdiosView> using data_type = typename Kokkos::View::value_type; data_type *pointer; adios2::MemorySpace mem_space; + adios2::ArrayOrdering m_layout; public: template @@ -44,11 +62,14 @@ class AdiosView> pointer = v.data(); mem_space = detail::memspace_kokkos_to_adios2::memory_space>::value; + m_layout = + detail::layout_kokkos_to_adios2::array_layout>::value; } data_type const *data() const { return pointer; } data_type *data() { return pointer; } adios2::MemorySpace memory_space() const { return mem_space; } + adios2::ArrayOrdering layout() const { return m_layout; } }; } diff --git a/bindings/CXX11/adios2/cxx11/Variable.cpp b/bindings/CXX11/adios2/cxx11/Variable.cpp index a6cb79330a..cba6187b35 100644 --- a/bindings/CXX11/adios2/cxx11/Variable.cpp +++ b/bindings/CXX11/adios2/cxx11/Variable.cpp @@ -37,7 +37,7 @@ namespace adios2 } \ \ template <> \ - MemorySpace Variable::GetDefaultMemorySpace() \ + MemorySpace Variable::GetMemorySpace() \ { \ return m_Variable->m_MemSpace; \ } \ @@ -266,6 +266,14 @@ ADIOS2_FOREACH_TYPE_1ARG(declare_template_instantiation) ADIOS2_FOREACH_PRIMITIVE_TYPE_1ARG(declare_template_instantiation) #undef declare_template_instantiation +#if defined(ADIOS2_HAVE_KOKKOS) || defined(ADIOS2_HAVE_GPU_SUPPORT) +#define declare_layout_template_instantiation(T) \ + template void Variable::SetArrayLayout(const ArrayOrdering layout); \ + template ArrayOrdering Variable::GetArrayLayout(); +ADIOS2_FOREACH_TYPE_1ARG(declare_layout_template_instantiation) +#undef declare_layout_template_instantiation +#endif + #define declare_template_instantiation(T) \ template std::vector::Info> Variable::ToBlocksInfoMin( \ const MinVarInfo *coreVarInfo) const; diff --git a/bindings/CXX11/adios2/cxx11/Variable.h b/bindings/CXX11/adios2/cxx11/Variable.h index 59f1061673..a6559bceec 100644 --- a/bindings/CXX11/adios2/cxx11/Variable.h +++ b/bindings/CXX11/adios2/cxx11/Variable.h @@ -158,7 +158,21 @@ class Variable * Get the memory space that was set by the application * @return the memory space stored in the Variable object */ - MemorySpace GetDefaultMemorySpace(); + MemorySpace GetMemorySpace(); + +#if defined(ADIOS2_HAVE_KOKKOS) || defined(ADIOS2_HAVE_GPU_SUPPORT) + /** + * Sets the for all following Puts + * to either host (default) or device (currently only CUDA supported) + * @param mem memory space where Put buffers are allocated + */ + void SetArrayLayout(const adios2::ArrayOrdering layout); + /** + * Get the memory space that was set by the application + * @return the memory space stored in the Variable object + */ + adios2::ArrayOrdering GetArrayLayout(); +#endif /** * Set new shape, care must be taken when reading back the variable for diff --git a/bindings/CXX11/adios2/cxx11/Variable.tcc b/bindings/CXX11/adios2/cxx11/Variable.tcc index cb212e3a61..e8b2854d9e 100644 --- a/bindings/CXX11/adios2/cxx11/Variable.tcc +++ b/bindings/CXX11/adios2/cxx11/Variable.tcc @@ -215,6 +215,20 @@ std::string ToString(const Variable &variable) return std::string("Variable<") + variable.Type() + ">(Name: \"" + variable.Name() + "\")"; } +#if defined(ADIOS2_HAVE_KOKKOS) || defined(ADIOS2_HAVE_GPU_SUPPORT) +template +void Variable::SetArrayLayout(const ArrayOrdering layout) +{ + m_Variable->SetArrayLayout(layout); +} + +template +adios2::ArrayOrdering Variable::GetArrayLayout() +{ + return m_Variable->GetArrayLayout(); +} +#endif + namespace detail { // Span diff --git a/source/adios2/core/IO.tcc b/source/adios2/core/IO.tcc index b4a68f43b3..bb340b78c5 100644 --- a/source/adios2/core/IO.tcc +++ b/source/adios2/core/IO.tcc @@ -62,6 +62,9 @@ Variable &IO::DefineVariable(const std::string &name, const Dims &shape, cons } } +#if defined(ADIOS2_HAVE_KOKKOS) || defined(ADIOS2_HAVE_GPU_SUPPORT) + variable.m_BaseLayout = m_ArrayOrder; +#endif return variable; } diff --git a/source/adios2/core/VariableBase.cpp b/source/adios2/core/VariableBase.cpp index 0481bf67d8..3a654ee777 100644 --- a/source/adios2/core/VariableBase.cpp +++ b/source/adios2/core/VariableBase.cpp @@ -42,21 +42,82 @@ VariableBase::VariableBase(const std::string &name, const DataType type, const s size_t VariableBase::TotalSize() const noexcept { return helper::GetTotalSize(m_Count); } +#if defined(ADIOS2_HAVE_KOKKOS) || defined(ADIOS2_HAVE_GPU_SUPPORT) +ArrayOrdering VariableBase::GetArrayLayout() { return m_ArrayLayout; } + +void VariableBase::SetArrayLayout(const ArrayOrdering layout) +{ + // first time the layout is being set + if (m_ArrayLayout == ArrayOrdering::Auto) + { + m_ArrayLayout = layout; + // adjust the variable dimensions for the given layout + UpdateLayout(m_Shape); + UpdateLayout(m_Count); + UpdateLayout(m_Start); + return; + } + if (m_ArrayLayout != layout) + { + std::string ExistingLayout("RightMajor"); + std::string NewLayout("RightMajor"); + if (m_ArrayLayout == ArrayOrdering::ColumnMajor) + ExistingLayout = "LeftMajor"; + if (layout == ArrayOrdering::ColumnMajor) + NewLayout = "LeftMajor"; + helper::Throw("Core", "VariableBase", "SetArrayLayout", + "Variable " + m_Name + " is " + ExistingLayout + + " and cannot received a " + NewLayout + " buffer"); + } +} +#endif + MemorySpace VariableBase::GetMemorySpace(const void *ptr) { +#if defined(ADIOS2_HAVE_KOKKOS) || defined(ADIOS2_HAVE_GPU_SUPPORT) + ArrayOrdering layout = m_BaseLayout; +#endif #ifdef ADIOS2_HAVE_GPU_SUPPORT + // first time the memory space is set if (m_MemSpace == MemorySpace::Detect) { if (helper::IsGPUbuffer(ptr)) + { m_MemSpace = MemorySpace::GPU; + layout = ArrayOrdering::ColumnMajor; + } else m_MemSpace = MemorySpace::Host; } +#endif +#if defined(ADIOS2_HAVE_KOKKOS) || defined(ADIOS2_HAVE_GPU_SUPPORT) + // set the layout based on the buffer memory space + // skipping throwing an exception for a mismatch + if (m_ArrayLayout == ArrayOrdering::Auto) + SetArrayLayout(layout); #endif return m_MemSpace; } -void VariableBase::SetMemorySpace(const MemorySpace mem) { m_MemSpace = mem; } +void VariableBase::SetMemorySpace(const MemorySpace mem) +{ +#ifdef ADIOS2_HAVE_GPU_SUPPORT + if (m_MemSpace != MemorySpace::Detect && m_MemSpace != mem) + { + std::string ExistingMemSpace("Host"); + std::string NewMemSpace("Host"); + if (m_MemSpace == MemorySpace::GPU) + ExistingMemSpace = "GPU"; + if (mem == MemorySpace::GPU) + NewMemSpace = "GPU"; + helper::Throw("Core", "VariableBase", "SetMemorySpace", + "Variable " + m_Name + " is set to " + + ExistingMemSpace + " and cannot received a " + + NewMemSpace + " buffer"); + } +#endif + m_MemSpace = mem; +} void VariableBase::SetShape(const adios2::Dims &shape) { @@ -91,6 +152,9 @@ void VariableBase::SetShape(const adios2::Dims &shape) } m_Shape = shape; +#if defined(ADIOS2_HAVE_KOKKOS) || defined(ADIOS2_HAVE_GPU_SUPPORT) + UpdateLayout(m_Shape); +#endif } void VariableBase::SetBlockSelection(const size_t blockID) @@ -146,6 +210,10 @@ void VariableBase::SetSelection(const Box &boxDims) m_Start = start; m_Count = count; m_SelectionType = SelectionType::BoundingBox; +#if defined(ADIOS2_HAVE_KOKKOS) || defined(ADIOS2_HAVE_GPU_SUPPORT) + UpdateLayout(m_Count); + UpdateLayout(m_Start); +#endif } void VariableBase::SetMemorySelection(const Box &memorySelection) @@ -200,6 +268,10 @@ void VariableBase::SetMemorySelection(const Box &memorySelection) m_MemoryStart = memorySelection.first; m_MemoryCount = memorySelection.second; +#if defined(ADIOS2_HAVE_KOKKOS) || defined(ADIOS2_HAVE_GPU_SUPPORT) + UpdateLayout(m_MemoryCount); + UpdateLayout(m_MemoryStart); +#endif } void VariableBase::SetAccuracy(const adios2::Accuracy &a) noexcept @@ -600,5 +672,16 @@ Dims VariableBase::Shape(const size_t step) const return m_Shape; } +#if defined(ADIOS2_HAVE_KOKKOS) || defined(ADIOS2_HAVE_GPU_SUPPORT) +inline void VariableBase::UpdateLayout(Dims &shape) +{ + // if the default execution layout for the memory space + // is not the same as the selected column/row major format + if (m_BaseLayout != m_ArrayLayout && m_ArrayLayout != ArrayOrdering::Auto) + { + std::reverse(shape.begin(), shape.end()); + } +} +#endif } // end namespace core } // end namespace adios2 diff --git a/source/adios2/core/VariableBase.h b/source/adios2/core/VariableBase.h index b777400f40..3b2505b1c7 100644 --- a/source/adios2/core/VariableBase.h +++ b/source/adios2/core/VariableBase.h @@ -57,6 +57,10 @@ class VariableBase #else MemorySpace m_MemSpace = MemorySpace::Host; #endif +#if defined(ADIOS2_HAVE_KOKKOS) || defined(ADIOS2_HAVE_GPU_SUPPORT) + ArrayOrdering m_BaseLayout; + ArrayOrdering m_ArrayLayout = ArrayOrdering::Auto; +#endif ShapeID m_ShapeID = ShapeID::Unknown; ///< see shape types in ADIOSTypes.h size_t m_BlockID = 0; ///< current block ID for local variables, global = 0 @@ -128,6 +132,20 @@ class VariableBase */ size_t TotalSize() const noexcept; +#if defined(ADIOS2_HAVE_KOKKOS) || defined(ADIOS2_HAVE_GPU_SUPPORT) + /** + * Get the layout used by the user buffers + * @return the layout used by the user buffers (RowMajor or ColumnMajor) + */ + ArrayOrdering GetArrayLayout(); + + /** + * Set the layout used by the user buffers + * @param the layout that will be used by future put/gets + */ + void SetArrayLayout(const ArrayOrdering layout); +#endif + /** * Get the memory space where a given buffers was allocated * @param pointer to the user data @@ -263,6 +281,10 @@ class VariableBase void CheckDimensionsCommon(const std::string hint) const; void CheckRandomAccess(const size_t step, const std::string hint) const; + +#if defined(ADIOS2_HAVE_KOKKOS) || defined(ADIOS2_HAVE_GPU_SUPPORT) + inline void UpdateLayout(Dims &shape); +#endif }; } // end namespace core From 265206a006425135d02e674b91deffb5b9dc8169 Mon Sep 17 00:00:00 2001 From: anagainaru Date: Mon, 22 Jan 2024 11:42:09 -0500 Subject: [PATCH 020/124] Allow the Shape function to receive a memory space or a layout --- bindings/CXX11/adios2/cxx11/Variable.cpp | 14 ++++++++++++++ bindings/CXX11/adios2/cxx11/Variable.h | 4 ++++ source/adios2/core/VariableBase.cpp | 18 ++++++++++++++++++ source/adios2/core/VariableBase.h | 2 ++ 4 files changed, 38 insertions(+) diff --git a/bindings/CXX11/adios2/cxx11/Variable.cpp b/bindings/CXX11/adios2/cxx11/Variable.cpp index cba6187b35..2b432112cf 100644 --- a/bindings/CXX11/adios2/cxx11/Variable.cpp +++ b/bindings/CXX11/adios2/cxx11/Variable.cpp @@ -126,6 +126,20 @@ namespace adios2 } \ \ template <> \ + Dims Variable::Shape(const ArrayOrdering layout, const size_t step) const \ + { \ + helper::CheckForNullptr(m_Variable, "in call to Variable::Shape"); \ + return m_Variable->Shape(step, MemorySpace::Host, layout); \ + } \ + \ + template <> \ + Dims Variable::Shape(const MemorySpace memSpace, const size_t step) const \ + { \ + helper::CheckForNullptr(m_Variable, "in call to Variable::Shape"); \ + return m_Variable->Shape(step, memSpace); \ + } \ + \ + template <> \ Dims Variable::Start() const \ { \ helper::CheckForNullptr(m_Variable, "in call to Variable::Start"); \ diff --git a/bindings/CXX11/adios2/cxx11/Variable.h b/bindings/CXX11/adios2/cxx11/Variable.h index a6559bceec..2dd77adf9a 100644 --- a/bindings/CXX11/adios2/cxx11/Variable.h +++ b/bindings/CXX11/adios2/cxx11/Variable.h @@ -269,6 +269,10 @@ class Variable * @return shape vector */ adios2::Dims Shape(const size_t step = adios2::EngineCurrentStep) const; + adios2::Dims Shape(const ArrayOrdering layout, + const size_t step = adios2::EngineCurrentStep) const; + adios2::Dims Shape(const MemorySpace memSpace, + const size_t step = adios2::EngineCurrentStep) const; /** * Inspects current start point diff --git a/source/adios2/core/VariableBase.cpp b/source/adios2/core/VariableBase.cpp index 3a654ee777..bd8058cd49 100644 --- a/source/adios2/core/VariableBase.cpp +++ b/source/adios2/core/VariableBase.cpp @@ -643,6 +643,24 @@ void VariableBase::CheckRandomAccess(const size_t step, const std::string hint) } } +Dims VariableBase::Shape(const size_t step, const MemorySpace memSpace, + const ArrayOrdering layout) const +{ + auto dims = Shape(step); +#if defined(ADIOS2_HAVE_KOKKOS) || defined(ADIOS2_HAVE_GPU_SUPPORT) + bool mismatchMemSpace = + (memSpace != MemorySpace::Host && m_BaseLayout != ArrayOrdering::ColumnMajor); + bool mismatchLayout = (layout != m_BaseLayout && layout != ArrayOrdering::Auto); + if (mismatchMemSpace || mismatchLayout) + { + Dims flipDims(dims.size()); + std::reverse_copy(dims.begin(), dims.end(), flipDims.begin()); + return flipDims; + } +#endif + return dims; +} + Dims VariableBase::Shape(const size_t step) const { CheckRandomAccess(step, "Shape"); diff --git a/source/adios2/core/VariableBase.h b/source/adios2/core/VariableBase.h index 3b2505b1c7..dc410ec78d 100644 --- a/source/adios2/core/VariableBase.h +++ b/source/adios2/core/VariableBase.h @@ -256,6 +256,8 @@ class VariableBase */ void CheckRandomAccessConflict(const std::string hint) const; + Dims Shape(const size_t step, const MemorySpace memSpace, + const ArrayOrdering layout = ArrayOrdering::Auto) const; Dims Shape(const size_t step = adios2::EngineCurrentStep) const; /** From 40e252ea8ef8fedd998e4859c05e6b1da5153105 Mon Sep 17 00:00:00 2001 From: anagainaru Date: Mon, 22 Jan 2024 14:51:44 -0500 Subject: [PATCH 021/124] Make all examples work with MPI and add an example for Kokkos Views with different layout combinations --- .../bpStepsWriteReadKokkos/CMakeLists.txt | 27 ++- .../bpStepsWriteReadKokkos.cpp | 152 +++++++++++------ .../bpWriteReadKokkosView.cpp | 156 ++++++++++++++++++ 3 files changed, 278 insertions(+), 57 deletions(-) create mode 100644 examples/hello/bpStepsWriteReadKokkos/bpWriteReadKokkosView.cpp diff --git a/examples/hello/bpStepsWriteReadKokkos/CMakeLists.txt b/examples/hello/bpStepsWriteReadKokkos/CMakeLists.txt index c420e6e762..22eff4fb0b 100644 --- a/examples/hello/bpStepsWriteReadKokkos/CMakeLists.txt +++ b/examples/hello/bpStepsWriteReadKokkos/CMakeLists.txt @@ -18,6 +18,14 @@ if(NOT TARGET adios2_core) set(CMAKE_CXX_COMPILER "${Kokkos_CXX_COMPILER}") endif() + find_package(MPI QUIET COMPONENTS ${_components}) + if(MPI_FOUND) + # Workaround for various MPI implementations forcing the link of C++ bindings + add_definitions(-DOMPI_SKIP_MPICXX -DMPICH_SKIP_MPICXX) + + list(APPEND _components MPI) + endif() + find_package(ADIOS2 REQUIRED COMPONENTS ${_components}) else() if(DEFINED Kokkos_CXX_COMPILER) @@ -25,9 +33,22 @@ else() endif() endif() -if(ADIOS2_HAVE_Kokkos) - add_executable(adios2_hello_bpStepsWriteReadKokkos bpStepsWriteReadKokkos.cpp) - kokkos_compilation(SOURCE bpStepsWriteReadKokkos.cpp) +add_executable(adios2_hello_bpStepsWriteReadKokkos bpStepsWriteReadKokkos.cpp) +kokkos_compilation(SOURCE bpStepsWriteReadKokkos.cpp) +if(ADIOS2_HAVE_MPI) + target_link_libraries(adios2_hello_bpStepsWriteReadKokkos adios2::cxx11_mpi MPI::MPI_C Kokkos::kokkos) +else() target_link_libraries(adios2_hello_bpStepsWriteReadKokkos adios2::cxx11 Kokkos::kokkos) +endif() install(TARGETS adios2_hello_bpStepsWriteReadKokkos RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) + +add_executable(adios2_hello_bpWriteReadKokkosView bpWriteReadKokkosView.cpp) +kokkos_compilation(SOURCE bpStepsWriteReadKokkos.cpp) +if(ADIOS2_HAVE_MPI) + target_link_libraries(adios2_hello_bpWriteReadKokkosView adios2::cxx11_mpi MPI::MPI_C Kokkos::kokkos) +else() + target_link_libraries(adios2_hello_bpWriteReadKokkosView adios2::cxx11 Kokkos::kokkos) +endif() + install(TARGETS adios2_hello_bpWriteReadKokkosView RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) + endif() diff --git a/examples/hello/bpStepsWriteReadKokkos/bpStepsWriteReadKokkos.cpp b/examples/hello/bpStepsWriteReadKokkos/bpStepsWriteReadKokkos.cpp index 9fd98b4027..10859eef15 100644 --- a/examples/hello/bpStepsWriteReadKokkos/bpStepsWriteReadKokkos.cpp +++ b/examples/hello/bpStepsWriteReadKokkos/bpStepsWriteReadKokkos.cpp @@ -7,31 +7,40 @@ */ #include #include +#include #include -#include - #include +#include +#include +#if ADIOS2_USE_MPI +#include +#endif template int BPWrite(const std::string fname, const size_t Nx, const size_t Ny, const size_t nSteps, - const std::string engine) + const std::string engine, int rank, int size) { // Initialize the simulation data Kokkos::View gpuSimData("simBuffer", Nx, Ny); static_assert(Kokkos::SpaceAccessibility::accessible, ""); - Kokkos::parallel_for("initBuffer", Kokkos::RangePolicy(0, Nx), KOKKOS_LAMBDA(int i) { - for (int j = 0; j < Ny; j++) - gpuSimData(i, j) = static_cast(i); - }); + Kokkos::parallel_for( + "initBuffer", Kokkos::RangePolicy(0, Nx), KOKKOS_LAMBDA(int i) { + for (int j = 0; j < Ny; j++) + gpuSimData(i, j) = static_cast(rank + i); + }); Kokkos::fence(); +#if ADIOS2_USE_MPI + adios2::ADIOS adios(MPI_COMM_WORLD); +#else adios2::ADIOS adios; +#endif adios2::IO io = adios.DeclareIO("WriteIO"); io.SetEngine(engine); - const adios2::Dims shape{Nx, Ny}; - const adios2::Dims start{0, 0}; + const adios2::Dims shape{static_cast(size * Nx), Ny}; + const adios2::Dims start{static_cast(rank * Nx), 0}; const adios2::Dims count{Nx, Ny}; auto data = io.DefineVariable("bpFloats", shape, start, count); @@ -40,7 +49,7 @@ int BPWrite(const std::string fname, const size_t Nx, const size_t Ny, const siz // Simulation steps for (unsigned int step = 0; step < nSteps; ++step) { - adios2::Box sel({0, 0}, {Nx, Ny}); + adios2::Box sel({static_cast(rank * Nx), 0}, {Nx, Ny}); data.SetSelection(sel); bpWriter.BeginStep(); @@ -48,41 +57,35 @@ int BPWrite(const std::string fname, const size_t Nx, const size_t Ny, const siz bpWriter.EndStep(); // Update values in the simulation data - Kokkos::parallel_for("updateBuffer", Kokkos::RangePolicy(0, Nx), - KOKKOS_LAMBDA(int i) { - for (int j = 0; j < Ny; j++) - gpuSimData(i, j) += 10; - }); + Kokkos::parallel_for( + "updateBuffer", Kokkos::RangePolicy(0, Nx), KOKKOS_LAMBDA(int i) { + for (int j = 0; j < Ny; j++) + gpuSimData(i, j) += 10; + }); Kokkos::fence(); } bpWriter.Close(); ExecSpace exe_space; - std::cout << "Done writing on memory space: " << exe_space.name() << std::endl; + if (rank == 0) + std::cout << "Done writing on memory space: " << exe_space.name() << std::endl; return 0; } -template -std::array GetDimenstions(adios2::Variable data) -{ - return {data.Shape()[1], data.Shape()[0]}; -} - -template <> -std::array GetDimenstions(adios2::Variable data) -{ - return {data.Shape()[0], data.Shape()[1]}; -} - template -int BPRead(const std::string fname, const std::string engine) +int BPRead(const std::string fname, const std::string engine, int rank, int size) { +#if ADIOS2_USE_MPI + adios2::ADIOS adios(MPI_COMM_WORLD); +#else adios2::ADIOS adios; +#endif adios2::IO io = adios.DeclareIO("ReadIO"); io.SetEngine(engine); ExecSpace exe_space; - std::cout << "Read on memory space: " << exe_space.name() << std::endl; + if (rank == 0) + std::cout << "Read on memory space: " << exe_space.name() << std::endl; adios2::Engine bpReader = io.Open(fname, adios2::Mode::Read); @@ -97,23 +100,44 @@ int BPRead(const std::string fname, const std::string engine) << step << "needs to have two dimensions" << std::endl; break; } - - const adios2::Dims start{0, 0}; - const adios2::Box sel(start, data.Shape()); - data.SetSelection(sel); - - auto dims = GetDimenstions(data); - size_t Nx = dims[0], Ny = dims[1]; + adios2::MemorySpace adiosMemSpace = adios2::MemorySpace::Host; +#ifdef ADIOS2_HAVE_GPU_SUPPORT + if (!std::is_same::value) + adiosMemSpace = adios2::MemorySpace::GPU; +#endif + auto dims = data.Shape(adiosMemSpace); + size_t Nx = dims[0]; + size_t Ny = dims[1]; + + if (Nx > Ny) + { + Nx = Nx / size; + const adios2::Dims start{static_cast(rank * Nx), 0}; + const adios2::Box sel(start, {Nx, Ny}); + data.SetSelection(sel); + } + else + { + Ny = Ny / size; + const adios2::Dims start{0, static_cast(rank * Ny)}; + const adios2::Box sel(start, {Nx, Ny}); + data.SetSelection(sel); + } Kokkos::View gpuSimData("simBuffer", Nx, Ny); - bpReader.Get(data, gpuSimData.data()); + + bpReader.Get(data, gpuSimData); bpReader.EndStep(); auto cpuData = Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace{}, gpuSimData); - for (size_t i = 0; i < Nx; i++) + if (rank == 0) { - for (size_t j = 0; j < Ny; j++) - std::cout << cpuData(i, j) << " "; - std::cout << std::endl; + std::cout << "Rank 0: " << std::endl; + for (size_t i = 0; i < Nx; i++) + { + for (size_t j = 0; j < Ny; j++) + std::cout << cpuData(i, j) << " "; + std::cout << std::endl; + } } } @@ -123,41 +147,61 @@ int BPRead(const std::string fname, const std::string engine) int main(int argc, char **argv) { + int rank, size; +#if ADIOS2_USE_MPI + int provided; + + // MPI_THREAD_MULTIPLE is only required if you enable the SST MPI_DP + MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &size); +#else + rank = 0; + size = 1; +#endif const std::string engine = argv[1] ? argv[1] : "BPFile"; - std::cout << "Using engine " << engine << std::endl; - const size_t Nx = 6, Ny = 10, nSteps = 1; + if (rank == 0) + std::cout << "Using engine " << engine << std::endl; + const size_t Nx = 3, Ny = 4, nSteps = 1; const std::string fromMemSpace = "Default"; const std::string toMemSpace = "Default"; const std::string filename = engine + "StepsWriteReadKokkos"; Kokkos::initialize(argc, argv); { - std::cout << "Using engine " << engine << std::endl; - if (fromMemSpace == "Default") { using mem_space = Kokkos::DefaultExecutionSpace::memory_space; - std::cout << "Writing on memory space: DefaultMemorySpace" << std::endl; + if (rank == 0) + std::cout << "Writing on memory space: DefaultMemorySpace" << std::endl; BPWrite(filename + ".bp", Nx, Ny, nSteps, - engine); + engine, rank, size); } else { - std::cout << "Writing on memory space: HostSpace" << std::endl; - BPWrite(filename + ".bp", nSteps, Nx, Ny, engine); + if (rank == 0) + std::cout << "Writing on memory space: HostSpace" << std::endl; + BPWrite(filename + ".bp", nSteps, Nx, Ny, engine, + rank, size); } if (toMemSpace == "Default") { using mem_space = Kokkos::DefaultExecutionSpace::memory_space; - std::cout << "Reading on memory space: DefaultMemorySpace" << std::endl; - BPRead(filename + ".bp", engine); + if (rank == 0) + std::cout << "Reading on memory space: DefaultMemorySpace" << std::endl; + BPRead(filename + ".bp", engine, rank, size); } else { - std::cout << "Reading on memory space: HostSpace" << std::endl; - BPRead(filename + ".bp", engine); + if (rank == 0) + std::cout << "Reading on memory space: HostSpace" << std::endl; + BPRead(filename + ".bp", engine, rank, size); } } Kokkos::finalize(); +#if ADIOS2_USE_MPI + MPI_Finalize(); +#endif + return 0; } diff --git a/examples/hello/bpStepsWriteReadKokkos/bpWriteReadKokkosView.cpp b/examples/hello/bpStepsWriteReadKokkos/bpWriteReadKokkosView.cpp new file mode 100644 index 0000000000..ac7f808be5 --- /dev/null +++ b/examples/hello/bpStepsWriteReadKokkos/bpWriteReadKokkosView.cpp @@ -0,0 +1,156 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * bpWriteReadKokkosView.cpp Simple example of writing and reading Kokkos Views through ADIOS2 BP + * engine using different combinations of Layouts for the write/read buffers + */ +#include +#include +#include +#include + +#include +#include +#include +#if ADIOS2_USE_MPI +#include +#endif + +template +int BPWrite(const std::string fname, const size_t Nx, const size_t Ny, const size_t nSteps, + int rank, int size) +{ + Kokkos::View gpuSimData("simBuffer", Nx, Ny); + Kokkos::parallel_for( + "initBuffer", Kokkos::MDRangePolicy>({0, 0}, {Nx, Ny}), + KOKKOS_LAMBDA(int i, int j) { gpuSimData(i, j) = static_cast(1 + i); }); + Kokkos::fence(); + +#if ADIOS2_USE_MPI + adios2::ADIOS adios(MPI_COMM_WORLD); +#else + adios2::ADIOS adios; +#endif + adios2::IO io = adios.DeclareIO("WriteIO"); + + const adios2::Dims shape{static_cast(size * Nx), Ny}; + const adios2::Dims start{static_cast(rank * Nx), 0}; + const adios2::Dims count{Nx, Ny}; + auto data = io.DefineVariable("bpFloats", shape, start, count); + + adios2::Engine bpWriter = io.Open(fname, adios2::Mode::Write); + + for (unsigned int step = 0; step < nSteps; ++step) + { + bpWriter.BeginStep(); + bpWriter.Put(data, gpuSimData); + bpWriter.EndStep(); + + Kokkos::parallel_for( + "updateBuffer", Kokkos::MDRangePolicy>({0, 0}, {Nx, Ny}), + KOKKOS_LAMBDA(int i, int j) { gpuSimData(i, j) += 10; }); + Kokkos::fence(); + } + + bpWriter.Close(); + return 0; +} + +template +int BPRead(const std::string fname, int rank, int size) +{ +#if ADIOS2_USE_MPI + adios2::ADIOS adios(MPI_COMM_WORLD); +#else + adios2::ADIOS adios; +#endif + adios2::IO io = adios.DeclareIO("ReadIO"); + adios2::Engine bpReader = io.Open(fname, adios2::Mode::Read); + + unsigned int step = 0; + for (; bpReader.BeginStep() == adios2::StepStatus::OK; ++step) + { + auto data = io.InquireVariable("bpFloats"); + if (data.Shape().size() != 2) + { + std::cout << "Error, the bpFloats variable on step " << step + << "needs to have two dimensions" << std::endl; + break; + } + + adios2::ArrayOrdering layout = adios2::ArrayOrdering::RowMajor; + if (std::is_same::value) + layout = adios2::ArrayOrdering::ColumnMajor; + auto dims = data.Shape(layout); + size_t Nx = dims[0]; + size_t Ny = dims[1]; + + Kokkos::View gpuSimData("simBuffer", Nx, Ny); + const adios2::Dims start{0, 0}; + const adios2::Box sel(start, {Nx, Ny}); + data.SetSelection(sel); + + bpReader.Get(data, gpuSimData); + bpReader.EndStep(); + + auto cpuData = Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace{}, gpuSimData); + // all processes are reading the entire global array, only rank 0 plots it + if (rank == 0) + { + std::cout << "Rank " << rank << " receieved data:" << std::endl; + for (size_t i = 0; i < Nx; i++) + { + for (size_t j = 0; j < Ny; j++) + std::cout << cpuData(i, j) << " "; + std::cout << std::endl; + } + } + } + + bpReader.Close(); + return 0; +} + +int main(int argc, char **argv) +{ + int rank, size; +#if ADIOS2_USE_MPI + int provided; + + // MPI_THREAD_MULTIPLE is only required if you enable the SST MPI_DP + MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &size); +#else + rank = 0; + size = 1; +#endif + const size_t Nx = 3, Ny = 4, nSteps = 1; + + const std::string filename = "KokkosViewWR"; + Kokkos::initialize(argc, argv); + { + std::cout << "Write on LayoutRight, read on LayoutRight" << std::endl; + BPWrite(filename + "_RR.bp", Nx, Ny, nSteps, rank, size); + BPRead(filename + "_RR.bp", rank, size); + + std::cout << "Write on LayoutRight, read on LayoutLeft" << std::endl; + BPWrite(filename + "_RL.bp", Nx, Ny, nSteps, rank, size); + BPRead(filename + "_RL.bp", rank, size); + + std::cout << "Write on LayoutLeft, read on LayoutLeft" << std::endl; + BPWrite(filename + "_LL.bp", Nx, Ny, nSteps, rank, size); + BPRead(filename + "_LL.bp", rank, size); + + std::cout << "Write on LayoutLeft, read on LayoutRight" << std::endl; + BPWrite(filename + "_LR.bp", Nx, Ny, nSteps, rank, size); + BPRead(filename + "_LR.bp", rank, size); + } + Kokkos::finalize(); +#if ADIOS2_USE_MPI + MPI_Finalize(); +#endif + + return 0; +} From 44c9624e3eee9d2932b84e65cbd1a1770b8901ed Mon Sep 17 00:00:00 2001 From: anagainaru Date: Mon, 22 Jan 2024 15:01:39 -0500 Subject: [PATCH 022/124] Fortran example using MPI and Kokkos --- .../bpStepsWriteReadKokkos/CMakeLists.txt | 29 ++++++- .../bpStepsWriteReadKokkos.F90 | 80 +++++++++++++++++++ .../hello/bpStepsWriteReadKokkos/view-cxx.cc | 41 ++++++++++ .../hello/bpStepsWriteReadKokkos/view-f.f90 | 39 +++++++++ 4 files changed, 187 insertions(+), 2 deletions(-) create mode 100644 examples/hello/bpStepsWriteReadKokkos/bpStepsWriteReadKokkos.F90 create mode 100644 examples/hello/bpStepsWriteReadKokkos/view-cxx.cc create mode 100644 examples/hello/bpStepsWriteReadKokkos/view-f.f90 diff --git a/examples/hello/bpStepsWriteReadKokkos/CMakeLists.txt b/examples/hello/bpStepsWriteReadKokkos/CMakeLists.txt index 22eff4fb0b..6f0025efde 100644 --- a/examples/hello/bpStepsWriteReadKokkos/CMakeLists.txt +++ b/examples/hello/bpStepsWriteReadKokkos/CMakeLists.txt @@ -18,6 +18,15 @@ if(NOT TARGET adios2_core) set(CMAKE_CXX_COMPILER "${Kokkos_CXX_COMPILER}") endif() + include(CheckLanguage) + check_language(Fortran) + if(CMAKE_Fortran_COMPILER) + enable_language(Fortran) + endif() + if(CMAKE_Fortran_COMPILER_LOADED) + list(APPEND _components Fortran) + endif() + find_package(MPI QUIET COMPONENTS ${_components}) if(MPI_FOUND) # Workaround for various MPI implementations forcing the link of C++ bindings @@ -33,6 +42,10 @@ else() endif() endif() +# flcl package needed for the Kokkos Fortran example +find_package(flcl QUIET) + +# C++ Kokkos example using default layouts and different memory spaces add_executable(adios2_hello_bpStepsWriteReadKokkos bpStepsWriteReadKokkos.cpp) kokkos_compilation(SOURCE bpStepsWriteReadKokkos.cpp) if(ADIOS2_HAVE_MPI) @@ -40,8 +53,9 @@ if(ADIOS2_HAVE_MPI) else() target_link_libraries(adios2_hello_bpStepsWriteReadKokkos adios2::cxx11 Kokkos::kokkos) endif() - install(TARGETS adios2_hello_bpStepsWriteReadKokkos RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) +install(TARGETS adios2_hello_bpStepsWriteReadKokkos RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) +# C++ Kokkos example using default memory space and layouts add_executable(adios2_hello_bpWriteReadKokkosView bpWriteReadKokkosView.cpp) kokkos_compilation(SOURCE bpStepsWriteReadKokkos.cpp) if(ADIOS2_HAVE_MPI) @@ -49,6 +63,17 @@ if(ADIOS2_HAVE_MPI) else() target_link_libraries(adios2_hello_bpWriteReadKokkosView adios2::cxx11 Kokkos::kokkos) endif() - install(TARGETS adios2_hello_bpWriteReadKokkosView RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) +install(TARGETS adios2_hello_bpWriteReadKokkosView RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) +# Fortran Kokkos example using default layouts and different memory spaces +if(ADIOS2_HAVE_MPI AND ADIOS2_HAVE_Fortran AND flcl_FOUND) + add_executable(adios2_hello_bpStepsWriteReadKokkos_f bpStepsWriteReadKokkos.F90 view-f.f90 view-cxx.cc) + target_link_libraries(adios2_hello_bpStepsWriteReadKokkos_f adios2::fortran_mpi MPI::MPI_Fortran flcl::flcl) + if (CMAKE_Fortran_COMPILER_ID STREQUAL "XL") + target_link_options(adios2_hello_bpStepsWriteReadKokkos_f PRIVATE LINKER:-lxlf90_r) + endif() + if (CMAKE_Fortran_COMPILER_ID STREQUAL "Intel" OR CMAKE_Fortran_COMPILER_ID STREQUAL "GNU") + set_target_properties(adios2_hello_bpStepsWriteReadKokkos_f PROPERTIES LINKER_LANGUAGE Fortran) + endif() + install(TARGETS adios2_hello_bpStepsWriteReadKokkos_f RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) endif() diff --git a/examples/hello/bpStepsWriteReadKokkos/bpStepsWriteReadKokkos.F90 b/examples/hello/bpStepsWriteReadKokkos/bpStepsWriteReadKokkos.F90 new file mode 100644 index 0000000000..cbffc5b353 --- /dev/null +++ b/examples/hello/bpStepsWriteReadKokkos/bpStepsWriteReadKokkos.F90 @@ -0,0 +1,80 @@ +program TestBPWriteReadHeatMap2D use mpi use adios2 + + implicit none + + integer(kind = 8)::sum_i1, + sum_i2 type(adios2_adios)::adios type(adios2_io)::ioPut, ioGet type(adios2_engine)::bpWriter, + bpReader type(adios2_variable), dimension(1)::var_g, + var_gIn + + integer(kind = 2), + dimension( + :, + :), + allocatable ::g, &sel_g integer(kind = 8), + dimension(2)::ishape, istart, icount integer(kind = 8), + dimension(2)::sel_start, sel_count integer ::ierr, irank, isize, + step_status integer ::in1, in2 integer ::i1, + i2 + + call MPI_INIT(ierr) call MPI_COMM_RANK(MPI_COMM_WORLD, irank, ierr) call + MPI_COMM_SIZE(MPI_COMM_WORLD, isize, ierr) + + in1 = 3 in2 = 4 icount = (/ in1, in2 /) istart = (/ 0, in2 *irank /) ishape = (/ in1, + in2 *isize /) + + allocate(g(in1, in2)) do i2 = 1, + in2 do i1 = 1, + in1 g(i1, i2) = irank + i1 end do end do + + !Start adios2 Writer call adios2_init(adios, MPI_COMM_WORLD, ierr) call + adios2_declare_io(ioPut, adios, 'WriteIO', ierr) + + call adios2_define_variable(var_g(1), ioPut, &'bpFloats', + adios2_type_integer2, &2, ishape, istart, + icount, &adios2_constant_dims, ierr) + + call + adios2_open(bpWriter, ioPut, 'BPFortranKokkos.bp', adios2_mode_write, &ierr) + + call adios2_put(bpWriter, var_g(1), g, ierr) + + call adios2_close(bpWriter, ierr) + + if (allocated(g)) deallocate(g) + + !Start adios2 Reader in rank 0 if (irank == 0) then + + call adios2_declare_io(ioGet, adios, 'ReadIO', ierr) + + call adios2_open(bpReader, ioGet, 'BPFortranKokkos.bp', + &adios2_mode_read, MPI_COMM_SELF, ierr) + + call + adios2_begin_step(bpReader, adios2_step_mode_read, -1., &step_status, ierr) + + call adios2_inquire_variable(var_gIn(1), ioGet, &'bpFloats', ierr) + + sel_start = (/ 0, 0 /) sel_count = (/ ishape(1), ishape(2) /) + + allocate(sel_g(ishape(1), ishape(2))) sel_g = 0 + + call adios2_set_selection(var_gIn(1), 2, sel_start, sel_count, &ierr) call + adios2_get(bpReader, var_gIn(1), sel_g, ierr) + + call adios2_end_step(bpReader, ierr) + + call adios2_close(bpReader, ierr) + + do i2 = 1, + INT(sel_count(2), 4) do i1 = 1, + INT(sel_count(1), 4) write(6, "(i8)", advance = "no") sel_g(i1, i2) end do write(6, *) end + do + + if (allocated(sel_g)) deallocate(sel_g) + + end if + + call adios2_finalize(adios, ierr) call MPI_Finalize(ierr) + + end program TestBPWriteReadHeatMap2D diff --git a/examples/hello/bpStepsWriteReadKokkos/view-cxx.cc b/examples/hello/bpStepsWriteReadKokkos/view-cxx.cc new file mode 100644 index 0000000000..c234aa37fb --- /dev/null +++ b/examples/hello/bpStepsWriteReadKokkos/view-cxx.cc @@ -0,0 +1,41 @@ +#include "flcl-cxx.hpp" +#include + +using view_type = flcl::view_i32_2d_t; + +extern "C" { + +void c_init_view(view_type **v_x, int *val) +{ + using flcl::view_from_ndarray; + + view_type x = **v_x; + int d_val = *val; + Kokkos::parallel_for("initBuffer", x.extent(1), KOKKOS_LAMBDA(const size_t idy) { + for (size_t idx = 0; idx < x.extent(0); idx++) + x(idx, idy) = d_val + idx; + }); + Kokkos::fence(); + + return; +} + +void c_print_view(view_type **v_x) +{ + using flcl::view_from_ndarray; + + view_type x = **v_x; + + Kokkos::parallel_for("printBuffer", 1, KOKKOS_LAMBDA(const size_t id) { + for (size_t idx = 0; idx < x.extent(0); idx++) + { + for (size_t idy = 0; idy < x.extent(1); idy++) + Kokkos::printf("%d ", x(idx, idy)); + Kokkos::printf("\n"); + } + }); + Kokkos::fence(); + + return; +} +} diff --git a/examples/hello/bpStepsWriteReadKokkos/view-f.f90 b/examples/hello/bpStepsWriteReadKokkos/view-f.f90 new file mode 100644 index 0000000000..18fe3070f4 --- /dev/null +++ b/examples/hello/bpStepsWriteReadKokkos/view-f.f90 @@ -0,0 +1,39 @@ +module view_f_mod use, intrinsic ::iso_c_binding use, + intrinsic ::iso_fortran_env + + use ::flcl_mod + + implicit none + + public + + interface subroutine f_init_view(x, val) & + &bind(c, name = 'c_init_view') use, + intrinsic ::iso_c_binding use ::flcl_mod type(c_ptr), intent(in)::x integer(c_int), + intent(in)::val end subroutine f_init_view + + subroutine f_print_view(x) & + &bind(c, name = 'c_print_view') use, + intrinsic ::iso_c_binding use ::flcl_mod type(c_ptr), + intent(in)::x end subroutine f_print_view end interface + + contains + + subroutine init_view(x, val) use, + intrinsic ::iso_c_binding use ::flcl_mod implicit none type(view_i32_2d_t), + intent(inout)::x integer(c_int), + intent(in)::val + + call f_init_view(x % ptr(), val) + + end subroutine init_view + + subroutine print_view(x) use, + intrinsic ::iso_c_binding use ::flcl_mod implicit none type(view_i32_2d_t), + intent(in)::x + + call f_print_view(x % ptr()) + + end subroutine print_view + + end module view_f_mod From 4f11b00eb53603058621ef447f28c3abbf9101e6 Mon Sep 17 00:00:00 2001 From: pnorbert Date: Wed, 24 Jan 2024 15:33:13 -0500 Subject: [PATCH 023/124] fix BP5 parameter parsing for size values with units (#4012) --- source/adios2/engine/bp5/BP5Engine.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/adios2/engine/bp5/BP5Engine.cpp b/source/adios2/engine/bp5/BP5Engine.cpp index 115d2ce688..0c49713a28 100644 --- a/source/adios2/engine/bp5/BP5Engine.cpp +++ b/source/adios2/engine/bp5/BP5Engine.cpp @@ -184,7 +184,7 @@ void BP5Engine::ParseParams(IO &io, struct BP5Params &Params) parameter = def; if (itKey != params_lowercase.end()) { - std::string value = itKey->second; + std::string value = helper::LowerCase(itKey->second); parameter = helper::StringToByteUnits(value, "for Parameter key=" + key + "in call to Open"); } From efc8e7ef4de062a2a93ac7ae4e2aff1c3dd811cb Mon Sep 17 00:00:00 2001 From: pnorbert Date: Thu, 25 Jan 2024 10:26:21 -0500 Subject: [PATCH 024/124] Merge 3769 to master (#4011) * Fix ChunkV maintaining CurOffset when downsizing current chunk in Allocate() and creating the next chunk. * Rename test. fix windows compile errors. * clang format --- .../toolkit/format/buffer/chunk/ChunkV.cpp | 25 ++- testing/adios2/CMakeLists.txt | 1 + testing/adios2/unit/CMakeLists.txt | 7 + testing/adios2/unit/TestChunkV.cpp | 150 ++++++++++++++++++ 4 files changed, 175 insertions(+), 8 deletions(-) create mode 100644 testing/adios2/unit/CMakeLists.txt create mode 100644 testing/adios2/unit/TestChunkV.cpp diff --git a/source/adios2/toolkit/format/buffer/chunk/ChunkV.cpp b/source/adios2/toolkit/format/buffer/chunk/ChunkV.cpp index 4f4f04fc05..32224298c8 100644 --- a/source/adios2/toolkit/format/buffer/chunk/ChunkV.cpp +++ b/source/adios2/toolkit/format/buffer/chunk/ChunkV.cpp @@ -57,8 +57,6 @@ size_t ChunkV::ChunkAlloc(Chunk &v, const size_t size) v.Ptr = (char *)((p + m_MemAlign - 1) & ~(m_MemAlign - 1)); } v.Size = actualsize; - /*std::cout << " ChunkAlloc: buf = " << (void *)v.Ptr - << " size = " << actualsize << std::endl;*/ return actualsize; } else @@ -105,15 +103,20 @@ size_t ChunkV::AddToVec(const size_t size, const void *buf, size_t align, bool C // realloc down to used size (helpful?) and set size in array /*std::cout << " downsize ptr = " << m_Chunks.back().Ptr << " to size = " << m_TailChunkPos << std::endl;*/ - size_t actualsize = ChunkAlloc(m_Chunks.back(), m_TailChunkPos); + Chunk &c = m_Chunks.back(); + size_t actualsize = ChunkAlloc(c, m_TailChunkPos); size_t alignment = actualsize - m_TailChunkPos; if (alignment) { - auto p = m_Chunks.back().Ptr + m_TailChunkPos; + auto p = c.Ptr + m_TailChunkPos; std::fill(p, p + alignment, 0); } retOffset += alignment; - DataV.back().Size = actualsize; + // Update entry in DataV as size and potentiall ptr has changed + // Learned from sanitizer: downsizing realloc still may change pointer + VecEntry &dv = DataV.back(); + dv.Size = actualsize; + dv.Base = c.Ptr; m_TailChunkPos = 0; m_TailChunk = nullptr; AppendPossible = false; @@ -183,14 +186,20 @@ BufferV::BufferPos ChunkV::Allocate(const size_t size, size_t align) { // No room in current chunk, close it out // realloc down to used size (helpful?) and set size in array - size_t actualsize = ChunkAlloc(m_Chunks.back(), m_TailChunkPos); + Chunk &c = m_Chunks.back(); + size_t actualsize = ChunkAlloc(c, m_TailChunkPos); size_t alignment = actualsize - m_TailChunkPos; if (alignment) { - auto p = m_Chunks.back().Ptr + m_TailChunkPos; + auto p = c.Ptr + m_TailChunkPos; std::fill(p, p + alignment, 0); + CurOffset += alignment; } - DataV.back().Size = actualsize; + // Update entry in DataV as size and potentiall ptr has changed + // Learned from sanitizer: downsizing realloc still may change pointer + VecEntry &dv = DataV.back(); + dv.Size = actualsize; + dv.Base = c.Ptr; m_TailChunkPos = 0; m_TailChunk = nullptr; AppendPossible = false; diff --git a/testing/adios2/CMakeLists.txt b/testing/adios2/CMakeLists.txt index 6e94ac1102..f6ff7c0afd 100644 --- a/testing/adios2/CMakeLists.txt +++ b/testing/adios2/CMakeLists.txt @@ -4,6 +4,7 @@ #------------------------------------------------------------------------------# add_subdirectory(interface) +add_subdirectory(unit) add_subdirectory(engine) add_subdirectory(transports) add_subdirectory(bindings) diff --git a/testing/adios2/unit/CMakeLists.txt b/testing/adios2/unit/CMakeLists.txt new file mode 100644 index 0000000000..de65ea8510 --- /dev/null +++ b/testing/adios2/unit/CMakeLists.txt @@ -0,0 +1,7 @@ +#------------------------------------------------------------------------------# +#Distributed under the OSI - approved Apache License, Version 2.0. See +#accompanying file Copyright.txt for details. +#------------------------------------------------------------------------------# + +gtest_add_tests_helper(ChunkV MPI_NONE "" Unit. "") + diff --git a/testing/adios2/unit/TestChunkV.cpp b/testing/adios2/unit/TestChunkV.cpp new file mode 100644 index 0000000000..6861e86f71 --- /dev/null +++ b/testing/adios2/unit/TestChunkV.cpp @@ -0,0 +1,150 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + */ +#include +#include +#include + +#include +#include +#include + +#include +#include +#include + +#include + +namespace adios2 +{ +namespace format +{ + +static void print_array(const std::string name, const uint8_t *ptr, const size_t size) +{ + std::cout << name << " ptr = " << (void *)ptr << "= [ "; + for (size_t n = 0; n < size; ++n) + { + std::cout << (unsigned int)ptr[n] << " "; + } + std::cout << "]\n"; +} + +TEST(ChunkV, AlignmentToMemBlockSize) +{ + /* Align chunks to a certain byte size, here 32 */ + size_t MemBlockSize = 32; + size_t ChunkSize = 137; + ChunkV b = ChunkV("test", false, 1, MemBlockSize, ChunkSize); + + /* Align "allocation" to a certain byte size, like type size, here 8 */ + size_t AlignnmentSize = 8; + /* Make allocation not aligned to either AlignmentSize nor the + * MemBlockSize*/ + size_t AllocSize = 51; + + { + adios2::format::BufferV::BufferPos pos = b.Allocate(AllocSize, AlignnmentSize); + ASSERT_EQ(pos.bufferIdx, 0); + ASSERT_EQ(pos.globalPos, 0); + ASSERT_EQ(pos.posInBuffer, 0); + uint8_t *ptr = reinterpret_cast(b.GetPtr(pos.bufferIdx, pos.posInBuffer)); + for (uint8_t n = 0; n < AllocSize; ++n) + { + ptr[n] = n; + } + print_array("array 1", ptr, AllocSize); + + std::vector vec = b.DataVec(); + ASSERT_EQ(vec.size(), 1); + /* First block is alone in first chunk, so only real length (51) should + * be here */ + ASSERT_EQ(vec[0].iov_len, 51); + } + { + /* second block should fit in existing chunk but aligned to 56 bytes + (first AllocSize aligned to AlignmentSize is 56) */ + adios2::format::BufferV::BufferPos pos = b.Allocate(AllocSize, AlignnmentSize); + ASSERT_EQ(pos.bufferIdx, 0); + ASSERT_EQ(pos.globalPos, 56); + ASSERT_EQ(pos.posInBuffer, 56); + uint8_t *ptr = reinterpret_cast(b.GetPtr(pos.bufferIdx, pos.posInBuffer)); + for (uint8_t n = 0; n < AllocSize; ++n) + { + ptr[n] = n; + } + print_array("array 2", ptr, AllocSize); + + std::vector vec = b.DataVec(); + ASSERT_EQ(vec.size(), 1); + /* Seconf block is aligned to 56, so real length should be 56+51 */ + ASSERT_EQ(vec[0].iov_len, 56 + 51); + } + { + /* - third block should NOT fit in existing chunk + - first chunk should be 128 bytes long (aligned to 32) + - globalPos should be 128 (0 in this second chunk) + */ + adios2::format::BufferV::BufferPos pos = b.Allocate(AllocSize, AlignnmentSize); + ASSERT_EQ(pos.bufferIdx, 1); + ASSERT_EQ(pos.globalPos, 128); + ASSERT_EQ(pos.posInBuffer, 0); + uint8_t *ptr = reinterpret_cast(b.GetPtr(pos.bufferIdx, pos.posInBuffer)); + for (uint8_t n = 0; n < AllocSize; ++n) + { + ptr[n] = n; + } + print_array("array 3", ptr, AllocSize); + + std::vector vec = b.DataVec(); + ASSERT_EQ(vec.size(), 2); + /* First chunk must be finalized and aligned to MemBlockSize, so should + * be 128 now */ + ASSERT_EQ(vec[0].iov_len, 128); + /* Third block is alone in second chunk, so only real length (51) should + * be here */ + ASSERT_EQ(vec[1].iov_len, 51); + + const uint8_t *chunk0 = reinterpret_cast(vec[0].iov_base); + const uint8_t *chunk1 = reinterpret_cast(vec[1].iov_base); + /* first array in chunk0[0..AllocSize-1]*/ + ASSERT_EQ(chunk0[0], 0); + ASSERT_EQ(chunk0[AllocSize - 1], AllocSize - 1); + /* alignment fill for 5 bytes*/ + ASSERT_EQ(chunk0[51], 0); + ASSERT_EQ(chunk0[52], 0); + ASSERT_EQ(chunk0[53], 0); + ASSERT_EQ(chunk0[54], 0); + ASSERT_EQ(chunk0[55], 0); + + /* second array in chunk0[56..107]*/ + ASSERT_EQ(chunk0[56], 0); + ASSERT_EQ(chunk0[56 + AllocSize - 1], AllocSize - 1); + /* alignment fill for 5 bytes*/ + ASSERT_EQ(chunk0[107], 0); + ASSERT_EQ(chunk0[108], 0); + ASSERT_EQ(chunk0[109], 0); + ASSERT_EQ(chunk0[110], 0); + ASSERT_EQ(chunk0[111], 0); + /* aligment fill to MemBlockSize for 16 bytes */ + ASSERT_EQ(chunk0[112], 0); + ASSERT_EQ(chunk0[127], 0); + + /* third array in chunk1[0..AllocSize-1]*/ + ASSERT_EQ(chunk1[1], 1); + ASSERT_EQ(chunk1[AllocSize - 1], AllocSize - 1); + } +} +} +} + +int main(int argc, char **argv) +{ + + int result; + ::testing::InitGoogleTest(&argc, argv); + result = RUN_ALL_TESTS(); + + return result; +} From 6b4b66f26ed7eb2e447e451be8cbb12e01505e4e Mon Sep 17 00:00:00 2001 From: pnorbert Date: Thu, 25 Jan 2024 12:56:07 -0500 Subject: [PATCH 025/124] Replace the only PERFSTUBS_SCOPED_TIMER_FUNC() call in BP5 to get rid of an ASAN error blocking local testing. (#4013) --- source/adios2/engine/bp5/BP5Writer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/adios2/engine/bp5/BP5Writer.cpp b/source/adios2/engine/bp5/BP5Writer.cpp index 8afeae2643..78c0646e6b 100644 --- a/source/adios2/engine/bp5/BP5Writer.cpp +++ b/source/adios2/engine/bp5/BP5Writer.cpp @@ -431,7 +431,7 @@ void BP5Writer::NotifyEngineAttribute(std::string name, AttributeBase *Attr, voi void BP5Writer::MarshalAttributes() { - PERFSTUBS_SCOPED_TIMER_FUNC(); + PERFSTUBS_SCOPED_TIMER("BP5Writer::MarshalAttributes"); const auto &attributes = m_IO.GetAttributes(); // if there are no new attributes, nothing to do From 85a7b646cc33500344d1eb28d6ca98741210373c Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Thu, 25 Jan 2024 13:04:13 -0500 Subject: [PATCH 026/124] Python support: - Allow Stream/FileReader constructor overloading and Stream.read()/write() overloading using @singledispatchmethod. - Return status from engine when using Stream.begin_step(). - Update examples with the changes to the API - Update python examples (hello_world, bpWriter, gray-scott, dataMan) - Change stream iterator to use engine BeginStep return status - Create python DataMan/SST writers and readers for both high-level and bindings APIs, and make them work with the C++ example versions. - Require python 3.8 as minimum, when @singledispatchmethod was introduced - Update some documentation, remove some io functions from stream that would be too late to call - Rename variable writer to io in testing consistently --- cmake/DetectOptions.cmake | 6 +- .../bpReader/bpReaderHeatMap2D-bindings.py | 79 +++++ examples/hello/bpReader/bpReaderHeatMap2D.py | 61 +--- examples/hello/bpWriter/bpWriter-bindings.py | 59 ++++ examples/hello/bpWriter/bpWriter.py | 39 +-- examples/hello/datamanReader/dataManReader.py | 39 ++- .../hello/datamanWriter/dataManWriter.cpp | 2 + examples/hello/datamanWriter/dataManWriter.py | 33 +- .../hello/helloWorld/hello-world-bindings.py | 57 ++++ examples/hello/helloWorld/hello-world-hl.py | 47 --- examples/hello/helloWorld/hello-world.py | 46 +-- examples/hello/sstReader/CMakeLists.txt | 10 +- .../hello/sstReader/sstReader-bindings.py | 44 +++ examples/hello/sstReader/sstReader.cpp | 73 +++-- examples/hello/sstReader/sstReader.py | 37 +++ examples/hello/sstWriter/CMakeLists.txt | 10 +- .../hello/sstWriter/sstWriter-bindings.py | 44 +++ examples/hello/sstWriter/sstWriter.cpp | 26 +- examples/hello/sstWriter/sstWriter.py | 40 ++- .../simulations/gray-scott/plot/gsplot.py | 10 +- .../simulations/gray-scott/plot/pdfplot.py | 7 +- python/adios2/adios.py | 4 +- python/adios2/file_reader.py | 24 +- python/adios2/io.py | 7 +- python/adios2/stream.py | 287 ++++++++++++------ python/adios2/variable.py | 12 +- testing/adios2/python/TestADIOS.py | 10 +- testing/adios2/python/TestAttribute.py | 12 +- testing/adios2/python/TestEngine.py | 32 +- testing/adios2/python/TestIO.py | 92 +++--- testing/adios2/python/TestOperator.py | 6 +- testing/adios2/python/TestVariable.py | 18 +- 32 files changed, 840 insertions(+), 433 deletions(-) create mode 100644 examples/hello/bpReader/bpReaderHeatMap2D-bindings.py create mode 100644 examples/hello/bpWriter/bpWriter-bindings.py create mode 100644 examples/hello/helloWorld/hello-world-bindings.py delete mode 100644 examples/hello/helloWorld/hello-world-hl.py create mode 100644 examples/hello/sstReader/sstReader-bindings.py create mode 100644 examples/hello/sstReader/sstReader.py create mode 100644 examples/hello/sstWriter/sstWriter-bindings.py diff --git a/cmake/DetectOptions.cmake b/cmake/DetectOptions.cmake index c8f8e948be..2b3b8cbe3d 100644 --- a/cmake/DetectOptions.cmake +++ b/cmake/DetectOptions.cmake @@ -400,15 +400,15 @@ if(NOT SHARED_LIBS_SUPPORTED) endif() if(ADIOS2_USE_PIP) - find_package(Python 3 REQUIRED COMPONENTS Interpreter Development.Module) + find_package(Python 3.8 REQUIRED COMPONENTS Interpreter Development.Module) set(ADIOS2_HAVE_PIP TRUE) elseif(ADIOS2_USE_Python STREQUAL AUTO) - find_package(Python 3 COMPONENTS Interpreter Development) + find_package(Python 3.8 COMPONENTS Interpreter Development) if(Python_FOUND AND ADIOS2_HAVE_MPI) find_package(PythonModule COMPONENTS mpi4py mpi4py/mpi4py.h) endif() elseif(ADIOS2_USE_Python) - find_package(Python 3 REQUIRED COMPONENTS Interpreter Development) + find_package(Python 3.8 REQUIRED COMPONENTS Interpreter Development) if(ADIOS2_HAVE_MPI) find_package(PythonModule REQUIRED COMPONENTS mpi4py mpi4py/mpi4py.h) endif() diff --git a/examples/hello/bpReader/bpReaderHeatMap2D-bindings.py b/examples/hello/bpReader/bpReaderHeatMap2D-bindings.py new file mode 100644 index 0000000000..b9a63c476a --- /dev/null +++ b/examples/hello/bpReader/bpReaderHeatMap2D-bindings.py @@ -0,0 +1,79 @@ +# +# Distributed under the OSI-approved Apache License, Version 2.0. See +# accompanying file Copyright.txt for details. +# +# bpReaderHeatMap2D-bindings.py +# +# +# Created on: Dec 5th, 2017 +# Author: William F Godoy godoywf@ornl.gov +# Norbert Podhorszki pnorbert@ornl.gov +# + +from mpi4py import MPI +import numpy +import adios2.bindings as adios2 + +# MPI +comm = MPI.COMM_WORLD +rank = comm.Get_rank() +size = comm.Get_size() + +# User data +Nx = 10 +Ny = 10 + +count = [Nx, Ny] +start = [rank * Nx, 0] +shape = [size * Nx, Ny] + +temperatures = numpy.zeros(Nx * Ny, dtype=int) + +for i in range(0, Nx): + iGlobal = start[0] + i + + for j in range(0, Ny): + value = iGlobal * shape[1] + j + temperatures[i * Nx + j] = value +# print(str(i) + "," + str(j) + " " + str(value)) + + +# ADIOS portion +adios = adios2.ADIOS(comm) +ioWrite = adios.DeclareIO("ioWriter") + +varTemperature = ioWrite.DefineVariable( + "temperature2D", temperatures, shape, start, count, adios2.ConstantDims +) + +obpStream = ioWrite.Open("HeatMap2D_py_bindings.bp", adios2.Mode.Write) +obpStream.BeginStep() +obpStream.Put(varTemperature, temperatures) +obpStream.EndStep() +obpStream.Close() + + +if rank == 0: + ioRead = adios.DeclareIO("ioReader") + + ibpStream = ioRead.Open("HeatMap2D_py_bindings.bp", adios2.Mode.Read, MPI.COMM_SELF) + + ibpStream.BeginStep() + + var_inTemperature = ioRead.InquireVariable("temperature2D") + + if var_inTemperature is not None: + var_inTemperature.SetSelection([[2, 2], [4, 4]]) + + inSize = var_inTemperature.SelectionSize() + print("Incoming size " + str(inSize)) + inTemperatures = numpy.zeros(var_inTemperature.Count(), dtype=int) + + ibpStream.Get(var_inTemperature, inTemperatures, adios2.Mode.Sync) + + print("Incoming temperature map") + for i in range(0, inTemperatures.shape[1]): + print(str(inTemperatures[i]) + " ") + + ibpStream.EndStep() + ibpStream.Close() diff --git a/examples/hello/bpReader/bpReaderHeatMap2D.py b/examples/hello/bpReader/bpReaderHeatMap2D.py index 7400b65e10..3208af23b1 100644 --- a/examples/hello/bpReader/bpReaderHeatMap2D.py +++ b/examples/hello/bpReader/bpReaderHeatMap2D.py @@ -7,11 +7,12 @@ # # Created on: Dec 5th, 2017 # Author: William F Godoy godoywf@ornl.gov +# Norbert Podhorszki pnorbert@ornl.gov # from mpi4py import MPI import numpy -import adios2 +from adios2 import Stream # MPI comm = MPI.COMM_WORLD @@ -26,7 +27,7 @@ start = [rank * Nx, 0] shape = [size * Nx, Ny] -temperatures = numpy.zeros(Nx * Ny, dtype=numpy.int) +temperatures = numpy.zeros(Nx * Ny, dtype=int) for i in range(0, Nx): iGlobal = start[0] + i @@ -34,49 +35,19 @@ for j in range(0, Ny): value = iGlobal * shape[1] + j temperatures[i * Nx + j] = value - print(str(i) + "," + str(j) + " " + str(value)) - - -# ADIOS portion -adios = adios2.ADIOS(comm) -ioWrite = adios.DeclareIO("ioWriter") - -varTemperature = ioWrite.DefineVariable( - "temperature2D", temperatures, shape, start, count, adios2.ConstantDims -) - -obpStream = ioWrite.Open("HeatMap2D_py.bp", adios2.Mode.Write) -obpStream.BeginStep() -obpStream.Put(varTemperature, temperatures) -obpStream.EndStep() -obpStream.Close() +# print(str(i) + "," + str(j) + " " + str(value)) +with Stream("HeatMap2D_py.bp", "w", comm) as obpStream: + obpStream.write("temperature2D", temperatures, shape, start, count) if rank == 0: - ioRead = adios.DeclareIO("ioReader") - - ibpStream = ioRead.Open("HeatMap2D_py.bp", adios2.Mode.Read, MPI.COMM_SELF) - - ibpStream.BeginStep() - - var_inTemperature = ioRead.InquireVariable("temperature2D") - - if var_inTemperature is not None: - var_inTemperature.SetSelection([[2, 2], [4, 4]]) - - inSize = var_inTemperature.SelectionSize() - print("Incoming size " + str(inSize)) - inTemperatures = numpy.zeros(inSize, dtype=numpy.int) - - ibpStream.Get(var_inTemperature, inTemperatures, adios2.Mode.Sync) - - print("Incoming temperature map") - - for i in range(0, inTemperatures.size): - print(str(inTemperatures[i]) + " ") - - if (i + 1) % 4 == 0: - print() - - ibpStream.EndStep() - ibpStream.Close() + with Stream("HeatMap2D_py.bp", "r", MPI.COMM_SELF) as ibpStream: + for _ in ibpStream.steps(): + var_inTemperature = ibpStream.inquire_variable("temperature2D") + if var_inTemperature is not None: + var_inTemperature.set_selection([[2, 2], [4, 4]]) + inTemperatures = ibpStream.read(var_inTemperature) + + print("Incoming temperature map") + for i in range(0, inTemperatures.shape[1]): + print(str(inTemperatures[i]) + " ") diff --git a/examples/hello/bpWriter/bpWriter-bindings.py b/examples/hello/bpWriter/bpWriter-bindings.py new file mode 100644 index 0000000000..118b2fe04e --- /dev/null +++ b/examples/hello/bpWriter/bpWriter-bindings.py @@ -0,0 +1,59 @@ +# +# Distributed under the OSI-approved Apache License, Version 2.0. See +# accompanying file Copyright.txt for details. +# +# bpWriter-bindings.py : Low-level Python API example +# Created on: Feb 2, 2017 +# Authors: William F Godoy godoywf@ornl.gov +# Norbert Podhorszki pnorbert@ornl.gov +# +# adios2.bindings gives access to the low level classes that are compiled from the C++ classes +# Only necessary to use this when the adios2 python API is insufficient for some reason +# This example doesn't use anything that is missing from the python API though. + +from mpi4py import MPI +import numpy +import adios2.bindings as adios2 + +# MPI +comm = MPI.COMM_WORLD +rank = comm.Get_rank() +size = comm.Get_size() + +# User data +myArray = numpy.array([0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0]) +myArray = myArray + (rank * 10) +Nx = myArray.size + +# ADIOS MPI Communicator +adios = adios2.ADIOS(comm) + +# ADIOS IO +bpIO = adios.DeclareIO("BPFile_N2N") +bpIO.SetEngine("BPFile") +# bpIO.SetParameters( {"Threads" : "2", "ProfileUnits" : "Microseconds", +# "InitialBufferSize" : "17Kb"} ) +bpIOParams = {} +bpIOParams["Threads"] = "2" +bpIOParams["ProfileUnits"] = "Microseconds" +bpIOParams["InitialBufferSize"] = "17Kb" +bpIO.SetParameters(bpIOParams) + +fileID = bpIO.AddTransport("File", {"Library": "fstream"}) + +# ADIOS Variable name, shape, start, offset, constant dims +ioArray = bpIO.DefineVariable( + "bpArray", myArray, [size * Nx], [rank * Nx], [Nx], adios2.ConstantDims +) + +# ADIOS Engine +bpFileWriter = bpIO.Open("bpWriter-py-bindings.bp", adios2.Mode.Write) +bpFileWriter.BeginStep() +bpFileWriter.Put(ioArray, myArray, adios2.Mode.Sync) +bpFileWriter.EndStep() +bpFileWriter.Close() + +# Read content: +# bpls -la bpWriter-py-bindings.bp +# bpls -la bpWriter-py-bindings.bp -d bpArray -n 10 +# bpls -la bpWriter-py-bindings.bp -d bpArray -n 10 -D diff --git a/examples/hello/bpWriter/bpWriter.py b/examples/hello/bpWriter/bpWriter.py index fe2272112b..dbfacc9dc7 100644 --- a/examples/hello/bpWriter/bpWriter.py +++ b/examples/hello/bpWriter/bpWriter.py @@ -4,7 +4,12 @@ # # bpWriter.py : only works with MPI version # Created on: Feb 2, 2017 -# Author: William F Godoy godoywf@ornl.gov +# Authors: William F Godoy godoywf@ornl.gov +# Norbert Podhorszki pnorbert@ornl.gov +# +# We use adios2.Adios and adios2.IO classes to set up the IO parameters. +# For default IO, it is sufficient to use the adios2.Stream class alone. + from mpi4py import MPI import numpy import adios2 @@ -16,32 +21,30 @@ # User data myArray = numpy.array([0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0]) +myArray = myArray + (rank * 10) Nx = myArray.size # ADIOS MPI Communicator -adios = adios2.ADIOS(comm) +adios = adios2.Adios(config_file=None, comm=comm) # ADIOS IO -bpIO = adios.DeclareIO("BPFile_N2N") -bpIO.SetEngine("BPFile") -# bpIO.SetParameters( {"Threads" : "2", "ProfileUnits" : "Microseconds", -# "InitialBufferSize" : "17Kb"} ) +bpIO = adios.declare_io("BPFile_N2N") + bpIOParams = {} bpIOParams["Threads"] = "2" bpIOParams["ProfileUnits"] = "Microseconds" bpIOParams["InitialBufferSize"] = "17Kb" -bpIO.SetParameters(bpIOParams) +bpIO.set_parameters(bpIOParams) -fileID = bpIO.AddTransport("File", {"Library": "fstream"}) +bpIO.add_transport("File", {"Library": "fstream"}) +bpIO.set_engine("BPFile") +a = bpIO.adios() -# ADIOS Variable name, shape, start, offset, constant dims -ioArray = bpIO.DefineVariable( - "bpArray", myArray, [size * Nx], [rank * Nx], [Nx], adios2.ConstantDims -) +# ADIOS output stream +with adios2.Stream(bpIO, "bpWriter-py.bp", "w", comm) as fh: + fh.write("bpArray", myArray, [size * Nx], [rank * Nx], [Nx]) -# ADIOS Engine -bpFileWriter = bpIO.Open("npArray.bp", adios2.Mode.Write) -bpFileWriter.BeginStep() -bpFileWriter.Put(ioArray, myArray, adios2.Mode.Sync) -bpFileWriter.EndStep() -bpFileWriter.Close() +# Read content: +# bpls -la bpWriter-py.bp +# bpls -la bpWriter-py.bp -d bpArray -n 10 +# bpls -la bpWriter-py.bp -d bpArray -n 10 -D diff --git a/examples/hello/datamanReader/dataManReader.py b/examples/hello/datamanReader/dataManReader.py index c773e84fb8..f3ddfe8e4d 100644 --- a/examples/hello/datamanReader/dataManReader.py +++ b/examples/hello/datamanReader/dataManReader.py @@ -10,31 +10,26 @@ from mpi4py import MPI import numpy as np -import adios2 +from adios2 import Adios, Stream +from adios2.bindings import StepStatus comm = MPI.COMM_WORLD rank = comm.Get_rank() size = comm.Get_size() -adios = adios2.ADIOS(comm) -io = adios.DeclareIO("whatever") -io.SetEngine("DataMan") -io.SetParameters({"IPAddress": "127.0.0.1", "Port": "12306", "Timeout": "5"}) +adios = Adios(comm) +io = adios.declare_io("whatever") +io.set_engine("DataMan") +io.set_parameters({"IPAddress": "127.0.0.1", "Port": "12306", "Timeout": "5"}) -engine = io.Open("HelloDataMan", adios2.Mode.Read, comm) - -while True: - stepStatus = engine.BeginStep() - if stepStatus == adios2.StepStatus.OK: - var = io.InquireVariable("FloatArray") - shape = var.Shape() - floatArray = np.zeros(np.prod(shape), dtype=np.float32) - engine.Get(var, floatArray, adios2.Mode.Sync) - currentStep = engine.CurrentStep() - engine.EndStep() - print("Step", currentStep, floatArray) - elif stepStatus == adios2.StepStatus.EndOfStream: - print("End of stream") - break - -engine.Close() +with Stream(io, "HelloDataMan", "r") as stream: + for _ in stream.steps(): + stepStatus = stream.step_status() + print(f"Step status = {stepStatus}") + var = io.inquire_variable("FloatArray") + shape = var.shape() + floatArray = stream.read(var) + currentStep = stream.current_step() + loopStep = stream.loop_index() + print("Loop index =", loopStep, "stream step =", currentStep, "data =") + print(floatArray) diff --git a/examples/hello/datamanWriter/dataManWriter.cpp b/examples/hello/datamanWriter/dataManWriter.cpp index fc106c2bb3..a615ef3661 100644 --- a/examples/hello/datamanWriter/dataManWriter.cpp +++ b/examples/hello/datamanWriter/dataManWriter.cpp @@ -9,6 +9,7 @@ */ #include +#include #include #include #include @@ -85,6 +86,7 @@ int main(int argc, char *argv[]) engine.Put(floatArrayVar, floatVector.data(), adios2::Mode::Sync); PrintData(floatVector, engine.CurrentStep()); engine.EndStep(); + std::this_thread::sleep_for(std::chrono::milliseconds(2000)); } engine.Close(); diff --git a/examples/hello/datamanWriter/dataManWriter.py b/examples/hello/datamanWriter/dataManWriter.py index 9035d0f42a..a212ef2ee9 100644 --- a/examples/hello/datamanWriter/dataManWriter.py +++ b/examples/hello/datamanWriter/dataManWriter.py @@ -10,7 +10,8 @@ from mpi4py import MPI import numpy as np -import adios2 +from time import sleep +from adios2 import Adios, Stream comm = MPI.COMM_WORLD rank = comm.Get_rank() @@ -30,22 +31,20 @@ for j in range(0, Ny): floatArray[i, j] = (start[0] + i) * shape[1] + (j + start[1]) -adios = adios2.ADIOS(comm) -io = adios.DeclareIO("whatever") -io.SetEngine("DataMan") -io.SetParameters({"IPAddress": "127.0.0.1", "Port": "12306", "Timeout": "5"}) +adios = Adios(comm) +io = adios.declare_io("whatever") +io.set_engine("DataMan") +io.set_parameters({"IPAddress": "127.0.0.1", "Port": "12306", "Timeout": "5"}) -var = io.DefineVariable( - "FloatArray", floatArray, shape, start, count, adios2.ConstantDims -) +var = io.define_variable("FloatArray", floatArray, shape, start, count, True) -engine = io.Open("HelloDataMan", adios2.Mode.Write) +with Stream(io, "HelloDataMan", "w") as stream: + for _ in stream.steps(steps): + # imitating costly computation + floatArray = floatArray + 1 + sleep(1.0) -for i in range(steps): - floatArray = floatArray + 1 - print("Step", i, floatArray) - engine.BeginStep() - engine.Put(var, floatArray) - engine.EndStep() - -engine.Close() + print("Step", stream.current_step(), floatArray) + stream.write(var, floatArray) + # Warning: the data of the current step is not published until + # the next loop entry or the exit of the loop diff --git a/examples/hello/helloWorld/hello-world-bindings.py b/examples/hello/helloWorld/hello-world-bindings.py new file mode 100644 index 0000000000..eeb3a205e5 --- /dev/null +++ b/examples/hello/helloWorld/hello-world-bindings.py @@ -0,0 +1,57 @@ +# +# Distributed under the OSI-approved Apache License, Version 2.0. See +# accompanying file Copyright.txt for details. +# +# hello-world.py : adios2 low-level API example to write and read a +# string Variable with a greeting +# +# Created on: 2/2/2021 +# Author: Dmitry Ganyushin ganyushindi@ornl.gov +# +import sys +from mpi4py import MPI +import adios2.bindings as adios2 + +DATA_FILENAME = "hello-world-py-bindings.bp" +# MPI +comm = MPI.COMM_WORLD +rank = comm.Get_rank() +size = comm.Get_size() + + +def writer(ad, greeting): + """write a string to a bp file""" + io = ad.DeclareIO("hello-world-writer") + var_greeting = io.DefineVariable("Greeting") + w = io.Open(DATA_FILENAME, adios2.Mode.Write) + w.BeginStep() + w.Put(var_greeting, greeting) + w.EndStep() + w.Close() + return 0 + + +def reader(ad): + """read a string from to a bp file""" + io = ad.DeclareIO("hello-world-reader") + r = io.Open(DATA_FILENAME, adios2.Mode.Read) + r.BeginStep() + var_greeting = io.InquireVariable("Greeting") + message = r.Get(var_greeting) + r.EndStep() + r.Close() + return message + + +def main(): + """driver function""" + ad = adios2.ADIOS(comm) + greeting = "Hello World from ADIOS2" + writer(ad, greeting) + message = reader(ad) + print("{}".format(message)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/examples/hello/helloWorld/hello-world-hl.py b/examples/hello/helloWorld/hello-world-hl.py deleted file mode 100644 index 825fb5282e..0000000000 --- a/examples/hello/helloWorld/hello-world-hl.py +++ /dev/null @@ -1,47 +0,0 @@ -# -# Distributed under the OSI-approved Apache License, Version 2.0. See -# accompanying file Copyright.txt for details. -# -# hello-world.py : adios2 high-level API example to write and read a -# string Variable with a greeting -# -# Created on: 2/2/2021 -# Author: Dmitry Ganyushin ganyushindi@ornl.gov -# -import sys -from mpi4py import MPI -import adios2 - -DATA_FILENAME = "hello-world-hl-py.bp" -# MPI -comm = MPI.COMM_WORLD -rank = comm.Get_rank() -size = comm.Get_size() - - -def writer(greeting): - """write a string to a bp file""" - with adios2.open(DATA_FILENAME, "w", comm) as fh: - fh.write("Greeting", greeting, end_step=True) - return 0 - - -def reader(): - """read a string from to a bp file""" - with adios2.open(DATA_FILENAME, "r", comm) as fh: - for fstep in fh: - message = fstep.read_string("Greeting") - return message - - -def main(): - """driver function""" - greeting = "Hello World from ADIOS2" - writer(greeting) - message = reader() - print("{}".format(message)) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/examples/hello/helloWorld/hello-world.py b/examples/hello/helloWorld/hello-world.py index 685d648597..a0f7204e7e 100644 --- a/examples/hello/helloWorld/hello-world.py +++ b/examples/hello/helloWorld/hello-world.py @@ -2,7 +2,7 @@ # Distributed under the OSI-approved Apache License, Version 2.0. See # accompanying file Copyright.txt for details. # -# hello-world.py : adios2 low-level API example to write and read a +# hello-world.py : adios2 Python API example to write and read a # string Variable with a greeting # # Created on: 2/2/2021 @@ -10,7 +10,7 @@ # import sys from mpi4py import MPI -import adios2 +from adios2 import Stream, FileReader DATA_FILENAME = "hello-world-py.bp" # MPI @@ -19,37 +19,37 @@ size = comm.Get_size() -def writer(ad, greeting): +def writer(greeting): """write a string to a bp file""" - io = ad.DeclareIO("hello-world-writer") - var_greeting = io.DefineVariable("Greeting") - w = io.Open(DATA_FILENAME, adios2.Mode.Write) - w.BeginStep() - w.Put(var_greeting, greeting) - w.EndStep() - w.Close() + with Stream(DATA_FILENAME, "w", comm) as fh: + fh.write("Greeting", greeting) return 0 -def reader(ad): - """read a string from to a bp file""" - io = ad.DeclareIO("hello-world-reader") - r = io.Open(DATA_FILENAME, adios2.Mode.Read) - r.BeginStep() - var_greeting = io.InquireVariable("Greeting") - message = r.Get(var_greeting) - r.EndStep() - r.Close() +def reader(): + """read a string from a bp file as Stream""" + message = f"variable Greeting not found in {DATA_FILENAME}" + with Stream(DATA_FILENAME, "r", comm) as fh: + for _ in fh.steps(): + message = fh.read("Greeting") + return message + + +def filereader(): + """read a string from a bp file using random access read mode""" + with FileReader(DATA_FILENAME, comm) as fh: + message = fh.read("Greeting") return message def main(): """driver function""" - ad = adios2.ADIOS(comm) greeting = "Hello World from ADIOS2" - writer(ad, greeting) - message = reader(ad) - print("{}".format(message)) + writer(greeting) + message = reader() + print("As read from adios2.Stream: {}".format(message)) + message2 = filereader() + print("As read from adios2.FileReader: {}".format(message2)) return 0 diff --git a/examples/hello/sstReader/CMakeLists.txt b/examples/hello/sstReader/CMakeLists.txt index 39a0b615c0..be46aab22c 100644 --- a/examples/hello/sstReader/CMakeLists.txt +++ b/examples/hello/sstReader/CMakeLists.txt @@ -26,12 +26,10 @@ endif() if(ADIOS2_HAVE_SST) add_executable(adios2_hello_sstReader sstReader.cpp) - target_link_libraries(adios2_hello_sstReader adios2::cxx11) - install(TARGETS adios2_hello_sstReader RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) - if(ADIOS2_HAVE_MPI) - add_executable(adios2_hello_sstReader_mpi sstReader.cpp) - target_link_libraries(adios2_hello_sstReader_mpi adios2::cxx11_mpi MPI::MPI_C) - install(TARGETS adios2_hello_sstReader_mpi RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) + target_link_libraries(adios2_hello_sstReader adios2::cxx11_mpi MPI::MPI_C) + else() + target_link_libraries(adios2_hello_sstReader adios2::cxx11) endif() + install(TARGETS adios2_hello_sstReader RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) endif() diff --git a/examples/hello/sstReader/sstReader-bindings.py b/examples/hello/sstReader/sstReader-bindings.py new file mode 100644 index 0000000000..38ed1ace7f --- /dev/null +++ b/examples/hello/sstReader/sstReader-bindings.py @@ -0,0 +1,44 @@ +from mpi4py import MPI +import numpy as np +import adios2.bindings as adios2 + +# MPI +comm = MPI.COMM_WORLD +rank = comm.Get_rank() +size = comm.Get_size() + +# ADIOS MPI Communicator +adios = adios2.ADIOS(comm) + +# ADIOS IO +sstIO = adios.DeclareIO("myIO") +sstIO.SetEngine("Sst") + + +sstReader = sstIO.Open("helloSst", adios2.Mode.Read) +loopStep = 0 +while True: + status = sstReader.BeginStep() + if not status == adios2.StepStatus.OK: + break + var = sstIO.InquireVariable("bpFloats") + shape = var.Shape() + count = int(shape[0] / size) + start = count * rank + if rank == size - 1: + count += shape[0] % size + var.SetSelection([[start], [count]]) + # note: we need to use np.float32 to be compatible with data from C++ writer + # using "float" works in Python only but leads to type mismatch with C++ + floatArray = np.zeros(count, dtype=np.float32) + currentStep = sstReader.CurrentStep() + sstReader.Get(var, floatArray) + sstReader.EndStep() + + print( + "Rank=", rank, "loop index =", loopStep, "stream step =", currentStep, "data =", floatArray + ) + loopStep = loopStep + 1 + +print(f"Exited loop with StepStatus = {status}") +sstReader.Close() diff --git a/examples/hello/sstReader/sstReader.cpp b/examples/hello/sstReader/sstReader.cpp index 56630b3eea..8dc397906a 100644 --- a/examples/hello/sstReader/sstReader.cpp +++ b/examples/hello/sstReader/sstReader.cpp @@ -20,6 +20,16 @@ #include #endif +template +void PrintData(const std::vector &data, const int rank, const size_t step) +{ + std::cout << "Rank: " << rank << " Step: " << step << " ["; + for (size_t i = 0; i < data.size(); ++i) + { + std::cout << data[i] << " "; + } + std::cout << "]" << std::endl; +} int main(int argc, char *argv[]) { @@ -53,26 +63,49 @@ int main(int argc, char *argv[]) sstIO.SetEngine("Sst"); adios2::Engine sstReader = sstIO.Open("helloSst", adios2::Mode::Read); - sstReader.BeginStep(); - adios2::Variable bpFloats = sstIO.InquireVariable("bpFloats"); - std::cout << "Incoming variable is of size " << bpFloats.Shape()[0] << "\n"; - const std::size_t total_size = bpFloats.Shape()[0]; - const std::size_t my_start = (total_size / size) * rank; - const std::size_t my_count = (total_size / size); - std::cout << "Reader rank " << rank << " reading " << my_count - << " floats starting at element " << my_start << "\n"; - - const adios2::Dims start{my_start}; - const adios2::Dims count{my_count}; - - const adios2::Box sel(start, count); - - std::vector myFloats; - myFloats.resize(my_count); - - bpFloats.SetSelection(sel); - sstReader.Get(bpFloats, myFloats.data()); - sstReader.EndStep(); + + while (true) + { + auto status = sstReader.BeginStep(); + if (status == adios2::StepStatus::EndOfStream) + { + break; + } + else if (status == adios2::StepStatus::OtherError) + { + std::cout << "ERROR in stream processing when calling BeginStep(). Quit" + << std::endl; + break; + } + + adios2::Variable bpFloats = sstIO.InquireVariable("bpFloats"); + // std::cout << "Incoming variable is of size " << bpFloats.Shape()[0] << "\n"; + const std::size_t total_size = bpFloats.Shape()[0]; + const std::size_t my_start = (total_size / size) * rank; + std::size_t my_count = (total_size / size); + if (rank == size - 1) + { + my_count += (total_size % size); + } + + // std::cout << "Reader rank " << rank << " reading " << my_count + // << " floats starting at element " << my_start << "\n"; + + const adios2::Dims start{my_start}; + const adios2::Dims count{my_count}; + + const adios2::Box sel(start, count); + + std::vector myFloats; + myFloats.resize(my_count); + + bpFloats.SetSelection(sel); + sstReader.Get(bpFloats, myFloats.data()); + sstReader.EndStep(); + // myfloats is filled ONLY AFTER EndStep!!! + // Use adios2.Mode.Sync extra parameter in Get() to get the data immediately + PrintData(myFloats, rank, sstReader.CurrentStep()); + } sstReader.Close(); } diff --git a/examples/hello/sstReader/sstReader.py b/examples/hello/sstReader/sstReader.py new file mode 100644 index 0000000000..93a7117ee3 --- /dev/null +++ b/examples/hello/sstReader/sstReader.py @@ -0,0 +1,37 @@ +from mpi4py import MPI +import numpy as np +from adios2 import Stream, Adios, bindings + +# MPI +comm = MPI.COMM_WORLD +rank = comm.Get_rank() +size = comm.Get_size() + +# ADIOS MPI Communicator +adios = Adios(comm) + +# ADIOS IO +io = adios.declare_io("myIO") +io.set_engine("Sst") + +with Stream(io, "helloSst", "r", comm) as stream: + for _ in stream.steps(): + var = stream.inquire_variable("bpFloats") + shape = var.shape() + count = int(shape[0] / size) + start = count * rank + if rank == size - 1: + count += shape[0] % size + floatArray = stream.read("bpFloats", [start], [count]) + currentStep = stream.current_step() + loopStep = stream.loop_index() + print( + "Rank=", + rank, + "loop index =", + loopStep, + "stream step =", + currentStep, + "data =", + floatArray, + ) diff --git a/examples/hello/sstWriter/CMakeLists.txt b/examples/hello/sstWriter/CMakeLists.txt index f5adca3e58..3636ddd5c7 100644 --- a/examples/hello/sstWriter/CMakeLists.txt +++ b/examples/hello/sstWriter/CMakeLists.txt @@ -26,12 +26,10 @@ endif() if(ADIOS2_HAVE_SST) add_executable(adios2_hello_sstWriter sstWriter.cpp) - target_link_libraries(adios2_hello_sstWriter adios2::cxx11) - install(TARGETS adios2_hello_sstWriter RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) - if(ADIOS2_HAVE_MPI) - add_executable(adios2_hello_sstWriter_mpi sstWriter.cpp) - target_link_libraries(adios2_hello_sstWriter_mpi adios2::cxx11_mpi MPI::MPI_C) - install(TARGETS adios2_hello_sstWriter_mpi RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) + target_link_libraries(adios2_hello_sstWriter adios2::cxx11_mpi MPI::MPI_C) + else() + target_link_libraries(adios2_hello_sstWriter adios2::cxx11) endif() + install(TARGETS adios2_hello_sstWriter RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) endif() diff --git a/examples/hello/sstWriter/sstWriter-bindings.py b/examples/hello/sstWriter/sstWriter-bindings.py new file mode 100644 index 0000000000..ede96042eb --- /dev/null +++ b/examples/hello/sstWriter/sstWriter-bindings.py @@ -0,0 +1,44 @@ +from mpi4py import MPI +import numpy as np +from time import sleep +import adios2.bindings as adios2 + +# MPI +comm = MPI.COMM_WORLD +rank = comm.Get_rank() +size = comm.Get_size() + +# ADIOS MPI Communicator +adios = adios2.ADIOS(comm) + +# ADIOS IO +sstIO = adios.DeclareIO("myIO") +sstIO.SetEngine("Sst") + +# Python MPI is usually not compatible with MPI used to build ADIOS +# Must use TCP based data transport, or RDMA (ucx, libfabric) if available +sstIO.SetParameter("DataTransport", "WAN") + +# note: we need to use np.float32 to be compatible with data from C++ writer +# using "float" works in Python only but leads to type mismatch with C++ +myArray = np.array([0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0], dtype=np.float32) +myArray = 10.0 * rank + myArray +nx = len(myArray) +increment = nx * size * 1.0 + +# ADIOS Variable +ioArray = sstIO.DefineVariable( + "bpFloats", myArray, [size * nx], [rank * nx], [nx], adios2.ConstantDims +) + +sstWriter = sstIO.Open("helloSst", adios2.Mode.Write) +for i in range(4): + print("Rank=", rank, "loop index =", i, "data =", myArray) + sstWriter.BeginStep() + sstWriter.Put(ioArray, myArray, adios2.Mode.Sync) + myArray += increment + # Warning: the data is not published until EndStep is called + sstWriter.EndStep() + sleep(1.0) + +sstWriter.Close() diff --git a/examples/hello/sstWriter/sstWriter.cpp b/examples/hello/sstWriter/sstWriter.cpp index 02d4981171..47e8c1b447 100644 --- a/examples/hello/sstWriter/sstWriter.cpp +++ b/examples/hello/sstWriter/sstWriter.cpp @@ -17,6 +17,17 @@ #include #endif +template +void PrintData(const std::vector &data, const int rank, const size_t step) +{ + std::cout << "Rank: " << rank << " Step: " << step << " ["; + for (size_t i = 0; i < data.size(); ++i) + { + std::cout << data[i] << " "; + } + std::cout << "]" << std::endl; +} + int main(int argc, char *argv[]) { @@ -42,6 +53,7 @@ int main(int argc, char *argv[]) static_cast(10.0 * rank + 6), static_cast(10.0 * rank + 7), static_cast(10.0 * rank + 8), static_cast(10.0 * rank + 9)}; const std::size_t Nx = myFloats.size(); + const float increment = Nx * size * 1.0; try { @@ -60,9 +72,17 @@ int main(int argc, char *argv[]) // Open returns a smart pointer to Engine containing the Derived class adios2::Engine sstWriter = sstIO.Open("helloSst", adios2::Mode::Write); - sstWriter.BeginStep(); - sstWriter.Put(bpFloats, myFloats.data()); - sstWriter.EndStep(); + for (size_t i = 0; i < 4; ++i) + { + PrintData(myFloats, rank, i); + sstWriter.BeginStep(); + sstWriter.Put(bpFloats, myFloats.data()); + sstWriter.EndStep(); + for (size_t k = 0; k < myFloats.size(); ++k) + { + myFloats[k] += increment; + } + } sstWriter.Close(); } catch (std::invalid_argument &e) diff --git a/examples/hello/sstWriter/sstWriter.py b/examples/hello/sstWriter/sstWriter.py index 08a7a8270a..f3a085a9cd 100644 --- a/examples/hello/sstWriter/sstWriter.py +++ b/examples/hello/sstWriter/sstWriter.py @@ -1,6 +1,7 @@ from mpi4py import MPI import numpy as np -import adios2 +from time import sleep +from adios2 import Stream, Adios, bindings # MPI comm = MPI.COMM_WORLD @@ -8,19 +9,32 @@ size = comm.Get_size() # ADIOS MPI Communicator -adios = adios2.ADIOS(comm) +adios = Adios(comm) # ADIOS IO -sstIO = adios.DeclareIO("myIO") -sstIO.SetEngine("Sst") -myArray = np.array([0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0], dtype="f") +io = adios.declare_io("myIO") +io.set_engine("Sst") + +# Python MPI is usually not compatible with MPI used to build ADIOS +# Must use TCP based data transport, or RDMA (ucx, libfabric) if available +io.set_parameter("DataTransport", "WAN") + +# note: we need to use np.float32 to be compatible with data from C++ writer +# using "float" works in Python only but leads to type mismatch with C++ +myArray = np.array([0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0], dtype=np.float32) myArray = 10.0 * rank + myArray nx = len(myArray) -ioArray = sstIO.DefineVariable( - "bpFloats", myArray, [size * nx], [rank * nx], [nx], adios2.ConstantDims -) -sstFileWriter = sstIO.Open("helloSst", adios2.Mode.Write) -sstFileWriter.BeginStep() -sstFileWriter.Put(ioArray, myArray, adios2.Mode.Sync) -sstFileWriter.EndStep() -sstFileWriter.Close() +increment = nx * size * 1.0 + +with Stream(io, "helloSst", "w", comm) as stream: + for _ in stream.steps(4): + currentStep = stream.current_step() + + # imitating computation + sleep(1.0) + + stream.write("bpFloats", myArray, [size * nx], [rank * nx], [nx]) + print("Rank=", rank, "loop index =", currentStep, "data =", myArray) + myArray += increment + # Warning: the data of the current step is not published until + # the next loop entry or the exit of the loop diff --git a/examples/simulations/gray-scott/plot/gsplot.py b/examples/simulations/gray-scott/plot/gsplot.py index 5573ffd996..5c1cd2d528 100644 --- a/examples/simulations/gray-scott/plot/gsplot.py +++ b/examples/simulations/gray-scott/plot/gsplot.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -from adios2 import Stream # pylint: disable=import-error +from adios2 import Adios, Stream # pylint: disable=import-error import argparse import numpy as np # pylint: disable=import-error import matplotlib.pyplot as plt # pylint: disable=import-error @@ -132,8 +132,9 @@ def read_data(args, fr, start_coord, size_dims): myrank = mpi.rank["app"] # Read the data from this object - fr = Stream(args.instream, "r", comm=mpi.comm_app, config_file="adios2.xml", - io_name="SimulationOutput") + adios = Adios("adios2.xml", mpi.comm_app) + io = adios.declare_io("SimulationOutput") + fr = Stream(io, args.instream, "r", mpi.comm_app) if args.outfile.endswith(".bp"): global writer @@ -144,7 +145,8 @@ def read_data(args, fr, start_coord, size_dims): # Read through the steps, one at a time plot_step = 0 for fr_step in fr.steps(): - # if fr_step.current_step() + # print(f"loop status = {fr_step.step_status()} " + # f"step = {fr.current_step()} counter={fr.loop_index()}") start, size, fullshape = mpi.Partition_3D_3D(fr, args) cur_step = fr_step.current_step() vars_info = fr.available_variables() diff --git a/examples/simulations/gray-scott/plot/pdfplot.py b/examples/simulations/gray-scott/plot/pdfplot.py index 4984da5ed2..345b98bccc 100644 --- a/examples/simulations/gray-scott/plot/pdfplot.py +++ b/examples/simulations/gray-scott/plot/pdfplot.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -from adios2 import Stream # pylint: disable=import-error +from adios2 import Adios, Stream # pylint: disable=import-error import argparse import numpy as np # pylint: disable=import-error import matplotlib.pyplot as plt # pylint: disable=import-error @@ -85,8 +85,9 @@ def read_data(args, fr, start_coord, size_dims): myrank = mpi.rank["app"] # Read the data from this object - fr = Stream(args.instream, "r", comm=mpi.comm_app, config_file="adios2.xml", - io_name="PDFAnalysisOutput") + adios = Adios("adios2.xml", mpi.comm_app) + io = adios.declare_io("PDFAnalysisOutput") + fr = Stream(io, args.instream, "r", mpi.comm_app) # Read through the steps, one at a time plot_step = 0 diff --git a/python/adios2/adios.py b/python/adios2/adios.py index 5deacad5a3..af39b7f78a 100644 --- a/python/adios2/adios.py +++ b/python/adios2/adios.py @@ -42,7 +42,7 @@ def declare_io(self, name): Args: name (str): IO instance name """ - return IO(self.impl.DeclareIO(name), name) + return IO(self.impl.DeclareIO(name), name, self) def at_io(self, name): """ @@ -56,7 +56,7 @@ def at_io(self, name): io = None io_instance = self.impl.AtIO(name) if io_instance: - io = IO(io_instance, name) + io = IO(io_instance, name, self) return io def remove_io(self, name): diff --git a/python/adios2/file_reader.py b/python/adios2/file_reader.py index 7b80c11b91..795cf9ab75 100644 --- a/python/adios2/file_reader.py +++ b/python/adios2/file_reader.py @@ -2,8 +2,8 @@ Distributed under the OSI-approved Apache License, Version 2.0. See accompanying file Copyright.txt for details. """ - -from adios2.stream import Stream +from functools import singledispatchmethod +from adios2 import Stream, IO class FileReader(Stream): @@ -12,15 +12,17 @@ class FileReader(Stream): def __repr__(self): return f"" - def __init__(self, path, *, comm=None, engine_type=None, config_file=None, io_name=None): - super().__init__( - path, - "rra", - comm=comm, - engine_type=engine_type, - config_file=config_file, - io_name=io_name, - ) + @singledispatchmethod + def __init__(self, path, comm=None): + super().__init__(path, "rra", comm) + + # e.g. FileReader(io: adios2.IO, path, mode) + # pylint: disable=E1121 + @__init__.register(IO) + def _(self, io: IO, path, mode, comm=None): + super().__init__(io, path, "rra", comm) + + # pylint: enable=E1121 def variables(self): """Returns the list of variables contained in the opened file""" diff --git a/python/adios2/io.py b/python/adios2/io.py index 6a5ef3a355..09d9c2520a 100644 --- a/python/adios2/io.py +++ b/python/adios2/io.py @@ -12,9 +12,10 @@ class IO: """High level representation of the IO class in the adios2.bindings""" - def __init__(self, impl, name): + def __init__(self, impl, name, adiosobj): self.impl = impl self._name = name + self._adios = adiosobj @property def impl(self): @@ -36,6 +37,10 @@ def __eq__(self, other): return self._name == other._name return False + def adios(self): + """Adios instance associated to this IO""" + return self._adios + def define_attribute( self, name, diff --git a/python/adios2/stream.py b/python/adios2/stream.py index 4ba245d545..abfbca3944 100644 --- a/python/adios2/stream.py +++ b/python/adios2/stream.py @@ -2,11 +2,10 @@ Distributed under the OSI-approved Apache License, Version 2.0. See accompanying file Copyright.txt for details. """ - -from adios2.adios import Adios -from adios2 import bindings - +from functools import singledispatchmethod +from sys import maxsize import numpy as np +from adios2 import bindings, Adios, IO, Variable def type_adios_to_numpy(name): @@ -26,58 +25,60 @@ def type_adios_to_numpy(name): }[name] +def string_to_mode(mode: str) -> [bindings.Mode, bool]: + """Convert high level open mode to adios2.bindings.Mode""" + read_mode = False + if mode == "r": + bmode = bindings.Mode.Read + read_mode = True + elif mode == "rra": + bmode = bindings.Mode.ReadRandomAccess + read_mode = True + elif mode == "w": + bmode = bindings.Mode.Write + elif mode == "a": + bmode = bindings.Mode.Append + else: + raise ValueError() + return bmode, read_mode + + +# pylint: disable=R0902 # Too many instance attributes class Stream: """High level implementation of the Stream class from the core API""" - def __init__( - self, path, mode="r", *, comm=None, engine_type=None, config_file=None, io_name=None - ): - + @singledispatchmethod + def __init__(self, path, mode, comm=None): # pylint: disable=R0912 # Too many branches if comm and not bindings.is_built_with_mpi: raise RuntimeError("Cannot use MPI since ADIOS2 was built without MPI support") - if config_file and engine_type: - raise RuntimeError("Arguments 'engine_type' and 'config_file' cannot be used together") - - if config_file and not io_name: - raise RuntimeError("Argument 'io_name' is required when using 'config_file'") - - if not engine_type: - engine_type = "File" - # pylint: disable=E1121 - if config_file: - if comm: - self._adios = Adios(config_file, comm) - else: - self._adios = Adios(config_file) - self._io_name = io_name - + if comm: + self._adios = Adios(comm) else: - if comm: - self._adios = Adios(comm) - else: - self._adios = Adios() - self._io_name = f"stream:{path}:engine_type:{engine_type}:mode:{mode}" + self._adios = Adios() + + self._io_name = f"stream:{path}:mode:{mode}" # pylint: enable=E1121 self._io = self._adios.declare_io(self._io_name) - if not config_file: - self._io.set_engine(engine_type) - - if mode == "r": - self._mode = bindings.Mode.Read - elif mode == "rra": - self._mode = bindings.Mode.ReadRandomAccess - elif mode == "w": - self._mode = bindings.Mode.Write - else: - raise ValueError() - + self._mode, self._read_mode = string_to_mode(mode) self._engine = self._io.open(path, self._mode) - self.index = 0 - self.max_steps = 0 + self.index = -1 + self.max_steps = maxsize + self._step_status = bindings.StepStatus.EndOfStream + + # e.g. Stream(io: adios2.IO, path, mode) + @__init__.register(IO) + def _(self, io: IO, path, mode, comm=None): + self._io = io + self._adios = io.adios() + self._mode, self._read_mode = string_to_mode(mode) + self._engine = self._io.open(path, self._mode, comm) + self.index = -1 + self.max_steps = maxsize + self._step_status = bindings.StepStatus.EndOfStream @property def mode(self): @@ -112,17 +113,27 @@ def __iter__(self): return self def __next__(self): - if self.index > 0: + if self.index >= 0 and self._step_status == bindings.StepStatus.OK: self.end_step() - if self.index == self.max_steps: - self.index = 0 - self.max_steps = 0 + if self.index == self.max_steps - 1: + self._step_status = bindings.StepStatus.EndOfStream raise StopIteration self.index += 1 - self.begin_step() + self._step_status = self.begin_step() + if self._step_status == bindings.StepStatus.EndOfStream: + raise StopIteration + + if self._step_status == bindings.StepStatus.OtherError: + print("ERROR: Stream returned an error. Ending the loop") + raise StopIteration + return self + def step_status(self): + """Inspect the stream status. Return adios2.bindings.StepStatus""" + return self._step_status + def set_parameters(self, **kwargs): """ Sets parameters using a dictionary. @@ -137,25 +148,6 @@ def set_parameters(self, **kwargs): """ self._io.set_parameters(**kwargs) - def set_transport(self, transport, parameters={}): - """ - Adds a transport and its parameters to current IO. Must be - supported by current engine type. - - Parameters - type - must be a supported transport type for current engine. - - parameters - acceptable parameters for a particular transport - CAN'T use the keywords "Transport" or "transport" in key - - Returns - transport_index - handler to added transport - """ - self._io.add_transport(transport, parameters) - def available_variables(self): """ @@ -210,7 +202,28 @@ def inquire_variable(self, name): """ return self._io.inquire_variable(name) - def write(self, name, content, shape=[], start=[], count=[], operations=None): + @singledispatchmethod + def write(self, variable: Variable, content): + """ + writes a variable + + Parameters + variable + adios2.Variable object to be written + Use variable.set_selection(), set_shape(), add_operation_string() + to prepare a write + + content + variable data values + """ + if isinstance(content, list): + content_np = np.array(content) + self._engine.put(variable, content_np, bindings.Mode.Sync) + else: + self._engine.put(variable, content, bindings.Mode.Sync) + + @write.register(str) + def _(self, name, content, shape=[], start=[], count=[], operations=None): """ writes a variable @@ -259,13 +272,59 @@ def write(self, name, content, shape=[], start=[], count=[], operations=None): for operation in operations: variable.add_operation_string(operation[0], operation[1]) - if isinstance(content, list): - content_np = np.array(content) - self._engine.put(variable, content_np, bindings.Mode.Sync) + self.write(variable, content) + + @singledispatchmethod + def read(self, variable: Variable): + """ + Random access read allowed to select steps, + only valid with Stream Engines + + Parameters + variable + adios2.Variable object to be read + Use variable.set_selection(), set_block_selection(), set_step_selection() + to prepare a read + Returns + array + resulting array from selection + """ + dtype = type_adios_to_numpy(variable.type()) + count = variable.count() + if count != []: + # array + # steps = variable.get_steps_from_step_selection() + # if steps == 0: + # steps = 1 + # Missing from C++ API: get the steps set for a variable. + # Calculate the steps from size of count and total size that + # we can get from the C++ API. + size_per_step = np.prod(count) + size_all_steps = variable.selection_size() + steps = int(size_all_steps / size_per_step) + if size_all_steps % size_per_step != 0: + print( + f"Stream read(), step calculation for array went horribly wrong " + f" variable name = {variable.name()}" + f" selection size = {size_all_steps} size per step = {size_per_step}" + ) + + output_shape = np.array(count) + output_shape[0] *= steps + print( + f"Stream.read variable {variable.name()} dtype = {dtype} " + f"shape = {output_shape}, steps = {variable.steps()}" + ) else: - self._engine.put(variable, content, bindings.Mode.Sync) + # scalar + output_shape = (variable.selection_size(),) - def read(self, name, start=[], count=[], block_id=None, step_selection=None): + output = np.zeros(output_shape, dtype=dtype) + self._engine.get(variable, output) + return output + + @read.register(str) + def _(self, name: str, start=[], count=[], block_id=None, step_selection=None): """ Random access read allowed to select steps, only valid with Stream Engines @@ -294,10 +353,11 @@ def read(self, name, start=[], count=[], block_id=None, step_selection=None): if not variable: raise ValueError() - if step_selection and not self.mode == bindings.Mode.ReadRandomAccess: + if step_selection and not self._mode == bindings.Mode.ReadRandomAccess: raise RuntimeError("step_selection parameter requires 'rra' mode") if step_selection: + print(f"Stream.read step selection = {step_selection}") variable.set_step_selection(step_selection) if block_id: @@ -309,18 +369,7 @@ def read(self, name, start=[], count=[], block_id=None, step_selection=None): if start != [] and count != []: variable.set_selection([start, count]) - output_shape = (variable.selection_size(),) - if count != []: - if step_selection: - output_shape = np.array(count) * step_selection[1] - else: - output_shape = count - - dtype = type_adios_to_numpy(variable.type()) - - output = np.zeros(output_shape, dtype=dtype) - self._engine.get(variable, output) - return output + return self.read(variable) def write_attribute(self, name, content, variable_name="", separator="/"): """ @@ -410,8 +459,9 @@ def begin_step(self): in file based engines) """ - if not self.engine.between_step_pairs(): - self.engine.begin_step() + if not self._engine.between_step_pairs(): + return self._engine.begin_step() + return bindings.StepStatus.OtherError def end_step(self): """ @@ -439,13 +489,30 @@ def close(self): def current_step(self): """ - Inspect current step when using for-in loops, read mode only + Inspect current step of the stream. The steps run from 0. + + Note that in a real stream, steps from the producer may be missed if + the consumer is slow and the producer is told to discard steps + when no one is reading them in time. You may see non-consecutive + numbers from this function call in this case. + + Use loop_index() to get a loop counter in a for ... .steps() loop. Returns current step """ return self._engine.current_step() + def loop_index(self): + """ + Inspect the loop counter when using for-in loops. This function returns + consecutive numbers from 0. + + Returns + the loop counter + """ + return self.index + def steps(self, num_steps=0): """ Returns an interator that can be use to itererate throught the steps. @@ -454,23 +521,43 @@ def steps(self, num_steps=0): Write Mode: num_steps is a mandatory argument and should specify the number of steps. - Read Mode: num_steps should not be used and there will be as much iterations - as steps exits. + Read Mode: if num_steps is not specified there will be as much iterations + as provided by the actual engine. If num_steps is given and there is not + that many steps in a file/stream, an error will occur. IMPORTANT NOTE: Do not use with ReadRandomAccess mode. """ + if not self._read_mode and num_steps == 0: + raise RuntimeError("Stream.steps() in write mode requires num_steps") + + if self._mode == bindings.Mode.ReadRandomAccess: + raise RuntimeError("Stream.steps() is not allowed in ReadRandomAccess mode") + + if self._mode == bindings.Mode.Read and not self.index < 0: + raise RuntimeError( + "Stream.steps() can only be called once in Read mode." + " Close the stream and reopen to run another iteration loop." + ) + if num_steps > 0: self.max_steps = num_steps else: - self.max_steps = self._engine.steps() + self.max_steps = maxsize # engine steps will limit the loop + + # in write mode we can run yet another loop + self.index = -1 - self.index = 0 return self def num_steps(self): - """READ MODE ONLY. Return the number of steps available.""" - if self.mode not in (bindings.Mode.ReadRandomAccess, bindings.Mode.Read): - raise RuntimeError("num_steps requires Read/ReadRandomOnly mode") + """ + READ MODE ONLY. Return the number of steps available. + Note that this is the steps of a file/stream. Each variable has + its own steps, which needs to inspected with var=stream.inquire_variable() and then + with var.steps() + """ + if not self._read_mode: + raise RuntimeError("num_steps requires Read/ReadRandomAccess mode") return self._engine.steps() @@ -485,10 +572,10 @@ def all_blocks_info(self, name): Returns: list of dictionaries with information of each step """ - if self.mode not in (bindings.Mode.ReadRandomAccess, bindings.Mode.Read): - raise RuntimeError("all_blocks_info requires Read/ReadRandomOnly mode") + if not self._read_mode: + raise RuntimeError("all_blocks_info requires Read/ReadRandomAccess mode") - if self.mode == bindings.Mode.Read: + if self._mode == bindings.Mode.Read: self.begin_step() - return self.engine.all_blocks_info(name) + return self._engine.all_blocks_info(name) diff --git a/python/adios2/variable.py b/python/adios2/variable.py index 18ccaaf3c3..865a550536 100644 --- a/python/adios2/variable.py +++ b/python/adios2/variable.py @@ -88,16 +88,18 @@ def set_step_selection(self, step_selection): """ self.impl.SetStepSelection(step_selection) - def shape(self, step=0): + def shape(self, step=None): """ Get the shape assigned to the given step for this variable. Args: - step (int): Desired step + step (int): Desired step. Only in ReadRandomAccess mode Returns: list: shape of the specified step in the form of [start, count]. """ + if step is None: + return self.impl.Shape() return self.impl.Shape(step) def shape_id(self): @@ -147,10 +149,12 @@ def start(self): def steps(self): """ - The current selected steps of the variable. + The number of steps of the variable. This is always 1 in a stream. + In ReadRandomAccess mode, this function returns the total number + of steps available, which can be used when selecting steps for read. Returns: - int: The current selected steps of the variable. + int: The avaialble steps of the variable. """ return self.impl.Steps() diff --git a/testing/adios2/python/TestADIOS.py b/testing/adios2/python/TestADIOS.py index 6450868bdf..489cd74ec6 100644 --- a/testing/adios2/python/TestADIOS.py +++ b/testing/adios2/python/TestADIOS.py @@ -19,17 +19,17 @@ def test_inquiry_operator(self): def test_declare_io(self): adios = Adios() - writer = adios.declare_io("BPWriter") - self.assertNotEqual(writer, None) + io = adios.declare_io("BPWriter") + self.assertNotEqual(io, None) def test_at_io(self): adios = Adios() - writer = adios.declare_io("BPWriter") + io = adios.declare_io("BPWriter") reader = adios.declare_io("BPReader") x = adios.at_io("BPReader") - self.assertNotEqual(writer, reader) + self.assertNotEqual(io, reader) self.assertEqual(reader, x) - self.assertNotEqual(writer, x) + self.assertNotEqual(io, x) def test_remove_io(self): adios = Adios() diff --git a/testing/adios2/python/TestAttribute.py b/testing/adios2/python/TestAttribute.py index cec72f321b..24fd79ee0d 100644 --- a/testing/adios2/python/TestAttribute.py +++ b/testing/adios2/python/TestAttribute.py @@ -9,23 +9,23 @@ class Test_attribute(unittest.TestCase): def test_create_write(self): adios = Adios() - with adios.declare_io("BPWriter") as writer: - ts = writer.define_attribute("timestamp", "20231122") + with adios.declare_io("BPWriter") as io: + ts = io.define_attribute("timestamp", "20231122") self.assertEqual(ts.name(), "timestamp") self.assertEqual(ts.data_string(), ["20231122"]) def test_create_reader(self): adios = Adios() - with adios.declare_io("BPWriter") as writer: - ts = writer.define_attribute("timestamp", "20231122") + with adios.declare_io("BPWriter") as io: + ts = io.define_attribute("timestamp", "20231122") self.assertEqual(ts.name(), "timestamp") self.assertEqual(ts.data_string(), ["20231122"]) def test_create_write_ndarray(self): adios = Adios() - with adios.declare_io("BPWriter") as writer: + with adios.declare_io("BPWriter") as io: arr = np.array([2023, 11, 22]) - ts = writer.define_attribute("timestamp", arr) + ts = io.define_attribute("timestamp", arr) self.assertEqual(ts.name(), "timestamp") self.assertTrue(np.array_equal(ts.data(), [2023, 11, 22])) diff --git a/testing/adios2/python/TestEngine.py b/testing/adios2/python/TestEngine.py index cd77951303..86ea86c670 100644 --- a/testing/adios2/python/TestEngine.py +++ b/testing/adios2/python/TestEngine.py @@ -9,32 +9,32 @@ class TestEngine(unittest.TestCase): def test_close(self): adios = Adios() - writer = adios.declare_io("BPWriter") - with writer.open("pythontestengine.bp", bindings.Mode.Write): + io = adios.declare_io("BPWriter") + with io.open("pythontestengine.bp", bindings.Mode.Write): pass def test_put(self): adios = Adios() - with adios.declare_io("BPWriter") as writer: - pressure = writer.define_variable("pressure") - temps = writer.define_variable("temps", np.empty([4], dtype=np.int64)) - with writer.open("pythontestengine.bp", bindings.Mode.Write) as engine: + with adios.declare_io("BPWriter") as io: + pressure = io.define_variable("pressure") + temps = io.define_variable("temps", np.empty([4], dtype=np.int64)) + with io.open("pythontestengine.bp", bindings.Mode.Write) as engine: engine.put(pressure, "35PSI") temps_measures = np.array([35, 40, 30, 45], dtype=np.int64) engine.put(temps, temps_measures) def test_get(self): adios = Adios() - with adios.declare_io("BPWriter") as writer: - pressure = writer.define_variable("pressure") - temps = writer.define_variable( + with adios.declare_io("BPWriter") as io: + pressure = io.define_variable("pressure") + temps = io.define_variable( name="temps", content=np.empty([4], dtype=np.int64), start=[0], shape=[4], count=[4], ) - with writer.open("pythontestengine.bp", bindings.Mode.Write) as engine: + with io.open("pythontestengine.bp", bindings.Mode.Write) as engine: engine.put(pressure, "35PSI") temps_measures = np.array([35, 40, 30, 45], dtype=np.int64) engine.put(temps, temps_measures) @@ -59,9 +59,9 @@ def test_get(self): def test_steps(self): adios = Adios() - with adios.declare_io("BPWriter") as writer: - pressure = writer.define_variable("pressure") - with writer.open("pythontestengine.bp", bindings.Mode.Write) as engine: + with adios.declare_io("BPWriter") as io: + pressure = io.define_variable("pressure") + with io.open("pythontestengine.bp", bindings.Mode.Write) as engine: for step in range(0, 10): engine.begin_step() self.assertTrue(engine.between_step_pairs()) @@ -80,9 +80,9 @@ def test_steps(self): def test_blockinfo(self): adios = Adios() - with adios.declare_io("BPWriter") as writer: - temps = writer.define_variable("temps", np.empty([4], dtype=np.int64)) - with writer.open("pythontestengine.bp", bindings.Mode.Write) as engine: + with adios.declare_io("BPWriter") as io: + temps = io.define_variable("temps", np.empty([4], dtype=np.int64)) + with io.open("pythontestengine.bp", bindings.Mode.Write) as engine: temps_measures = np.array([35, 40, 30, 45], dtype=np.int64) engine.put(temps, temps_measures) diff --git a/testing/adios2/python/TestIO.py b/testing/adios2/python/TestIO.py index 8b7d1ccf3f..6da95c47f1 100644 --- a/testing/adios2/python/TestIO.py +++ b/testing/adios2/python/TestIO.py @@ -11,90 +11,90 @@ def test_io_empty(self): def test_io_define_attribute(self): adios = Adios() - writer = adios.declare_io("BPWriter") - ts = writer.define_attribute("timestamp", "20231122") + io = adios.declare_io("BPWriter") + ts = io.define_attribute("timestamp", "20231122") self.assertIsNot(ts, None) def test_io_inquire_attribute(self): adios = Adios() - writer = adios.declare_io("BPWriter") - ts = writer.define_attribute("timestamp", "20231122") - coords = writer.define_attribute("coords", "43N74W") - x = writer.inquire_attribute("coords") + io = adios.declare_io("BPWriter") + ts = io.define_attribute("timestamp", "20231122") + coords = io.define_attribute("coords", "43N74W") + x = io.inquire_attribute("coords") self.assertNotEqual(ts, coords) self.assertNotEqual(ts, x) self.assertEqual(coords, x) def test_available_attribute(self): adios = Adios() - writer = adios.declare_io("BPWriter") - writer.define_attribute("timestamp", "20231122") - writer.inquire_attribute("timestamp") - self.assertIs(writer.inquire_attribute("coords"), None) + io = adios.declare_io("BPWriter") + io.define_attribute("timestamp", "20231122") + io.inquire_attribute("timestamp") + self.assertIs(io.inquire_attribute("coords"), None) def test_remove_attribute(self): adios = Adios() - writer = adios.declare_io("BPWriter") - writer.define_attribute("timestamp", "20231122") - writer.remove_attribute("timestamp") - self.assertIs(writer.inquire_attribute("timestamp"), None) + io = adios.declare_io("BPWriter") + io.define_attribute("timestamp", "20231122") + io.remove_attribute("timestamp") + self.assertIs(io.inquire_attribute("timestamp"), None) def test_remove_all_attribute(self): adios = Adios() - writer = adios.declare_io("BPWriter") - writer.define_attribute("timestamp", "20231122") - writer.define_attribute("coords", "43N74W") - writer.remove_all_attributes() - self.assertIs(writer.inquire_attribute("timestamp"), None) - self.assertIs(writer.inquire_attribute("coords"), None) + io = adios.declare_io("BPWriter") + io.define_attribute("timestamp", "20231122") + io.define_attribute("coords", "43N74W") + io.remove_all_attributes() + self.assertIs(io.inquire_attribute("timestamp"), None) + self.assertIs(io.inquire_attribute("coords"), None) def test_io_define_variable(self): adios = Adios() - writer = adios.declare_io("BPWriter") - temp = writer.define_variable("temp") + io = adios.declare_io("BPWriter") + temp = io.define_variable("temp") self.assertNotEqual(temp, None) def test_io_inquire_variable(self): adios = Adios() - writer = adios.declare_io("BPWriter") - temp = writer.define_variable("temp") - presure = writer.define_variable("pressure") - x = writer.inquire_variable("pressure") + io = adios.declare_io("BPWriter") + temp = io.define_variable("temp") + presure = io.define_variable("pressure") + x = io.inquire_variable("pressure") self.assertNotEqual(temp, presure) self.assertNotEqual(temp, x) self.assertEqual(presure, x) def test_available_variable(self): adios = Adios() - writer = adios.declare_io("BPWriter") - writer.define_variable("temp") - writer.inquire_variable("temp") - self.assertIs(writer.inquire_attribute("pressure"), None) + io = adios.declare_io("BPWriter") + io.define_variable("temp") + io.inquire_variable("temp") + self.assertIs(io.inquire_attribute("pressure"), None) def test_remove_variable(self): adios = Adios() - writer = adios.declare_io("BPWriter") - writer.define_variable("temp") - writer.remove_variable("temp") - self.assertIs(writer.inquire_attribute("temp"), None) + io = adios.declare_io("BPWriter") + io.define_variable("temp") + io.remove_variable("temp") + self.assertIs(io.inquire_attribute("temp"), None) def test_remove_all_variable(self): adios = Adios() - writer = adios.declare_io("BPWriter") - writer.define_variable("temp") - writer.define_variable("pressure") - writer.remove_all_variables() - self.assertIs(writer.inquire_attribute("pressure"), None) - self.assertIs(writer.inquire_attribute("temp"), None) + io = adios.declare_io("BPWriter") + io.define_variable("temp") + io.define_variable("pressure") + io.remove_all_variables() + self.assertIs(io.inquire_attribute("pressure"), None) + self.assertIs(io.inquire_attribute("temp"), None) def test_open_engine(self): adios = Adios() - writer = adios.declare_io("BPWriter") - writer.set_engine("BPFile") - writer.set_parameter("threads", "2") - writer.set_parameters({"AsyncOpen": "On", "MaxOpenFilesAtOnce": "512"}) - writer.add_transport("File", {"Library": "POSIX"}) - engine = writer.open("pythontest.bp", bindings.Mode.Write) + io = adios.declare_io("BPWriter") + io.set_engine("BPFile") + io.set_parameter("threads", "2") + io.set_parameters({"AsyncOpen": "On", "MaxOpenFilesAtOnce": "512"}) + io.add_transport("File", {"Library": "POSIX"}) + engine = io.open("pythontest.bp", bindings.Mode.Write) self.assertNotEqual(engine, None) diff --git a/testing/adios2/python/TestOperator.py b/testing/adios2/python/TestOperator.py index c37467538d..7237c45833 100644 --- a/testing/adios2/python/TestOperator.py +++ b/testing/adios2/python/TestOperator.py @@ -10,10 +10,10 @@ class TestOperator(unittest.TestCase): def test_operator_basic(self): adios = Adios() op1 = adios.define_operator("noop", "null") - with adios.declare_io("BPWriter") as writer: - temps = writer.define_variable("temps", np.empty([4], dtype=np.int64)) + with adios.declare_io("BPWriter") as io: + temps = io.define_variable("temps", np.empty([4], dtype=np.int64)) temps.add_operation(op1) - with writer.open("pythontestvariable.bp", bindings.Mode.Write) as engine: + with io.open("pythontestvariable.bp", bindings.Mode.Write) as engine: temps_measures = np.array([35, 40, 30, 45], dtype=np.int64) engine.put(temps, temps_measures) diff --git a/testing/adios2/python/TestVariable.py b/testing/adios2/python/TestVariable.py index 098ca636d0..9a8dccd7eb 100644 --- a/testing/adios2/python/TestVariable.py +++ b/testing/adios2/python/TestVariable.py @@ -9,9 +9,9 @@ class TestVariable(unittest.TestCase): def test_create_write(self): adios = Adios() - with adios.declare_io("BPWriter") as writer: - temps = writer.define_variable("temps", np.empty([4], dtype=np.int64)) - with writer.open("pythontestvariable.bp", bindings.Mode.Write) as engine: + with adios.declare_io("BPWriter") as io: + temps = io.define_variable("temps", np.empty([4], dtype=np.int64)) + with io.open("pythontestvariable.bp", bindings.Mode.Write) as engine: temps_measures = np.array([35, 40, 30, 45], dtype=np.int64) engine.put(temps, temps_measures) self.assertEqual(temps.name(), "temps") @@ -22,9 +22,9 @@ def test_create_write(self): def test_create_reader(self): adios = Adios() - with adios.declare_io("BPWriter") as writer: - temps = writer.define_variable("temps", np.empty([4], dtype=np.int64)) - with writer.open("pythontestvariable.bp", bindings.Mode.Write) as engine: + with adios.declare_io("BPWriter") as io: + temps = io.define_variable("temps", np.empty([4], dtype=np.int64)) + with io.open("pythontestvariable.bp", bindings.Mode.Write) as engine: temps_measures = np.array([35, 40, 30, 45], dtype=np.int64) engine.put(temps, temps_measures) @@ -44,10 +44,10 @@ def test_create_reader(self): def test_operators(self): adios = Adios() op1 = adios.define_operator("noop", "null") - with adios.declare_io("BPWriter") as writer: - temps = writer.define_variable("temps", np.empty([4], dtype=np.int64)) + with adios.declare_io("BPWriter") as io: + temps = io.define_variable("temps", np.empty([4], dtype=np.int64)) temps.add_operation(op1) - with writer.open("pythontestvariable.bp", bindings.Mode.Write) as engine: + with io.open("pythontestvariable.bp", bindings.Mode.Write) as engine: temps_measures = np.array([35, 40, 30, 45], dtype=np.int64) engine.put(temps, temps_measures) From feeb100c85179b7c7af0f7cc7dbed916eaaa4a19 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Fri, 26 Jan 2024 14:31:56 -0500 Subject: [PATCH 027/124] format --- python/adios2/stream.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/adios2/stream.py b/python/adios2/stream.py index abfbca3944..5e34f04c93 100644 --- a/python/adios2/stream.py +++ b/python/adios2/stream.py @@ -552,7 +552,7 @@ def steps(self, num_steps=0): def num_steps(self): """ READ MODE ONLY. Return the number of steps available. - Note that this is the steps of a file/stream. Each variable has + Note that this is the steps of a file/stream. Each variable has its own steps, which needs to inspected with var=stream.inquire_variable() and then with var.steps() """ From 11983037537bcfb5e8ef715cc3b076e6b834ae35 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Fri, 26 Jan 2024 15:55:48 -0500 Subject: [PATCH 028/124] turn off python for el8 images. Update dockerfile for el8 to install python39 but actual images is not built. --- scripts/ci/cmake/ci-el8-icc-mpich.cmake | 2 +- scripts/ci/cmake/ci-el8-icc-ompi.cmake | 2 +- scripts/ci/cmake/ci-el8-icc-serial.cmake | 2 +- scripts/ci/cmake/ci-el8-oneapi-mpich.cmake | 2 +- scripts/ci/cmake/ci-el8-oneapi-ompi.cmake | 2 +- scripts/ci/cmake/ci-el8-oneapi-serial.cmake | 2 +- scripts/ci/images/Dockerfile.ci-el8-intel | 4 ++-- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/scripts/ci/cmake/ci-el8-icc-mpich.cmake b/scripts/ci/cmake/ci-el8-icc-mpich.cmake index cf1bf07c66..8f461e4e3b 100644 --- a/scripts/ci/cmake/ci-el8-icc-mpich.cmake +++ b/scripts/ci/cmake/ci-el8-icc-mpich.cmake @@ -17,7 +17,7 @@ ADIOS2_USE_DataSpaces:BOOL=OFF ADIOS2_USE_Fortran:BOOL=OFF ADIOS2_USE_HDF5:BOOL=ON ADIOS2_USE_MPI:BOOL=ON -ADIOS2_USE_Python:BOOL=ON +ADIOS2_USE_Python:BOOL=OFF ADIOS2_USE_SZ:BOOL=ON ADIOS2_USE_ZeroMQ:STRING=ON ADIOS2_USE_ZFP:BOOL=ON diff --git a/scripts/ci/cmake/ci-el8-icc-ompi.cmake b/scripts/ci/cmake/ci-el8-icc-ompi.cmake index 6f8803e44d..785ea2ea3b 100644 --- a/scripts/ci/cmake/ci-el8-icc-ompi.cmake +++ b/scripts/ci/cmake/ci-el8-icc-ompi.cmake @@ -13,7 +13,7 @@ ADIOS2_USE_DataSpaces:BOOL=OFF ADIOS2_USE_Fortran:BOOL=OFF ADIOS2_USE_HDF5:BOOL=ON ADIOS2_USE_MPI:BOOL=ON -ADIOS2_USE_Python:BOOL=ON +ADIOS2_USE_Python:BOOL=OFF ADIOS2_USE_SZ:BOOL=ON ADIOS2_USE_ZeroMQ:STRING=ON ADIOS2_USE_ZFP:BOOL=ON diff --git a/scripts/ci/cmake/ci-el8-icc-serial.cmake b/scripts/ci/cmake/ci-el8-icc-serial.cmake index 17e860765c..a769ffbbef 100644 --- a/scripts/ci/cmake/ci-el8-icc-serial.cmake +++ b/scripts/ci/cmake/ci-el8-icc-serial.cmake @@ -13,7 +13,7 @@ ADIOS2_USE_DataSpaces:BOOL=OFF ADIOS2_USE_Fortran:BOOL=OFF ADIOS2_USE_HDF5:BOOL=ON ADIOS2_USE_MPI:BOOL=OFF -ADIOS2_USE_Python:BOOL=ON +ADIOS2_USE_Python:BOOL=OFF ADIOS2_USE_SZ:BOOL=ON ADIOS2_USE_ZeroMQ:STRING=ON ADIOS2_USE_ZFP:BOOL=ON diff --git a/scripts/ci/cmake/ci-el8-oneapi-mpich.cmake b/scripts/ci/cmake/ci-el8-oneapi-mpich.cmake index 7a837a984a..49ed45643c 100644 --- a/scripts/ci/cmake/ci-el8-oneapi-mpich.cmake +++ b/scripts/ci/cmake/ci-el8-oneapi-mpich.cmake @@ -17,7 +17,7 @@ ADIOS2_USE_DataSpaces:BOOL=OFF ADIOS2_USE_Fortran:BOOL=OFF ADIOS2_USE_HDF5:BOOL=ON ADIOS2_USE_MPI:BOOL=ON -ADIOS2_USE_Python:BOOL=ON +ADIOS2_USE_Python:BOOL=OFF ADIOS2_USE_SZ:BOOL=ON ADIOS2_USE_ZeroMQ:STRING=ON ADIOS2_USE_ZFP:BOOL=ON diff --git a/scripts/ci/cmake/ci-el8-oneapi-ompi.cmake b/scripts/ci/cmake/ci-el8-oneapi-ompi.cmake index 8a1c5e5fa9..bf9ffa5f74 100644 --- a/scripts/ci/cmake/ci-el8-oneapi-ompi.cmake +++ b/scripts/ci/cmake/ci-el8-oneapi-ompi.cmake @@ -13,7 +13,7 @@ ADIOS2_USE_DataSpaces:BOOL=OFF ADIOS2_USE_Fortran:BOOL=OFF ADIOS2_USE_HDF5:BOOL=ON ADIOS2_USE_MPI:BOOL=ON -ADIOS2_USE_Python:BOOL=ON +ADIOS2_USE_Python:BOOL=OFF ADIOS2_USE_SZ:BOOL=ON ADIOS2_USE_ZeroMQ:STRING=ON ADIOS2_USE_ZFP:BOOL=ON diff --git a/scripts/ci/cmake/ci-el8-oneapi-serial.cmake b/scripts/ci/cmake/ci-el8-oneapi-serial.cmake index 8e971a994d..0b7a95acec 100644 --- a/scripts/ci/cmake/ci-el8-oneapi-serial.cmake +++ b/scripts/ci/cmake/ci-el8-oneapi-serial.cmake @@ -13,7 +13,7 @@ ADIOS2_USE_DataSpaces:BOOL=OFF ADIOS2_USE_Fortran:BOOL=OFF ADIOS2_USE_HDF5:BOOL=ON ADIOS2_USE_MPI:BOOL=OFF -ADIOS2_USE_Python:BOOL=ON +ADIOS2_USE_Python:BOOL=OFF ADIOS2_USE_SZ:BOOL=ON ADIOS2_USE_ZeroMQ:STRING=ON ADIOS2_USE_ZFP:BOOL=ON diff --git a/scripts/ci/images/Dockerfile.ci-el8-intel b/scripts/ci/images/Dockerfile.ci-el8-intel index 4470f3a703..c68f6ff517 100644 --- a/scripts/ci/images/Dockerfile.ci-el8-intel +++ b/scripts/ci/images/Dockerfile.ci-el8-intel @@ -26,8 +26,7 @@ RUN dnf install -y \ mpich-devel \ patch \ patchelf \ - python3-devel \ - python3-mpi4py-mpich \ + python39-devel \ python3-pip \ tar \ tcl \ @@ -41,6 +40,7 @@ RUN dnf install -y \ # Install the compilers from an external repo COPY oneAPI.repo /etc/yum.repos.d/ RUN pip3 install numpy && \ + pip3 install mpi4py-mpich && \ dnf install -y \ intel-oneapi-compiler-dpcpp-cpp-2023.2.1 \ intel-oneapi-compiler-fortran-2023.2.1 \ From 7a632687cdd473f13e185b5959df3860c77dafe2 Mon Sep 17 00:00:00 2001 From: Greg Eisenhauer Date: Mon, 29 Jan 2024 10:12:11 -0500 Subject: [PATCH 029/124] ffs 2023-10-27 (e8989c26) (#4016) Code extracted from: https://github.com/GTkorvo/ffs.git at commit e8989c262d380ab305e99bd4056073729b7d304d (master). Upstream Shortlog ----------------- Co-authored-by: ffs Upstream --- thirdparty/ffs/ffs/fm/fm_formats.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/thirdparty/ffs/ffs/fm/fm_formats.c b/thirdparty/ffs/ffs/fm/fm_formats.c index 83414a45a5..83ff1604cf 100755 --- a/thirdparty/ffs/ffs/fm/fm_formats.c +++ b/thirdparty/ffs/ffs/fm/fm_formats.c @@ -1982,7 +1982,7 @@ generate_format3_server_ID(server_ID_type *server_ID, ((version_3_format_ID *) server_ID->value)->rep_len = htons((short)(server_format_rep_length >> 2)); // Mod length by 4 ((version_3_format_ID *) server_ID->value)->top_byte_rep_len = (unsigned char) - htons((short)(server_format_rep_length >> 18)); // Essentially, we capture the top 26 bytes of the server length + 0xff & (server_format_rep_length >> 18); ((version_3_format_ID *) server_ID->value)->hash1 = htonl(hash1); ((version_3_format_ID *) server_ID->value)->hash2 = htonl(hash2); } @@ -3953,11 +3953,12 @@ stringify_server_ID(unsigned char *ID, char *buffer, int len) break; } case 2:{ - version_2_format_ID *id2 = (version_2_format_ID*)ID; + version_3_format_ID *id3 = (version_3_format_ID*)ID; if (len < 3+3+6+10+6+50) /* approx size */ return; - snprintf(buffer, len, "\n", - id2->version, id2->unused, ntohs(id2->rep_len) << 2, - ntohl(id2->hash1), ntohl(id2->hash2)); + int rep_len = get_rep_len_format_ID(ID); + snprintf(buffer, len, "\n", + id3->version, rep_len, + ntohl(id3->hash1), ntohl(id3->hash2)); break; } default: @@ -4021,11 +4022,10 @@ get_rep_len_format_ID(void *format_ID) { switch (version_of_format_ID(format_ID)) { case 2:{ - version_2_format_ID *id2 = (version_2_format_ID *) format_ID; - short tmp; - memcpy(&tmp, &id2->rep_len, 2); - tmp = ntohs(tmp); - return tmp << 2; + version_3_format_ID *id3 = (version_3_format_ID *) format_ID; + int rep_len = htons(id3->rep_len); + rep_len += (id3->top_byte_rep_len << 16); + return rep_len << 2; } case 0: case 1: From c1367bdfd7b18e2eded09365c5c76ab6f2cb2aed Mon Sep 17 00:00:00 2001 From: Greg Eisenhauer Date: Tue, 30 Jan 2024 15:35:56 -0500 Subject: [PATCH 030/124] WIP: Posix transport should fail if you try to read past EOF (#3997) * Add test for local operator, tweak posix transport to fail on EOF when desired, add a unit test * Disable posix test on Windows --- source/adios2/engine/bp5/BP5Reader.cpp | 11 ++ source/adios2/engine/bp5/BP5Reader.h | 2 + .../toolkit/transport/file/FilePOSIX.cpp | 37 ++++- .../adios2/toolkit/transport/file/FilePOSIX.h | 5 +- .../toolkit/transportman/TransportMan.cpp | 22 ++- .../toolkit/transportman/TransportMan.h | 9 +- .../bp/operations/TestBPWriteReadBZIP2.cpp | 129 ++++++++++++++++++ testing/adios2/unit/CMakeLists.txt | 3 + testing/adios2/unit/TestPosixTransport.cpp | 101 ++++++++++++++ 9 files changed, 309 insertions(+), 10 deletions(-) create mode 100644 testing/adios2/unit/TestPosixTransport.cpp diff --git a/source/adios2/engine/bp5/BP5Reader.cpp b/source/adios2/engine/bp5/BP5Reader.cpp index 00209ff1f4..cb0b9aa570 100644 --- a/source/adios2/engine/bp5/BP5Reader.cpp +++ b/source/adios2/engine/bp5/BP5Reader.cpp @@ -225,6 +225,12 @@ std::pair BP5Reader::ReadData(adios2::transportman::TransportMan } FileManager.OpenFileID(subFileName, SubfileNum, Mode::Read, m_IO.m_TransportsParameters[0], /*{{"transport", "File"}},*/ false); + if (!m_WriterIsActive) + { + Params transportParameters; + transportParameters["FailOnEOF"] = "true"; + FileManager.SetParameters(transportParameters, -1); + } } TP endSubfile = NOW(); double timeSubfile = DURATION(startSubfile, endSubfile); @@ -300,6 +306,11 @@ void BP5Reader::PerformLocalGets() m_WriterMap[m_WriterMapIndex[r2.Timestep]].RankToSubfile[r2.WriterRank]); }; + if (!m_InitialWriterActiveCheckDone) + { + CheckWriterActive(); + m_InitialWriterActiveCheckDone = true; + } // TP start = NOW(); PERFSTUBS_SCOPED_TIMER("BP5Reader::PerformGets"); m_JSONProfiler.Start("DataRead"); diff --git a/source/adios2/engine/bp5/BP5Reader.h b/source/adios2/engine/bp5/BP5Reader.h index 3c45b1db2c..cc73944c2f 100644 --- a/source/adios2/engine/bp5/BP5Reader.h +++ b/source/adios2/engine/bp5/BP5Reader.h @@ -113,6 +113,8 @@ class BP5Reader : public BP5Engine, public Engine Minifooter m_Minifooter; + bool m_InitialWriterActiveCheckDone = false; + void Init(); void InitParameters(); void InitTransports(); diff --git a/source/adios2/toolkit/transport/file/FilePOSIX.cpp b/source/adios2/toolkit/transport/file/FilePOSIX.cpp index 20f98cb179..fedb934bf3 100644 --- a/source/adios2/toolkit/transport/file/FilePOSIX.cpp +++ b/source/adios2/toolkit/transport/file/FilePOSIX.cpp @@ -2,13 +2,12 @@ * Distributed under the OSI-approved Apache License, Version 2.0. See * accompanying file Copyright.txt for details. * - * FileDescriptor.cpp file I/O using POSIX I/O library + * FilePOSIX.cpp file I/O using POSIX I/O library * - * Created on: Oct 6, 2016 - * Author: William F Godoy godoywf@ornl.gov */ #include "FilePOSIX.h" #include "adios2/helper/adiosLog.h" +#include "adios2/helper/adiosString.h" #ifdef ADIOS2_HAVE_O_DIRECT #ifndef _GNU_SOURCE @@ -22,7 +21,8 @@ #include // open #include // open, fstat #include // open -#include // write, close, ftruncate +#include +#include // write, close, ftruncate /// \cond EXCLUDE_FROM_DOXYGEN #include //std::ios_base::failure @@ -398,6 +398,7 @@ void FilePOSIX::WriteV(const core::iovec *iov, const int iovcnt, size_t start) void FilePOSIX::Read(char *buffer, size_t size, size_t start) { auto lf_Read = [&](char *buffer, size_t size) { + size_t backoff_ns = 20; while (size > 0) { ProfilerStart("read"); @@ -417,6 +418,25 @@ void FilePOSIX::Read(char *buffer, size_t size, size_t start) "Toolkit", "transport::file::FilePOSIX", "Read", "couldn't read from file " + m_Name + " " + SysErrMsg()); } + else if (readSize == 0) + { + if (m_FailOnEOF) + { + helper::Throw( + "Toolkit", "transport::file::FilePOSIX", "Read", + "Read past end of file on " + m_Name + " " + SysErrMsg()); + } + else + { + // read past EOF, but we're to wait for data. Exponential backoff with a limit + // of .5 sec (500,000,000 nanosec) + std::this_thread::sleep_for(std::chrono::nanoseconds(backoff_ns)); + constexpr size_t backoff_limit = 500 * 1000 * 1000; + backoff_ns *= 2; + if (backoff_ns > backoff_limit) + backoff_ns = backoff_limit; + } + } buffer += readSize; size -= readSize; @@ -595,5 +615,14 @@ void FilePOSIX::Truncate(const size_t length) void FilePOSIX::MkDir(const std::string &fileName) {} +void FilePOSIX::SetParameters(const Params ¶ms) +{ + // Parameters are set from config parameters if present + // Otherwise, they are set from environment if present + // Otherwise, they remain at their default value + + helper::GetParameter(params, "FailOnEOF", m_FailOnEOF); +} + } // end namespace transport } // end namespace adios2 diff --git a/source/adios2/toolkit/transport/file/FilePOSIX.h b/source/adios2/toolkit/transport/file/FilePOSIX.h index 394aa9ea15..667d63cd0a 100644 --- a/source/adios2/toolkit/transport/file/FilePOSIX.h +++ b/source/adios2/toolkit/transport/file/FilePOSIX.h @@ -4,8 +4,6 @@ * * FileDescriptor.h wrapper of POSIX library functions for file I/O * - * Created on: Oct 6, 2016 - * Author: William F Godoy godoywf@ornl.gov */ #ifndef ADIOS2_TOOLKIT_TRANSPORT_FILE_FILEDESCRIPTOR_H_ @@ -68,10 +66,13 @@ class FilePOSIX : public Transport void MkDir(const std::string &fileName) final; + void SetParameters(const Params ¶ms) final; + private: /** POSIX file handle returned by Open */ int m_FileDescriptor = -1; int m_Errno = 0; + bool m_FailOnEOF = false; // default to false for historic reasons bool m_IsOpening = false; std::future m_OpenFuture; bool m_DirectIO = false; diff --git a/source/adios2/toolkit/transportman/TransportMan.cpp b/source/adios2/toolkit/transportman/TransportMan.cpp index bb7ff8666d..2e35e73f8b 100644 --- a/source/adios2/toolkit/transportman/TransportMan.cpp +++ b/source/adios2/toolkit/transportman/TransportMan.cpp @@ -4,8 +4,6 @@ * * TransportMan.cpp * - * Created on: May 23, 2017 - * Author: William F Godoy godoywf@ornl.gov */ #include "TransportMan.h" @@ -410,6 +408,26 @@ void TransportMan::ReadFile(char *buffer, const size_t size, const size_t start, itTransport->second->Read(buffer, size, start); } +void TransportMan::SetParameters(const Params ¶ms, const int transportIndex) +{ + if (transportIndex == -1) + { + for (auto &transportPair : m_Transports) + { + auto &transport = transportPair.second; + + transport->SetParameters(params); + } + } + else + { + auto itTransport = m_Transports.find(transportIndex); + CheckFile(itTransport, + ", in call to SetParameters with index " + std::to_string(transportIndex)); + itTransport->second->SetParameters(params); + } +} + void TransportMan::FlushFiles(const int transportIndex) { if (transportIndex == -1) diff --git a/source/adios2/toolkit/transportman/TransportMan.h b/source/adios2/toolkit/transportman/TransportMan.h index acf3ac1cd2..adc1d46d13 100644 --- a/source/adios2/toolkit/transportman/TransportMan.h +++ b/source/adios2/toolkit/transportman/TransportMan.h @@ -4,8 +4,6 @@ * * TransportMan.h : manages a vector of transports * - * Created on: May 23, 2017 - * Author: William F Godoy godoywf@ornl.gov */ #ifndef ADIOS2_TOOLKIT_TRANSPORT_TRANSPORTMANAGER_H_ @@ -210,6 +208,13 @@ class TransportMan */ bool FileExists(const std::string &name, const Params ¶meters, const bool profile); + /** + * Set Transport Paramers + * @param params + * @param transportIndex + */ + void SetParameters(const Params ¶ms, const int transportIndex = -1); + protected: core::IO &m_IO; helper::Comm const &m_Comm; diff --git a/testing/adios2/engine/bp/operations/TestBPWriteReadBZIP2.cpp b/testing/adios2/engine/bp/operations/TestBPWriteReadBZIP2.cpp index 4e1f7f86fb..dcbafcccf0 100644 --- a/testing/adios2/engine/bp/operations/TestBPWriteReadBZIP2.cpp +++ b/testing/adios2/engine/bp/operations/TestBPWriteReadBZIP2.cpp @@ -148,6 +148,134 @@ void BZIP2Accuracy1D(const std::string accuracy) } } +void BZIP2Accuracy1DLocal(const std::string accuracy) +{ + // Each process would write a 1x8 array and all processes would + // write a Nx 1D array + const std::string fname("BPWR_BZIP2_1D_Local_" + accuracy + ".bp"); + + int mpiRank = 0; + // Number of rows + const size_t Nx = 1000; + + // Number of steps + const size_t NSteps = 1; + + std::vector r32s(Nx); + std::vector r64s(Nx); + + // range 0 to 999 + std::iota(r32s.begin(), r32s.end(), 0.f); + std::iota(r64s.begin(), r64s.end(), 0.); + +#if ADIOS2_USE_MPI + MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); +#endif + +#if ADIOS2_USE_MPI + adios2::ADIOS adios(MPI_COMM_WORLD); +#else + adios2::ADIOS adios; +#endif + { + adios2::IO io = adios.DeclareIO("TestIO"); + + if (!engineName.empty()) + { + io.SetEngine(engineName); + } + else + { + // Create the BP Engine + io.SetEngine("BPFile"); + } + + const adios2::Dims shape{}; + const adios2::Dims start{}; + const adios2::Dims count{Nx}; + + adios2::Variable var_r32 = + io.DefineVariable("r32", shape, start, count, adios2::ConstantDims); + adios2::Variable var_r64 = + io.DefineVariable("r64", shape, start, count, adios2::ConstantDims); + + // add operations + adios2::Operator BZIP2Op = + adios.DefineOperator("BZIP2Compressor", adios2::ops::LosslessBZIP2); + + var_r32.AddOperation(BZIP2Op, {{adios2::ops::bzip2::key::blockSize100k, accuracy}}); + var_r64.AddOperation(BZIP2Op, {{adios2::ops::bzip2::key::blockSize100k, accuracy}}); + + adios2::Engine bpWriter = io.Open(fname, adios2::Mode::Write); + + for (size_t step = 0; step < NSteps; ++step) + { + bpWriter.BeginStep(); + bpWriter.Put("r32", r32s.data()); + bpWriter.Put("r64", r64s.data()); + bpWriter.EndStep(); + } + + bpWriter.Close(); + } + + { + adios2::IO io = adios.DeclareIO("ReadIO"); + + if (!engineName.empty()) + { + io.SetEngine(engineName); + } + else + { + // Create the BP Engine + io.SetEngine("BPFile"); + } + + adios2::Engine bpReader = io.Open(fname, adios2::Mode::Read); + + unsigned int t = 0; + std::vector decompressedR32s; + std::vector decompressedR64s; + + while (bpReader.BeginStep() == adios2::StepStatus::OK) + { + auto var_r32 = io.InquireVariable("r32"); + EXPECT_TRUE(var_r32); + ASSERT_EQ(var_r32.ShapeID(), adios2::ShapeID::LocalArray); + ASSERT_EQ(var_r32.Steps(), NSteps); + + auto var_r64 = io.InquireVariable("r64"); + EXPECT_TRUE(var_r64); + ASSERT_EQ(var_r64.ShapeID(), adios2::ShapeID::LocalArray); + ASSERT_EQ(var_r64.Steps(), NSteps); + auto r32_info = bpReader.BlocksInfo(var_r32, -1); + auto r64_info = bpReader.BlocksInfo(var_r64, -1); + + var_r32.SetBlockSelection(mpiRank); + var_r64.SetBlockSelection(mpiRank); + + bpReader.Get(var_r32, decompressedR32s); + bpReader.Get(var_r64, decompressedR64s); + bpReader.EndStep(); + + for (size_t i = 0; i < Nx; ++i) + { + std::stringstream ss; + ss << "t=" << t << " i=" << i << " rank=" << mpiRank; + std::string msg = ss.str(); + ASSERT_EQ(decompressedR32s[i], r32s[i]) << msg; + ASSERT_EQ(decompressedR64s[i], r64s[i]) << msg; + } + ++t; + } + + EXPECT_EQ(t, NSteps); + + bpReader.Close(); + } +} + void BZIP2Accuracy2D(const std::string accuracy) { // Each process would write a 1x8 array and all processes would @@ -830,6 +958,7 @@ class BPWriteReadBZIP2 : public ::testing::TestWithParam }; TEST_P(BPWriteReadBZIP2, ADIOS2BPWriteReadBZIP21D) { BZIP2Accuracy1D(GetParam()); } +TEST_P(BPWriteReadBZIP2, ADIOS2BPWriteReadBZIP21DLocal) { BZIP2Accuracy1DLocal(GetParam()); } TEST_P(BPWriteReadBZIP2, ADIOS2BPWriteReadBZIP22D) { BZIP2Accuracy2D(GetParam()); } TEST_P(BPWriteReadBZIP2, ADIOS2BPWriteReadBZIP23D) { BZIP2Accuracy3D(GetParam()); } TEST_P(BPWriteReadBZIP2, ADIOS2BPWriteReadBZIP21DSel) { BZIP2Accuracy1DSel(GetParam()); } diff --git a/testing/adios2/unit/CMakeLists.txt b/testing/adios2/unit/CMakeLists.txt index de65ea8510..c38e0d300f 100644 --- a/testing/adios2/unit/CMakeLists.txt +++ b/testing/adios2/unit/CMakeLists.txt @@ -4,4 +4,7 @@ #------------------------------------------------------------------------------# gtest_add_tests_helper(ChunkV MPI_NONE "" Unit. "") +if(UNIX) + gtest_add_tests_helper(PosixTransport MPI_NONE "" Unit. "") +endif() diff --git a/testing/adios2/unit/TestPosixTransport.cpp b/testing/adios2/unit/TestPosixTransport.cpp new file mode 100644 index 0000000000..c2d565087e --- /dev/null +++ b/testing/adios2/unit/TestPosixTransport.cpp @@ -0,0 +1,101 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + */ +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "adios2/helper/adiosLog.h" +#include "adios2/helper/adiosString.h" +#include +#include +#include +#include + +#include + +namespace adios2 +{ +namespace format +{ + +TEST(FileTransport, FailOnEOF) +{ + { + std::vector b(256, 0xef); + helper::Comm comm = helper::CommDummy(); + std::unique_ptr w = + std::unique_ptr(new transport::FilePOSIX(comm)); + + w->Open("FailOnEOF", Mode::Write); + w->Write((char *)b.data(), b.size()); + w->Close(); + } + { + std::vector b(256); + helper::Comm comm = helper::CommDummy(); + std::unique_ptr r = + std::unique_ptr(new transport::FilePOSIX(comm)); + + r->Open("FailOnEOF", Mode::Read); + Params p = {{"FailOnEOF", "true"}}; + r->SetParameters(p); + EXPECT_THROW(r->Read((char *)b.data(), b.size() * 2), std::ios_base::failure); + r->Close(); + } +} + +TEST(FileTransport, WaitForData) +{ + constexpr int size = 256; + std::vector b(size, 0xef); + helper::Comm comm = helper::CommDummy(); + std::unique_ptr w = + std::unique_ptr(new transport::FilePOSIX(comm)); + + w->Open("FailOnEOF", Mode::Write); + w->Write((char *)b.data(), b.size()); + { + auto lf_WriteMore = [&](const transport::FilePOSIX *) { + std::vector b2(size, 0xfe); + std::this_thread::sleep_for(std::chrono::seconds(2)); + w->Write((char *)b2.data(), size); + std::cout << "Wrote data" << std::endl; + }; + + // write more data soon + auto h = std::async(std::launch::async, lf_WriteMore, w.get()); + + std::vector b(size * 2); + helper::Comm comm = helper::CommDummy(); + std::unique_ptr r = + std::unique_ptr(new transport::FilePOSIX(comm)); + + r->Open("FailOnEOF", Mode::Read); + r->Read((char *)b.data(), size * 2); + ASSERT_EQ(b[0], 0xef); + ASSERT_EQ(b[size], 0xfe); + r->Close(); + } + w->Close(); +} +} +} + +int main(int argc, char **argv) +{ + + int result; + ::testing::InitGoogleTest(&argc, argv); + result = RUN_ALL_TESTS(); + + return result; +} From 85c2db739fc8a59cf1abe55d858f41a1fe55e622 Mon Sep 17 00:00:00 2001 From: Greg Eisenhauer Date: Thu, 1 Feb 2024 19:39:24 -0500 Subject: [PATCH 031/124] Make POSIX transport work on Windows, default to it (#4023) --- source/adios2/CMakeLists.txt | 5 ++- source/adios2/common/ADIOSTypes.h | 4 -- .../toolkit/transport/file/FilePOSIX.cpp | 37 +++++++++++++------ .../toolkit/transportman/TransportMan.cpp | 4 -- 4 files changed, 29 insertions(+), 21 deletions(-) diff --git a/source/adios2/CMakeLists.txt b/source/adios2/CMakeLists.txt index d02c6eeab4..c63ffa6140 100644 --- a/source/adios2/CMakeLists.txt +++ b/source/adios2/CMakeLists.txt @@ -187,8 +187,9 @@ target_link_libraries(adios2_core PRIVATE target_link_libraries(adios2_core PUBLIC ${CMAKE_THREAD_LIBS_INIT}) target_compile_features(adios2_core PUBLIC "$") -if(UNIX) - target_sources(adios2_core PRIVATE toolkit/transport/file/FilePOSIX.cpp) + +target_sources(adios2_core PRIVATE toolkit/transport/file/FilePOSIX.cpp) +if(UNIX) target_sources(adios2_core PRIVATE toolkit/transport/file/FileHTTP.cpp) endif() diff --git a/source/adios2/common/ADIOSTypes.h b/source/adios2/common/ADIOSTypes.h index 21e9eae120..5085ccf295 100644 --- a/source/adios2/common/ADIOSTypes.h +++ b/source/adios2/common/ADIOSTypes.h @@ -243,11 +243,7 @@ struct MinVarInfo void PrintMVI(std::ostream &os, const MinVarInfo &mvi); // adios defaults -#ifdef _WIN32 -const std::string DefaultFileLibrary("fstream"); -#else const std::string DefaultFileLibrary("POSIX"); -#endif const std::string DefaultTimeUnit("Microseconds"); constexpr TimeUnit DefaultTimeUnitEnum(TimeUnit::Microseconds); diff --git a/source/adios2/toolkit/transport/file/FilePOSIX.cpp b/source/adios2/toolkit/transport/file/FilePOSIX.cpp index fedb934bf3..d3abbaed21 100644 --- a/source/adios2/toolkit/transport/file/FilePOSIX.cpp +++ b/source/adios2/toolkit/transport/file/FilePOSIX.cpp @@ -22,7 +22,18 @@ #include // open, fstat #include // open #include +#ifndef _MSC_VER #include // write, close, ftruncate +#define O_BINARY 0 +#else +#include +#define close _close +#define open _open +#define lseek(a, b, c) _lseek(a, (long)b, c) +#define write(a, b, c) _write(a, b, (unsigned int)c) +#define read(a, b, c) _read(a, b, (unsigned int)c) +#define ftruncate _chsize +#endif /// \cond EXCLUDE_FROM_DOXYGEN #include //std::ios_base::failure @@ -77,7 +88,7 @@ void FilePOSIX::Open(const std::string &name, const Mode openMode, const bool as auto lf_AsyncOpenWrite = [&](const std::string &name, const bool directio) -> int { ProfilerStart("open"); errno = 0; - int flag = __GetOpenFlag(O_WRONLY | O_CREAT | O_TRUNC, directio); + int flag = __GetOpenFlag(O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, directio); int FD = open(m_Name.c_str(), flag, 0666); m_Errno = errno; ProfilerStop("open"); @@ -102,7 +113,8 @@ void FilePOSIX::Open(const std::string &name, const Mode openMode, const bool as ProfilerStart("open"); errno = 0; m_FileDescriptor = - open(m_Name.c_str(), __GetOpenFlag(O_WRONLY | O_CREAT | O_TRUNC, directio), 0666); + open(m_Name.c_str(), + __GetOpenFlag(O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, directio), 0666); m_Errno = errno; ProfilerStop("open"); } @@ -111,7 +123,8 @@ void FilePOSIX::Open(const std::string &name, const Mode openMode, const bool as case Mode::Append: ProfilerStart("open"); errno = 0; - m_FileDescriptor = open(m_Name.c_str(), __GetOpenFlag(O_RDWR | O_CREAT, directio), 0777); + m_FileDescriptor = + open(m_Name.c_str(), __GetOpenFlag(O_RDWR | O_CREAT | O_BINARY, directio), 0777); lseek(m_FileDescriptor, 0, SEEK_END); m_Errno = errno; ProfilerStop("open"); @@ -120,7 +133,7 @@ void FilePOSIX::Open(const std::string &name, const Mode openMode, const bool as case Mode::Read: ProfilerStart("open"); errno = 0; - m_FileDescriptor = open(m_Name.c_str(), O_RDONLY); + m_FileDescriptor = open(m_Name.c_str(), O_RDONLY | O_BINARY); m_Errno = errno; ProfilerStop("open"); break; @@ -142,7 +155,7 @@ void FilePOSIX::OpenChain(const std::string &name, Mode openMode, const helper:: auto lf_AsyncOpenWrite = [&](const std::string &name, const bool directio) -> int { ProfilerStart("open"); errno = 0; - int flag = __GetOpenFlag(O_WRONLY | O_CREAT | O_TRUNC, directio); + int flag = __GetOpenFlag(O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, directio); int FD = open(m_Name.c_str(), flag, 0666); m_Errno = errno; ProfilerStop("open"); @@ -177,12 +190,14 @@ void FilePOSIX::OpenChain(const std::string &name, Mode openMode, const helper:: errno = 0; if (chainComm.Rank() == 0) { - m_FileDescriptor = open( - m_Name.c_str(), __GetOpenFlag(O_WRONLY | O_CREAT | O_TRUNC, directio), 0666); + m_FileDescriptor = + open(m_Name.c_str(), + __GetOpenFlag(O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, directio), 0666); } else { - m_FileDescriptor = open(m_Name.c_str(), __GetOpenFlag(O_WRONLY, directio), 0666); + m_FileDescriptor = + open(m_Name.c_str(), __GetOpenFlag(O_WRONLY | O_BINARY, directio), 0666); lseek(m_FileDescriptor, 0, SEEK_SET); } m_Errno = errno; @@ -196,11 +211,11 @@ void FilePOSIX::OpenChain(const std::string &name, Mode openMode, const helper:: if (chainComm.Rank() == 0) { m_FileDescriptor = - open(m_Name.c_str(), __GetOpenFlag(O_RDWR | O_CREAT, directio), 0666); + open(m_Name.c_str(), __GetOpenFlag(O_RDWR | O_CREAT | O_BINARY, directio), 0666); } else { - m_FileDescriptor = open(m_Name.c_str(), __GetOpenFlag(O_RDWR, directio)); + m_FileDescriptor = open(m_Name.c_str(), __GetOpenFlag(O_RDWR | O_BINARY, directio)); } lseek(m_FileDescriptor, 0, SEEK_END); m_Errno = errno; @@ -210,7 +225,7 @@ void FilePOSIX::OpenChain(const std::string &name, Mode openMode, const helper:: case Mode::Read: ProfilerStart("open"); errno = 0; - m_FileDescriptor = open(m_Name.c_str(), O_RDONLY); + m_FileDescriptor = open(m_Name.c_str(), O_RDONLY | O_BINARY); m_Errno = errno; ProfilerStop("open"); break; diff --git a/source/adios2/toolkit/transportman/TransportMan.cpp b/source/adios2/toolkit/transportman/TransportMan.cpp index 2e35e73f8b..4d646d8879 100644 --- a/source/adios2/toolkit/transportman/TransportMan.cpp +++ b/source/adios2/toolkit/transportman/TransportMan.cpp @@ -15,9 +15,7 @@ #include "adios2/helper/adiosFunctions.h" //CreateDirectory /// transports -#ifndef _WIN32 #include "adios2/toolkit/transport/file/FilePOSIX.h" -#endif #ifdef ADIOS2_HAVE_DAOS #include "adios2/toolkit/transport/file/FileDaos.h" #endif @@ -574,7 +572,6 @@ std::shared_ptr TransportMan::OpenFileTransport(const std::string &fi transport->SetBuffer(nullptr, 0); } } -#ifndef _WIN32 else if (library == "posix") { transport = std::make_shared(m_Comm); @@ -585,7 +582,6 @@ std::shared_ptr TransportMan::OpenFileTransport(const std::string &fi library + " transport does not support buffered I/O."); } } -#endif #ifdef ADIOS2_HAVE_DAOS else if (library == "daos") { From 0e120946e6be4b0397ffaf92e8b0a7484cf395af Mon Sep 17 00:00:00 2001 From: Dmitry Ganyushin Date: Wed, 31 Jan 2024 22:33:09 -0500 Subject: [PATCH 032/124] WIP --- source/adios2/engine/campaign/CampaignData.cpp | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/source/adios2/engine/campaign/CampaignData.cpp b/source/adios2/engine/campaign/CampaignData.cpp index 13a0ebd8c3..72996210d2 100644 --- a/source/adios2/engine/campaign/CampaignData.cpp +++ b/source/adios2/engine/campaign/CampaignData.cpp @@ -195,11 +195,16 @@ int inflateToFile(const unsigned char *source, const size_t blobsize, std::ofstr strm.avail_in = (uInt)(blobsize > CHUNK ? CHUNK : blobsize); strm.next_in = p; + std::cout<<"avail_in = "<< strm.avail_in <(iBlobsize); std::ofstream f; From e2a783e869c1f78aa09bbbf7b20d7d7d15124aa4 Mon Sep 17 00:00:00 2001 From: Dmitry Ganyushin Date: Thu, 1 Feb 2024 21:33:31 -0500 Subject: [PATCH 033/124] Added position increment. Added exception for writing operation. --- .../adios2/engine/campaign/CampaignData.cpp | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/source/adios2/engine/campaign/CampaignData.cpp b/source/adios2/engine/campaign/CampaignData.cpp index 72996210d2..f95739177d 100644 --- a/source/adios2/engine/campaign/CampaignData.cpp +++ b/source/adios2/engine/campaign/CampaignData.cpp @@ -168,7 +168,8 @@ void ReadCampaignData(sqlite3 *db, CampaignData &cd) allocated for processing, Z_DATA_ERROR if the deflate data is invalid or incomplete, Z_VERSION_ERROR if the version of zlib.h and the version of the library linked do not match, or Z_ERRNO if there - is an error reading or writing the files. */ + is an error reading or writing the files. + http://www.zlib.net/zlib_how.html */ int inflateToFile(const unsigned char *source, const size_t blobsize, std::ofstream *dest) { constexpr size_t CHUNK = 16777216; @@ -190,21 +191,19 @@ int inflateToFile(const unsigned char *source, const size_t blobsize, std::ofstr /* decompress until deflate stream ends or end of file */ unsigned char *p = const_cast(source); + uInt pos = 0; do { - strm.avail_in = (uInt)(blobsize > CHUNK ? CHUNK : blobsize); - strm.next_in = p; + uInt CHUNK_SIZE = static_cast(blobsize > CHUNK ? CHUNK : blobsize); + strm.avail_in = CHUNK_SIZE; - std::cout<<"avail_in = "<< strm.avail_in <("Core", "Campaign", "Inflate", + "error writing file "); } } while (strm.avail_out == 0); - + pos += CHUNK_SIZE; /* done when inflate() says it's done */ } while (ret != Z_STREAM_END); From 62b4405a40ad4379461deab68ea1e74804ee35ab Mon Sep 17 00:00:00 2001 From: dmitry-ganyushin Date: Fri, 2 Feb 2024 10:47:33 -0500 Subject: [PATCH 034/124] Formatting --- source/adios2/engine/campaign/CampaignData.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/source/adios2/engine/campaign/CampaignData.cpp b/source/adios2/engine/campaign/CampaignData.cpp index f95739177d..fa6dda7770 100644 --- a/source/adios2/engine/campaign/CampaignData.cpp +++ b/source/adios2/engine/campaign/CampaignData.cpp @@ -319,12 +319,10 @@ void SaveToFile(sqlite3 *db, const std::string &path, const CampaignBPFile &bpfi int iBlobsize = sqlite3_column_bytes(statement, 0); const void *p = sqlite3_column_blob(statement, 0); - std::cout << "-- Retrieved from DB data of " << bpfile.name - << " size = " << iBlobsize + std::cout << "-- Retrieved from DB data of " << bpfile.name << " size = " << iBlobsize << " compressed = " << bpfile.compressed << " compressed size = " << bpfile.lengthCompressed - << " original size = " << bpfile.lengthOriginal << " blob = " << p - << "\n"; + << " original size = " << bpfile.lengthOriginal << " blob = " << p << "\n"; size_t blobsize = static_cast(iBlobsize); std::ofstream f; From 1e63aaa46bfb0acd467615f6cd4f81963a9935b2 Mon Sep 17 00:00:00 2001 From: Vicente Adolfo Bolea Sanchez Date: Tue, 6 Feb 2024 15:24:56 -0500 Subject: [PATCH 035/124] readthedocs: enable query docs --- docs/user_guide/source/api_full/python.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/user_guide/source/api_full/python.rst b/docs/user_guide/source/api_full/python.rst index fc92952c62..828017bb00 100644 --- a/docs/user_guide/source/api_full/python.rst +++ b/docs/user_guide/source/api_full/python.rst @@ -37,3 +37,8 @@ Operator class -------------- .. autoclass:: adios2.bindings.adios2_bindings::Operator :members: + +Query class +-------------- +.. autoclass:: adios2.bindings.adios2_bindings::Query + :members: From 271b8097e845c7b3e0e3a8fa45f9571c154e6b86 Mon Sep 17 00:00:00 2001 From: Greg Eisenhauer Date: Fri, 9 Feb 2024 07:25:00 -0500 Subject: [PATCH 036/124] Don't force initialization for buffers whose contents we will immediately destroy (#4031) * BufferMalloc --- source/adios2/CMakeLists.txt | 1 + source/adios2/engine/bp5/BP5Reader.cpp | 23 +- source/adios2/engine/bp5/BP5Reader.h | 5 +- source/adios2/helper/adiosMemory.h | 9 +- source/adios2/helper/adiosMemory.inl | 399 ++++++++---------- source/adios2/toolkit/format/bp/BPBase.tcc | 8 +- .../toolkit/format/bp5/BP5Serializer.cpp | 22 +- .../format/buffer/heap/BufferMalloc.cpp | 101 +++++ .../toolkit/format/buffer/heap/BufferMalloc.h | 52 +++ .../format/buffer/heap/BufferMalloc.tcc | 54 +++ 10 files changed, 420 insertions(+), 254 deletions(-) create mode 100644 source/adios2/toolkit/format/buffer/heap/BufferMalloc.cpp create mode 100644 source/adios2/toolkit/format/buffer/heap/BufferMalloc.h create mode 100644 source/adios2/toolkit/format/buffer/heap/BufferMalloc.tcc diff --git a/source/adios2/CMakeLists.txt b/source/adios2/CMakeLists.txt index c63ffa6140..ff96f8e2c1 100644 --- a/source/adios2/CMakeLists.txt +++ b/source/adios2/CMakeLists.txt @@ -80,6 +80,7 @@ add_library(adios2_core toolkit/format/buffer/chunk/ChunkV.cpp toolkit/format/buffer/ffs/BufferFFS.cpp toolkit/format/buffer/heap/BufferSTL.cpp + toolkit/format/buffer/heap/BufferMalloc.cpp toolkit/format/buffer/malloc/MallocV.cpp toolkit/format/bp/BPBase.cpp toolkit/format/bp/BPBase.tcc diff --git a/source/adios2/engine/bp5/BP5Reader.cpp b/source/adios2/engine/bp5/BP5Reader.cpp index cb0b9aa570..636f67ace4 100644 --- a/source/adios2/engine/bp5/BP5Reader.cpp +++ b/source/adios2/engine/bp5/BP5Reader.cpp @@ -70,8 +70,8 @@ void BP5Reader::InstallMetadataForTimestep(size_t Step) { // variable metadata for timestep size_t ThisMDSize = - helper::ReadValue(m_Metadata.m_Buffer, Position, m_Minifooter.IsLittleEndian); - char *ThisMD = m_Metadata.m_Buffer.data() + MDPosition; + helper::ReadValue(m_Metadata.Data(), Position, m_Minifooter.IsLittleEndian); + char *ThisMD = m_Metadata.Data() + MDPosition; if (m_OpenMode == Mode::ReadRandomAccess) { m_BP5Deserializer->InstallMetaData(ThisMD, ThisMDSize, WriterRank, Step); @@ -86,8 +86,8 @@ void BP5Reader::InstallMetadataForTimestep(size_t Step) { // attribute metadata for timestep size_t ThisADSize = - helper::ReadValue(m_Metadata.m_Buffer, Position, m_Minifooter.IsLittleEndian); - char *ThisAD = m_Metadata.m_Buffer.data() + MDPosition; + helper::ReadValue(m_Metadata.Data(), Position, m_Minifooter.IsLittleEndian); + char *ThisAD = m_Metadata.Data() + MDPosition; if (ThisADSize > 0) m_BP5Deserializer->InstallAttributeData(ThisAD, ThisADSize); MDPosition += ThisADSize; @@ -820,8 +820,7 @@ void BP5Reader::UpdateBuffer(const TimePoint &timeoutInstant, const Seconds &pol for (auto p : m_FilteredMetadataInfo) { m_JSONProfiler.AddBytes("metadataread", p.second); - m_MDFileManager.ReadFile(m_Metadata.m_Buffer.data() + mempos, p.second, - p.first); + m_MDFileManager.ReadFile(m_Metadata.Data() + mempos, p.second, p.first); mempos += p.second; } m_MDFileAlreadyReadSize = expectedMinFileSize; @@ -863,14 +862,20 @@ void BP5Reader::UpdateBuffer(const TimePoint &timeoutInstant, const Seconds &pol } } - // broadcast buffer to all ranks from zero - m_Comm.BroadcastVector(m_Metadata.m_Buffer); - // broadcast metadata index buffer to all ranks from zero m_Comm.BroadcastVector(m_MetaMetadata.m_Buffer); InstallMetaMetaData(m_MetaMetadata); + size_t inputSize = m_Comm.BroadcastValue(m_Metadata.Size(), 0); + + if (m_Comm.Rank() != 0) + { + m_Metadata.Resize(inputSize, "metadata broadcast"); + } + + m_Comm.Bcast(m_Metadata.Data(), inputSize, 0); + if (m_OpenMode == Mode::ReadRandomAccess) { for (size_t Step = 0; Step < m_MetadataIndexTable.size(); Step++) diff --git a/source/adios2/engine/bp5/BP5Reader.h b/source/adios2/engine/bp5/BP5Reader.h index cc73944c2f..440158e509 100644 --- a/source/adios2/engine/bp5/BP5Reader.h +++ b/source/adios2/engine/bp5/BP5Reader.h @@ -4,8 +4,6 @@ * * BP5Reader.h * - * Created on: Aug 1, 2018 - * Author: Lipeng Wan wanl@ornl.gov */ #ifndef ADIOS2_ENGINE_BP5_BP5READER_H_ @@ -18,6 +16,7 @@ #include "adios2/helper/adiosComm.h" #include "adios2/helper/adiosRangeFilter.h" #include "adios2/toolkit/format/bp5/BP5Deserializer.h" +#include "adios2/toolkit/format/buffer/heap/BufferMalloc.h" #include "adios2/toolkit/remote/Remote.h" #include "adios2/toolkit/transportman/TransportMan.h" @@ -225,7 +224,7 @@ class BP5Reader : public BP5Engine, public Engine format::BufferSTL m_MetadataIndex; format::BufferSTL m_MetaMetadata; - format::BufferSTL m_Metadata; + format::BufferMalloc m_Metadata; void InstallMetaMetaData(format::BufferSTL MetaMetadata); void InstallMetadataForTimestep(size_t Step); diff --git a/source/adios2/helper/adiosMemory.h b/source/adios2/helper/adiosMemory.h index 10076fd8c7..7c56dd9823 100644 --- a/source/adios2/helper/adiosMemory.h +++ b/source/adios2/helper/adiosMemory.h @@ -5,8 +5,6 @@ * adiosMemory.h : Memory copy operations functions using std::copy std::insert * and std::memcpy * - * Created on: May 17, 2017 - * Author: William F Godoy godoywf@ornl.gov */ #ifndef ADIOS2_HELPER_ADIOSMEMORY_H_ @@ -83,7 +81,7 @@ void CopyToBufferThreads(std::vector &buffer, size_t &position, const T *s const size_t elements = 1, const unsigned int threads = 1) noexcept; template -void ReverseCopyFromBuffer(const std::vector &buffer, size_t &position, T *destination, +void ReverseCopyFromBuffer(const char *buffer, size_t &position, T *destination, const size_t elements = 1) noexcept; /** @@ -95,7 +93,7 @@ void ReverseCopyFromBuffer(const std::vector &buffer, size_t &position, T * @param elements number of elements of destination type */ template -void CopyFromBuffer(const std::vector &buffer, size_t &position, T *destination, +void CopyFromBuffer(const char *buffer, size_t &position, T *destination, const size_t elements = 1) noexcept; /** @@ -110,6 +108,9 @@ template T ReadValue(const std::vector &buffer, size_t &position, const bool isLittleEndian = true) noexcept; +template +T ReadValue(const char *buffer, size_t &position, const bool isLittleEndian = true) noexcept; + /** Read in 'nElems' elements from buffer into output array * output must be pre-allocated. */ diff --git a/source/adios2/helper/adiosMemory.inl b/source/adios2/helper/adiosMemory.inl index 7854056c37..25928198c7 100644 --- a/source/adios2/helper/adiosMemory.inl +++ b/source/adios2/helper/adiosMemory.inl @@ -4,8 +4,6 @@ * * adiosMemory.inl definition of template functions in adiosMemory.h * - * Created on: May 17, 2017 - * Author: William F Godoy godoywf@ornl.gov */ #ifndef ADIOS2_HELPER_ADIOSMEMORY_INL_ @@ -33,8 +31,7 @@ namespace helper #ifdef ADIOS2_HAVE_ENDIAN_REVERSE template -inline void CopyEndianReverse(const char *src, const size_t payloadStride, - T *dest) +inline void CopyEndianReverse(const char *src, const size_t payloadStride, T *dest) { if (sizeof(T) == 1) { @@ -47,8 +44,7 @@ inline void CopyEndianReverse(const char *src, const size_t payloadStride, } template <> -inline void CopyEndianReverse>(const char *src, - const size_t payloadStride, +inline void CopyEndianReverse>(const char *src, const size_t payloadStride, std::complex *dest) { std::reverse_copy(src, src + payloadStride, reinterpret_cast(dest)); @@ -57,8 +53,7 @@ inline void CopyEndianReverse>(const char *src, } template <> -inline void CopyEndianReverse>(const char *src, - const size_t payloadStride, +inline void CopyEndianReverse>(const char *src, const size_t payloadStride, std::complex *dest) { std::reverse_copy(src, src + payloadStride, reinterpret_cast(dest)); @@ -68,26 +63,23 @@ inline void CopyEndianReverse>(const char *src, #endif template -void InsertToBuffer(std::vector &buffer, const T *source, - const size_t elements) noexcept +void InsertToBuffer(std::vector &buffer, const T *source, const size_t elements) noexcept { const char *src = reinterpret_cast(source); buffer.insert(buffer.end(), src, src + elements * sizeof(T)); } template -void CopyFromGPUToBuffer(std::vector &dest, size_t &position, - const T *GPUbuffer, MemorySpace memSpace, - const size_t elements) noexcept +void CopyFromGPUToBuffer(std::vector &dest, size_t &position, const T *GPUbuffer, + MemorySpace memSpace, const size_t elements) noexcept { - CopyFromGPUToBuffer(dest.data(), position, GPUbuffer, memSpace, - elements * sizeof(T)); + CopyFromGPUToBuffer(dest.data(), position, GPUbuffer, memSpace, elements * sizeof(T)); position += elements * sizeof(T); } template -void CopyFromGPUToBuffer(char *dest, size_t position, const T *GPUbuffer, - MemorySpace memSpace, const size_t size) noexcept +void CopyFromGPUToBuffer(char *dest, size_t position, const T *GPUbuffer, MemorySpace memSpace, + const size_t size) noexcept { #ifdef ADIOS2_HAVE_GPU_SUPPORT if (memSpace == MemorySpace::GPU) @@ -99,8 +91,8 @@ void CopyFromGPUToBuffer(char *dest, size_t position, const T *GPUbuffer, } template -void CopyFromBufferToGPU(T *GPUbuffer, size_t position, const char *source, - MemorySpace memSpace, const size_t size) noexcept +void CopyFromBufferToGPU(T *GPUbuffer, size_t position, const char *source, MemorySpace memSpace, + const size_t size) noexcept { #ifdef ADIOS2_HAVE_GPU_SUPPORT if (memSpace == MemorySpace::GPU) @@ -111,9 +103,8 @@ void CopyFromBufferToGPU(T *GPUbuffer, size_t position, const char *source, #endif } -static inline void NdCopyGPU(const char *&inOvlpBase, char *&outOvlpBase, - CoreDims &inOvlpGapSize, CoreDims &outOvlpGapSize, - CoreDims &ovlpCount, size_t minContDim, +static inline void NdCopyGPU(const char *&inOvlpBase, char *&outOvlpBase, CoreDims &inOvlpGapSize, + CoreDims &outOvlpGapSize, CoreDims &ovlpCount, size_t minContDim, size_t blockSize, MemorySpace memSpace) { DimsArray pos(ovlpCount.size(), (size_t)0); @@ -152,9 +143,8 @@ void CopyToBuffer(std::vector &buffer, size_t &position, const T *source, } template -void CopyToBufferThreads(std::vector &buffer, size_t &position, - const T *source, const size_t elements, - const unsigned int threads) noexcept +void CopyToBufferThreads(std::vector &buffer, size_t &position, const T *source, + const size_t elements, const unsigned int threads) noexcept { if (elements == 0) { @@ -182,9 +172,8 @@ void CopyToBufferThreads(std::vector &buffer, size_t &position, const size_t srcStart = stride * t * sizeof(T); if (t == threads - 1) // last thread takes stride + remainder { - copyThreads.push_back(std::thread(std::memcpy, &buffer[bufferStart], - &src[srcStart], - last * sizeof(T))); + copyThreads.push_back( + std::thread(std::memcpy, &buffer[bufferStart], &src[srcStart], last * sizeof(T))); // std::copy not working properly with std::thread...why? // copyThreads.push_back(std::thread(std::copy, // &src[srcStart], @@ -195,9 +184,8 @@ void CopyToBufferThreads(std::vector &buffer, size_t &position, } else { - copyThreads.push_back(std::thread(std::memcpy, &buffer[bufferStart], - &src[srcStart], - stride * sizeof(T))); + copyThreads.push_back( + std::thread(std::memcpy, &buffer[bufferStart], &src[srcStart], stride * sizeof(T))); // std::copy not working properly with std::thread...why? // copyThreads.push_back(std::thread( // std::copy, &src[srcStart], &src[srcStart] + stride @@ -215,22 +203,18 @@ void CopyToBufferThreads(std::vector &buffer, size_t &position, } template -inline void ReverseCopyFromBuffer(const std::vector &buffer, - size_t &position, T *destination, +inline void ReverseCopyFromBuffer(const char *buffer, size_t &position, T *destination, const size_t elements) noexcept { - std::reverse_copy(buffer.begin() + position, - buffer.begin() + position + sizeof(T) * elements, + std::reverse_copy(buffer + position, buffer + position + sizeof(T) * elements, reinterpret_cast(destination)); position += elements * sizeof(T); } template -void CopyFromBuffer(const std::vector &buffer, size_t &position, - T *destination, size_t elements) noexcept +void CopyFromBuffer(const char *buffer, size_t &position, T *destination, size_t elements) noexcept { - std::copy(buffer.begin() + position, - buffer.begin() + position + sizeof(T) * elements, + std::copy(buffer + position, buffer + position + sizeof(T) * elements, reinterpret_cast(destination)); position += elements * sizeof(T); } @@ -248,6 +232,26 @@ inline T ReadValue(const std::vector &buffer, size_t &position, { T value; +#ifdef ADIOS2_HAVE_ENDIAN_REVERSE + if (IsLittleEndian() != isLittleEndian) + { + ReverseCopyFromBuffer(buffer.data(), position, &value); + } + else + { + CopyFromBuffer(buffer.data(), position, &value); + } +#else + CopyFromBuffer(buffer.data(), position, &value); +#endif + return value; +} + +template +inline T ReadValue(const char *buffer, size_t &position, const bool isLittleEndian) noexcept +{ + T value; + #ifdef ADIOS2_HAVE_ENDIAN_REVERSE if (IsLittleEndian() != isLittleEndian) { @@ -264,129 +268,120 @@ inline T ReadValue(const std::vector &buffer, size_t &position, } template <> -inline std::complex -ReadValue>(const std::vector &buffer, - size_t &position, - const bool isLittleEndian) noexcept +inline std::complex ReadValue>(const std::vector &buffer, + size_t &position, + const bool isLittleEndian) noexcept { std::complex value; #ifdef ADIOS2_HAVE_ENDIAN_REVERSE if (IsLittleEndian() != isLittleEndian) { - ReverseCopyFromBuffer(buffer, position, &value); + ReverseCopyFromBuffer(buffer.data(), position, &value); return std::complex(value.imag(), value.real()); } else { - CopyFromBuffer(buffer, position, &value); + CopyFromBuffer(buffer.data(), position, &value); } #else - CopyFromBuffer(buffer, position, &value); + CopyFromBuffer(buffer.data(), position, &value); #endif return value; } template <> -inline std::complex -ReadValue>(const std::vector &buffer, - size_t &position, - const bool isLittleEndian) noexcept +inline std::complex ReadValue>(const std::vector &buffer, + size_t &position, + const bool isLittleEndian) noexcept { std::complex value; #ifdef ADIOS2_HAVE_ENDIAN_REVERSE if (IsLittleEndian() != isLittleEndian) { - ReverseCopyFromBuffer(buffer, position, &value); + ReverseCopyFromBuffer(buffer.data(), position, &value); return std::complex(value.imag(), value.real()); } else { - CopyFromBuffer(buffer, position, &value); + CopyFromBuffer(buffer.data(), position, &value); } #else - CopyFromBuffer(buffer, position, &value); + CopyFromBuffer(buffer.data(), position, &value); #endif return value; } template -inline void ReadArray(const std::vector &buffer, size_t &position, - T *output, const size_t nElems, - const bool isLittleEndian) noexcept +inline void ReadArray(const std::vector &buffer, size_t &position, T *output, + const size_t nElems, const bool isLittleEndian) noexcept { #ifdef ADIOS2_HAVE_ENDIAN_REVERSE if (IsLittleEndian() != isLittleEndian) { - ReverseCopyFromBuffer(buffer, position, output, nElems); + ReverseCopyFromBuffer(buffer.data(), position, output, nElems); } else { - CopyFromBuffer(buffer, position, output, nElems); + CopyFromBuffer(buffer.data(), position, output, nElems); } #else - CopyFromBuffer(buffer, position, output, nElems); + CopyFromBuffer(buffer.data(), position, output, nElems); #endif } template <> -inline void ReadArray>(const std::vector &buffer, - size_t &position, - std::complex *output, - const size_t nElems, +inline void ReadArray>(const std::vector &buffer, size_t &position, + std::complex *output, const size_t nElems, const bool isLittleEndian) noexcept { #ifdef ADIOS2_HAVE_ENDIAN_REVERSE if (IsLittleEndian() != isLittleEndian) { - ReverseCopyFromBuffer(buffer, position, output, nElems); + ReverseCopyFromBuffer(buffer.data(), position, output, nElems); } else { - CopyFromBuffer(buffer, position, output, nElems); + CopyFromBuffer(buffer.data(), position, output, nElems); } #else - CopyFromBuffer(buffer, position, output, nElems); + CopyFromBuffer(buffer.data(), position, output, nElems); #endif } template <> -inline void ReadArray>(const std::vector &buffer, - size_t &position, - std::complex *output, - const size_t nElems, +inline void ReadArray>(const std::vector &buffer, size_t &position, + std::complex *output, const size_t nElems, const bool isLittleEndian) noexcept { #ifdef ADIOS2_HAVE_ENDIAN_REVERSE if (IsLittleEndian() != isLittleEndian) { - ReverseCopyFromBuffer(buffer, position, output, nElems); + ReverseCopyFromBuffer(buffer.data(), position, output, nElems); } else { - CopyFromBuffer(buffer, position, output, nElems); + CopyFromBuffer(buffer.data(), position, output, nElems); } #else - CopyFromBuffer(buffer, position, output, nElems); + CopyFromBuffer(buffer.data(), position, output, nElems); #endif } template -void ClipVector(std::vector &vec, const size_t start, - const size_t end) noexcept +void ClipVector(std::vector &vec, const size_t start, const size_t end) noexcept { vec.resize(end); vec.erase(vec.begin(), vec.begin() + start); } template -void CopyMemoryBlock(T *dest, const Dims &destStart, const Dims &destCount, - const bool destRowMajor, const U *src, - const Dims &srcStart, const Dims &srcCount, - const bool srcRowMajor, const bool endianReverse, - const Dims &destMemStart, const Dims &destMemCount, - const Dims &srcMemStart, const Dims &srcMemCount) noexcept +void CopyMemoryBlock(T *dest, const Dims &destStart, const Dims &destCount, const bool destRowMajor, + const U *src, const Dims &srcStart, const Dims &srcCount, + const bool srcRowMajor, const bool endianReverse, const Dims &destMemStart, + const Dims &destMemCount, const Dims &srcMemStart, + const Dims &srcMemCount) noexcept { // transform everything to payload dims const Dims destStartPayload = PayloadDims(destStart, destRowMajor); @@ -399,28 +394,23 @@ void CopyMemoryBlock(T *dest, const Dims &destStart, const Dims &destCount, const Dims srcMemStartPayload = PayloadDims(srcMemStart, srcRowMajor); const Dims srcMemCountPayload = PayloadDims(srcMemCount, srcRowMajor); - CopyPayload(reinterpret_cast(dest), destStartPayload, - destCountPayload, destRowMajor, - reinterpret_cast(src), srcStartPayload, - srcCountPayload, srcRowMajor, destMemStartPayload, - destMemCountPayload, srcMemStartPayload, srcMemCountPayload, + CopyPayload(reinterpret_cast(dest), destStartPayload, destCountPayload, destRowMajor, + reinterpret_cast(src), srcStartPayload, srcCountPayload, srcRowMajor, + destMemStartPayload, destMemCountPayload, srcMemStartPayload, srcMemCountPayload, endianReverse, GetDataType()); } template void ClipContiguousMemory(T *dest, const Dims &destStart, const Dims &destCount, - const char *contiguousMemory, - const Box &blockBox, - const Box &intersectionBox, - const bool isRowMajor, const bool reverseDimensions, - const bool endianReverse, const MemorySpace memSpace) + const char *contiguousMemory, const Box &blockBox, + const Box &intersectionBox, const bool isRowMajor, + const bool reverseDimensions, const bool endianReverse, + const MemorySpace memSpace) { auto lf_ClipRowMajor = - [](T *dest, const Dims &destStart, const Dims &destCount, - const char *contiguousMemory, const Box &blockBox, - const Box &intersectionBox, const bool isRowMajor, - const bool reverseDimensions, const bool endianReverse, - const MemorySpace memSpace) + [](T *dest, const Dims &destStart, const Dims &destCount, const char *contiguousMemory, + const Box &blockBox, const Box &intersectionBox, const bool isRowMajor, + const bool reverseDimensions, const bool endianReverse, const MemorySpace memSpace) { const Dims &istart = intersectionBox.first; @@ -429,8 +419,7 @@ void ClipContiguousMemory(T *dest, const Dims &destStart, const Dims &destCount, Dims currentPoint(istart); // current point for memory copy // convert selection to EndBox and reverse if we are inside a // column-major reader - const Box selectionBox = - helper::StartEndBox(destStart, destCount, reverseDimensions); + const Box selectionBox = helper::StartEndBox(destStart, destCount, reverseDimensions); const size_t dimensions = istart.size(); @@ -441,14 +430,10 @@ void ClipContiguousMemory(T *dest, const Dims &destStart, const Dims &destCount, */ size_t nContDim = 1; while (nContDim <= dimensions - 1 && - blockBox.first[dimensions - nContDim] == - istart[dimensions - nContDim] && - blockBox.second[dimensions - nContDim] == - iend[dimensions - nContDim] && - blockBox.first[dimensions - nContDim] == - selectionBox.first[dimensions - nContDim] && - blockBox.second[dimensions - nContDim] == - selectionBox.second[dimensions - nContDim]) + blockBox.first[dimensions - nContDim] == istart[dimensions - nContDim] && + blockBox.second[dimensions - nContDim] == iend[dimensions - nContDim] && + blockBox.first[dimensions - nContDim] == selectionBox.first[dimensions - nContDim] && + blockBox.second[dimensions - nContDim] == selectionBox.second[dimensions - nContDim]) { ++nContDim; } @@ -462,22 +447,19 @@ void ClipContiguousMemory(T *dest, const Dims &destStart, const Dims &destCount, const size_t stride = nContElems * sizeof(T); const size_t intersectionStart = - helper::LinearIndex(blockBox, intersectionBox.first, true) * - sizeof(T); + helper::LinearIndex(blockBox, intersectionBox.first, true) * sizeof(T); bool run = true; while (run) { // here copy current linear memory between currentPoint and end const size_t contiguousStart = - helper::LinearIndex(blockBox, currentPoint, true) * sizeof(T) - - intersectionStart; + helper::LinearIndex(blockBox, currentPoint, true) * sizeof(T) - intersectionStart; - const size_t variableStart = - helper::LinearIndex(selectionBox, currentPoint, true); + const size_t variableStart = helper::LinearIndex(selectionBox, currentPoint, true); - CopyContiguousMemory(contiguousMemory + contiguousStart, stride, - dest + variableStart, endianReverse, memSpace); + CopyContiguousMemory(contiguousMemory + contiguousStart, stride, dest + variableStart, + endianReverse, memSpace); // Here update each non-contiguous dim recursively if (nContDim >= dimensions) @@ -513,11 +495,9 @@ void ClipContiguousMemory(T *dest, const Dims &destStart, const Dims &destCount, }; auto lf_ClipColumnMajor = - [](T *dest, const Dims &destStart, const Dims &destCount, - const char *contiguousMemory, const Box &blockBox, - const Box &intersectionBox, const bool isRowMajor, - const bool reverseDimensions, const bool endianReverse, - const MemorySpace memSpace) + [](T *dest, const Dims &destStart, const Dims &destCount, const char *contiguousMemory, + const Box &blockBox, const Box &intersectionBox, const bool isRowMajor, + const bool reverseDimensions, const bool endianReverse, const MemorySpace memSpace) { const Dims &istart = intersectionBox.first; @@ -525,8 +505,7 @@ void ClipContiguousMemory(T *dest, const Dims &destStart, const Dims &destCount, Dims currentPoint(istart); // current point for memory copy - const Box selectionBox = - helper::StartEndBox(destStart, destCount, reverseDimensions); + const Box selectionBox = helper::StartEndBox(destStart, destCount, reverseDimensions); const size_t dimensions = istart.size(); /* Determine how many dimensions we can copy at once. @@ -535,12 +514,10 @@ void ClipContiguousMemory(T *dest, const Dims &destStart, const Dims &destCount, nContDim = 1: line by line */ size_t nContDim = 1; - while ( - nContDim <= dimensions - 1 && - blockBox.first[nContDim - 1] == istart[nContDim - 1] && - blockBox.second[nContDim - 1] == iend[nContDim - 1] && - blockBox.first[nContDim - 1] == selectionBox.first[nContDim - 1] && - blockBox.second[nContDim - 1] == selectionBox.second[nContDim - 1]) + while (nContDim <= dimensions - 1 && blockBox.first[nContDim - 1] == istart[nContDim - 1] && + blockBox.second[nContDim - 1] == iend[nContDim - 1] && + blockBox.first[nContDim - 1] == selectionBox.first[nContDim - 1] && + blockBox.second[nContDim - 1] == selectionBox.second[nContDim - 1]) { ++nContDim; } @@ -554,22 +531,19 @@ void ClipContiguousMemory(T *dest, const Dims &destStart, const Dims &destCount, const size_t stride = nContElems * sizeof(T); const size_t intersectionStart = - helper::LinearIndex(blockBox, intersectionBox.first, false) * - sizeof(T); + helper::LinearIndex(blockBox, intersectionBox.first, false) * sizeof(T); bool run = true; while (run) { // here copy current linear memory between currentPoint and end const size_t contiguousStart = - helper::LinearIndex(blockBox, currentPoint, false) * sizeof(T) - - intersectionStart; + helper::LinearIndex(blockBox, currentPoint, false) * sizeof(T) - intersectionStart; - const size_t variableStart = - helper::LinearIndex(selectionBox, currentPoint, false); + const size_t variableStart = helper::LinearIndex(selectionBox, currentPoint, false); - CopyContiguousMemory(contiguousMemory + contiguousStart, stride, - dest + variableStart, endianReverse, memSpace); + CopyContiguousMemory(contiguousMemory + contiguousStart, stride, dest + variableStart, + endianReverse, memSpace); // Here update each non-contiguous dim recursively. if (nContDim >= dimensions) @@ -613,37 +587,33 @@ void ClipContiguousMemory(T *dest, const Dims &destStart, const Dims &destCount, const Dims &end = intersectionBox.second; const size_t stride = (end.back() - start.back() + 1) * sizeof(T); - CopyContiguousMemory(contiguousMemory, stride, dest + normalizedStart, - endianReverse, memSpace); + CopyContiguousMemory(contiguousMemory, stride, dest + normalizedStart, endianReverse, + memSpace); return; } if (isRowMajor) // stored with C, C++, Python { - lf_ClipRowMajor(dest, destStart, destCount, contiguousMemory, blockBox, - intersectionBox, isRowMajor, reverseDimensions, - endianReverse, memSpace); + lf_ClipRowMajor(dest, destStart, destCount, contiguousMemory, blockBox, intersectionBox, + isRowMajor, reverseDimensions, endianReverse, memSpace); } else // stored with Fortran, R { - lf_ClipColumnMajor(dest, destStart, destCount, contiguousMemory, - blockBox, intersectionBox, isRowMajor, - reverseDimensions, endianReverse, memSpace); + lf_ClipColumnMajor(dest, destStart, destCount, contiguousMemory, blockBox, intersectionBox, + isRowMajor, reverseDimensions, endianReverse, memSpace); } } template void ClipContiguousMemory(T *dest, const Dims &destStart, const Dims &destCount, - const std::vector &contiguousMemory, - const Box &blockBox, - const Box &intersectionBox, - const bool isRowMajor, const bool reverseDimensions, - const bool endianReverse, const MemorySpace memSpace) + const std::vector &contiguousMemory, const Box &blockBox, + const Box &intersectionBox, const bool isRowMajor, + const bool reverseDimensions, const bool endianReverse, + const MemorySpace memSpace) { - ClipContiguousMemory(dest, destStart, destCount, contiguousMemory.data(), - blockBox, intersectionBox, isRowMajor, - reverseDimensions, endianReverse, memSpace); + ClipContiguousMemory(dest, destStart, destCount, contiguousMemory.data(), blockBox, + intersectionBox, isRowMajor, reverseDimensions, endianReverse, memSpace); } template @@ -681,8 +651,7 @@ void CopyContiguousMemory(const char *src, const size_t payloadStride, T *dest, } template -void Resize(std::vector &vec, const size_t dataSize, const std::string hint, - T value) +void Resize(std::vector &vec, const size_t dataSize, const std::string hint, T value) { try { @@ -692,10 +661,9 @@ void Resize(std::vector &vec, const size_t dataSize, const std::string hint, } catch (...) { - helper::ThrowNested( - "Helper", "adiosMemory", "Resize", - "buffer overflow when resizing to " + std::to_string(dataSize) + - " bytes, " + hint); + helper::ThrowNested("Helper", "adiosMemory", "Resize", + "buffer overflow when resizing to " + + std::to_string(dataSize) + " bytes, " + hint); } } @@ -710,11 +678,10 @@ void Resize(std::vector &vec, const size_t dataSize, const std::string hint, // functions) and copies to the output buffer in blocks. the memory address // calculation complexity for copying each block is minimized to O(1), which is // independent of the number of dimensions. -static inline void -NdCopyRecurDFSeqPadding(size_t curDim, const char *&inOvlpBase, - char *&outOvlpBase, CoreDims &inOvlpGapSize, - CoreDims &outOvlpGapSize, CoreDims &ovlpCount, - size_t &minContDim, size_t &blockSize) +static inline void NdCopyRecurDFSeqPadding(size_t curDim, const char *&inOvlpBase, + char *&outOvlpBase, CoreDims &inOvlpGapSize, + CoreDims &outOvlpGapSize, CoreDims &ovlpCount, + size_t &minContDim, size_t &blockSize) { // note: all elements in and below this node are contiguous on input and // output @@ -732,9 +699,8 @@ NdCopyRecurDFSeqPadding(size_t curDim, const char *&inOvlpBase, { for (size_t i = 0; i < ovlpCount[curDim]; i++) { - NdCopyRecurDFSeqPadding(curDim + 1, inOvlpBase, outOvlpBase, - inOvlpGapSize, outOvlpGapSize, ovlpCount, - minContDim, blockSize); + NdCopyRecurDFSeqPadding(curDim + 1, inOvlpBase, outOvlpBase, inOvlpGapSize, + outOvlpGapSize, ovlpCount, minContDim, blockSize); } // the gap between current node and the next needs to be padded so that // next contigous block starts at the correct position for both input @@ -754,12 +720,11 @@ NdCopyRecurDFSeqPadding(size_t curDim, const char *&inOvlpBase, // each element is minimized to average O(1), which is independent of // the number of dimensions. -static inline void -NdCopyRecurDFSeqPaddingRevEndian(size_t curDim, const char *&inOvlpBase, - char *&outOvlpBase, CoreDims &inOvlpGapSize, - CoreDims &outOvlpGapSize, CoreDims &ovlpCount, - size_t minCountDim, size_t blockSize, - size_t elmSize, size_t numElmsPerBlock) +static inline void NdCopyRecurDFSeqPaddingRevEndian(size_t curDim, const char *&inOvlpBase, + char *&outOvlpBase, CoreDims &inOvlpGapSize, + CoreDims &outOvlpGapSize, CoreDims &ovlpCount, + size_t minCountDim, size_t blockSize, + size_t elmSize, size_t numElmsPerBlock) { if (curDim == minCountDim) { @@ -780,10 +745,9 @@ NdCopyRecurDFSeqPaddingRevEndian(size_t curDim, const char *&inOvlpBase, { for (size_t i = 0; i < ovlpCount[curDim]; i++) { - NdCopyRecurDFSeqPaddingRevEndian( - curDim + 1, inOvlpBase, outOvlpBase, inOvlpGapSize, - outOvlpGapSize, ovlpCount, minCountDim, blockSize, elmSize, - numElmsPerBlock); + NdCopyRecurDFSeqPaddingRevEndian(curDim + 1, inOvlpBase, outOvlpBase, inOvlpGapSize, + outOvlpGapSize, ovlpCount, minCountDim, blockSize, + elmSize, numElmsPerBlock); } } inOvlpBase += inOvlpGapSize[curDim]; @@ -795,11 +759,10 @@ NdCopyRecurDFSeqPaddingRevEndian(size_t curDim, const char *&inOvlpBase, // used for buffer of Column major // the memory address calculation complexity for copying each element is // minimized to average O(1), which is independent of the number of dimensions. -static inline void -NdCopyRecurDFNonSeqDynamic(size_t curDim, const char *inBase, char *outBase, - CoreDims &inRltvOvlpSPos, CoreDims &outRltvOvlpSPos, - CoreDims &inStride, CoreDims &outStride, - CoreDims &ovlpCount, size_t elmSize) +static inline void NdCopyRecurDFNonSeqDynamic(size_t curDim, const char *inBase, char *outBase, + CoreDims &inRltvOvlpSPos, CoreDims &outRltvOvlpSPos, + CoreDims &inStride, CoreDims &outStride, + CoreDims &ovlpCount, size_t elmSize) { if (curDim == inStride.size()) { @@ -810,11 +773,9 @@ NdCopyRecurDFNonSeqDynamic(size_t curDim, const char *inBase, char *outBase, for (size_t i = 0; i < ovlpCount[curDim]; i++) { NdCopyRecurDFNonSeqDynamic( - curDim + 1, - inBase + (inRltvOvlpSPos[curDim] + i) * inStride[curDim], - outBase + (outRltvOvlpSPos[curDim] + i) * outStride[curDim], - inRltvOvlpSPos, outRltvOvlpSPos, inStride, outStride, ovlpCount, - elmSize); + curDim + 1, inBase + (inRltvOvlpSPos[curDim] + i) * inStride[curDim], + outBase + (outRltvOvlpSPos[curDim] + i) * outStride[curDim], inRltvOvlpSPos, + outRltvOvlpSPos, inStride, outStride, ovlpCount, elmSize); } } } @@ -825,10 +786,11 @@ NdCopyRecurDFNonSeqDynamic(size_t curDim, const char *inBase, char *outBase, // The memory address calculation complexity for copying each element is // minimized to average O(1), which is independent of the number of dimensions. -static inline void NdCopyRecurDFNonSeqDynamicRevEndian( - size_t curDim, const char *inBase, char *outBase, CoreDims &inRltvOvlpSPos, - CoreDims &outRltvOvlpSPos, CoreDims &inStride, CoreDims &outStride, - CoreDims &ovlpCount, size_t elmSize) +static inline void NdCopyRecurDFNonSeqDynamicRevEndian(size_t curDim, const char *inBase, + char *outBase, CoreDims &inRltvOvlpSPos, + CoreDims &outRltvOvlpSPos, + CoreDims &inStride, CoreDims &outStride, + CoreDims &ovlpCount, size_t elmSize) { if (curDim == inStride.size()) { @@ -842,19 +804,16 @@ static inline void NdCopyRecurDFNonSeqDynamicRevEndian( for (size_t i = 0; i < ovlpCount[curDim]; i++) { NdCopyRecurDFNonSeqDynamicRevEndian( - curDim + 1, - inBase + (inRltvOvlpSPos[curDim] + i) * inStride[curDim], - outBase + (outRltvOvlpSPos[curDim] + i) * outStride[curDim], - inRltvOvlpSPos, outRltvOvlpSPos, inStride, outStride, ovlpCount, - elmSize); + curDim + 1, inBase + (inRltvOvlpSPos[curDim] + i) * inStride[curDim], + outBase + (outRltvOvlpSPos[curDim] + i) * outStride[curDim], inRltvOvlpSPos, + outRltvOvlpSPos, inStride, outStride, ovlpCount, elmSize); } } } -static inline void -NdCopyIterDFSeqPadding(const char *&inOvlpBase, char *&outOvlpBase, - CoreDims &inOvlpGapSize, CoreDims &outOvlpGapSize, - CoreDims &ovlpCount, size_t minContDim, size_t blockSize) +static inline void NdCopyIterDFSeqPadding(const char *&inOvlpBase, char *&outOvlpBase, + CoreDims &inOvlpGapSize, CoreDims &outOvlpGapSize, + CoreDims &ovlpCount, size_t minContDim, size_t blockSize) { DimsArray pos(ovlpCount.size(), (size_t)0); size_t curDim = 0; @@ -882,10 +841,11 @@ NdCopyIterDFSeqPadding(const char *&inOvlpBase, char *&outOvlpBase, } } -static inline void NdCopyIterDFSeqPaddingRevEndian( - const char *&inOvlpBase, char *&outOvlpBase, CoreDims &inOvlpGapSize, - CoreDims &outOvlpGapSize, CoreDims &ovlpCount, size_t minContDim, - size_t blockSize, size_t elmSize, size_t numElmsPerBlock) +static inline void NdCopyIterDFSeqPaddingRevEndian(const char *&inOvlpBase, char *&outOvlpBase, + CoreDims &inOvlpGapSize, + CoreDims &outOvlpGapSize, CoreDims &ovlpCount, + size_t minContDim, size_t blockSize, + size_t elmSize, size_t numElmsPerBlock) { DimsArray pos(ovlpCount.size(), (size_t)0); size_t curDim = 0; @@ -918,11 +878,9 @@ static inline void NdCopyIterDFSeqPaddingRevEndian( } while (pos[curDim] == ovlpCount[curDim]); } } -static inline void NdCopyIterDFDynamic(const char *inBase, char *outBase, - CoreDims &inRltvOvlpSPos, - CoreDims &outRltvOvlpSPos, - CoreDims &inStride, CoreDims &outStride, - CoreDims &ovlpCount, size_t elmSize) +static inline void NdCopyIterDFDynamic(const char *inBase, char *outBase, CoreDims &inRltvOvlpSPos, + CoreDims &outRltvOvlpSPos, CoreDims &inStride, + CoreDims &outStride, CoreDims &ovlpCount, size_t elmSize) { size_t curDim = 0; DimsArray pos(ovlpCount.size() + 1, (size_t)0); @@ -935,11 +893,9 @@ static inline void NdCopyIterDFDynamic(const char *inBase, char *outBase, while (curDim != inStride.size()) { inAddr[curDim + 1] = - inAddr[curDim] + - (inRltvOvlpSPos[curDim] + pos[curDim]) * inStride[curDim]; + inAddr[curDim] + (inRltvOvlpSPos[curDim] + pos[curDim]) * inStride[curDim]; outAddr[curDim + 1] = - outAddr[curDim] + - (outRltvOvlpSPos[curDim] + pos[curDim]) * outStride[curDim]; + outAddr[curDim] + (outRltvOvlpSPos[curDim] + pos[curDim]) * outStride[curDim]; pos[curDim]++; curDim++; } @@ -956,10 +912,10 @@ static inline void NdCopyIterDFDynamic(const char *inBase, char *outBase, } } -static inline void NdCopyIterDFDynamicRevEndian( - const char *inBase, char *outBase, CoreDims &inRltvOvlpSPos, - CoreDims &outRltvOvlpSPos, CoreDims &inStride, CoreDims &outStride, - CoreDims &ovlpCount, size_t elmSize) +static inline void NdCopyIterDFDynamicRevEndian(const char *inBase, char *outBase, + CoreDims &inRltvOvlpSPos, CoreDims &outRltvOvlpSPos, + CoreDims &inStride, CoreDims &outStride, + CoreDims &ovlpCount, size_t elmSize) { size_t curDim = 0; DimsArray pos(ovlpCount.size() + 1, (size_t)0); @@ -972,11 +928,9 @@ static inline void NdCopyIterDFDynamicRevEndian( while (curDim != inStride.size()) { inAddr[curDim + 1] = - inAddr[curDim] + - (inRltvOvlpSPos[curDim] + pos[curDim]) * inStride[curDim]; + inAddr[curDim] + (inRltvOvlpSPos[curDim] + pos[curDim]) * inStride[curDim]; outAddr[curDim + 1] = - outAddr[curDim] + - (outRltvOvlpSPos[curDim] + pos[curDim]) * outStride[curDim]; + outAddr[curDim] + (outRltvOvlpSPos[curDim] + pos[curDim]) * outStride[curDim]; pos[curDim]++; curDim++; } @@ -999,8 +953,8 @@ static inline void NdCopyIterDFDynamicRevEndian( template size_t PayloadSize(const T * /*data*/, const Dims &count) noexcept { - const bool isZeros = std::all_of(count.begin(), count.end(), - [](const size_t i) { return i == 0; }); + const bool isZeros = + std::all_of(count.begin(), count.end(), [](const size_t i) { return i == 0; }); if (isZeros) { @@ -1011,8 +965,7 @@ size_t PayloadSize(const T * /*data*/, const Dims &count) noexcept } template <> -inline size_t PayloadSize(const std::string *data, - const Dims & /*count*/) noexcept +inline size_t PayloadSize(const std::string *data, const Dims & /*count*/) noexcept { return data->size() + 2; // 2 bytes for the string size } diff --git a/source/adios2/toolkit/format/bp/BPBase.tcc b/source/adios2/toolkit/format/bp/BPBase.tcc index 19b8a0d31c..fd73fc8d36 100644 --- a/source/adios2/toolkit/format/bp/BPBase.tcc +++ b/source/adios2/toolkit/format/bp/BPBase.tcc @@ -232,17 +232,17 @@ inline void BPBase::ParseCharacteristics(const std::vector &buffer, size_t #ifdef ADIOS2_HAVE_ENDIAN_REVERSE if (helper::IsLittleEndian() != isLittleEndian) { - helper::ReverseCopyFromBuffer(buffer, position, + helper::ReverseCopyFromBuffer(buffer.data(), position, characteristics.Statistics.Values.data(), size); } else { - helper::CopyFromBuffer(buffer, position, + helper::CopyFromBuffer(buffer.data(), position, characteristics.Statistics.Values.data(), size); } #else - helper::CopyFromBuffer(buffer, position, characteristics.Statistics.Values.data(), - size); + helper::CopyFromBuffer(buffer.data(), position, + characteristics.Statistics.Values.data(), size); #endif } break; diff --git a/source/adios2/toolkit/format/bp5/BP5Serializer.cpp b/source/adios2/toolkit/format/bp5/BP5Serializer.cpp index 2f618089dc..2030023d79 100644 --- a/source/adios2/toolkit/format/bp5/BP5Serializer.cpp +++ b/source/adios2/toolkit/format/bp5/BP5Serializer.cpp @@ -1567,13 +1567,13 @@ std::vector BP5Serializer::BreakoutContiguousMetadata( for (size_t Rank = 0; Rank < Counts.size(); Rank++) { uint64_t NMMBCount, MBCount, ABCount, DSCount, WDPCount; - helper::CopyFromBuffer(Aggregate, Position, &NMMBCount); + helper::CopyFromBuffer(Aggregate.data(), Position, &NMMBCount); for (uint64_t i = 0; i < NMMBCount; i++) { uint64_t IDLen; uint64_t InfoLen; - helper::CopyFromBuffer(Aggregate, Position, &IDLen); - helper::CopyFromBuffer(Aggregate, Position, &InfoLen); + helper::CopyFromBuffer(Aggregate.data(), Position, &IDLen); + helper::CopyFromBuffer(Aggregate.data(), Position, &InfoLen); uint64_t IDPosition = Position; uint64_t InfoPosition = Position + IDLen; Position = InfoPosition + InfoLen; @@ -1592,33 +1592,33 @@ std::vector BP5Serializer::BreakoutContiguousMetadata( UniqueMetaMetaBlocks.push_back(New); } } - helper::CopyFromBuffer(Aggregate, Position, &MBCount); + helper::CopyFromBuffer(Aggregate.data(), Position, &MBCount); for (uint64_t i = 0; i < MBCount; ++i) { uint64_t MEBSize; - helper::CopyFromBuffer(Aggregate, Position, &MEBSize); + helper::CopyFromBuffer(Aggregate.data(), Position, &MEBSize); MetadataBlocks.push_back({Aggregate.data() + Position, MEBSize}); Position += MEBSize; } - helper::CopyFromBuffer(Aggregate, Position, &ABCount); + helper::CopyFromBuffer(Aggregate.data(), Position, &ABCount); for (uint64_t i = 0; i < ABCount; ++i) { uint64_t AEBSize; - helper::CopyFromBuffer(Aggregate, Position, &AEBSize); + helper::CopyFromBuffer(Aggregate.data(), Position, &AEBSize); AttributeBlocks.push_back({Aggregate.data() + Position, AEBSize}); Position += AEBSize; } uint64_t element; - helper::CopyFromBuffer(Aggregate, Position, &DSCount); + helper::CopyFromBuffer(Aggregate.data(), Position, &DSCount); for (uint64_t i = 0; i < DSCount; ++i) { - helper::CopyFromBuffer(Aggregate, Position, &element); + helper::CopyFromBuffer(Aggregate.data(), Position, &element); DataSizes.push_back(element); } - helper::CopyFromBuffer(Aggregate, Position, &WDPCount); + helper::CopyFromBuffer(Aggregate.data(), Position, &WDPCount); for (uint64_t i = 0; i < WDPCount; ++i) { - helper::CopyFromBuffer(Aggregate, Position, &element); + helper::CopyFromBuffer(Aggregate.data(), Position, &element); WriterDataPositions.push_back(element); } } diff --git a/source/adios2/toolkit/format/buffer/heap/BufferMalloc.cpp b/source/adios2/toolkit/format/buffer/heap/BufferMalloc.cpp new file mode 100644 index 0000000000..f68a73ccd0 --- /dev/null +++ b/source/adios2/toolkit/format/buffer/heap/BufferMalloc.cpp @@ -0,0 +1,101 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * BufferMalloc.cpp + * + */ + +#include "BufferMalloc.h" +#include "BufferMalloc.tcc" +#include "adios2/helper/adiosLog.h" +#include "time.h" +#include +#include +#include + +namespace adios2 +{ +namespace format +{ + +BufferMalloc::BufferMalloc() : Buffer("BufferMalloc") {} +BufferMalloc::~BufferMalloc() +{ + if (m_size) + free(m_buffer); +} + +char *BufferMalloc::Data() noexcept { return (char *)m_buffer; } + +const char *BufferMalloc::Data() const noexcept { return (const char *)m_buffer; } + +void BufferMalloc::Resize(const size_t size, const std::string hint) +{ + if (size == 0) + return; + if (m_size == 0) + { + m_buffer = (char *)malloc(size); + if (m_buffer == NULL) + helper::ThrowNested( + "Toolkit::Format", "buffer::heap::BufferMalloc", "BufferSystemV", + "buffer overflow when resizing to " + std::to_string(size) + " bytes, " + hint); + } + else + { + char *tmp = (char *)realloc(m_buffer, size); + if (tmp == NULL) + helper::ThrowNested( + "Toolkit::Format", "buffer::heap::BufferMalloc", "BufferSystemV", + "buffer overflow when resizing to " + std::to_string(size) + " bytes, " + hint); + m_buffer = tmp; + } + m_size = size; +} + +void BufferMalloc::Reset(const bool resetAbsolutePosition, const bool zeroInitialize) +{ + m_Position = 0; + if (resetAbsolutePosition) + { + m_AbsolutePosition = 0; + } + if (zeroInitialize) + { + // std::fill(m_Buffer.begin(), m_Buffer.end(), 0); + } + else + { + // just zero out the first and last 1kb + const size_t bufsize = m_size; + size_t s = (bufsize < 1024 ? bufsize : 1024); + std::fill_n(m_buffer, s, 0); + if (bufsize > 1024) + { + size_t pos = bufsize - 1024; + if (pos < 1024) + { + pos = 1024; + } + s = bufsize - pos; + std::fill_n(m_buffer + bufsize - pos, s, 0); + } + } +} + +size_t BufferMalloc::GetAvailableSize() const { return m_size - m_Position; } + +void BufferMalloc::Delete() +{ + if (m_size) + free(m_buffer); + m_size = 0; +} + +size_t BufferMalloc::DebugGetSize() const { return m_size; }; + +size_t BufferMalloc::Size() const { return m_size; }; + +} // end namespace format +} // end namespace adios2 diff --git a/source/adios2/toolkit/format/buffer/heap/BufferMalloc.h b/source/adios2/toolkit/format/buffer/heap/BufferMalloc.h new file mode 100644 index 0000000000..0f64a786c1 --- /dev/null +++ b/source/adios2/toolkit/format/buffer/heap/BufferMalloc.h @@ -0,0 +1,52 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * BufferMalloc.h + * + */ + +#ifndef ADIOS2_TOOLKIT_FORMAT_BUFFER_HEAP_BUFFERMALLOC_H_ +#define ADIOS2_TOOLKIT_FORMAT_BUFFER_HEAP_BUFFERMALLOC_H_ + +#include "adios2/toolkit/format/buffer/Buffer.h" + +#include "adios2/common/ADIOSMacros.h" + +namespace adios2 +{ +namespace format +{ + +class BufferMalloc : public Buffer +{ +public: + BufferMalloc(); + ~BufferMalloc(); + + char *Data() noexcept final; + const char *Data() const noexcept final; + + void Resize(const size_t size, const std::string hint) final; + + void Reset(const bool resetAbsolutePosition, const bool zeroInitialize) final; + + size_t GetAvailableSize() const final; + + template + size_t Align() const noexcept; + + void Delete(); + + size_t DebugGetSize() const; + size_t Size() const; + +private: + size_t m_size = 0; + char *m_buffer = NULL; +}; + +} // end namespace format +} // end namespace adios2 + +#endif /* ADIOS2_TOOLKIT_FORMAT_BUFFER_HEAP_BUFFERMALLOC_H_ */ diff --git a/source/adios2/toolkit/format/buffer/heap/BufferMalloc.tcc b/source/adios2/toolkit/format/buffer/heap/BufferMalloc.tcc new file mode 100644 index 0000000000..c2cbf63fb2 --- /dev/null +++ b/source/adios2/toolkit/format/buffer/heap/BufferMalloc.tcc @@ -0,0 +1,54 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * BufferMalloc.tcc + * + */ + +#ifndef ADIOS2_TOOLKIT_FORMAT_BUFFER_HEAP_BUFFERMALLOC_TCC_ +#define ADIOS2_TOOLKIT_FORMAT_BUFFER_HEAP_BUFFERMALLOC_TCC_ + +#include "BufferMalloc.h" + +#include + +#ifdef _WIN32 +#pragma warning(disable : 4146) // Windows complains about unsigned minus +#endif + +namespace adios2 +{ +namespace format +{ + +template +size_t BufferMalloc::Align() const noexcept +{ + // std::align implementation from llvm libc++ + // needed due to bug in gcc 4.8 + auto lf_align = [](const size_t alignment, const size_t size, void *&ptr, size_t &space) { + if (size <= space) + { + const char *p1 = static_cast(ptr); + const char *p2 = reinterpret_cast( + reinterpret_cast(p1 + (alignment - 1)) & -alignment); + const size_t d = static_cast(p2 - p1); + if (d <= space - size) + { + space -= d; + } + } + }; + + void *currentAddress = + reinterpret_cast(const_cast((char *)Data() + m_Position)); + size_t size = GetAvailableSize(); + lf_align(alignof(T), sizeof(T), currentAddress, size); + return GetAvailableSize() - size; +} + +} // end namespace format +} // end namespace adios2 + +#endif /* ADIOS2_TOOLKIT_FORMAT_BUFFER_HEAP_BUFFERMALLOC_TCC_ */ From 2da6d4d51b192b4a63b98114cf51946867d9a1b1 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Wed, 31 Jan 2024 15:07:51 -0500 Subject: [PATCH 037/124] fix IO.available_attributes(), remove a debug print from Stream, fix examples printing, fix el8 python3 build --- examples/hello/sstReader/sstReader-bindings.py | 10 +++++++++- examples/hello/sstReader/sstReader.py | 1 + examples/hello/sstWriter/sstWriter-bindings.py | 2 +- examples/hello/sstWriter/sstWriter.py | 2 +- python/adios2/io.py | 5 +---- python/adios2/stream.py | 4 ---- scripts/ci/images/Dockerfile.ci-el8-intel | 2 +- 7 files changed, 14 insertions(+), 12 deletions(-) diff --git a/examples/hello/sstReader/sstReader-bindings.py b/examples/hello/sstReader/sstReader-bindings.py index 38ed1ace7f..efaa06e352 100644 --- a/examples/hello/sstReader/sstReader-bindings.py +++ b/examples/hello/sstReader/sstReader-bindings.py @@ -36,7 +36,15 @@ sstReader.EndStep() print( - "Rank=", rank, "loop index =", loopStep, "stream step =", currentStep, "data =", floatArray + "Rank=", + rank, + "loop index =", + loopStep, + "stream step =", + currentStep, + "data =", + floatArray, + flush=True, ) loopStep = loopStep + 1 diff --git a/examples/hello/sstReader/sstReader.py b/examples/hello/sstReader/sstReader.py index 93a7117ee3..afd16820da 100644 --- a/examples/hello/sstReader/sstReader.py +++ b/examples/hello/sstReader/sstReader.py @@ -34,4 +34,5 @@ currentStep, "data =", floatArray, + flush=True, ) diff --git a/examples/hello/sstWriter/sstWriter-bindings.py b/examples/hello/sstWriter/sstWriter-bindings.py index ede96042eb..c28e63d7a6 100644 --- a/examples/hello/sstWriter/sstWriter-bindings.py +++ b/examples/hello/sstWriter/sstWriter-bindings.py @@ -33,7 +33,7 @@ sstWriter = sstIO.Open("helloSst", adios2.Mode.Write) for i in range(4): - print("Rank=", rank, "loop index =", i, "data =", myArray) + print("Rank=", rank, "loop index =", i, "data =", myArray, flush=True) sstWriter.BeginStep() sstWriter.Put(ioArray, myArray, adios2.Mode.Sync) myArray += increment diff --git a/examples/hello/sstWriter/sstWriter.py b/examples/hello/sstWriter/sstWriter.py index f3a085a9cd..3cace4fff6 100644 --- a/examples/hello/sstWriter/sstWriter.py +++ b/examples/hello/sstWriter/sstWriter.py @@ -34,7 +34,7 @@ sleep(1.0) stream.write("bpFloats", myArray, [size * nx], [rank * nx], [nx]) - print("Rank=", rank, "loop index =", currentStep, "data =", myArray) + print("Rank=", rank, "loop index =", currentStep, "data =", myArray, flush=True) myArray += increment # Warning: the data of the current step is not published until # the next loop entry or the exit of the loop diff --git a/python/adios2/io.py b/python/adios2/io.py index 09d9c2520a..c8f265ea38 100644 --- a/python/adios2/io.py +++ b/python/adios2/io.py @@ -103,10 +103,7 @@ def available_attributes(self): value attribute information dictionary """ - attributes = {} - for name, attr in self.impl.AvailableAttributes(): - attributes[name] = Attribute(attr, name) - return attributes + return self.impl.AvailableAttributes() def remove_attribute(self, name): """ diff --git a/python/adios2/stream.py b/python/adios2/stream.py index 5e34f04c93..99cffccd30 100644 --- a/python/adios2/stream.py +++ b/python/adios2/stream.py @@ -311,10 +311,6 @@ def read(self, variable: Variable): output_shape = np.array(count) output_shape[0] *= steps - print( - f"Stream.read variable {variable.name()} dtype = {dtype} " - f"shape = {output_shape}, steps = {variable.steps()}" - ) else: # scalar output_shape = (variable.selection_size(),) diff --git a/scripts/ci/images/Dockerfile.ci-el8-intel b/scripts/ci/images/Dockerfile.ci-el8-intel index c68f6ff517..23801ce56d 100644 --- a/scripts/ci/images/Dockerfile.ci-el8-intel +++ b/scripts/ci/images/Dockerfile.ci-el8-intel @@ -27,7 +27,7 @@ RUN dnf install -y \ patch \ patchelf \ python39-devel \ - python3-pip \ + python39-pip \ tar \ tcl \ unzip \ From e46f1f95386d8edafdcbd1fecacc792da26c3e78 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Mon, 5 Feb 2024 14:22:21 -0500 Subject: [PATCH 038/124] Extend python tests with string and string list inputs. Fix IO.define_attribute() to handle list of strings as well as single values (converted to numpy arrays) --- python/adios2/io.py | 10 +++++++++- .../bindings/python/TestBPWriteReadTypes.py | 13 +++++++++++-- testing/adios2/python/TestIO.py | 16 +++++++++++++--- 3 files changed, 33 insertions(+), 6 deletions(-) diff --git a/python/adios2/io.py b/python/adios2/io.py index c8f265ea38..229b6b8cf9 100644 --- a/python/adios2/io.py +++ b/python/adios2/io.py @@ -66,7 +66,15 @@ def define_attribute( e.g. variable_name + separator + name ("var/attr") Not used if variable_name is empty """ - return Attribute(self.impl, name, content, variable_name, separator) + + # string or list of strings passed on as is + if isinstance(content, list) and isinstance(content[0], str): + return Attribute(self.impl, name, content, variable_name, separator) + if isinstance(content, str): + return Attribute(self.impl, name, content, variable_name, separator) + + # python values (single or list) needs to be passed as a numpy array + return Attribute(self.impl, name, np.asarray(content), variable_name, separator) def inquire_attribute(self, name, variable_name="", separator="/"): """ diff --git a/testing/adios2/bindings/python/TestBPWriteReadTypes.py b/testing/adios2/bindings/python/TestBPWriteReadTypes.py index 829ace519d..399758bdb1 100644 --- a/testing/adios2/bindings/python/TestBPWriteReadTypes.py +++ b/testing/adios2/bindings/python/TestBPWriteReadTypes.py @@ -38,7 +38,7 @@ def check_array(np1, np2, hint): Nx = 8 # list of tested attributes and variables -attr_names = ["attrString", "attrI8", "attrI16", "attrI32", "attrI64", +attr_names = ["attrString", "attrStringArray", "attrI8", "attrI16", "attrI32", "attrI64", "attrU8", "attrU16", "attrU32", "attrU64", "attrR32", "attrR64"] var_names = ["varStr", "varI8", "varI16", "varI32", "varI64", "varU8", "varU16", "varU32", "varU64", @@ -82,7 +82,8 @@ def check_array(np1, np2, hint): varR64 = ioWriter.DefineVariable( "varR64", data.R64, shape, start, count, adios2.ConstantDims) -attString = ioWriter.DefineAttribute("attrString", ["one", "two", "three"]) +attString = ioWriter.DefineAttribute("attrString", "one") +attStringArray = ioWriter.DefineAttribute("attrStringArray", ["one", "two", "three"]) attI8 = ioWriter.DefineAttribute("attrI8", data.I8) attI16 = ioWriter.DefineAttribute("attrI16", data.I16) attI32 = ioWriter.DefineAttribute("attrI32", data.I32) @@ -146,6 +147,7 @@ def check_array(np1, np2, hint): reader = ioReader.Open("npTypes.bp", adios2.Mode.ReadRandomAccess) attrString = ioReader.InquireAttribute("attrString") +attrStringArray = ioReader.InquireAttribute("attrStringArray") attrI8 = ioReader.InquireAttribute("attrI8") attrI16 = ioReader.InquireAttribute("attrI16") attrI32 = ioReader.InquireAttribute("attrI32") @@ -158,6 +160,7 @@ def check_array(np1, np2, hint): attrR64 = ioReader.InquireAttribute("attrR64") check_object(attrString, "attrString") +check_object(attrStringArray, "attrStringArray") check_object(attrI8, "attrI8") check_object(attrI16, "attrI16") check_object(attrI32, "attrI32") @@ -170,6 +173,12 @@ def check_array(np1, np2, hint): check_object(attrR64, "attrR64") attrStringData = attrString.DataString() +print(f"attrString = {attrStringData}", flush=True) +if attrStringData[0] != "one": + raise ValueError('attrString failed') + +attrStringData = attrStringArray.DataString() +print(f"attrStringArray = {attrStringData}", flush=True) if attrStringData[0] != "one": raise ValueError('attrStringData[0] failed') if attrStringData[1] != "two": diff --git a/testing/adios2/python/TestIO.py b/testing/adios2/python/TestIO.py index 6da95c47f1..5462d3a9f4 100644 --- a/testing/adios2/python/TestIO.py +++ b/testing/adios2/python/TestIO.py @@ -24,13 +24,22 @@ def test_io_inquire_attribute(self): self.assertNotEqual(ts, coords) self.assertNotEqual(ts, x) self.assertEqual(coords, x) + self.assertIs(io.inquire_attribute("notanattribute"), None) - def test_available_attribute(self): + def test_available_attributes(self): adios = Adios() io = adios.declare_io("BPWriter") io.define_attribute("timestamp", "20231122") - io.inquire_attribute("timestamp") - self.assertIs(io.inquire_attribute("coords"), None) + io.define_attribute("stringarray", ["one", "two", "three"]) + io.define_attribute("afloat", 3.14) + io.define_attribute("floatarray", [3.14, 6.28]) + attrs = io.available_attributes() + print("Available attributes:") + for aname, ainfo in attrs.items(): + print(f" {aname}:\t{ainfo}") + self.assertEqual(attrs['timestamp']['Value'], '"20231122"') + self.assertEqual(attrs['stringarray']['Value'], '{ "one", "two", "three" }') + self.assertFalse('coords' in attrs) def test_remove_attribute(self): adios = Adios() @@ -96,6 +105,7 @@ def test_open_engine(self): io.add_transport("File", {"Library": "POSIX"}) engine = io.open("pythontest.bp", bindings.Mode.Write) self.assertNotEqual(engine, None) + engine.close() if __name__ == "__main__": From 4222f1f5b3060686adfc0ec3864a38e5d567a9c2 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Mon, 5 Feb 2024 14:27:12 -0500 Subject: [PATCH 039/124] format --- pyproject.toml | 3 +++ testing/adios2/python/TestIO.py | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index be271f2e9e..ad420b5e97 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,6 +48,9 @@ Documentation = "https://adios2.readthedocs.io/" Discussions = "https://github.com/ornladios/ADIOS2/discussions" Changelog = "https://github.com/ornladios/ADIOS2/releases" +[project.entry-points."xarray.backends"] +adios = "adios2.xarraybackend:AdiosBackendEntrypoint" + [tool.cibuildwheel] # Trigger an install of the package, and run a basic test test-command = "python -m unittest adios2.test.simple_read_write.TestSimpleReadWrite" diff --git a/testing/adios2/python/TestIO.py b/testing/adios2/python/TestIO.py index 5462d3a9f4..76d369c66e 100644 --- a/testing/adios2/python/TestIO.py +++ b/testing/adios2/python/TestIO.py @@ -37,9 +37,9 @@ def test_available_attributes(self): print("Available attributes:") for aname, ainfo in attrs.items(): print(f" {aname}:\t{ainfo}") - self.assertEqual(attrs['timestamp']['Value'], '"20231122"') - self.assertEqual(attrs['stringarray']['Value'], '{ "one", "two", "three" }') - self.assertFalse('coords' in attrs) + self.assertEqual(attrs["timestamp"]["Value"], '"20231122"') + self.assertEqual(attrs["stringarray"]["Value"], '{ "one", "two", "three" }') + self.assertFalse("coords" in attrs) def test_remove_attribute(self): adios = Adios() From 65612a49091b956cfe11e43a02eacd57ff53e95b Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Thu, 8 Feb 2024 15:33:49 -0500 Subject: [PATCH 040/124] fix FileReader init, no need for mode. Add Stream.inquire_attribute() --- python/adios2/file_reader.py | 2 +- python/adios2/stream.py | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/python/adios2/file_reader.py b/python/adios2/file_reader.py index 795cf9ab75..8a9345f053 100644 --- a/python/adios2/file_reader.py +++ b/python/adios2/file_reader.py @@ -19,7 +19,7 @@ def __init__(self, path, comm=None): # e.g. FileReader(io: adios2.IO, path, mode) # pylint: disable=E1121 @__init__.register(IO) - def _(self, io: IO, path, mode, comm=None): + def _(self, io: IO, path, comm=None): super().__init__(io, path, "rra", comm) # pylint: enable=E1121 diff --git a/python/adios2/stream.py b/python/adios2/stream.py index 99cffccd30..d737864290 100644 --- a/python/adios2/stream.py +++ b/python/adios2/stream.py @@ -202,6 +202,27 @@ def inquire_variable(self, name): """ return self._io.inquire_variable(name) + def inquire_attribute(self, name, variable_name="", separator="/"): + """ + Inquire an attribute + + Parameters + name + attribute name + + variable_name + if attribute is associated with a variable + + separator + concatenation string between variable_name and attribute + e.g. variable_name + separator + name ("var/attr") + Not used if variable_name is empty + + Returns + The attribute if it is defined, otherwise None + """ + return self._io.inquire_attribute(name, variable_name, separator) + @singledispatchmethod def write(self, variable: Variable, content): """ From 886a19ca887d049c7addc51c848a3e9ead8352a3 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Thu, 8 Feb 2024 15:50:55 -0500 Subject: [PATCH 041/124] Fix check in io.py. Disable pyling W0221 in file_reader.py since we deliberately decrease the number of arguments for the sake of usability. --- python/adios2/file_reader.py | 2 +- python/adios2/io.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/adios2/file_reader.py b/python/adios2/file_reader.py index 8a9345f053..ac40791a6f 100644 --- a/python/adios2/file_reader.py +++ b/python/adios2/file_reader.py @@ -5,7 +5,7 @@ from functools import singledispatchmethod from adios2 import Stream, IO - +# pylint: disable=W0221 class FileReader(Stream): """High level implementation of the FileReader class for read Random access mode""" diff --git a/python/adios2/io.py b/python/adios2/io.py index 229b6b8cf9..b493de72bc 100644 --- a/python/adios2/io.py +++ b/python/adios2/io.py @@ -68,7 +68,7 @@ def define_attribute( """ # string or list of strings passed on as is - if isinstance(content, list) and isinstance(content[0], str): + if isinstance(content, list) and len(content) > 0 and isinstance(content[0], str): return Attribute(self.impl, name, content, variable_name, separator) if isinstance(content, str): return Attribute(self.impl, name, content, variable_name, separator) From df3a6bb9624248e86f4810cad35436e414031505 Mon Sep 17 00:00:00 2001 From: Dmitry Ganyushin Date: Mon, 19 Feb 2024 17:38:15 -0500 Subject: [PATCH 042/124] If table exists, it will be updated --- source/adios2/engine/campaign/CampaignManager.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/adios2/engine/campaign/CampaignManager.cpp b/source/adios2/engine/campaign/CampaignManager.cpp index 5a218b68b8..0289a07a0b 100644 --- a/source/adios2/engine/campaign/CampaignManager.cpp +++ b/source/adios2/engine/campaign/CampaignManager.cpp @@ -30,7 +30,7 @@ int CMapToSqlite(const CampaignRecordMap &cmap, const int rank, std::string name { sqlite3 *db; int rc; - char *zErrMsg = 0; + char *zErrMsg = nullptr; std::string sqlcmd; std::string db_name = name + ".db"; rc = sqlite3_open(db_name.c_str(), &db); @@ -43,7 +43,7 @@ int CMapToSqlite(const CampaignRecordMap &cmap, const int rank, std::string name "SQL error on writing records:"); sqlite3_free(zErrMsg); } - sqlcmd = "CREATE TABLE bpfiles (name);"; + sqlcmd = "CREATE TABLE if not exists bpfiles (name);"; rc = sqlite3_exec(db, sqlcmd.c_str(), 0, 0, &zErrMsg); if (rc != SQLITE_OK) { @@ -57,8 +57,8 @@ int CMapToSqlite(const CampaignRecordMap &cmap, const int rank, std::string name size_t rowid = 1000; for (auto &r : cmap) { - sqlcmd = "INSERT INTO bpfiles (rowid, name)\n"; - sqlcmd += "VALUES(" + std::to_string(rowid) + "," + "'" + r.first + "'" + ");"; + sqlcmd = "INSERT INTO bpfiles (name)\n"; + sqlcmd += "VALUES('" + r.first + "');"; rowid++; rc = sqlite3_exec(db, sqlcmd.c_str(), 0, 0, &zErrMsg); if (rc != SQLITE_OK) From 13d83fcaa6502c0e0445dce8ef5d6d2faf5d1c43 Mon Sep 17 00:00:00 2001 From: dmitry-ganyushin Date: Tue, 20 Feb 2024 09:45:50 -0500 Subject: [PATCH 043/124] Fixed campaign python script. Clean-up removed rowid --- source/adios2/engine/campaign/CampaignManager.cpp | 2 -- .../adios_campaign_manager/adios2_campaign_manager.py | 7 +++---- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/source/adios2/engine/campaign/CampaignManager.cpp b/source/adios2/engine/campaign/CampaignManager.cpp index 0289a07a0b..2b963f5907 100644 --- a/source/adios2/engine/campaign/CampaignManager.cpp +++ b/source/adios2/engine/campaign/CampaignManager.cpp @@ -54,12 +54,10 @@ int CMapToSqlite(const CampaignRecordMap &cmap, const int rank, std::string name sqlite3_free(zErrMsg); } - size_t rowid = 1000; for (auto &r : cmap) { sqlcmd = "INSERT INTO bpfiles (name)\n"; sqlcmd += "VALUES('" + r.first + "');"; - rowid++; rc = sqlite3_exec(db, sqlcmd.c_str(), 0, 0, &zErrMsg); if (rc != SQLITE_OK) { diff --git a/source/utils/adios_campaign_manager/adios2_campaign_manager.py b/source/utils/adios_campaign_manager/adios2_campaign_manager.py index 7122287767..c752d95aac 100755 --- a/source/utils/adios_campaign_manager/adios2_campaign_manager.py +++ b/source/utils/adios_campaign_manager/adios2_campaign_manager.py @@ -2,7 +2,6 @@ import argparse import glob -import json import sqlite3 import zlib from datetime import datetime @@ -145,7 +144,7 @@ def AddDatasetToArchive(args: dict, dataset: str, cur: sqlite3.Cursor, hostID: i print(f"WARNING: Dataset {dataset} is not an ADIOS dataset. Skip") -def ProcessJsonFile(args: dict, jsonlist: list, cur: sqlite3.Cursor, hostID: int, dirID: int): +def ProcessDBFile(args: dict, jsonlist: list, cur: sqlite3.Cursor, hostID: int, dirID: int): for entry in jsonlist: # print(f"Process entry {entry}:") if isinstance(entry, dict): @@ -217,7 +216,7 @@ def Update(args: dict, cur: sqlite3.Cursor): # (shortHostName, longHostName)) # hostID = curHost.lastrowid - curDir = cur.execute('insert into directory values (?, ?)', + curDir = cur.execute('insert or replace into directory values (?, ?)', (hostID, rootdir)) dirID = curDir.lastrowid con.commit() @@ -225,7 +224,7 @@ def Update(args: dict, cur: sqlite3.Cursor): db_list = MergeDBFiles(dbFileList) # print(f"Merged json = {jsonlist}") - ProcessJsonFile(args, db_list, cur, hostID, dirID) + ProcessDBFile(args, db_list, cur, hostID, dirID) con.commit() From 7e88591ab8a50383e08150bb70fcfc96e3924a5b Mon Sep 17 00:00:00 2001 From: Greg Eisenhauer Date: Tue, 20 Feb 2024 14:42:20 -0500 Subject: [PATCH 044/124] Fix Windows Testing build (#4027) --- cmake/ADIOSFunctions.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/ADIOSFunctions.cmake b/cmake/ADIOSFunctions.cmake index 3f8fa8a8e5..b0e4f070d8 100644 --- a/cmake/ADIOSFunctions.cmake +++ b/cmake/ADIOSFunctions.cmake @@ -91,7 +91,7 @@ function(python_add_test) set_property(TEST ${ARGS_NAME} PROPERTY ENVIRONMENT "PYTHONPATH=${ADIOS2_BINARY_DIR}/${CMAKE_INSTALL_PYTHONDIR};$ENV{PYTHONPATH}" - "PATH=$;$ENV{PATH}" + "PATH=$;$ENV{PATH}" ) endif() endfunction() From eb2cee8a36c4be5c98bbf77c90939c2dd156ceec Mon Sep 17 00:00:00 2001 From: Greg Eisenhauer Date: Tue, 20 Feb 2024 14:46:49 -0500 Subject: [PATCH 045/124] Tweak POSIX fail on EOF to kill heisenbug (#4035) Tweak POSIX fail on EOF to kill heisenbug --- .../toolkit/transport/file/FilePOSIX.cpp | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/source/adios2/toolkit/transport/file/FilePOSIX.cpp b/source/adios2/toolkit/transport/file/FilePOSIX.cpp index d3abbaed21..3029667313 100644 --- a/source/adios2/toolkit/transport/file/FilePOSIX.cpp +++ b/source/adios2/toolkit/transport/file/FilePOSIX.cpp @@ -1,6 +1,4 @@ /* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. * * FilePOSIX.cpp file I/O using POSIX I/O library * @@ -437,9 +435,19 @@ void FilePOSIX::Read(char *buffer, size_t size, size_t start) { if (m_FailOnEOF) { - helper::Throw( - "Toolkit", "transport::file::FilePOSIX", "Read", - "Read past end of file on " + m_Name + " " + SysErrMsg()); + // we got an EOF on data that *should* be present, + // but maybe we've got filesystem consistency + // issues. We'll wait, but if the backoff time + // reaches 30 seconds (nearly 45 seconds total + // wait time) and we still don't have data, treat + // this as a real failure and throw an exception. + std::this_thread::sleep_for(std::chrono::nanoseconds(backoff_ns)); + backoff_ns *= 2; + if (std::chrono::nanoseconds(backoff_ns) > std::chrono::seconds(30)) + helper::Throw( + "Toolkit", "transport::file::FilePOSIX", "Read", + "Read past end of file on " + m_Name + " trying to read " + + std::to_string(size) + " bytes " + SysErrMsg()); } else { From 532d514adf5fa4b5a1100ed3598b1ba42d477ec2 Mon Sep 17 00:00:00 2001 From: Vicente Adolfo Bolea Sanchez Date: Tue, 20 Feb 2024 17:51:22 -0500 Subject: [PATCH 046/124] ci: add macos static build --- .github/workflows/everything.yml | 24 +++++---- ...ci-macos-11-xcode13_0-shared-serial.cmake} | 0 ...-macos-12-xcode13_4_1-shared-serial.cmake} | 0 ...i-macos-12-xcode13_4_1-static-serial.cmake | 52 +++++++++++++++++++ .../cmake/ci-macos1015-xcode1211-make.cmake | 24 --------- .../ci/cmake/ci-macos11-xcode131-ninja.cmake | 28 ---------- 6 files changed, 65 insertions(+), 63 deletions(-) rename scripts/ci/cmake/{ci-macos11-xcode13_0-serial.cmake => ci-macos-11-xcode13_0-shared-serial.cmake} (100%) rename scripts/ci/cmake/{ci-macos12-xcode13_4_1-serial.cmake => ci-macos-12-xcode13_4_1-shared-serial.cmake} (100%) create mode 100644 scripts/ci/cmake/ci-macos-12-xcode13_4_1-static-serial.cmake delete mode 100644 scripts/ci/cmake/ci-macos1015-xcode1211-make.cmake delete mode 100644 scripts/ci/cmake/ci-macos11-xcode131-ninja.cmake diff --git a/.github/workflows/everything.yml b/.github/workflows/everything.yml index e45b8a5f7e..be966def07 100644 --- a/.github/workflows/everything.yml +++ b/.github/workflows/everything.yml @@ -270,9 +270,9 @@ jobs: runs-on: ${{ matrix.image }} env: - GH_YML_JOBNAME: ${{ matrix.os }}-${{ matrix.compiler }}-${{ matrix.parallel }} + GH_YML_JOBNAME: ${{ matrix.image }}-${{ matrix.compiler }}-${{ matrix.shared }}-${{ matrix.parallel }} GH_YML_BASE_OS: macOS - GH_YML_MATRIX_OS: ${{ matrix.os }} + GH_YML_MATRIX_OS: ${{ matrix.image }} GH_YML_MATRIX_COMPILER: ${{ matrix.compiler }} GH_YML_MATRIX_PARALLEL: ${{ matrix.parallel }} CCACHE_BASEDIR: "${GITHUB_WORKSPACE}" @@ -283,15 +283,17 @@ jobs: strategy: fail-fast: false matrix: - os: [macos11, macos12] - parallel: [serial] + image: [macos-11, macos-12] + shared: [static, shared] include: - - os: macos11 - image: macos-11 + - parallel: serial + - image: macos-11 compiler: xcode13_0 - - os: macos12 - image: macos-12 + - image: macos-12 compiler: xcode13_4_1 + exclude: + - image: macos-11 + shared: static steps: - uses: actions/checkout@v4 @@ -308,9 +310,9 @@ jobs: id: restore-cache with: path: .ccache - key: ccache-${{ matrix.os }}-${{ matrix.compiler }}-${{ matrix.parallel }}-${{ github.sha }} + key: ccache-${{ matrix.image}}-${{ matrix.compiler }}-${{ matrix.parallel }}-${{ github.sha }} restore-keys: | - ccache-${{ matrix.os }}-${{ matrix.compiler }}-${{ matrix.parallel }} + ccache-${{ matrix.image }}-${{ matrix.compiler }}-${{ matrix.parallel }} - name: Configure cache run: ccache -z - name: Update @@ -327,7 +329,7 @@ jobs: id: save-cache with: path: .ccache - key: ccache-${{ matrix.os }}-${{ matrix.compiler }}-${{ matrix.parallel }}-${{ github.sha }} + key: ccache-${{ matrix.image }}-${{ matrix.compiler }}-${{ matrix.parallel }}-${{ github.sha }} - name: Test run: gha/scripts/ci/gh-actions/run.sh test diff --git a/scripts/ci/cmake/ci-macos11-xcode13_0-serial.cmake b/scripts/ci/cmake/ci-macos-11-xcode13_0-shared-serial.cmake similarity index 100% rename from scripts/ci/cmake/ci-macos11-xcode13_0-serial.cmake rename to scripts/ci/cmake/ci-macos-11-xcode13_0-shared-serial.cmake diff --git a/scripts/ci/cmake/ci-macos12-xcode13_4_1-serial.cmake b/scripts/ci/cmake/ci-macos-12-xcode13_4_1-shared-serial.cmake similarity index 100% rename from scripts/ci/cmake/ci-macos12-xcode13_4_1-serial.cmake rename to scripts/ci/cmake/ci-macos-12-xcode13_4_1-shared-serial.cmake diff --git a/scripts/ci/cmake/ci-macos-12-xcode13_4_1-static-serial.cmake b/scripts/ci/cmake/ci-macos-12-xcode13_4_1-static-serial.cmake new file mode 100644 index 0000000000..1c10c3a157 --- /dev/null +++ b/scripts/ci/cmake/ci-macos-12-xcode13_4_1-static-serial.cmake @@ -0,0 +1,52 @@ +# Client maintainer: vicente.bolea@kitware.com +set(ENV{CC} clang) +set(ENV{CXX} clang++) +set(ENV{FC} gfortran-11) + +set(dashboard_cache " +BUILD_SHARED_LIBS=OFF +CMAKE_BUILD_TYPE:STRING=Release +BUILD_TESTING:BOOL=ON +ADIOS2_BUILD_EXAMPLES:BOOL=OFF + +ADIOS2_USE_AWSSDK:STRING=OFF +ADIOS2_USE_Blosc2:STRING=OFF +ADIOS2_USE_Blosc:BOOL=OFF +ADIOS2_USE_BZip2:BOOL=OFF +ADIOS2_USE_Catalyst:STRING=OFF +ADIOS2_USE_CUDA:STRING=OFF +ADIOS2_USE_DAOS:STRING=OFF +ADIOS2_USE_DataMan:BOOL=OFF +ADIOS2_USE_DataSpaces:STRING=OFF +ADIOS2_USE_Fortran:BOOL=OFF +ADIOS2_USE_HDF5:BOOL=OFF +ADIOS2_USE_HDF5_VOL:STRING=OFF +ADIOS2_USE_IME:STRING=OFF +ADIOS2_USE_Kokkos:STRING=OFF +ADIOS2_USE_LIBPRESSIO:STRING=OFF +ADIOS2_USE_MGARD:STRING=OFF +ADIOS2_USE_MHS:STRING=OFF +ADIOS2_USE_MPI:BOOL=OFF +ADIOS2_USE_PNG:BOOL=OFF +ADIOS2_USE_Python:BOOL=OFF +ADIOS2_USE_SSC:BOOL=OFF +ADIOS2_USE_SST:BOOL=OFF +ADIOS2_USE_SZ:BOOL=OFF +ADIOS2_USE_ZeroMQ:STRING=OFF +ADIOS2_USE_ZFP:BOOL=OFF + +CMAKE_C_COMPILER_LAUNCHER=ccache +CMAKE_CXX_COMPILER_LAUNCHER=ccache +CMAKE_C_FLAGS:STRING=-Wall +CMAKE_CXX_FLAGS:STRING=-Wall +CMAKE_Fortran_FLAGS:STRING=-Wall +") + +set(ENV{MACOSX_DEPLOYMENT_TARGET} "12.1") +set(CTEST_CMAKE_GENERATOR "Ninja") +list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") +set(CTEST_TEST_ARGS + # Unclear why these tests currently die. Disabling until it can be addressed. + EXCLUDE "Install.Make.*" +) +include(${CMAKE_CURRENT_LIST_DIR}/ci-common.cmake) diff --git a/scripts/ci/cmake/ci-macos1015-xcode1211-make.cmake b/scripts/ci/cmake/ci-macos1015-xcode1211-make.cmake deleted file mode 100644 index 3bed8581cf..0000000000 --- a/scripts/ci/cmake/ci-macos1015-xcode1211-make.cmake +++ /dev/null @@ -1,24 +0,0 @@ -# Client maintainer: chuck.atkins@kitware.com -set(ENV{CC} clang) -set(ENV{CXX} clang++) -set(ENV{FC} gfortran) - -set(dashboard_cache " -BUILD_TESTING:BOOL=ON -ADIOS2_BUILD_EXAMPLES:BOOL=ON - -ADIOS2_USE_Fortran:BOOL=ON -ADIOS2_USE_MPI:BOOL=OFF -ADISO2_USE_Python:BOOL=ON - -CMAKE_C_COMPILER_LAUNCHER=ccache -CMAKE_CXX_COMPILER_LAUNCHER=ccache -CMAKE_C_FLAGS:STRING=-Wall -CMAKE_CXX_FLAGS:STRING=-Wall -CMAKE_Fortran_FLAGS:STRING=-Wall -") - -set(ENV{MACOSX_DEPLOYMENT_TARGET} "10.15") -set(CTEST_CMAKE_GENERATOR "Unix Makefiles") -list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") -include(${CMAKE_CURRENT_LIST_DIR}/ci-common.cmake) diff --git a/scripts/ci/cmake/ci-macos11-xcode131-ninja.cmake b/scripts/ci/cmake/ci-macos11-xcode131-ninja.cmake deleted file mode 100644 index cf9b3b9d3c..0000000000 --- a/scripts/ci/cmake/ci-macos11-xcode131-ninja.cmake +++ /dev/null @@ -1,28 +0,0 @@ -# Client maintainer: chuck.atkins@kitware.com -set(ENV{CC} clang) -set(ENV{CXX} clang++) -set(ENV{FC} gfortran) - -set(dashboard_cache " -BUILD_TESTING:BOOL=ON -ADIOS2_BUILD_EXAMPLES:BOOL=ON - -ADIOS2_USE_Fortran:BOOL=ON -ADIOS2_USE_MPI:BOOL=OFF -ADIOS2_USE_Python:BOOL=ON - -CMAKE_C_COMPILER_LAUNCHER=ccache -CMAKE_CXX_COMPILER_LAUNCHER=ccache -CMAKE_C_FLAGS:STRING=-Wall -CMAKE_CXX_FLAGS:STRING=-Wall -CMAKE_Fortran_FLAGS:STRING=-Wall -") - -set(ENV{MACOSX_DEPLOYMENT_TARGET} "12.0") -set(CTEST_CMAKE_GENERATOR "Ninja") -list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") -set(CTEST_TEST_ARGS - # Unclear why these tests currently die. Disabling until it can be addressed. - EXCLUDE "Install.Make.*" -) -include(${CMAKE_CURRENT_LIST_DIR}/ci-common.cmake) From 70be9a5f2e3945e4294a7b32714beb99b34ce7f9 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Thu, 15 Feb 2024 12:04:01 -0500 Subject: [PATCH 047/124] Python API: support writing Python numeric variables and lists (of one type) as ADIOS variables and attributes. Add extra tests for them and some in the bpWrite and bpReaderHeatMap2D examples. - Add glue to accept lists and single python value for variables and attributes. - Add test for lists and values (int, float and complex) to API tests. - Turn on HDF5 Python test (serial version) --- bindings/Python/py11Engine.cpp | 34 ++++- bindings/Python/py11Engine.h | 6 + bindings/Python/py11IO.cpp | 116 +++++++++++++- bindings/Python/py11IO.h | 21 +++ bindings/Python/py11glue.cpp | 64 ++++++++ examples/hello/bpReader/bpReaderHeatMap2D.py | 34 +++-- examples/hello/bpWriter/bpWriter-bindings.py | 5 + examples/hello/bpWriter/bpWriter.py | 3 + examples/useCases/CMakeLists.txt | 4 +- python/adios2/engine.py | 9 +- python/adios2/file_reader.py | 1 + python/adios2/io.py | 19 ++- python/adios2/stream.py | 33 ++-- .../bindings/python/TestBPBlocksInfo.py | 16 +- .../bindings/python/TestBPReadMultisteps.py | 77 +++++----- .../bindings/python/TestBPSelectSteps.py | 20 +-- .../bindings/python/TestBPWriteRead2D.py | 19 ++- .../bindings/python/TestBPWriteReadString.py | 13 +- .../bindings/python/TestBPWriteReadTypes.py | 120 ++++++++------- .../python/TestBPWriteReadTypes_nompi.py | 36 ++--- .../bindings/python/TestGetException_nompi.py | 21 ++- .../adios2/bindings/python/TestNullEngine.py | 21 +-- testing/adios2/bindings/python/TestQuery.py | 70 +++++---- .../bindings/python/TestQueryLocalArray.py | 57 +++---- .../adios2/bindings/python/adios2NPTypes.py | 75 ++++++--- testing/adios2/python/CMakeLists.txt | 5 + .../python/TestBPWriteTypesHighLevelAPI.py | 143 +++++++++++++----- .../TestBPWriteTypesHighLevelAPILocal.py | 56 +++---- .../TestBPWriteTypesHighLevelAPI_HDF5.py | 37 ++--- testing/adios2/python/TestEngine.py | 4 +- testing/adios2/python/adios2NPTypes.py | 59 +++++++- 31 files changed, 814 insertions(+), 384 deletions(-) diff --git a/bindings/Python/py11Engine.cpp b/bindings/Python/py11Engine.cpp index f4bdafef27..faa4d26716 100644 --- a/bindings/Python/py11Engine.cpp +++ b/bindings/Python/py11Engine.cpp @@ -70,12 +70,44 @@ void Engine::Put(Variable variable, const pybind11::array &array, const Mode lau else { throw std::invalid_argument("ERROR: for variable " + variable.Name() + - " numpy array type is not supported or " + " numpy array type " + variable.Type() + + " is not supported (found type " + ToString(type) + + ") or " "is not memory contiguous " ", in call to Put\n"); } } +void Engine::Put(Variable variable, const std::vector &ints, const Mode launch) +{ + helper::CheckForNullptr(m_Engine, "in call to Engine::Put list of ints"); + helper::CheckForNullptr(variable.m_VariableBase, + "for variable, in call to Engine::Put list of ints"); + + m_Engine->Put(*dynamic_cast *>(variable.m_VariableBase), + reinterpret_cast(ints.data()), launch); +} + +void Engine::Put(Variable variable, const std::vector &floats, const Mode launch) +{ + helper::CheckForNullptr(m_Engine, "in call to Engine::Put list of floats"); + helper::CheckForNullptr(variable.m_VariableBase, + "for variable, in call to Engine::Put list of floats"); + + m_Engine->Put(*dynamic_cast *>(variable.m_VariableBase), + reinterpret_cast(floats.data()), launch); +} + +void Engine::Put(Variable variable, const std::vector> &complexes, + const Mode launch) +{ + helper::CheckForNullptr(m_Engine, "in call to Engine::Put list of complexes"); + helper::CheckForNullptr(variable.m_VariableBase, + "for variable, in call to Engine::Put list of complexes"); + m_Engine->Put(*dynamic_cast> *>(variable.m_VariableBase), + reinterpret_cast *>(complexes.data()), launch); +} + void Engine::Put(Variable variable, const std::string &string) { helper::CheckForNullptr(m_Engine, "for engine, in call to Engine::Put string"); diff --git a/bindings/Python/py11Engine.h b/bindings/Python/py11Engine.h index 3fb286f2de..617efd1d55 100644 --- a/bindings/Python/py11Engine.h +++ b/bindings/Python/py11Engine.h @@ -49,6 +49,12 @@ class Engine StepStatus BeginStep(); void Put(Variable variable, const pybind11::array &array, const Mode launch = Mode::Deferred); + void Put(Variable variable, const std::vector &ints, + const Mode launch = Mode::Deferred); + void Put(Variable variable, const std::vector &doubles, + const Mode launch = Mode::Deferred); + void Put(Variable variable, const std::vector> &complexes, + const Mode launch = Mode::Deferred); void Put(Variable variable, const std::string &string); void PerformPuts(); void PerformDataWrite(); diff --git a/bindings/Python/py11IO.cpp b/bindings/Python/py11IO.cpp index b5931219d5..34466e09aa 100644 --- a/bindings/Python/py11IO.cpp +++ b/bindings/Python/py11IO.cpp @@ -101,6 +101,51 @@ Variable IO::DefineVariable(const std::string &name, const pybind11::array &arra return Variable(variable); } +Variable IO::DefineVariable(const std::string &name, const pybind11::object &value, + const Dims &shape, const Dims &start, const Dims &count, + const bool isConstantDims) +{ + helper::CheckForNullptr(m_IO, "for variable " + name + ", in call to IO::DefineVariable"); + core::VariableBase *variable = nullptr; + const auto t = value.get_type(); + const auto ts = pybind11::str(t); + const auto tss = pybind11::cast(ts); + if (pybind11::isinstance(value)) + { + variable = &m_IO->DefineVariable(name); + } + else if (pybind11::isinstance(value)) + { + variable = &m_IO->DefineVariable(name, shape, start, count, isConstantDims); + } + else if (pybind11::isinstance(value)) + { + variable = &m_IO->DefineVariable(name, shape, start, count, isConstantDims); + } + else if (tss == "") + { + variable = + &m_IO->DefineVariable>(name, shape, start, count, isConstantDims); + } + else if (tss == "") + { + variable = + &m_IO->DefineVariable>(name, shape, start, count, isConstantDims); + } + else if (tss == "") + { + variable = + &m_IO->DefineVariable>(name, shape, start, count, isConstantDims); + } + else + { + throw std::invalid_argument("ERROR: variable " + name + + " can't be defined with an object with type " + tss + + ", in call to DefineVariable\n"); + } + return Variable(variable); +} + Variable IO::InquireVariable(const std::string &name) { helper::CheckForNullptr(m_IO, "for variable " + name + ", in call to IO::InquireVariable"); @@ -126,7 +171,6 @@ Attribute IO::DefineAttribute(const std::string &name, const pybind11::array &ar const std::string &variableName, const std::string separator) { helper::CheckForNullptr(m_IO, "for attribute " + name + ", in call to IO::DefineAttribute"); - core::AttributeBase *attribute = nullptr; if (false) @@ -156,7 +200,6 @@ Attribute IO::DefineAttribute(const std::string &name, const std::string &string const std::string &variableName, const std::string separator) { helper::CheckForNullptr(m_IO, "for attribute " + name + ", in call to IO::DefineAttribute"); - return Attribute( &m_IO->DefineAttribute(name, stringValue, variableName, separator)); } @@ -165,11 +208,78 @@ Attribute IO::DefineAttribute(const std::string &name, const std::vectorDefineAttribute(name, strings.data(), strings.size(), variableName, separator)); } +Attribute IO::DefineAttribute(const std::string &name, const std::vector &ints, + const std::string &variableName, const std::string separator) +{ + helper::CheckForNullptr(m_IO, "for attribute " + name + ", in call to IO::DefineAttribute"); + return Attribute( + &m_IO->DefineAttribute(name, ints.data(), ints.size(), variableName, separator)); +} + +Attribute IO::DefineAttribute(const std::string &name, const std::vector &doubles, + const std::string &variableName, const std::string separator) +{ + helper::CheckForNullptr(m_IO, "for attribute " + name + ", in call to IO::DefineAttribute"); + return Attribute(&m_IO->DefineAttribute(name, doubles.data(), doubles.size(), + variableName, separator)); +} + +Attribute IO::DefineAttribute(const std::string &name, + const std::vector> &complexdoubles, + const std::string &variableName, const std::string separator) +{ + helper::CheckForNullptr(m_IO, "for attribute " + name + ", in call to IO::DefineAttribute"); + return Attribute(&m_IO->DefineAttribute>( + name, complexdoubles.data(), complexdoubles.size(), variableName, separator)); +} + +Attribute IO::DefineAttribute(const std::string &name, const pybind11::object &value, + const std::string &variableName, const std::string separator) +{ + helper::CheckForNullptr(m_IO, "for attribute " + name + ", in call to IO::DefineAttribute"); + + core::AttributeBase *attribute = nullptr; + const auto t = value.get_type(); + const auto ts = pybind11::str(t); + const auto tss = pybind11::cast(ts); + if (pybind11::isinstance(value)) + { + auto v = pybind11::cast(value); + attribute = &m_IO->DefineAttribute(name, v, variableName, separator); + } + else if (pybind11::isinstance(value)) + { + auto v = pybind11::cast(value); + attribute = &m_IO->DefineAttribute(name, v, variableName, separator); + } + else if (tss == "") + { + auto v = pybind11::cast>(value); + attribute = &m_IO->DefineAttribute(name, v, variableName, separator); + } + else if (tss == "") + { + auto v = pybind11::cast>(value); + attribute = &m_IO->DefineAttribute(name, v, variableName, separator); + } + else if (tss == "") + { + auto v = pybind11::cast>(value); + attribute = &m_IO->DefineAttribute(name, v, variableName, separator); + } + else + { + throw std::invalid_argument("ERROR: attribute " + name + + " can't be defined with an object with type " + tss + + ", in call to DefineAttribute\n"); + } + return Attribute(attribute); +} + Attribute IO::InquireAttribute(const std::string &name, const std::string &variableName, const std::string separator) { diff --git a/bindings/Python/py11IO.h b/bindings/Python/py11IO.h index 9308af270d..ef290874e6 100644 --- a/bindings/Python/py11IO.h +++ b/bindings/Python/py11IO.h @@ -13,6 +13,7 @@ #include +#include #include #include "py11Attribute.h" @@ -57,6 +58,10 @@ class IO const Dims &shape, const Dims &start, const Dims &count, const bool isConstantDims); + Variable DefineVariable(const std::string &name, const pybind11::object &value, + const Dims &shape, const Dims &start, const Dims &count, + const bool isConstantDims); + Variable InquireVariable(const std::string &name); Attribute DefineAttribute(const std::string &name, const pybind11::array &array, @@ -71,6 +76,22 @@ class IO const std::string &variableName = "", const std::string separator = "/"); + Attribute DefineAttribute(const std::string &name, const std::vector &ints, + const std::string &variableName = "", + const std::string separator = "/"); + + Attribute DefineAttribute(const std::string &name, const std::vector &doubles, + const std::string &variableName = "", + const std::string separator = "/"); + + Attribute DefineAttribute(const std::string &name, + const std::vector> &complexdoubles, + const std::string &variableName = "", + const std::string separator = "/"); + + Attribute DefineAttribute(const std::string &name, const pybind11::object &value, + const std::string &variableName, const std::string separator); + Attribute InquireAttribute(const std::string &name, const std::string &variableName = "", const std::string separator = "/"); diff --git a/bindings/Python/py11glue.cpp b/bindings/Python/py11glue.cpp index 90b12ba1e9..2f0c1742f0 100644 --- a/bindings/Python/py11glue.cpp +++ b/bindings/Python/py11glue.cpp @@ -11,6 +11,7 @@ #include #include +#include #include #include @@ -203,6 +204,15 @@ PYBIND11_MODULE(ADIOS2_PYTHON_MODULE_NAME, m) pybind11::arg("shape") = adios2::Dims(), pybind11::arg("start") = adios2::Dims(), pybind11::arg("count") = adios2::Dims(), pybind11::arg("isConstantDims") = false) + .def("DefineVariable", + (adios2::py11::Variable(adios2::py11::IO::*)( + const std::string &, const pybind11::object &, const adios2::Dims &, + const adios2::Dims &, const adios2::Dims &, const bool)) & + adios2::py11::IO::DefineVariable, + pybind11::return_value_policy::move, pybind11::arg("name"), pybind11::arg("value"), + pybind11::arg("shape") = adios2::Dims(), pybind11::arg("start") = adios2::Dims(), + pybind11::arg("count") = adios2::Dims(), pybind11::arg("isConstantDims") = false) + .def("DefineVariable", (adios2::py11::Variable(adios2::py11::IO::*)(const std::string &)) & adios2::py11::IO::DefineVariable, @@ -243,6 +253,38 @@ PYBIND11_MODULE(ADIOS2_PYTHON_MODULE_NAME, m) pybind11::arg("name"), pybind11::arg("strings"), pybind11::arg("variable_name") = "", pybind11::arg("separator") = "/", pybind11::return_value_policy::move) + .def("DefineAttribute", + (adios2::py11::Attribute(adios2::py11::IO::*)( + const std::string &, const std::vector &, const std::string &, + const std::string)) & + adios2::py11::IO::DefineAttribute, + pybind11::arg("name"), pybind11::arg("ints"), pybind11::arg("variable_name") = "", + pybind11::arg("separator") = "/", pybind11::return_value_policy::move) + + .def("DefineAttribute", + (adios2::py11::Attribute(adios2::py11::IO::*)( + const std::string &, const std::vector &, const std::string &, + const std::string)) & + adios2::py11::IO::DefineAttribute, + pybind11::arg("name"), pybind11::arg("doubles"), pybind11::arg("variable_name") = "", + pybind11::arg("separator") = "/", pybind11::return_value_policy::move) + + .def("DefineAttribute", + (adios2::py11::Attribute(adios2::py11::IO::*)( + const std::string &, const std::vector> &, + const std::string &, const std::string)) & + adios2::py11::IO::DefineAttribute, + pybind11::arg("name"), pybind11::arg("complexes"), pybind11::arg("variable_name") = "", + pybind11::arg("separator") = "/", pybind11::return_value_policy::move) + + .def("DefineAttribute", + (adios2::py11::Attribute(adios2::py11::IO::*)( + const std::string &, const pybind11::object &, const std::string &, + const std::string)) & + adios2::py11::IO::DefineAttribute, + pybind11::arg("name"), pybind11::arg("value"), pybind11::arg("variable_name") = "", + pybind11::arg("separator") = "/", pybind11::return_value_policy::move) + .def("Open", (adios2::py11::Engine(adios2::py11::IO::*)(const std::string &, const int)) & adios2::py11::IO::Open) #if ADIOS2_USE_MPI @@ -385,6 +427,28 @@ PYBIND11_MODULE(ADIOS2_PYTHON_MODULE_NAME, m) .def("Put", (void(adios2::py11::Engine::*)(adios2::py11::Variable, const std::string &)) & adios2::py11::Engine::Put) + .def("Put", + (void(adios2::py11::Engine::*)(adios2::py11::Variable, const std::vector &, + const adios2::Mode launch)) & + adios2::py11::Engine::Put, + pybind11::arg("variable"), pybind11::arg("ints"), + pybind11::arg("launch") = adios2::Mode::Sync) + + .def("Put", + (void(adios2::py11::Engine::*)(adios2::py11::Variable, const std::vector &, + const adios2::Mode launch)) & + adios2::py11::Engine::Put, + pybind11::arg("variable"), pybind11::arg("floats"), + pybind11::arg("launch") = adios2::Mode::Sync) + + .def("Put", + (void(adios2::py11::Engine::*)(adios2::py11::Variable, + const std::vector> &, + const adios2::Mode launch)) & + adios2::py11::Engine::Put, + pybind11::arg("variable"), pybind11::arg("complexes"), + pybind11::arg("launch") = adios2::Mode::Sync) + .def("PerformPuts", &adios2::py11::Engine::PerformPuts) .def("PerformDataWrite", &adios2::py11::Engine::PerformDataWrite) diff --git a/examples/hello/bpReader/bpReaderHeatMap2D.py b/examples/hello/bpReader/bpReaderHeatMap2D.py index 3208af23b1..0f7dfe70ea 100644 --- a/examples/hello/bpReader/bpReaderHeatMap2D.py +++ b/examples/hello/bpReader/bpReaderHeatMap2D.py @@ -12,7 +12,7 @@ from mpi4py import MPI import numpy -from adios2 import Stream +from adios2 import Stream, FileReader # MPI comm = MPI.COMM_WORLD @@ -39,15 +39,23 @@ with Stream("HeatMap2D_py.bp", "w", comm) as obpStream: obpStream.write("temperature2D", temperatures, shape, start, count) - -if rank == 0: - with Stream("HeatMap2D_py.bp", "r", MPI.COMM_SELF) as ibpStream: - for _ in ibpStream.steps(): - var_inTemperature = ibpStream.inquire_variable("temperature2D") - if var_inTemperature is not None: - var_inTemperature.set_selection([[2, 2], [4, 4]]) - inTemperatures = ibpStream.read(var_inTemperature) - - print("Incoming temperature map") - for i in range(0, inTemperatures.shape[1]): - print(str(inTemperatures[i]) + " ") + if not rank: + obpStream.write("N", [size, Nx, Ny]) # will be an array in output + obpStream.write("Nx", numpy.array(Nx)) # will be a scalar in output + obpStream.write("Ny", Ny) # will be a scalar in output + obpStream.write_attribute("dimensions", [size * Nx, Ny], "temperature2D") + +if not rank: + with FileReader("HeatMap2D_py.bp", MPI.COMM_SELF) as ibpFile: + var_inTemperature = ibpFile.inquire_variable("temperature2D") + if var_inTemperature is not None: + var_inTemperature.set_selection([[2, 2], [4, 4]]) + inTemperatures = ibpFile.read(var_inTemperature) + + in_nx = ibpFile.read("Nx") # scalar is read as a numpy array with 1 element + in_ny = ibpFile.read("Ny") # scalar is read as a numpy array with 1 element + print(f"Incoming nx, ny = {in_nx}, {in_ny}") + + print("Incoming temperature map") + for i in range(0, inTemperatures.shape[1]): + print(str(inTemperatures[i])) diff --git a/examples/hello/bpWriter/bpWriter-bindings.py b/examples/hello/bpWriter/bpWriter-bindings.py index 118b2fe04e..4c98371e79 100644 --- a/examples/hello/bpWriter/bpWriter-bindings.py +++ b/examples/hello/bpWriter/bpWriter-bindings.py @@ -46,10 +46,15 @@ "bpArray", myArray, [size * Nx], [rank * Nx], [Nx], adios2.ConstantDims ) +varNx = bpIO.DefineVariable("Nx", numpy.array(Nx)) # type is derived from numpy array type +bpIO.DefineAttribute("size", Nx, "bpArray") +bpIO.DefineAttribute("dimensions", ["Nx"], "bpArray") + # ADIOS Engine bpFileWriter = bpIO.Open("bpWriter-py-bindings.bp", adios2.Mode.Write) bpFileWriter.BeginStep() bpFileWriter.Put(ioArray, myArray, adios2.Mode.Sync) +bpFileWriter.Put(varNx, numpy.array(Nx), adios2.Mode.Sync) bpFileWriter.EndStep() bpFileWriter.Close() diff --git a/examples/hello/bpWriter/bpWriter.py b/examples/hello/bpWriter/bpWriter.py index dbfacc9dc7..96daec8ece 100644 --- a/examples/hello/bpWriter/bpWriter.py +++ b/examples/hello/bpWriter/bpWriter.py @@ -43,6 +43,9 @@ # ADIOS output stream with adios2.Stream(bpIO, "bpWriter-py.bp", "w", comm) as fh: fh.write("bpArray", myArray, [size * Nx], [rank * Nx], [Nx]) + fh.write("Nx", Nx) + fh.write_attribute("size", Nx, "bpArray") + fh.write_attribute("dimensions", ["Nx"], "bpArray") # Read content: # bpls -la bpWriter-py.bp diff --git a/examples/useCases/CMakeLists.txt b/examples/useCases/CMakeLists.txt index e53d930f80..d49e897634 100644 --- a/examples/useCases/CMakeLists.txt +++ b/examples/useCases/CMakeLists.txt @@ -1,7 +1,7 @@ -#------------------------------------------------------------------------------# +# ------------------------------------------------------------------------------# # Distributed under the OSI-approved Apache License, Version 2.0. See # accompanying file Copyright.txt for details. -#------------------------------------------------------------------------------# +# ------------------------------------------------------------------------------# add_subdirectory(fidesOneCell) add_subdirectory(insituGlobalArrays) diff --git a/python/adios2/engine.py b/python/adios2/engine.py index eb091cae50..9954392807 100644 --- a/python/adios2/engine.py +++ b/python/adios2/engine.py @@ -102,8 +102,15 @@ def put(self, variable, content, mode=bindings.Mode.Deferred): """ if isinstance(content, np.ndarray): self.impl.Put(variable.impl, content, mode) - else: + elif isinstance(content, str): + self.impl.Put(variable.impl, content) + elif isinstance(content, list): self.impl.Put(variable.impl, content) + elif not hasattr(content, "__len__"): + content = np.array([content]) + self.impl.Put(variable.impl, content) + else: + raise ValueError def perform_puts(self): """Perform the puts calls""" diff --git a/python/adios2/file_reader.py b/python/adios2/file_reader.py index ac40791a6f..ee351e7e26 100644 --- a/python/adios2/file_reader.py +++ b/python/adios2/file_reader.py @@ -5,6 +5,7 @@ from functools import singledispatchmethod from adios2 import Stream, IO + # pylint: disable=W0221 class FileReader(Stream): """High level implementation of the FileReader class for read Random access mode""" diff --git a/python/adios2/io.py b/python/adios2/io.py index b493de72bc..974addee7f 100644 --- a/python/adios2/io.py +++ b/python/adios2/io.py @@ -67,13 +67,13 @@ def define_attribute( Not used if variable_name is empty """ - # string or list of strings passed on as is - if isinstance(content, list) and len(content) > 0 and isinstance(content[0], str): + if isinstance(content, (np.ndarray, str, list)): return Attribute(self.impl, name, content, variable_name, separator) - if isinstance(content, str): + + if not hasattr(content, "__len__") and isinstance(content, (int, float, complex)): return Attribute(self.impl, name, content, variable_name, separator) - # python values (single or list) needs to be passed as a numpy array + # Anything else, try to pass as a numpy array (and get an error here if that fails) return Attribute(self.impl, name, np.asarray(content), variable_name, separator) def inquire_attribute(self, name, variable_name="", separator="/"): @@ -161,13 +161,18 @@ def define_variable( Whether dimensions are constant """ var_impl = None - if isinstance(content, np.ndarray): + if isinstance(content, list) and len(content) > 0: var_impl = self.impl.DefineVariable( - name, content, shape, start, count, is_constant_dims + name, content[0], shape, start, count, is_constant_dims ) + elif content is None: + var_impl = self.impl.DefineVariable(name, "", shape, start, count, is_constant_dims) + else: - var_impl = self.impl.DefineVariable(name) + var_impl = self.impl.DefineVariable( + name, content, shape, start, count, is_constant_dims + ) return Variable(var_impl) diff --git a/python/adios2/stream.py b/python/adios2/stream.py index d737864290..63490b4b3f 100644 --- a/python/adios2/stream.py +++ b/python/adios2/stream.py @@ -22,6 +22,8 @@ def type_adios_to_numpy(name): "uint64_t": np.uint64, "float": np.float32, "double": np.float64, + "complex": np.complex64, + "double complex": np.complex128, }[name] @@ -187,6 +189,7 @@ def available_attributes(self): def define_variable(self, name): """ Define new variable without specifying its type and content. + This only works for string output variables """ return self._io.define_variable(name) @@ -237,11 +240,7 @@ def write(self, variable: Variable, content): content variable data values """ - if isinstance(content, list): - content_np = np.array(content) - self._engine.put(variable, content_np, bindings.Mode.Sync) - else: - self._engine.put(variable, content, bindings.Mode.Sync) + self._engine.put(variable, content, bindings.Mode.Sync) @write.register(str) def _(self, name, content, shape=[], start=[], count=[], operations=None): @@ -270,15 +269,20 @@ def _(self, name, content, shape=[], start=[], count=[], operations=None): variable = self._io.inquire_variable(name) if not variable: - # Sequences variables + # Sequence variables if isinstance(content, np.ndarray): variable = self._io.define_variable(name, content, shape, start, count) elif isinstance(content, list): - content_np = np.array(content) - variable = self._io.define_variable(name, content_np, shape, start, count) - # Scalars variables - elif isinstance(content, str) or not hasattr(content, "__len__"): - variable = self.define_variable(name) + if shape == [] and count == []: + shape = [len(content)] + count = shape + start = [0] + variable = self._io.define_variable(name, content, shape, start, count) + # Scalar variables + elif isinstance(content, str): + variable = self._io.define_variable(name, content) + elif not hasattr(content, "__len__"): + variable = self._io.define_variable(name, content, [], [], []) else: raise ValueError @@ -370,14 +374,13 @@ def _(self, name: str, start=[], count=[], block_id=None, step_selection=None): if not variable: raise ValueError() - if step_selection and not self._mode == bindings.Mode.ReadRandomAccess: + if step_selection is not None and not self._mode == bindings.Mode.ReadRandomAccess: raise RuntimeError("step_selection parameter requires 'rra' mode") - if step_selection: - print(f"Stream.read step selection = {step_selection}") + if step_selection is not None: variable.set_step_selection(step_selection) - if block_id: + if block_id is not None: variable.set_block_selection(block_id) if variable.type() == "string" and variable.single_value() is True: diff --git a/testing/adios2/bindings/python/TestBPBlocksInfo.py b/testing/adios2/bindings/python/TestBPBlocksInfo.py index 6968ae2ba8..fd9eee5c35 100644 --- a/testing/adios2/bindings/python/TestBPBlocksInfo.py +++ b/testing/adios2/bindings/python/TestBPBlocksInfo.py @@ -38,22 +38,22 @@ adios = adios2.ADIOS(comm) ioWrite = adios.DeclareIO("ioWriter") -varTemperature = ioWrite.DefineVariable("temperature2D", temperatures, shape, - start, count, adios2.ConstantDims) +varTemperature = ioWrite.DefineVariable( + "temperature2D", temperatures, shape, start, count, adios2.ConstantDims +) -obpStream = ioWrite.Open('HeatMap2D_py.bp', adios2.Mode.Write) +obpStream = ioWrite.Open("HeatMap2D_py.bp", adios2.Mode.Write) obpStream.Put(varTemperature, temperatures) obpStream.Close() if rank == 0: ioRead = adios.DeclareIO("ioReader") - ibpStream = ioRead.Open('HeatMap2D_py.bp', adios2.Mode.ReadRandomAccess, - MPI.COMM_SELF) + ibpStream = ioRead.Open("HeatMap2D_py.bp", adios2.Mode.ReadRandomAccess, MPI.COMM_SELF) var_inTemperature = ioRead.InquireVariable("temperature2D") info = ibpStream.BlocksInfo("temperature2D", 0) assert info is not None - assert info[0]['Start'] == '0,0' - assert info[0]['Count'] == '10,10' - assert info[0]['WriterID'] == '0' + assert info[0]["Start"] == "0,0" + assert info[0]["Count"] == "10,10" + assert info[0]["WriterID"] == "0" diff --git a/testing/adios2/bindings/python/TestBPReadMultisteps.py b/testing/adios2/bindings/python/TestBPReadMultisteps.py index 0b0b1133cf..c165ab8b84 100644 --- a/testing/adios2/bindings/python/TestBPReadMultisteps.py +++ b/testing/adios2/bindings/python/TestBPReadMultisteps.py @@ -17,12 +17,12 @@ def check_object(adios2_object, name): if adios2_object is None: - raise ValueError(str(name) + ' not found') + raise ValueError(str(name) + " not found") def check_name(name, name_list): if name not in name_list: - raise ValueError(str(name) + ' not found in list') + raise ValueError(str(name) + " not found in list") # MPI @@ -43,29 +43,19 @@ def check_name(name, name_list): # ADIOS Variable name, shape, start, offset, constant dims # All local variables -varI8 = ioWriter.DefineVariable( - "varI8", data.I8, shape, start, count, adios2.ConstantDims) -varI16 = ioWriter.DefineVariable( - "varI16", data.I16, shape, start, count, adios2.ConstantDims) -varI32 = ioWriter.DefineVariable( - "varI32", data.I32, shape, start, count, adios2.ConstantDims) -varI64 = ioWriter.DefineVariable( - "varI64", data.I64, shape, start, count, adios2.ConstantDims) - -varU8 = ioWriter.DefineVariable( - "varU8", data.U8, shape, start, count, adios2.ConstantDims) -varU16 = ioWriter.DefineVariable( - "varU16", data.U16, shape, start, count, adios2.ConstantDims) -varU32 = ioWriter.DefineVariable( - "varU32", data.U32, shape, start, count, adios2.ConstantDims) -varU64 = ioWriter.DefineVariable( - "varU64", data.U64, shape, start, count, adios2.ConstantDims) - -varR32 = ioWriter.DefineVariable( - "varR32", data.R32, shape, start, count, adios2.ConstantDims) - -varR64 = ioWriter.DefineVariable( - "varR64", data.R64, shape, start, count, adios2.ConstantDims) +varI8 = ioWriter.DefineVariable("varI8", data.I8, shape, start, count, adios2.ConstantDims) +varI16 = ioWriter.DefineVariable("varI16", data.I16, shape, start, count, adios2.ConstantDims) +varI32 = ioWriter.DefineVariable("varI32", data.I32, shape, start, count, adios2.ConstantDims) +varI64 = ioWriter.DefineVariable("varI64", data.I64, shape, start, count, adios2.ConstantDims) + +varU8 = ioWriter.DefineVariable("varU8", data.U8, shape, start, count, adios2.ConstantDims) +varU16 = ioWriter.DefineVariable("varU16", data.U16, shape, start, count, adios2.ConstantDims) +varU32 = ioWriter.DefineVariable("varU32", data.U32, shape, start, count, adios2.ConstantDims) +varU64 = ioWriter.DefineVariable("varU64", data.U64, shape, start, count, adios2.ConstantDims) + +varR32 = ioWriter.DefineVariable("varR32", data.R32, shape, start, count, adios2.ConstantDims) + +varR64 = ioWriter.DefineVariable("varR64", data.R64, shape, start, count, adios2.ConstantDims) attString = ioWriter.DefineAttribute("attrString", ["hello attribute"]) attI8 = ioWriter.DefineAttribute("attrI8", data.I8) @@ -74,7 +64,6 @@ def check_name(name, name_list): writer = ioWriter.Open("npTypes.bp", adios2.Mode.Write) for i in range(0, 3): - npi8 = np.full((Nx), i, dtype=np.int8) npi16 = np.full((Nx), i, dtype=np.int16) npi32 = np.full((Nx), i, dtype=np.int32) @@ -138,9 +127,19 @@ def check_name(name, name_list): attr_names = ["attrString", "attrI8"] -var_names = ["varStr", "varI8", "varI16", "varI32", "varI64", - "varU8", "varU16", "varU32", "varU64", - "varR32", "varR64"] +var_names = [ + "varStr", + "varI8", + "varI16", + "varI32", + "varI64", + "varU8", + "varU16", + "varU32", + "varU64", + "varR32", + "varR64", +] attributesInfo = ioReader.AvailableAttributes() for name, info in attributesInfo.items(): @@ -226,34 +225,34 @@ def check_name(name, name_list): for i in range(0, 3): for j in range(0, Nx): if inI8[i][j] != i: - raise ValueError('failed reading I8') + raise ValueError("failed reading I8") if inI16[i][j] != i: - raise ValueError('failed reading I16') + raise ValueError("failed reading I16") if inI32[i][j] != i: - raise ValueError('failed reading I32') + raise ValueError("failed reading I32") if inI64[i][j] != i: - raise ValueError('failed reading I64') + raise ValueError("failed reading I64") if inU8[i][j] != i: - raise ValueError('failed reading U8') + raise ValueError("failed reading U8") if inU16[i][j] != i: - raise ValueError('failed reading U16') + raise ValueError("failed reading U16") if inU32[i][j] != i: - raise ValueError('failed reading U32') + raise ValueError("failed reading U32") if inU64[i][j] != i: - raise ValueError('failed reading U64') + raise ValueError("failed reading U64") if inR32[i][j] != i: - raise ValueError('failed reading R32') + raise ValueError("failed reading R32") if inR64[i][j] != i: - raise ValueError('failed reading R64') + raise ValueError("failed reading R64") # here tests reader data diff --git a/testing/adios2/bindings/python/TestBPSelectSteps.py b/testing/adios2/bindings/python/TestBPSelectSteps.py index 803ee45ccd..a0d209001d 100644 --- a/testing/adios2/bindings/python/TestBPSelectSteps.py +++ b/testing/adios2/bindings/python/TestBPSelectSteps.py @@ -25,13 +25,11 @@ class TestAdiosSelectSteps(unittest.TestCase): - def setUp(self): total_steps = 10 with adios2.open(TESTDATA_FILENAME, "w", comm) as fh: for i in range(total_steps): - fh.write("step", np.full((Nx), i, dtype=np.int32), - shape, start, count) + fh.write("step", np.full((Nx), i, dtype=np.int32), shape, start, count) fh.end_step() def test_select_steps_reading_fullAPI(self): @@ -40,18 +38,22 @@ def test_select_steps_reading_fullAPI(self): adios = adios2.ADIOS() ioReadBP = adios.DeclareIO("hellopy") ioReadBP.SetParameter(TESTDATA_FILENAME, param_string) - fh = ioReadBP.Open(TESTDATA_FILENAME, - adios2.Mode.ReadRandomAccess, comm) + fh = ioReadBP.Open(TESTDATA_FILENAME, adios2.Mode.ReadRandomAccess, comm) var = ioReadBP.InquireVariable("step") var.SetSelection([[0], [size * Nx]]) var.SetStepSelection([0, len(selected_steps)]) data = np.zeros((len(selected_steps), size * Nx), dtype=np.int32) fh.Get(var, data) fh.PerformGets() - self.assertTrue(all( - [list(data[i]) == [selected_steps[i] for x in range(len(data[i]))] - for i in range(len(selected_steps))])) + self.assertTrue( + all( + [ + list(data[i]) == [selected_steps[i] for x in range(len(data[i]))] + for i in range(len(selected_steps)) + ] + ) + ) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/testing/adios2/bindings/python/TestBPWriteRead2D.py b/testing/adios2/bindings/python/TestBPWriteRead2D.py index 17a04a2c8e..f8be1a4758 100644 --- a/testing/adios2/bindings/python/TestBPWriteRead2D.py +++ b/testing/adios2/bindings/python/TestBPWriteRead2D.py @@ -38,10 +38,11 @@ adios = adios2.ADIOS(comm) ioWrite = adios.DeclareIO("ioWriter") -varTemperature = ioWrite.DefineVariable("temperature2D", temperatures, shape, - start, count, adios2.ConstantDims) +varTemperature = ioWrite.DefineVariable( + "temperature2D", temperatures, shape, start, count, adios2.ConstantDims +) -obpStream = ioWrite.Open('HeatMap2D_py.bp', adios2.Mode.Write) +obpStream = ioWrite.Open("HeatMap2D_py.bp", adios2.Mode.Write) obpStream.Put(varTemperature, temperatures) obpStream.Close() @@ -49,12 +50,11 @@ if rank == 0: # ADIOS2 read ioRead = adios.DeclareIO("ioReader") - ibpStream = ioRead.Open('HeatMap2D_py.bp', adios2.Mode.ReadRandomAccess, - MPI.COMM_SELF) + ibpStream = ioRead.Open("HeatMap2D_py.bp", adios2.Mode.ReadRandomAccess, MPI.COMM_SELF) var_inTemperature = ioRead.InquireVariable("temperature2D") if var_inTemperature is False: - raise ValueError('var_inTemperature is False') + raise ValueError("var_inTemperature is False") assert var_inTemperature is not None readOffset = [2, 2] @@ -66,8 +66,7 @@ ibpStream.Close() # print('Incoming temperature map\n', inTemperatures) - expected = np.array([[22, 23, 24, 25], - [32, 33, 34, 35], - [42, 43, 44, 45], - [52, 53, 54, 55]], np.int32) + expected = np.array( + [[22, 23, 24, 25], [32, 33, 34, 35], [42, 43, 44, 45], [52, 53, 54, 55]], np.int32 + ) assert np.array_equal(inTemperatures, expected) diff --git a/testing/adios2/bindings/python/TestBPWriteReadString.py b/testing/adios2/bindings/python/TestBPWriteReadString.py index 355347c7a4..5b1d104545 100644 --- a/testing/adios2/bindings/python/TestBPWriteReadString.py +++ b/testing/adios2/bindings/python/TestBPWriteReadString.py @@ -15,14 +15,13 @@ class TestAdiosWriteReadStringfullAPI(unittest.TestCase): - def test_write_read_string_fullAPI(self): comm = MPI.COMM_WORLD - theString = 'hello adios' - bpFilename = 'string_test_fullAPI.bp' - varname = 'mystringvar' + theString = "hello adios" + bpFilename = "string_test_fullAPI.bp" + varname = "mystringvar" adios = adios2.ADIOS(comm) - ioWrite = adios.DeclareIO('ioWriter') + ioWrite = adios.DeclareIO("ioWriter") adEngine = ioWrite.Open(bpFilename, adios2.Mode.Write) varMyString = ioWrite.DefineVariable(varname) for step in range(N_STEPS): @@ -31,7 +30,7 @@ def test_write_read_string_fullAPI(self): adEngine.EndStep() adEngine.Close() - ioRead = adios.DeclareIO('ioReader') + ioRead = adios.DeclareIO("ioReader") adEngine = ioRead.Open(bpFilename, adios2.Mode.Read) for step in range(N_STEPS): adEngine.BeginStep() @@ -42,5 +41,5 @@ def test_write_read_string_fullAPI(self): adEngine.Close() -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/testing/adios2/bindings/python/TestBPWriteReadTypes.py b/testing/adios2/bindings/python/TestBPWriteReadTypes.py index 399758bdb1..04a0a22c05 100644 --- a/testing/adios2/bindings/python/TestBPWriteReadTypes.py +++ b/testing/adios2/bindings/python/TestBPWriteReadTypes.py @@ -16,19 +16,19 @@ def check_object(adios2_object, name): if adios2_object is False: - raise ValueError(str(name) + ' not found') + raise ValueError(str(name) + " not found") def check_name(name, name_list): if name not in name_list: - raise ValueError(str(name) + ' not found in list') + raise ValueError(str(name) + " not found in list") def check_array(np1, np2, hint): if (np1 == np2).all() is False: print("InData: " + str(np1)) print("Data: " + str(np2)) - raise ValueError('Array read failed ' + str(hint)) + raise ValueError("Array read failed " + str(hint)) # MPI @@ -38,11 +38,33 @@ def check_array(np1, np2, hint): Nx = 8 # list of tested attributes and variables -attr_names = ["attrString", "attrStringArray", "attrI8", "attrI16", "attrI32", "attrI64", - "attrU8", "attrU16", "attrU32", "attrU64", "attrR32", "attrR64"] -var_names = ["varStr", "varI8", "varI16", "varI32", "varI64", - "varU8", "varU16", "varU32", "varU64", - "varR32", "varR64"] +attr_names = [ + "attrString", + "attrStringArray", + "attrI8", + "attrI16", + "attrI32", + "attrI64", + "attrU8", + "attrU16", + "attrU32", + "attrU64", + "attrR32", + "attrR64", +] +var_names = [ + "varStr", + "varI8", + "varI16", + "varI32", + "varI64", + "varU8", + "varU16", + "varU32", + "varU64", + "varR32", + "varR64", +] # Start ADIOS adios = adios2.ADIOS(comm) @@ -58,29 +80,19 @@ def check_array(np1, np2, hint): # All local variables varStr = ioWriter.DefineVariable("varStr") -varI8 = ioWriter.DefineVariable( - "varI8", data.I8, shape, start, count, adios2.ConstantDims) -varI16 = ioWriter.DefineVariable( - "varI16", data.I16, shape, start, count, adios2.ConstantDims) -varI32 = ioWriter.DefineVariable( - "varI32", data.I32, shape, start, count, adios2.ConstantDims) -varI64 = ioWriter.DefineVariable( - "varI64", data.I64, shape, start, count, adios2.ConstantDims) - -varU8 = ioWriter.DefineVariable( - "varU8", data.U8, shape, start, count, adios2.ConstantDims) -varU16 = ioWriter.DefineVariable( - "varU16", data.U16, shape, start, count, adios2.ConstantDims) -varU32 = ioWriter.DefineVariable( - "varU32", data.U32, shape, start, count, adios2.ConstantDims) -varU64 = ioWriter.DefineVariable( - "varU64", data.U64, shape, start, count, adios2.ConstantDims) - -varR32 = ioWriter.DefineVariable( - "varR32", data.R32, shape, start, count, adios2.ConstantDims) - -varR64 = ioWriter.DefineVariable( - "varR64", data.R64, shape, start, count, adios2.ConstantDims) +varI8 = ioWriter.DefineVariable("varI8", data.I8, shape, start, count, adios2.ConstantDims) +varI16 = ioWriter.DefineVariable("varI16", data.I16, shape, start, count, adios2.ConstantDims) +varI32 = ioWriter.DefineVariable("varI32", data.I32, shape, start, count, adios2.ConstantDims) +varI64 = ioWriter.DefineVariable("varI64", data.I64, shape, start, count, adios2.ConstantDims) + +varU8 = ioWriter.DefineVariable("varU8", data.U8, shape, start, count, adios2.ConstantDims) +varU16 = ioWriter.DefineVariable("varU16", data.U16, shape, start, count, adios2.ConstantDims) +varU32 = ioWriter.DefineVariable("varU32", data.U32, shape, start, count, adios2.ConstantDims) +varU64 = ioWriter.DefineVariable("varU64", data.U64, shape, start, count, adios2.ConstantDims) + +varR32 = ioWriter.DefineVariable("varR32", data.R32, shape, start, count, adios2.ConstantDims) + +varR64 = ioWriter.DefineVariable("varR64", data.R64, shape, start, count, adios2.ConstantDims) attString = ioWriter.DefineAttribute("attrString", "one") attStringArray = ioWriter.DefineAttribute("attrStringArray", ["one", "two", "three"]) @@ -96,13 +108,12 @@ def check_array(np1, np2, hint): attR64 = ioWriter.DefineAttribute("attrR64", data.R64) ioWriter.SetEngine("BPFile") -ioParams = {'Threads': '1', 'InitialBufferSize': '17Kb'} +ioParams = {"Threads": "1", "InitialBufferSize": "17Kb"} ioWriter.SetParameters(ioParams) engineType = ioWriter.EngineType() if engineType != "BPFile": - raise ValueError(str(engineType) + - ' incorrect engine type, should be BPFile') + raise ValueError(str(engineType) + " incorrect engine type, should be BPFile") ioWriter.SetParameter("profileunits", "microseconds") ioWriter.AddTransport("file") @@ -120,7 +131,6 @@ def check_array(np1, np2, hint): writer.LockWriterDefinitions() for i in range(0, nsteps): - data.update(rank, i, size) writer.BeginStep() @@ -175,16 +185,16 @@ def check_array(np1, np2, hint): attrStringData = attrString.DataString() print(f"attrString = {attrStringData}", flush=True) if attrStringData[0] != "one": - raise ValueError('attrString failed') + raise ValueError("attrString failed") attrStringData = attrStringArray.DataString() print(f"attrStringArray = {attrStringData}", flush=True) if attrStringData[0] != "one": - raise ValueError('attrStringData[0] failed') + raise ValueError("attrStringData[0] failed") if attrStringData[1] != "two": - raise ValueError('attrStringData[1] failed') + raise ValueError("attrStringData[1] failed") if attrStringData[2] != "three": - raise ValueError('attrStringData[2] failed') + raise ValueError("attrStringData[2] failed") attrI8Data = attrI8.Data() attrI16Data = attrI16.Data() @@ -197,16 +207,16 @@ def check_array(np1, np2, hint): attrR32Data = attrR32.Data() attrR64Data = attrR64.Data() -check_array(attrI8Data, data.I8, 'I8') -check_array(attrI16Data, data.I16, 'I16') -check_array(attrI32Data, data.I32, 'I32') -check_array(attrI64Data, data.I64, 'I64') -check_array(attrU8Data, data.U8, 'U8') -check_array(attrU16Data, data.U16, 'U16') -check_array(attrU32Data, data.U32, 'U32') -check_array(attrU64Data, data.U64, 'U64') -check_array(attrR32Data, data.R32, 'R32') -check_array(attrR64Data, data.R64, 'R64') +check_array(attrI8Data, data.I8, "I8") +check_array(attrI16Data, data.I16, "I16") +check_array(attrI32Data, data.I32, "I32") +check_array(attrI64Data, data.I64, "I64") +check_array(attrU8Data, data.U8, "U8") +check_array(attrU16Data, data.U16, "U16") +check_array(attrU32Data, data.U32, "U32") +check_array(attrU64Data, data.U64, "U64") +check_array(attrR32Data, data.R32, "R32") +check_array(attrR64Data, data.R64, "R64") attributesInfo = ioReader.AvailableAttributes() for name, info in attributesInfo.items(): @@ -251,11 +261,11 @@ def check_array(np1, np2, hint): print("\n") -result = adios.RemoveIO('writer') +result = adios.RemoveIO("writer") if result is False: - raise ValueError('Could not remove IO writer') + raise ValueError("Could not remove IO writer") -assert (reader.Steps() == nsteps) +assert reader.Steps() == nsteps reader.Close() @@ -263,10 +273,10 @@ def check_array(np1, np2, hint): ioReader.RemoveAllVariables() varStr = ioReader.InquireVariable("varStr") if varStr is True: - raise ValueError('Could remove reader variables') + raise ValueError("Could remove reader variables") adios.RemoveAllIOs() try: - ioWriter = adios.DeclareIO('reader') + ioWriter = adios.DeclareIO("reader") except ValueError: - raise ValueError('Could not re-Declare IO reader') + raise ValueError("Could not re-Declare IO reader") diff --git a/testing/adios2/bindings/python/TestBPWriteReadTypes_nompi.py b/testing/adios2/bindings/python/TestBPWriteReadTypes_nompi.py index edee61f1c1..1df907a4d5 100644 --- a/testing/adios2/bindings/python/TestBPWriteReadTypes_nompi.py +++ b/testing/adios2/bindings/python/TestBPWriteReadTypes_nompi.py @@ -22,29 +22,19 @@ # ADIOS Variable name, shape, start, offset, constant dims # All local variables -varI8 = bpIO.DefineVariable( - "varI8", data.I8, [], [], [data.I8.size], adios2.ConstantDims) -varI16 = bpIO.DefineVariable( - "varI16", data.I16, [], [], [data.I16.size], adios2.ConstantDims) -varI32 = bpIO.DefineVariable( - "varI32", data.I32, [], [], [data.I32.size], adios2.ConstantDims) -varI64 = bpIO.DefineVariable( - "varI64", data.I64, [], [], [data.I64.size], adios2.ConstantDims) - -varU8 = bpIO.DefineVariable( - "varUI8", data.U8, [], [], [data.U8.size], adios2.ConstantDims) -varU16 = bpIO.DefineVariable( - "varUI16", data.U16, [], [], [data.U16.size], adios2.ConstantDims) -varU32 = bpIO.DefineVariable( - "varUI32", data.U32, [], [], [data.U32.size], adios2.ConstantDims) -varU64 = bpIO.DefineVariable( - "varUI64", data.U64, [], [], [data.U64.size], adios2.ConstantDims) - -varR32 = bpIO.DefineVariable( - "varR32", data.R32, [], [], [data.R32.size], adios2.ConstantDims) - -varR64 = bpIO.DefineVariable( - "varR64", data.R64, [], [], [data.R64.size], adios2.ConstantDims) +varI8 = bpIO.DefineVariable("varI8", data.I8, [], [], [data.I8.size], adios2.ConstantDims) +varI16 = bpIO.DefineVariable("varI16", data.I16, [], [], [data.I16.size], adios2.ConstantDims) +varI32 = bpIO.DefineVariable("varI32", data.I32, [], [], [data.I32.size], adios2.ConstantDims) +varI64 = bpIO.DefineVariable("varI64", data.I64, [], [], [data.I64.size], adios2.ConstantDims) + +varU8 = bpIO.DefineVariable("varUI8", data.U8, [], [], [data.U8.size], adios2.ConstantDims) +varU16 = bpIO.DefineVariable("varUI16", data.U16, [], [], [data.U16.size], adios2.ConstantDims) +varU32 = bpIO.DefineVariable("varUI32", data.U32, [], [], [data.U32.size], adios2.ConstantDims) +varU64 = bpIO.DefineVariable("varUI64", data.U64, [], [], [data.U64.size], adios2.ConstantDims) + +varR32 = bpIO.DefineVariable("varR32", data.R32, [], [], [data.R32.size], adios2.ConstantDims) + +varR64 = bpIO.DefineVariable("varR64", data.R64, [], [], [data.R64.size], adios2.ConstantDims) # ADIOS Engine diff --git a/testing/adios2/bindings/python/TestGetException_nompi.py b/testing/adios2/bindings/python/TestGetException_nompi.py index 6e2d8a333d..86b2dd235d 100644 --- a/testing/adios2/bindings/python/TestGetException_nompi.py +++ b/testing/adios2/bindings/python/TestGetException_nompi.py @@ -1,26 +1,22 @@ - import numpy as np import logging import adios2.bindings as adios2 -if __name__ == '__main__': +if __name__ == "__main__": __spec__ = None def main(): - print("====================================") format = "%(asctime)s: %(message)s" - logging.basicConfig(format=format, level=logging.INFO, - datefmt="%H:%M:%S") + logging.basicConfig(format=format, level=logging.INFO, datefmt="%H:%M:%S") writer() reader_fail() reader_success() def writer(): - logging.info(" Writer: initiating writing") data = np.arange(3, dtype=np.float32) logging.info(f" data dtypes: {data.dtype!s}") @@ -33,8 +29,7 @@ def writer(): IO = adios_io.DeclareIO("writer") writer = IO.Open("testdatafile", adios2.Mode.Write) - writebuffer = IO.DefineVariable("np_data", data, shape, start, - count, adios2.ConstantDims) + writebuffer = IO.DefineVariable("np_data", data, shape, start, count, adios2.ConstantDims) if writebuffer: writer.BeginStep() writer.Put(writebuffer, data, adios2.Mode.Sync) @@ -48,7 +43,6 @@ def writer(): def reader_fail(): - adios_io = adios2.ADIOS() io = adios_io.DeclareIO("reader") @@ -85,11 +79,12 @@ def reader_fail(): raise StopIteration(f"next step failed to initiate {stepStatus!s}") reader.EndStep() reader.Close() - logging.info(" Reader: finished reading",) + logging.info( + " Reader: finished reading", + ) def reader_success(): - adios_io = adios2.ADIOS() io = adios_io.DeclareIO("reader") @@ -126,7 +121,9 @@ def reader_success(): raise StopIteration(f"next step failed to initiate {stepStatus!s}") reader.EndStep() reader.Close() - logging.info(" Reader: finished reading",) + logging.info( + " Reader: finished reading", + ) if __name__ == "__main__": diff --git a/testing/adios2/bindings/python/TestNullEngine.py b/testing/adios2/bindings/python/TestNullEngine.py index ac5fb1796a..9b88b01afc 100644 --- a/testing/adios2/bindings/python/TestNullEngine.py +++ b/testing/adios2/bindings/python/TestNullEngine.py @@ -36,16 +36,17 @@ adios = adios2.ADIOS(comm) ioWrite = adios.DeclareIO("ioWriter") -varTemperature = ioWrite.DefineVariable("temperature2D", temperatures, shape, - start, count, adios2.ConstantDims) +varTemperature = ioWrite.DefineVariable( + "temperature2D", temperatures, shape, start, count, adios2.ConstantDims +) ioWrite.SetEngine("NULL") -nullWriter = ioWrite.Open('NULL_py.bp', adios2.Mode.Write) +nullWriter = ioWrite.Open("NULL_py.bp", adios2.Mode.Write) -assert (nullWriter.Type() == "NullWriter") +assert nullWriter.Type() == "NullWriter" status = nullWriter.BeginStep() -assert (status == adios2.StepStatus.OK) +assert status == adios2.StepStatus.OK nullWriter.Put(varTemperature, temperatures) nullWriter.EndStep() @@ -54,19 +55,19 @@ # ADIOS2 read ioRead = adios.DeclareIO("ioReader") ioRead.SetEngine("null") -nullReader = ioRead.Open('NULL_py.bp', adios2.Mode.Read, MPI.COMM_SELF) +nullReader = ioRead.Open("NULL_py.bp", adios2.Mode.Read, MPI.COMM_SELF) -assert (nullReader.Type() == "NullReader") +assert nullReader.Type() == "NullReader" inTemperatures = np.zeros(1, dtype=np.int32) status = nullReader.BeginStep() -assert (status == adios2.StepStatus.EndOfStream) +assert status == adios2.StepStatus.EndOfStream var_inTemperature = ioRead.InquireVariable("temperature2D") -if (var_inTemperature is True): - raise ValueError('var_inTemperature is not False') +if var_inTemperature is True: + raise ValueError("var_inTemperature is not False") # nullReader.Get(var_inTemperature, inTemperatures) diff --git a/testing/adios2/bindings/python/TestQuery.py b/testing/adios2/bindings/python/TestQuery.py index d0ee554d0f..a1b1cd59e3 100644 --- a/testing/adios2/bindings/python/TestQuery.py +++ b/testing/adios2/bindings/python/TestQuery.py @@ -13,29 +13,29 @@ # # usage: [bp4 | bp5=default] ## # ####################################### numSteps = 5 -queryFile = 'query.xml' -targetVarName = 'var0' +queryFile = "query.xml" +targetVarName = "var0" # User data -myArray = np.array([0, 1., 2., 3., 4., 5., 6., 7., 8., 9.]) +myArray = np.array([0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0]) Nx = myArray.size # ADIOS MPI Communicator adios = adios2.ADIOS(comm) -supportedEngines = ['bp5', 'bp4'] -engineType = 'bp5' -if (len(sys.argv) > 1): +supportedEngines = ["bp5", "bp4"] +engineType = "bp5" +if len(sys.argv) > 1: engineType = sys.argv[1].lower() -if (engineType in supportedEngines): - if (rank == 0): - print('Using engine type:', engineType) +if engineType in supportedEngines: + if rank == 0: + print("Using engine type:", engineType) else: - sys.exit('specified engine does not exist') + sys.exit("specified engine does not exist") -dataFileName = 'test_' + engineType + '.bp' +dataFileName = "test_" + engineType + ".bp" def writeDataFile(): @@ -43,8 +43,8 @@ def writeDataFile(): bpIO.SetEngine(engineType) ioArray = bpIO.DefineVariable( - targetVarName, myArray, [size * Nx], [rank * Nx], - [Nx], adios2.ConstantDims) + targetVarName, myArray, [size * Nx], [rank * Nx], [Nx], adios2.ConstantDims + ) bpFileWriter = bpIO.Open(dataFileName, adios2.Mode.Write) @@ -59,29 +59,31 @@ def writeDataFile(): def createQueryFile(): print(".. Writing query file to: ", queryFile) - file1 = open(queryFile, 'w') + file1 = open(queryFile, "w") queryContent = [ - "\n", "\n", - " \n" - " \n", - " \n", - " \n", - " \n", " \n", - " \n", " \n", "\n" + '\n', + "\n", + ' \n' ' \n', + ' \n', + ' \n', + ' \n', + " \n", + " \n", + " \n", + "\n", ] file1.writelines(queryContent) file1.close() def doAnalysis(reader, touched_blocks, varList): - print(" Step: ", reader.CurrentStep(), - " num touched blocks: ", len(touched_blocks)) - if (0 == reader.CurrentStep()): - assert (len(touched_blocks) == min(size, 2)) - if (1 == reader.CurrentStep()): - assert (len(touched_blocks) == size) - if (1 < reader.CurrentStep()): - assert (len(touched_blocks) == 0) + print(" Step: ", reader.CurrentStep(), " num touched blocks: ", len(touched_blocks)) + if 0 == reader.CurrentStep(): + assert len(touched_blocks) == min(size, 2) + if 1 == reader.CurrentStep(): + assert len(touched_blocks) == size + if 1 < reader.CurrentStep(): + assert len(touched_blocks) == 0 values = [] data = {} @@ -89,7 +91,7 @@ def doAnalysis(reader, touched_blocks, varList): for var in varList: data[var] = [] - if (len(touched_blocks) > 0): + if len(touched_blocks) > 0: for n in touched_blocks: for var in varList: values = np.zeros(n[1], dtype=np.double) @@ -109,14 +111,14 @@ def queryDataFile(): print("Num steps: ", reader.Steps()) - while (reader.BeginStep() == adios2.StepStatus.OK): + while reader.BeginStep() == adios2.StepStatus.OK: # bp5 loads metadata after beginstep(), # therefore query has to be called per step w = adios2.Query(queryFile, reader) # assume only rank 0 wants to process result var = [queryIO.InquireVariable(targetVarName)] - if (rank == 0): + if rank == 0: touched_blocks = w.GetResult() doAnalysis(reader, touched_blocks, var) @@ -127,10 +129,12 @@ def queryDataFile(): def cleanUp(): import os import shutil + os.remove(queryFile) shutil.rmtree(dataFileName) print(" Cleanup generated files: ", queryFile, dataFileName) + # # actual setup: # @@ -138,7 +142,7 @@ def cleanUp(): writeDataFile() -if (0 == rank): +if 0 == rank: createQueryFile() queryDataFile() cleanUp() diff --git a/testing/adios2/bindings/python/TestQueryLocalArray.py b/testing/adios2/bindings/python/TestQueryLocalArray.py index 4eab11acde..b78dd7a2e3 100644 --- a/testing/adios2/bindings/python/TestQueryLocalArray.py +++ b/testing/adios2/bindings/python/TestQueryLocalArray.py @@ -13,29 +13,29 @@ # # usage: [bp4 | bp5=default] ## # ####################################### numSteps = 5 -queryFile = 'query.xml' -targetVarName = 'var0' +queryFile = "query.xml" +targetVarName = "var0" # User data -myArray = np.array([0, 1., 2., 3., 4., 5., 6., 7., 8., 9.]) +myArray = np.array([0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0]) Nx = myArray.size # ADIOS MPI Communicator adios = adios2.ADIOS(comm) -supportedEngines = ['bp5', 'bp4'] -engineType = 'bp5' -if (len(sys.argv) > 1): +supportedEngines = ["bp5", "bp4"] +engineType = "bp5" +if len(sys.argv) > 1: engineType = sys.argv[1].lower() -if (engineType in supportedEngines): - if (rank == 0): - print('Using engine type:', engineType) +if engineType in supportedEngines: + if rank == 0: + print("Using engine type:", engineType) else: - sys.exit('specified engine does not exist') + sys.exit("specified engine does not exist") -dataFileName = 'test_' + engineType + '.bp' +dataFileName = "test_" + engineType + ".bp" def writeDataFile(): @@ -50,8 +50,7 @@ def writeDataFile(): for i in range(numSteps): bpFileWriter.BeginStep() - bpFileWriter.Put(ioArray, i * 10.0 + myArray / (rank + 1), - adios2.Mode.Sync) + bpFileWriter.Put(ioArray, i * 10.0 + myArray / (rank + 1), adios2.Mode.Sync) bpFileWriter.EndStep() bpFileWriter.Close() @@ -60,23 +59,25 @@ def writeDataFile(): def createQueryFile(): print(".. Writing query file to: ", queryFile) - file1 = open(queryFile, 'w') + file1 = open(queryFile, "w") queryContent = [ - "\n", "\n", - " \n" - " \n", - " \n", - " \n", - " \n", " \n", - " \n", " \n", "\n" + '\n', + "\n", + ' \n' ' \n', + ' \n', + ' \n', + ' \n', + " \n", + " \n", + " \n", + "\n", ] file1.writelines(queryContent) file1.close() def doAnalysis(reader, touched_blocks, varList): - print(" Step: ", reader.CurrentStep(), - " num touched blocks: ", len(touched_blocks)) + print(" Step: ", reader.CurrentStep(), " num touched blocks: ", len(touched_blocks)) values = [] data = {} @@ -84,7 +85,7 @@ def doAnalysis(reader, touched_blocks, varList): for var in varList: data[var] = [] - if (len(touched_blocks) > 0): + if len(touched_blocks) > 0: for n in touched_blocks: for var in varList: values = np.zeros(10, dtype=np.double) @@ -104,14 +105,14 @@ def queryDataFile(): print("Num steps: ", reader.Steps()) - while (reader.BeginStep() == adios2.StepStatus.OK): + while reader.BeginStep() == adios2.StepStatus.OK: # bp5 loads metadata after beginstep(), # therefore query has to be called per step w = adios2.Query(queryFile, reader) # assume only rank 0 wants to process result var = [queryIO.InquireVariable(targetVarName)] - if (rank == 0): + if rank == 0: touched_blocks = w.GetBlockIDs() doAnalysis(reader, touched_blocks, var) @@ -122,10 +123,12 @@ def queryDataFile(): def cleanUp(): import os import shutil + os.remove(queryFile) shutil.rmtree(dataFileName) print(" Cleanup generated files: ", queryFile, dataFileName) + # # actual setup: # @@ -134,7 +137,7 @@ def cleanUp(): writeDataFile() -if (0 == rank): +if 0 == rank: createQueryFile() queryDataFile() cleanUp() diff --git a/testing/adios2/bindings/python/adios2NPTypes.py b/testing/adios2/bindings/python/adios2NPTypes.py index d0cc81cf6f..d35f73e644 100644 --- a/testing/adios2/bindings/python/adios2NPTypes.py +++ b/testing/adios2/bindings/python/adios2NPTypes.py @@ -10,45 +10,70 @@ class SmallTestData: - def __init__(self): self.Nx = 10 self.Str = "Hello ADIOS2 Python" self.I8 = np.array([0, 1, -2, 3, -4, 5, -6, 7, -8, 9], dtype=np.int8) - self.I16 = np.array( - [512, 513, -510, 515, -508, 517, -506, 519, -504, 521], - dtype=np.int16) + self.I16 = np.array([512, 513, -510, 515, -508, 517, -506, 519, -504, 521], dtype=np.int16) self.I32 = np.array( - [131072, 131073, -131070, 131075, -131068, - 131077, -131066, 131079, -131064, 131081], - dtype=np.int32) + [131072, 131073, -131070, 131075, -131068, 131077, -131066, 131079, -131064, 131081], + dtype=np.int32, + ) self.I64 = np.array( - [8589934592, 8589934593, -8589934590, 8589934595, -8589934588, - 8589934597, -8589934586, 8589934599, -8589934584, 8589934601], - dtype=np.int64) + [ + 8589934592, + 8589934593, + -8589934590, + 8589934595, + -8589934588, + 8589934597, + -8589934586, + 8589934599, + -8589934584, + 8589934601, + ], + dtype=np.int64, + ) - self.U8 = np.array( - [128, 129, 130, 131, 132, 133, 134, 135, 136, 137], dtype=np.uint8) + self.U8 = np.array([128, 129, 130, 131, 132, 133, 134, 135, 136, 137], dtype=np.uint8) self.U16 = np.array( - [32768, 32769, 32770, 32771, 32772, 32773, 32774, 32775, 32776, - 32777], - dtype=np.uint16) + [32768, 32769, 32770, 32771, 32772, 32773, 32774, 32775, 32776, 32777], dtype=np.uint16 + ) self.U32 = np.array( - [2147483648, 2147483649, 2147483650, 2147483651, 2147483652, - 2147483653, 2147483654, 2147483655, 2147483656, 2147483657], - dtype=np.uint32) + [ + 2147483648, + 2147483649, + 2147483650, + 2147483651, + 2147483652, + 2147483653, + 2147483654, + 2147483655, + 2147483656, + 2147483657, + ], + dtype=np.uint32, + ) self.U64 = np.array( - [9223372036854775808, 9223372036854775809, 9223372036854775810, - 9223372036854775811, 9223372036854775812, 9223372036854775813, - 9223372036854775814, 9223372036854775815, 9223372036854775816, - 9223372036854775817], dtype=np.uint64) + [ + 9223372036854775808, + 9223372036854775809, + 9223372036854775810, + 9223372036854775811, + 9223372036854775812, + 9223372036854775813, + 9223372036854775814, + 9223372036854775815, + 9223372036854775816, + 9223372036854775817, + ], + dtype=np.uint64, + ) self.R32 = np.array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9], dtype=np.float32) - self.R64 = np.array([0, -1, -2, -3, -4, -5, -6, -7, -8, -9], - dtype=np.float64) + self.R64 = np.array([0, -1, -2, -3, -4, -5, -6, -7, -8, -9], dtype=np.float64) def update(self, rank, step, size): - self.I8 += 1 self.I16 += 1 self.I32 += 1 diff --git a/testing/adios2/python/CMakeLists.txt b/testing/adios2/python/CMakeLists.txt index a553327269..84ebb9b49d 100644 --- a/testing/adios2/python/CMakeLists.txt +++ b/testing/adios2/python/CMakeLists.txt @@ -32,4 +32,9 @@ if(ADIOS2_HAVE_MPI) if(ADIOS2_HAVE_ZFP) add_python_mpi_test(BPZfpHighLevelAPI) endif() + if(ADIOS2_HAVE_HDF5) + python_add_test(NAME Api.Python.BPWriteTypesHighLevelAPI_HDF5 SCRIPT TestBPWriteTypesHighLevelAPI_HDF5.py) + # MPI version currently fails in H5Fclose. + # add_python_mpi_test(BPWriteTypesHighLevelAPI_HDF5) + endif() endif() diff --git a/testing/adios2/python/TestBPWriteTypesHighLevelAPI.py b/testing/adios2/python/TestBPWriteTypesHighLevelAPI.py index e6da7abb6f..231032fde8 100644 --- a/testing/adios2/python/TestBPWriteTypesHighLevelAPI.py +++ b/testing/adios2/python/TestBPWriteTypesHighLevelAPI.py @@ -19,7 +19,7 @@ size = comm.Get_size() # Test data -data = SmallTestData() +data = SmallTestData(rank) nx = data.Nx shape = [size * nx] @@ -33,6 +33,7 @@ s.write("rank", np.array(rank), shape=[LocalValueDim]) if rank == 0 and step.current_step() == 0: s.write("tag", "Testing ADIOS2 high-level API") + s.write("nx", data.Nx) s.write("gvarI8", np.array(data.i8[0])) s.write("gvarI16", np.array(data.i16[0])) s.write("gvarI32", np.array(data.i32[0])) @@ -43,9 +44,21 @@ s.write("gvarU64", np.array(data.u64[0])) s.write("gvarR32", np.array(data.r32[0])) s.write("gvarR64", np.array(data.r64[0])) + s.write("gvarC64", np.array(data.c64[0])) + i = data.int_list[0] + f = data.float_list[0] + c = data.complex_list[0] + print(f"type of i = {type(i)}, type of f = {type(f)}, type of c = {type(c)}") + s.write("an_int_value", i) + s.write("a_float_value", f) + s.write("a_complex_value", c) + s.write("an_int_list", data.int_list) + s.write("a_float_list", data.float_list) + s.write("a_complex_list", data.complex_list) # single value attributes s.write_attribute("attrStr", "Testing single string attribute") + s.write_attribute("attrNx", data.Nx) s.write_attribute("attrI8", np.array(data.i8[0])) s.write_attribute("attrI16", np.array(data.i16[0])) s.write_attribute("attrI32", np.array(data.i32[0])) @@ -56,6 +69,15 @@ s.write_attribute("attrU64", np.array(data.u64[0])) s.write_attribute("attrR32", np.array(data.r32[0])) s.write_attribute("attrR64", np.array(data.r64[0])) + s.write_attribute("attrC64", np.array(data.c64[0])) + + s.write_attribute("attr_int_value", i) + s.write_attribute("attr_float_value", f) + s.write_attribute("attr_complex_value", c) + + s.write_attribute("attr_int_list", data.int_list) + s.write_attribute("attr_float_list", data.float_list) + s.write_attribute("attr_complex_list", data.complex_list) s.write_attribute("attrStrArray", ["string1", "string2", "string3"]) s.write_attribute("attrI8Array", data.i8) @@ -68,6 +90,7 @@ s.write_attribute("attrU64Array", data.u64) s.write_attribute("attrR32Array", data.r32) s.write_attribute("attrR64Array", data.r64) + s.write_attribute("attrC64Array", data.c64) s.write("steps", "Step:" + str(step.current_step())) s.write("varI8", data.i8, shape, start, count) @@ -80,6 +103,7 @@ s.write("varU64", data.u64, shape, start, count) s.write("varR32", data.r32, shape, start, count) s.write("varR64", data.r64, shape, start, count) + s.write("varC64", data.c64, shape, start, count) if rank == 0 and step.current_step() == 0: s.write_attribute("varattrStrArray", ["varattr1", "varattr2", "varattr3"], "steps") @@ -98,7 +122,7 @@ comm.Barrier() # Reader -data = SmallTestData() +data = SmallTestData(rank) with Stream("types_np.bp", "r", comm=comm) as fr: # file only @@ -116,8 +140,11 @@ # print("\t" + key + ": " + value) # print("\n") - if step == 0: + if rank == 0 and step == 0: inTag = fr_step.read("tag") + nx = fr_step.read("nx") + print(f"nx = {nx}") + assert nx == data.Nx inI8 = fr_step.read("gvarI8") inI16 = fr_step.read("gvarI16") inI32 = fr_step.read("gvarI32") @@ -165,6 +192,7 @@ # attributes inTag = fr_step.read_attribute("attrStr") + inNx = fr_step.read_attribute("attrNx") inI8 = fr_step.read_attribute("attrI8") inI16 = fr_step.read_attribute("attrI16") inI32 = fr_step.read_attribute("attrI32") @@ -179,6 +207,9 @@ if inTag[0] != "Testing single string attribute": raise ValueError("attr string read failed") + if inNx != data.Nx: + raise ValueError("attrNx read failed") + if inI8[0] != data.i8[0]: raise ValueError("attrI8 read failed") @@ -209,6 +240,37 @@ if inR64[0] != data.r64[0]: raise ValueError("attrR64 read failed") + in_an_int_value = fr_step.read("an_int_value") + print(f"an_int_value = {in_an_int_value} of type {type(in_an_int_value)}") + assert in_an_int_value == data.int_list[0] + + in_a_float_value = fr_step.read("a_float_value") + print(f"a_float_value = {in_a_float_value} of type {type(in_a_float_value)}") + assert in_a_float_value == data.float_list[0] + + inC64 = fr_step.read("gvarC64") + print(f"inC64 = {inC64} of type {type(inC64)}") + assert inC64 == data.c64[0] + + in_a_complex_value = fr_step.read("a_complex_value") + print(f"a_complex_value = {in_a_complex_value} of type {type(in_a_complex_value)}") + assert in_a_complex_value == data.complex_list[0] + + an_int_list = fr_step.read("an_int_list") + if not (an_int_list == data.int_list).all(): + raise ValueError("an_int_list array read failed") + print(f"an_int_list = {an_int_list} of type {type(an_int_list)}") + + a_float_list = fr_step.read("a_float_list") + if not (a_float_list == data.float_list).all(): + raise ValueError("a_float_list array read failed") + print(f"a_float_list = {a_float_list} of type {type(a_float_list)}") + + a_complex_list = fr_step.read("a_complex_list") + if not (a_complex_list == data.complex_list).all(): + raise ValueError("a_complex_list array read failed") + print(f"a_complex_list = {a_complex_list} of type {type(a_complex_list)}") + # Array attribute inTag = fr_step.read_attribute_string("attrStrArray") inI8 = fr_step.read_attribute("attrI8Array") @@ -225,36 +287,49 @@ if inTag != ["string1", "string2", "string3"]: raise ValueError("attrStrArray read failed") - if (inI8 == data.i8).all() is False: + if not (inI8 == data.i8).all(): raise ValueError("attrI8 array read failed") - if (inI16 == data.i16).all() is False: + if not (inI16 == data.i16).all(): raise ValueError("attrI16 array read failed") - if (inI32 == data.i32).all() is False: + if not (inI32 == data.i32).all(): raise ValueError("attrI32 array read failed") - if (inI64 == data.i64).all() is False: + if not (inI64 == data.i64).all(): raise ValueError("attrI64 array read failed") - if (inU8 == data.u8).all() is False: + if not (inU8 == data.u8).all(): raise ValueError("attrU8 array read failed") - if (inU16 == data.u16).all() is False: + if not (inU16 == data.u16).all(): raise ValueError("attrU16 array read failed") - if (inU32 == data.u32).all() is False: + if not (inU32 == data.u32).all(): raise ValueError("attrU32 array read failed") - if (inU64 == data.u64).all() is False: + if not (inU64 == data.u64).all(): raise ValueError("attrU64 array read failed") - if (inR32 == data.r32).all() is False: + if not (inR32 == data.r32).all(): raise ValueError("attrR32 array read failed") - if (inR64 == data.r64).all() is False: + if not (inR64 == data.r64).all(): raise ValueError("attrR64 array read failed") + # Array attributes (written as List) + in_attr_float_list = fr_step.read_attribute("attr_float_list") + in_attr_int_list = fr_step.read_attribute("attr_int_list") + + print(f"attr_float_list = {in_attr_float_list}") + print(f"attr_int_list = {in_attr_int_list}") + + if not (in_attr_float_list == np.array(data.float_list)).all(): + raise ValueError("attr_float_list array read failed") + + if not (in_attr_int_list == np.array(data.int_list)).all(): + raise ValueError("attr_int_list array read failed") + inTags = fr_step.read_attribute_string("varattrStrArray", "steps") inI8 = fr_step.read_attribute("varattrI8Array", "varI8") in16 = fr_step.read_attribute("varattrI16Array", "varI16") @@ -271,34 +346,34 @@ print(inTags) raise ValueError("var attrStrArray read failed") - if (inI8 == data.i8).all() is False: + if not (inI8 == data.i8).all(): raise ValueError("var attrI8 array read failed") - if (inI16 == data.i16).all() is False: + if not (inI16 == data.i16).all(): raise ValueError("var attrI16 array read failed") - if (inI32 == data.i32).all() is False: + if not (inI32 == data.i32).all(): raise ValueError("var attrI32 array read failed") - if (inI64 == data.i64).all() is False: + if not (inI64 == data.i64).all(): raise ValueError("var attrI64 array read failed") - if (inU8 == data.u8).all() is False: + if not (inU8 == data.u8).all(): raise ValueError("var attrU8 array read failed") - if (inU16 == data.u16).all() is False: + if not (inU16 == data.u16).all(): raise ValueError("var attrU16 array read failed") - if (inU32 == data.u32).all() is False: + if not (inU32 == data.u32).all(): raise ValueError("var attrU32 array read failed") - if (inU64 == data.u64).all() is False: + if not (inU64 == data.u64).all(): raise ValueError("var attrU64 array read failed") - if (inR32 == data.r32).all() is False: + if not (inR32 == data.r32).all(): raise ValueError("var attrR32 array read failed") - if (inR64 == data.r64).all() is False: + if not (inR64 == data.r64).all(): raise ValueError("var attrR64 array read failed") stepStr = "Step:" + str(step) @@ -309,7 +384,7 @@ indataRanks = fr_step.read("rank", [0], [size]) dataRanks = np.arange(0, size) - if (indataRanks == dataRanks).all() is False: + if not (indataRanks == dataRanks).all(): raise ValueError("Ranks read failed") indataI8 = fr_step.read("varI8", start, count) @@ -323,32 +398,32 @@ indataR32 = fr_step.read("varR32", start, count) indataR64 = fr_step.read("varR64", start, count) - if (indataI8 == data.i8).all() is False: + if not (indataI8 == data.i8).all(): raise ValueError("i8 array read failed") - if (indataI16 == data.i16).all() is False: + if not (indataI16 == data.i16).all(): raise ValueError("i16 array read failed") - if (indataI32 == data.i32).all() is False: + if not (indataI32 == data.i32).all(): raise ValueError("i32 array read failed") - if (indataI64 == data.i64).all() is False: + if not (indataI64 == data.i64).all(): raise ValueError("i64 array read failed") - if (indataU8 == data.u8).all() is False: + if not (indataU8 == data.u8).all(): raise ValueError("u8 array read failed") - if (indataU16 == data.u16).all() is False: + if not (indataU16 == data.u16).all(): raise ValueError("u16 array read failed") - if (indataU32 == data.u32).all() is False: + if not (indataU32 == data.u32).all(): raise ValueError("u32 array read failed") - if (indataU64 == data.u64).all() is False: + if not (indataU64 == data.u64).all(): raise ValueError("u64 array read failed") - if (indataR32 == data.r32).all() is False: + if not (indataR32 == data.r32).all(): raise ValueError("r32 array read failed") - if (indataR64 == data.r64).all() is False: + if not (indataR64 == data.r64).all(): raise ValueError("r64 array read failed") diff --git a/testing/adios2/python/TestBPWriteTypesHighLevelAPILocal.py b/testing/adios2/python/TestBPWriteTypesHighLevelAPILocal.py index 6a719b3ba9..d4eafcb259 100644 --- a/testing/adios2/python/TestBPWriteTypesHighLevelAPILocal.py +++ b/testing/adios2/python/TestBPWriteTypesHighLevelAPILocal.py @@ -14,7 +14,7 @@ def check_array(np1, np2, hint): - if (np1 == np2).all() is False: + if not (np1 == np2).all(): print("InData: " + str(np1)) print("Data: " + str(np2)) raise ValueError("Array read failed " + str(hint)) @@ -25,13 +25,14 @@ def check_array(np1, np2, hint): size = comm.Get_size() # Test data -data = SmallTestData() +data = SmallTestData(rank) nx = data.Nx shape = [] start = [] count = [nx] + # Writer with Stream("types_np_local.bp", "w", comm=comm) as s: for step in s.steps(5): @@ -46,35 +47,38 @@ def check_array(np1, np2, hint): s.write("varU64", data.u64, shape, start, count) s.write("varR32", data.r32, shape, start, count) s.write("varR64", data.r64, shape, start, count) - + s.write("an_int_list", data.int_list, shape, start, count) + s.write("a_float_list", data.float_list, shape, start, count) + s.write("a_complex_list", data.complex_list, shape, start, count) # Reader -data = SmallTestData() +data = SmallTestData(rank) with Stream("types_np_local.bp", "r", comm=comm) as s: for fr_step in s.steps(): step = fr_step.current_step() + data.update(rank, step, size) + indataI8 = fr_step.read("varI8", block_id=rank) + indataI16 = fr_step.read("varI16", block_id=rank) + indataI32 = fr_step.read("varI32", block_id=rank) + indataI64 = fr_step.read("varI64", block_id=rank) + indataU8 = fr_step.read("varU8", block_id=rank) + indataU16 = fr_step.read("varU16", block_id=rank) + indataU32 = fr_step.read("varU32", block_id=rank) + indataU64 = fr_step.read("varU64", block_id=rank) + indataR32 = fr_step.read("varR32", block_id=rank) + indataR64 = fr_step.read("varR64", block_id=rank) - for b in range(0, size): - data.update(b, step, size) + in_int_list = fr_step.read("an_int_list", block_id=rank) - indataI8 = fr_step.read("varI8", b) - indataI16 = fr_step.read("varI16", b) - indataI32 = fr_step.read("varI32", b) - indataI64 = fr_step.read("varI64", b) - indataU8 = fr_step.read("varU8", b) - indataU16 = fr_step.read("varU16", b) - indataU32 = fr_step.read("varU32", b) - indataU64 = fr_step.read("varU64", b) - indataR32 = fr_step.read("varR32", b) - indataR64 = fr_step.read("varR64", b) + print(f"step {step} rank {rank} I16={indataI16} data.I16 = {data.i16}", flush=True) - check_array(indataI8, data.i8, "i8") - check_array(indataI16, data.i16, "i16") - check_array(indataI32, data.i32, "i32") - check_array(indataI64, data.i64, "i64") - check_array(indataU8, data.u8, "u8") - check_array(indataU16, data.u16, "u16") - check_array(indataU32, data.u32, "u32") - check_array(indataU64, data.u64, "u64") - check_array(indataR32, data.r32, "r32") - check_array(indataR64, data.r64, "r64") + check_array(indataI8, data.i8, "i8") + check_array(indataI16, data.i16, "i16") + check_array(indataI32, data.i32, "i32") + check_array(indataI64, data.i64, "i64") + check_array(indataU8, data.u8, "u8") + check_array(indataU16, data.u16, "u16") + check_array(indataU32, data.u32, "u32") + check_array(indataU64, data.u64, "u64") + check_array(indataR32, data.r32, "r32") + check_array(indataR64, data.r64, "r64") diff --git a/testing/adios2/python/TestBPWriteTypesHighLevelAPI_HDF5.py b/testing/adios2/python/TestBPWriteTypesHighLevelAPI_HDF5.py index 00e72110a2..d95c160bc9 100644 --- a/testing/adios2/python/TestBPWriteTypesHighLevelAPI_HDF5.py +++ b/testing/adios2/python/TestBPWriteTypesHighLevelAPI_HDF5.py @@ -10,24 +10,26 @@ # Author: William F Godoy godoywf@ornl.gov from adios2NPTypes import SmallTestData -from mpi4py import MPI import numpy as np -import adios2.bindings as adios2 +from adios2 import Adios, Stream -comm = MPI.COMM_WORLD -rank = comm.Get_rank() -size = comm.Get_size() +rank = 0 +size = 1 # Test data -data = SmallTestData() +data = SmallTestData(rank) nx = data.Nx shape = [size * nx] start = [rank * nx] count = [nx] +adios = Adios() +io = adios.declare_io("writeh5") +io.set_engine("HDF5") + # Writer -with adios2.open("types_np.h5", "w", comm, "HDF5") as fw: +with Stream(io, "types_np.h5", "w") as fw: for i in range(0, 5): data.update(rank, i, size) @@ -97,12 +99,13 @@ fw.end_step() -comm.Barrier() - # Reader -data = SmallTestData() +data = SmallTestData(rank) + +io = adios.declare_io("readh5") +io.set_engine("HDF5") -with adios2.open("types_np.h5", "r", comm, "HDF5") as fr: +with Stream("types_np.h5", "r") as fr: for fr_step in fr: step = fr_step.current_step() data.update(rank, step, size) @@ -115,8 +118,8 @@ # print("\t" + key + ": " + value) # print("\n") - if step == 0: - inTag = fr_step.read_string("tag") + if rank == 0 and step == 0: + inTag = fr_step.read("tag") inI8 = fr_step.read("gvarI8") inI16 = fr_step.read("gvarI16") inI32 = fr_step.read("gvarI32") @@ -128,7 +131,7 @@ inR32 = fr_step.read("gvarR32") inR64 = fr_step.read("gvarR64") - if inTag[0] != "Testing ADIOS2 high-level API": + if inTag != "Testing ADIOS2 high-level API": print("InTag: " + str(inTag)) raise ValueError("tag variable read failed") @@ -163,7 +166,7 @@ raise ValueError("gvarR64 read failed") # attributes - inTag = fr_step.read_attribute_string("attrStr") + inTag = fr_step.read_attribute("attrStr") inI8 = fr_step.read_attribute("attrI8") inI16 = fr_step.read_attribute("attrI16") inI32 = fr_step.read_attribute("attrI32") @@ -302,8 +305,8 @@ stepStr = "Step:" + str(step) - instepStr = fr_step.read_string("steps") - if instepStr[0] != stepStr: + instepStr = fr_step.read("steps") + if instepStr != stepStr: raise ValueError("steps variable read failed: " + instepStr + " " + stepStr) indataI8 = fr_step.read("varI8", start, count) diff --git a/testing/adios2/python/TestEngine.py b/testing/adios2/python/TestEngine.py index 86ea86c670..5e48536e7a 100644 --- a/testing/adios2/python/TestEngine.py +++ b/testing/adios2/python/TestEngine.py @@ -17,7 +17,7 @@ def test_put(self): adios = Adios() with adios.declare_io("BPWriter") as io: pressure = io.define_variable("pressure") - temps = io.define_variable("temps", np.empty([4], dtype=np.int64)) + temps = io.define_variable("temps", np.empty([1], dtype=np.int64)) with io.open("pythontestengine.bp", bindings.Mode.Write) as engine: engine.put(pressure, "35PSI") temps_measures = np.array([35, 40, 30, 45], dtype=np.int64) @@ -29,7 +29,7 @@ def test_get(self): pressure = io.define_variable("pressure") temps = io.define_variable( name="temps", - content=np.empty([4], dtype=np.int64), + content=np.empty([1], dtype=np.int64), start=[0], shape=[4], count=[4], diff --git a/testing/adios2/python/adios2NPTypes.py b/testing/adios2/python/adios2NPTypes.py index df6ca200b7..711dd1a74a 100644 --- a/testing/adios2/python/adios2NPTypes.py +++ b/testing/adios2/python/adios2NPTypes.py @@ -7,10 +7,11 @@ # Author: William F Godoy godoywf@ornl.gov import numpy as np +import cmath class SmallTestData: - def __init__(self): + def __init__(self, rank=0): self.Nx = 10 self.Str = "Hello ADIOS2 Python" self.i8 = np.array([0, 1, -2, 3, -4, 5, -6, 7, -8, 9], dtype=np.int8) @@ -73,16 +74,64 @@ def __init__(self): self.r32 = np.array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9], dtype=np.float32) self.r64 = np.array([0, -1, -2, -3, -4, -5, -6, -7, -8, -9], dtype=np.float64) + self.c64 = np.array( + [ + complex(0, 1), + complex(2, 3), + complex(4, 5), + complex(6, 7), + complex(8, 9), + complex(0, -1), + complex(-2, -3), + complex(-4, -5), + complex(-6, -7), + complex(-8, -9), + ], + dtype=np.complex128, + ) + + self.int_list = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + self.float_list = [0.0, 1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8, 9.9] + self.complex_list = [ + complex(0.0, rank), + complex(1.0, rank), + complex(2.0, rank), + complex(3.0, rank), + complex(4.0, rank), + complex(5.0, rank), + complex(6.0, rank), + complex(7.0, rank), + complex(8.0, rank), + complex(9.0, rank), + ] + + self.i8 += rank + self.i16 += rank + self.i32 += rank + self.i64 += rank + self.u8 += rank + self.u16 += rank + self.u32 += rank + self.u64 += rank + self.r32 += rank + self.r64 += rank + self.c64 += complex(rank, rank) + self.int_list = [i + rank for i in self.int_list] + self.float_list = [f + rank for f in self.float_list] + self.complex_list = [c + complex(rank, 0) for c in self.complex_list] + def update(self, rank, step, size): self.i8 += 1 self.i16 += 1 self.i32 += 1 self.i64 += 1 - self.u8 += 1 self.u16 += 1 self.u32 += 1 self.u64 += 1 - - self.r32 += 1 - self.r64 += 1 + self.r32 += 1.0 + self.r64 += 1.0 + self.c64 += complex(1.0, 1.0) + self.int_list = [i + 1 for i in self.int_list] + self.float_list = [i + 1.0 for i in self.float_list] + self.complex_list = [i + complex(1, 0) for i in self.complex_list] From 96dd3233f8d9c265a27882a938cfa424f6f3f30b Mon Sep 17 00:00:00 2001 From: Greg Eisenhauer Date: Wed, 21 Feb 2024 17:07:55 -0500 Subject: [PATCH 048/124] Add a static windows build to CI (#4037) - Upstream EVpath, FFS and dill. - Actually add the CI Co-authored-by: Vicente Bolea --- .github/workflows/everything.yml | 8 +- .../ci-win2022-vs2022-static-serial.cmake | 28 + thirdparty/EVPath/EVPath/CMakeLists.txt | 2 +- thirdparty/EVPath/EVPath/cmenet.c | 2 +- thirdparty/EVPath/EVPath/cmepoll.c | 128 +---- thirdparty/EVPath/EVPath/cmmulticast.c | 2 +- thirdparty/EVPath/EVPath/cmselect.c | 2 +- thirdparty/EVPath/EVPath/cmsockets.c | 2 +- thirdparty/EVPath/EVPath/cmudp.c | 2 +- .../EVPath/EVPath/dfg_tests/auto_tree_test.c | 7 + .../EVPath/EVPath/dfg_tests/fail_chain_test.c | 7 + .../EVPath/dfg_tests/self_reconfig_test.c | 7 + .../EVPath/EVPath/dfg_tests/tree_test.c | 7 + thirdparty/EVPath/EVPath/ev_dfg.c | 5 +- thirdparty/EVPath/EVPath/evp.c | 6 +- thirdparty/EVPath/EVPath/mtests/cmconn.c | 3 +- thirdparty/EVPath/EVPath/response.c | 2 +- thirdparty/EVPath/EVPath/tests/extract_test.c | 2 +- thirdparty/dill/dill/CMakeLists.txt | 5 + thirdparty/dill/dill/base.ops | 11 +- .../ffs/.github/workflows/build-and-test.yml | 3 + thirdparty/ffs/ffs/CMakeLists.txt | 4 + thirdparty/ffs/ffs/cod/cg.c | 4 + thirdparty/ffs/ffs/cod/cod.h | 50 +- thirdparty/ffs/ffs/cod/cod.y | 4 +- .../ffs/ffs/cod/pregen_source/Windows/cod.l | 63 --- .../ffs/cod/pregen_source/Windows/cod.tab.c | 518 +++++++----------- .../ffs/cod/pregen_source/Windows/cod.tab.h | 320 +++++------ .../ffs/ffs/cod/pregen_source/Windows/cod.y | 6 +- .../ffs/cod/pregen_source/Windows/lex.yy.c | 63 --- thirdparty/ffs/ffs/cod/standard.c | 6 +- thirdparty/ffs/ffs/cod/tests/CMakeLists.txt | 5 +- thirdparty/ffs/ffs/cod/tests/general.ops | 1 + thirdparty/ffs/ffs/cod/tests/t1.c | 2 +- thirdparty/ffs/ffs/ffs/ffs.c | 3 +- thirdparty/ffs/ffs/ffs/ffs.h.in | 94 ++-- thirdparty/ffs/ffs/fm/fm.h | 148 +++-- thirdparty/ffs/ffs/fm/fm_dump.c | 2 +- thirdparty/ffs/ffs/fm/io_interface.h | 2 +- thirdparty/ffs/ffs/fm/nt_io.c | 13 +- thirdparty/ffs/ffs/fm/tests/scale_test.c | 2 +- .../ffs/scripts/ci/cmake/windows-common.cmake | 5 + .../windows2022-vs2022-msvc-static.cmake | 8 + 43 files changed, 631 insertions(+), 933 deletions(-) create mode 100644 scripts/ci/cmake/ci-win2022-vs2022-static-serial.cmake create mode 100644 thirdparty/ffs/ffs/scripts/ci/cmake/windows2022-vs2022-msvc-static.cmake diff --git a/.github/workflows/everything.yml b/.github/workflows/everything.yml index be966def07..4bd22408f5 100644 --- a/.github/workflows/everything.yml +++ b/.github/workflows/everything.yml @@ -339,7 +339,7 @@ jobs: runs-on: ${{ matrix.image }} env: - GH_YML_JOBNAME: ${{ matrix.os }}-${{ matrix.compiler }}-${{ matrix.parallel }} + GH_YML_JOBNAME: ${{ matrix.os }}-${{ matrix.compiler }}${{ matrix.shared == 'static' && '-static' || ''}}-${{ matrix.parallel }} GH_YML_BASE_OS: Windows GH_YML_MATRIX_OS: ${{ matrix.os }} GH_YML_MATRIX_COMPILER: ${{ matrix.compiler }} @@ -349,6 +349,7 @@ jobs: fail-fast: false matrix: os: [win2019, win2022] + shared: [shared] parallel: [serial, ompi] include: - os: win2019 @@ -357,6 +358,11 @@ jobs: - os: win2022 image: windows-2022 compiler: vs2022 + - os: win2022 + image: windows-2022 + shared: static + compiler: vs2022 + parallel: serial defaults: run: diff --git a/scripts/ci/cmake/ci-win2022-vs2022-static-serial.cmake b/scripts/ci/cmake/ci-win2022-vs2022-static-serial.cmake new file mode 100644 index 0000000000..605be8e69b --- /dev/null +++ b/scripts/ci/cmake/ci-win2022-vs2022-static-serial.cmake @@ -0,0 +1,28 @@ +set(ENV{CC} cl) +set(ENV{CXX} cl) +set(ENV{CFLAGS} /WX) +set(ENV{CXXFLAGS} /WX) + +# Tests need to find hdf5.dll +# set(ENV{PATH} "$ENV{PATH};C:/hdf5/HDF5-1.14.2.1-win64/bin") + +set(dashboard_cache " +CMAKE_MSVC_RUNTIME_LIBRARY=MultiThreadedDebug +BUILD_SHARED_LIBS=OFF +gtest_force_shared_crt=ON +ADIOS2_USE_EXTERNAL_GTEST=OFF +BUILD_TESTING:BOOL=ON +ADIOS2_BUILD_EXAMPLES:BOOL=ON + +ADIOS2_USE_Fortran:BOOL=OFF +ADIOS2_USE_MPI:BOOL=OFF +ADIOS2_USE_HDF5:STRING=OFF +ADIOS2_USE_Python:BOOL=OFF +ADIOS2_USE_HDF5_VOL:STRING=OFF +# HDF5_ROOT:PATH=C:/hdf5/HDF5-1.14.2.1-win64 +") + +set(CTEST_CMAKE_GENERATOR "Visual Studio 17 2022") +set(CTEST_CMAKE_GENERATOR_PLATFORM "x64") +list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") +include(${CMAKE_CURRENT_LIST_DIR}/ci-common.cmake) diff --git a/thirdparty/EVPath/EVPath/CMakeLists.txt b/thirdparty/EVPath/EVPath/CMakeLists.txt index 003f31325f..132c650619 100644 --- a/thirdparty/EVPath/EVPath/CMakeLists.txt +++ b/thirdparty/EVPath/EVPath/CMakeLists.txt @@ -376,7 +376,7 @@ if (MSVC) set(EVPATH_USE_ZPL_ENET FALSE) endif() if(NOT (DEFINED EVPATH_USE_ZPL_ENET)) - option(EVPATH_USE_ZPL_ENET "Build the enet transport" "ON") + option(EVPATH_USE_ZPL_ENET "Build the enet transport" "OFF") endif() if(EVPATH_USE_ZPL_ENET) set(RUN_ZPL_ENET_TESTS TRUE) diff --git a/thirdparty/EVPath/EVPath/cmenet.c b/thirdparty/EVPath/EVPath/cmenet.c index eeac56f3e5..b33989de52 100644 --- a/thirdparty/EVPath/EVPath/cmenet.c +++ b/thirdparty/EVPath/EVPath/cmenet.c @@ -1414,7 +1414,7 @@ int err; * NT Sux. */ -int +static int pipe(filedes) SOCKET filedes[2]; { diff --git a/thirdparty/EVPath/EVPath/cmepoll.c b/thirdparty/EVPath/EVPath/cmepoll.c index b28fca2790..84d95470c9 100644 --- a/thirdparty/EVPath/EVPath/cmepoll.c +++ b/thirdparty/EVPath/EVPath/cmepoll.c @@ -114,10 +114,7 @@ static char*WSAerror_str(int err); #endif static void -init_select_data(svc, sdp, cm) -CMtrans_services svc; -select_data_ptr *sdp; -CManager cm; +init_select_data(CMtrans_services svc, select_data_ptr *sdp, CManager cm) { select_data_ptr sd = malloc(sizeof(struct select_data)); *sdp = sd; @@ -157,9 +154,7 @@ typedef struct _periodic_task { } task_handle_s; static void -free_epoll_data(svc, sdp) -CMtrans_services svc; -select_data_ptr *sdp; +free_epoll_data(CMtrans_services svc, select_data_ptr *sdp) { periodic_task_handle tasks; select_data_ptr sd = *sdp; @@ -184,10 +179,7 @@ select_data_ptr *sdp; ((tvp)->tv_usec cmp (uvp)->tv_usec)))) static void -set_soonest_timeout(timeout, task_list, now) -struct timeval *timeout; -periodic_task_handle task_list; -struct timeval now; +set_soonest_timeout(struct timeval *timeout, periodic_task_handle task_list, struct timeval now) { struct timeval this_delay; if (task_list == NULL) return; @@ -210,10 +202,7 @@ struct timeval now; } static void -increment_time(time, increment_sec, increment_usec) -struct timeval *time; -int increment_sec; -int increment_usec; +increment_time(struct timeval *time, int increment_sec, int increment_usec) { time->tv_usec += increment_usec; time->tv_sec += increment_sec; @@ -227,11 +216,7 @@ static void shutdown_wake_mechanism(select_data_ptr sd); static void -socket_select(svc, sd, timeout_sec, timeout_usec) -CMtrans_services svc; -select_data_ptr sd; -int timeout_sec; -int timeout_usec; +socket_select(CMtrans_services svc, select_data_ptr sd, int timeout_sec, int timeout_usec) { int i, res; int fd; @@ -475,13 +460,7 @@ int timeout_usec; } extern void -libcmepoll_LTX_add_select(svc, sdp, fd, func, arg1, arg2) -CMtrans_services svc; -select_data_ptr *sdp; -int fd; -select_list_func func; -void *arg1; -void *arg2; +libcmepoll_LTX_add_select(CMtrans_services svc, select_data_ptr *sdp, int fd, select_list_func func, void *arg1, void *arg2) { select_data_ptr sd = *((select_data_ptr *)sdp); struct epoll_event ep_event; @@ -540,13 +519,7 @@ void *arg2; } extern void -libcmepoll_LTX_write_select(svc, sdp, fd, func, arg1, arg2) -CMtrans_services svc; -select_data_ptr *sdp; -int fd; -select_list_func func; -void *arg1; -void *arg2; +libcmepoll_LTX_write_select(CMtrans_services svc, select_data_ptr *sdp, int fd, select_list_func func, void *arg1, void *arg2) { select_data_ptr sd = *((select_data_ptr *)sdp); struct epoll_event ep_event; @@ -616,15 +589,7 @@ void *arg2; } extern periodic_task_handle -libcmepoll_LTX_add_periodic(svc, sdp, interval_sec, interval_usec, - func, arg1, arg2) -CMtrans_services svc; -select_data_ptr *sdp; -int interval_sec; -int interval_usec; -select_list_func func; -void *arg1; -void *arg2; +libcmepoll_LTX_add_periodic(CMtrans_services svc, select_data_ptr *sdp, int interval_sec, int interval_usec, select_list_func func, void *arg1, void *arg2) { select_data_ptr sd = *((select_data_ptr *)sdp); periodic_task_handle handle = malloc(sizeof(struct _periodic_task)); @@ -671,15 +636,7 @@ void *arg2; extern periodic_task_handle -libcmepoll_LTX_add_delayed_task(svc, sdp, delay_sec, delay_usec, - func, arg1, arg2) -CMtrans_services svc; -select_data_ptr *sdp; -int delay_sec; -int delay_usec; -select_list_func func; -void *arg1; -void *arg2; +libcmepoll_LTX_add_delayed_task(CMtrans_services svc, select_data_ptr *sdp, int delay_sec, int delay_usec, select_list_func func, void *arg1, void *arg2) { select_data_ptr sd = *((select_data_ptr *)sdp); periodic_task_handle handle = malloc(sizeof(struct _periodic_task)); @@ -725,9 +682,7 @@ void *arg2; } static int -remove_periodic_task(sd, handle) -select_data_ptr sd; -periodic_task_handle handle; +remove_periodic_task(select_data_ptr sd, periodic_task_handle handle) { periodic_task_handle list, last = NULL; list = sd->periodic_task_list; @@ -765,10 +720,7 @@ periodic_task_handle handle; extern void -libcmepoll_LTX_remove_periodic(svc, sdp, handle) -CMtrans_services svc; -select_data_ptr *sdp; -periodic_task_handle handle; +libcmepoll_LTX_remove_periodic(CMtrans_services svc, select_data_ptr *sdp, periodic_task_handle handle) { select_data_ptr sd = *((select_data_ptr *)sdp); if (sd == NULL) return; @@ -778,10 +730,7 @@ periodic_task_handle handle; } extern void -libcmepoll_LTX_remove_select(svc, sdp, fd) -CMtrans_services svc; -select_data_ptr *sdp; -int fd; +libcmepoll_LTX_remove_select(CMtrans_services svc, select_data_ptr *sdp, int fd) { select_data_ptr sd = *((select_data_ptr *)sdp); @@ -812,8 +761,7 @@ int fd; } static void -shutdown_wake_mechanism(sd) -select_data_ptr sd; +shutdown_wake_mechanism(select_data_ptr sd) { if (sd->wake_read_fd == -1) return; close(sd->wake_read_fd); @@ -821,9 +769,7 @@ select_data_ptr sd; sd->wake_read_fd = sd->wake_write_fd = -1; } -static void read_wake_fd(fd_as_ptr, junk) -void *fd_as_ptr; -void *junk; +static void read_wake_fd(void *fd_as_ptr, void *junk) { char buffer; int fd = (int) (long)fd_as_ptr; @@ -838,8 +784,7 @@ void *junk; #ifdef HAVE_WINDOWS_H static char* -WSAerror_str(err) -int err; +WSAerror_str(int err) { switch(err) { case WSAEINTR: return "WSAEINTR"; @@ -900,9 +845,8 @@ int err; * NT Sux. */ -int -pipe(filedes) -int filedes[2]; +static int +pipe(int filedes[2]) { int length; @@ -990,9 +934,7 @@ int filedes[2]; #endif static void -setup_wake_mechanism(svc, sdp) -CMtrans_services svc; -select_data_ptr *sdp; +setup_wake_mechanism(CMtrans_services svc, select_data_ptr *sdp) { int filedes[2]; @@ -1015,9 +957,7 @@ select_data_ptr *sdp; } extern void -libcmepoll_LTX_wake_function(svc, sdp) -CMtrans_services svc; -select_data_ptr *sdp; +libcmepoll_LTX_wake_function(CMtrans_services svc, select_data_ptr *sdp) { if (*sdp != NULL) { wake_server_thread(*sdp); @@ -1025,8 +965,7 @@ select_data_ptr *sdp; } static void -wake_server_thread(sd) -select_data_ptr sd; +wake_server_thread(select_data_ptr sd) { static char buffer = 'W'; /* doesn't matter what we write */ if (sd->wake_write_fd != -1) { @@ -1041,9 +980,7 @@ select_data_ptr sd; } extern void -libcmepoll_LTX_blocking_function(svc, client_data) -CMtrans_services svc; -void *client_data; +libcmepoll_LTX_blocking_function(CMtrans_services svc, void *client_data) { select_data_ptr sd = *((select_data_ptr *)client_data); if (sd == NULL) { @@ -1058,9 +995,7 @@ void *client_data; } extern void -libcmepoll_LTX_polling_function(svc, client_data) -CMtrans_services svc; -void *client_data; +libcmepoll_LTX_polling_function(CMtrans_services svc, void *client_data) { select_data_ptr sd = *((select_data_ptr *)client_data); if (sd == NULL) { @@ -1075,10 +1010,7 @@ void *client_data; } extern void -libcmepoll_LTX_select_initialize(svc, cm, client_data) -CMtrans_services svc; -CManager cm; -void *client_data; +libcmepoll_LTX_select_initialize(CMtrans_services svc, CManager cm, void *client_data) { if (*((select_data_ptr *)client_data) == NULL) { init_select_data(svc, (select_data_ptr*)client_data, cm); @@ -1086,10 +1018,7 @@ void *client_data; } extern void -libcmepoll_LTX_select_shutdown(svc, cm, client_data) -CMtrans_services svc; -CManager cm; -void *client_data; +libcmepoll_LTX_select_shutdown(CMtrans_services svc, CManager cm, void *client_data) { select_data_ptr *sdp = client_data; select_data_ptr sd = *sdp; @@ -1103,10 +1032,7 @@ void *client_data; } extern void -libcmepoll_LTX_select_free(svc, cm, client_data) -CMtrans_services svc; -CManager cm; -void *client_data; +libcmepoll_LTX_select_free(CMtrans_services svc, CManager cm, void *client_data) { select_data_ptr *sdp = client_data; select_data_ptr sd = *sdp; @@ -1119,9 +1045,7 @@ void *client_data; } extern void -libcmepoll_LTX_select_stop(svc, client_data) -CMtrans_services svc; -void *client_data; +libcmepoll_LTX_select_stop(CMtrans_services svc, void *client_data) { if (*((select_data_ptr *)client_data) != NULL) { (*((select_data_ptr*)client_data))->closed = 1; diff --git a/thirdparty/EVPath/EVPath/cmmulticast.c b/thirdparty/EVPath/EVPath/cmmulticast.c index 03acd641e7..3d0e1f0897 100644 --- a/thirdparty/EVPath/EVPath/cmmulticast.c +++ b/thirdparty/EVPath/EVPath/cmmulticast.c @@ -449,7 +449,7 @@ libcmmulticast_LTX_writev_func(CMtrans_services svc, mcast_conn_data_ptr mcd, st } #ifdef HAVE_WINDOWS_H -int socket_global_init = 0; +static int socket_global_init = 0; /* Winsock init stuff, ask for ver 1.1 */ static WORD wVersionRequested = MAKEWORD(1, 1); static WSADATA wsaData; diff --git a/thirdparty/EVPath/EVPath/cmselect.c b/thirdparty/EVPath/EVPath/cmselect.c index baf9409df6..ab0ad46a19 100644 --- a/thirdparty/EVPath/EVPath/cmselect.c +++ b/thirdparty/EVPath/EVPath/cmselect.c @@ -879,7 +879,7 @@ int err; * NT Sux. */ -int +static int pipe(SOCKET *filedes) { diff --git a/thirdparty/EVPath/EVPath/cmsockets.c b/thirdparty/EVPath/EVPath/cmsockets.c index a4bfb79688..5a70aded91 100644 --- a/thirdparty/EVPath/EVPath/cmsockets.c +++ b/thirdparty/EVPath/EVPath/cmsockets.c @@ -1135,7 +1135,7 @@ libcmsockets_LTX_NBwritev_func(CMtrans_services svc, socket_conn_data_ptr scd, v return init_bytes - left; } -int socket_global_init = 0; +static int socket_global_init = 0; #ifdef HAVE_WINDOWS_H /* Winsock init stuff, ask for ver 2.2 */ diff --git a/thirdparty/EVPath/EVPath/cmudp.c b/thirdparty/EVPath/EVPath/cmudp.c index d865034dde..4c6815f2ad 100644 --- a/thirdparty/EVPath/EVPath/cmudp.c +++ b/thirdparty/EVPath/EVPath/cmudp.c @@ -646,7 +646,7 @@ libcmudp_LTX_writev_func(CMtrans_services svc, udp_conn_data_ptr ucd, struct iov } #ifdef HAVE_WINDOWS_H -int socket_global_init = 0; +static int socket_global_init = 0; /* Winsock init stuff, ask for ver 1.1 */ static WORD wVersionRequested = MAKEWORD(1, 1); static WSADATA wsaData; diff --git a/thirdparty/EVPath/EVPath/dfg_tests/auto_tree_test.c b/thirdparty/EVPath/EVPath/dfg_tests/auto_tree_test.c index d4b1d781b6..97705efd4b 100644 --- a/thirdparty/EVPath/EVPath/dfg_tests/auto_tree_test.c +++ b/thirdparty/EVPath/EVPath/dfg_tests/auto_tree_test.c @@ -5,6 +5,7 @@ #ifdef HAVE_UNISTD_H #include #endif +#include #include #include "cod.h" @@ -71,6 +72,12 @@ be_test_master(int argc, char **argv) #ifdef HAVE_WINDOWS_H SetTimer(NULL, 5, 1000, (TIMERPROC) fail_and_die); #else + struct sigaction sigact; + sigact.sa_flags = 0; + sigact.sa_handler = fail_and_die; + sigemptyset(&sigact.sa_mask); + sigaddset(&sigact.sa_mask, SIGALRM); + sigaction(SIGALRM, &sigact, NULL); alarm(240); /* reset time limit to 4 minutes */ #endif if (argc == 1) { diff --git a/thirdparty/EVPath/EVPath/dfg_tests/fail_chain_test.c b/thirdparty/EVPath/EVPath/dfg_tests/fail_chain_test.c index a3673e6db4..e68cae7962 100755 --- a/thirdparty/EVPath/EVPath/dfg_tests/fail_chain_test.c +++ b/thirdparty/EVPath/EVPath/dfg_tests/fail_chain_test.c @@ -6,6 +6,7 @@ #ifdef HAVE_UNISTD_H #include #endif +#include #include "evpath.h" #include "ev_dfg.h" #include "test_support.h" @@ -140,6 +141,12 @@ be_test_master(int argc, char **argv) #ifdef HAVE_WINDOWS_H SetTimer(NULL, 5, 1000, (TIMERPROC) fail_and_die); #else + struct sigaction sigact; + sigact.sa_flags = 0; + sigact.sa_handler = fail_and_die; + sigemptyset(&sigact.sa_mask); + sigaddset(&sigact.sa_mask, SIGALRM); + sigaction(SIGALRM, &sigact, NULL); alarm(240); /* reset time limit to 4 minutes */ #endif if (argc == 1) { diff --git a/thirdparty/EVPath/EVPath/dfg_tests/self_reconfig_test.c b/thirdparty/EVPath/EVPath/dfg_tests/self_reconfig_test.c index 4970d8d050..6484c79c2f 100644 --- a/thirdparty/EVPath/EVPath/dfg_tests/self_reconfig_test.c +++ b/thirdparty/EVPath/EVPath/dfg_tests/self_reconfig_test.c @@ -6,6 +6,7 @@ #ifdef HAVE_UNISTD_H #include #endif +#include #include "evpath.h" #include "ev_dfg.h" #include "test_support.h" @@ -168,6 +169,12 @@ be_test_master(int argc, char **argv) #ifdef HAVE_WINDOWS_H SetTimer(NULL, 5, 1000, (TIMERPROC) fail_and_die); #else + struct sigaction sigact; + sigact.sa_flags = 0; + sigact.sa_handler = fail_and_die; + sigemptyset(&sigact.sa_mask); + sigaddset(&sigact.sa_mask, SIGALRM); + sigaction(SIGALRM, &sigact, NULL); alarm(240); /* reset time limit to 4 minutes */ #endif if (argc == 1) { diff --git a/thirdparty/EVPath/EVPath/dfg_tests/tree_test.c b/thirdparty/EVPath/EVPath/dfg_tests/tree_test.c index d157647986..5770de6eb4 100644 --- a/thirdparty/EVPath/EVPath/dfg_tests/tree_test.c +++ b/thirdparty/EVPath/EVPath/dfg_tests/tree_test.c @@ -6,6 +6,7 @@ #ifdef HAVE_UNISTD_H #include #endif +#include #include "ev_dfg.h" #include "test_support.h" @@ -56,6 +57,12 @@ be_test_master(int argc, char **argv) #ifdef HAVE_WINDOWS_H SetTimer(NULL, 5, 1000, (TIMERPROC) fail_and_die); #else + struct sigaction sigact; + sigact.sa_flags = 0; + sigact.sa_handler = fail_and_die; + sigemptyset(&sigact.sa_mask); + sigaddset(&sigact.sa_mask, SIGALRM); + sigaction(SIGALRM, &sigact, NULL); alarm(300); #endif if (argc == 1) { diff --git a/thirdparty/EVPath/EVPath/ev_dfg.c b/thirdparty/EVPath/EVPath/ev_dfg.c index 3493e0a431..3720d401f8 100644 --- a/thirdparty/EVPath/EVPath/ev_dfg.c +++ b/thirdparty/EVPath/EVPath/ev_dfg.c @@ -1583,13 +1583,14 @@ INT_EVdfg_create(EVmaster master) extern char *INT_EVmaster_get_contact_list(EVmaster master) { attr_list contact_list = NULL; - atom_t CM_TRANSPORT = attr_atom_from_string("CM_TRANSPORT"); - atom_t CM_ENET_CONN_TIMEOUT = attr_atom_from_string("CM_ENET_CONN_TIMEOUT"); CManager cm = master->cm; char *tmp = NULL; /* use enet transport if available */ #if defined(ENET_FOUND) || defined(ZPL_ENET_AVAILABLE) + atom_t CM_ENET_CONN_TIMEOUT = attr_atom_from_string("CM_ENET_CONN_TIMEOUT"); + atom_t CM_TRANSPORT = attr_atom_from_string("CM_TRANSPORT"); + (void) CM_TRANSPORT; attr_list listen_list = create_attr_list(); #if defined(ENET_FOUND) add_string_attr(listen_list, CM_TRANSPORT, strdup("enet")); diff --git a/thirdparty/EVPath/EVPath/evp.c b/thirdparty/EVPath/EVPath/evp.c index f72ddb0cf8..d38ae3cc7c 100644 --- a/thirdparty/EVPath/EVPath/evp.c +++ b/thirdparty/EVPath/EVPath/evp.c @@ -3700,7 +3700,7 @@ INT_EVextract_stone_events(CManager cm, EVstone stone_id) stone_type stone; EVevent_list list = malloc(sizeof(list[0])); - list[0].length = -1; + list[0].length = (size_t)-1; stone = stone_struct(evp, stone_id); if (!stone) return NULL; list = extract_events_from_queue(cm, stone->queue, list); @@ -3764,7 +3764,7 @@ extract_events_from_queue(CManager cm, queue_ptr que, EVevent_list list) first = que->queue_head; last = que->queue_tail; - while (list[num_of_elements].length != -1) num_of_elements++; + while (list[num_of_elements].length != (size_t)-1) num_of_elements++; while(first != NULL && last != NULL) { list = (EVevent_list) realloc (list, (num_of_elements + 2) * sizeof(list[0])); current_entry = &list[num_of_elements]; @@ -3779,7 +3779,7 @@ extract_events_from_queue(CManager cm, queue_ptr que, EVevent_list list) num_of_elements++; first = first->next; } - list[num_of_elements].length = -1; + list[num_of_elements].length = (size_t)-1; return list; } diff --git a/thirdparty/EVPath/EVPath/mtests/cmconn.c b/thirdparty/EVPath/EVPath/mtests/cmconn.c index a2a3cafdd3..6b723b4fda 100644 --- a/thirdparty/EVPath/EVPath/mtests/cmconn.c +++ b/thirdparty/EVPath/EVPath/mtests/cmconn.c @@ -26,9 +26,8 @@ #include #endif #include "evpath.h" -#ifdef HAVE_SYS_WAIT_H #include -#endif + #ifdef _MSC_VER #define drand48() (((double)rand())/((double)RAND_MAX)) #define lrand48() rand() diff --git a/thirdparty/EVPath/EVPath/response.c b/thirdparty/EVPath/EVPath/response.c index fae5c44294..fad8f71cac 100644 --- a/thirdparty/EVPath/EVPath/response.c +++ b/thirdparty/EVPath/EVPath/response.c @@ -1974,7 +1974,7 @@ extern sm_ref cod_build_param_node(const char *id, sm_ref typ, int param_num); extern void cod_add_decl_to_parse_context(const char *name, sm_ref item, cod_parse_context context); -extern FFS_DECLSPEC void +extern void cod_add_param(const char *id, const char *typ, int param_num, cod_parse_context context); diff --git a/thirdparty/EVPath/EVPath/tests/extract_test.c b/thirdparty/EVPath/EVPath/tests/extract_test.c index 5ef1816aed..90247c7abd 100644 --- a/thirdparty/EVPath/EVPath/tests/extract_test.c +++ b/thirdparty/EVPath/EVPath/tests/extract_test.c @@ -321,7 +321,7 @@ main(int argc, char **argv) EVdrain_stone(cm, term1); events = EVextract_stone_events(cm, term1); count = 0; - while (events && (events[count].length != -1)) { + while (events && (events[count].length != (size_t)-1)) { EVsubmit_encoded(cm, term0, events[count].buffer, events[count].length, attrs); count++; } diff --git a/thirdparty/dill/dill/CMakeLists.txt b/thirdparty/dill/dill/CMakeLists.txt index b90f757087..0ac720b7fd 100644 --- a/thirdparty/dill/dill/CMakeLists.txt +++ b/thirdparty/dill/dill/CMakeLists.txt @@ -117,6 +117,9 @@ if(CMAKE_SYSTEM_PROCESSOR MATCHES "i.86|x86_64|AMD64") if(NOT (CMAKE_SIZEOF_VOID_P EQUAL SIZEOF_LONG)) set(BASE_OPS_ARG "-msvc_long") endif() + if(BUILD_SHARED_LIBS) + set(BASE_OPS_ARG "${BASE_OPS_ARG} -build_shared") + endif() elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "arm.5.*") set(NATIVE_ARCH arm5) set(HOST_ARM5 1) @@ -281,6 +284,8 @@ add_custom_command( DEPENDS base.ops ) +message (STATUS "base.ops argument is ${BASE_OPS_ARG}") + option(DILL_MULTI_TARGET "Build all available target architectures" OFF) if(DILL_MULTI_TARGET) diff --git a/thirdparty/dill/dill/base.ops b/thirdparty/dill/dill/base.ops index cd7e3bf3c0..003b5b5138 100755 --- a/thirdparty/dill/dill/base.ops +++ b/thirdparty/dill/dill/base.ops @@ -6,9 +6,12 @@ $MSVC_LONG = 0; while ($_ = $ARGV[0]) { shift; last if /^--$/; - if (/^-msvc_long/) {$MSVC_LONG = 1;} + if (/-msvc_long/) {$MSVC_LONG = 1;} + if (/-build_shared/) {$BUILD_SHARED = 1;} } +print "BUILD_SHARED is $BUILD_SHARED\n"; + sub upperc { local($_) = pop(@_); tr/[a-z]/[A-Z]/; @@ -65,7 +68,11 @@ print HOUT "#define dill_jalp(s, return_addr_reg, target) (s->j->jal)(s, return_ print HOUT "#define dill_special(s, type, param) if (s->j->special) (s->j->special)(s, type, param)\n"; print HOUT "#if defined(_MSC_VER) && !defined(DILL_SRC)\n"; -print HOUT "#define DILL_DECLSPEC __declspec(dllimport)\n"; +if ($BUILD_SHARED) { + print HOUT "#define DILL_DECLSPEC __declspec(dllimport)\n"; +} else { + print HOUT "#define DILL_DECLSPEC\n"; +} print HOUT "#else\n"; print HOUT "#define DILL_DECLSPEC\n"; print HOUT "#endif\n"; diff --git a/thirdparty/ffs/ffs/.github/workflows/build-and-test.yml b/thirdparty/ffs/ffs/.github/workflows/build-and-test.yml index 0b9926ae9e..6614bcfe79 100644 --- a/thirdparty/ffs/ffs/.github/workflows/build-and-test.yml +++ b/thirdparty/ffs/ffs/.github/workflows/build-and-test.yml @@ -88,12 +88,15 @@ jobs: jobname: [ windows2019-vs2019-clang, windows2022-vs2022-msvc, + windows2022-vs2022-msvc-static, macos-clang ] include: - jobname: windows2019-vs2019-clang vm: windows-2019 - jobname: windows2022-vs2022-msvc vm: windows-2022 + - jobname: windows2022-vs2022-msvc-static + vm: windows-2022 - jobname: macos-clang vm: macos-latest diff --git a/thirdparty/ffs/ffs/CMakeLists.txt b/thirdparty/ffs/ffs/CMakeLists.txt index 7d8589f95e..c9f516988d 100644 --- a/thirdparty/ffs/ffs/CMakeLists.txt +++ b/thirdparty/ffs/ffs/CMakeLists.txt @@ -186,6 +186,10 @@ add_custom_command( add_custom_target(docs ALL DEPENDS "cod_node.c") +if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + SET_SOURCE_FILES_PROPERTIES(cod.tab.c PROPERTIES COMPILE_FLAGS -Wno-unused-function) +endif() + add_custom_target(generated DEPENDS cod_node.c ${BISON_CODParser_OUTPUT_SOURCE}) list(APPEND COD_MASTER_SRC_LIST diff --git a/thirdparty/ffs/ffs/cod/cg.c b/thirdparty/ffs/ffs/cod/cg.c index 1b41d8d3af..e4642c9edc 100644 --- a/thirdparty/ffs/ffs/cod/cg.c +++ b/thirdparty/ffs/ffs/cod/cg.c @@ -148,6 +148,7 @@ cg_required_align(dill_stream s, sm_ref node) required_align = dill_type_align(s, DILL_P); break; default: + required_align = 0; //silence warning assert(0); } return required_align; @@ -1750,6 +1751,7 @@ execute_operator_cg(dill_stream s, operator_t op, int op_type, dill_reg result, } else if (typ->node_type == cod_array_type_decl) { size = typ->node.array_type_decl.cg_element_size; } else { + size = 1; // avoid warning assert(0); } } else if ((ptr->node_type == cod_operator) && (ptr->node.operator.op == op_address)) { @@ -1758,6 +1760,7 @@ execute_operator_cg(dill_stream s, operator_t op, int op_type, dill_reg result, if (cod_expr_is_string(ptr)) { size = 1; } else { + size = 1; // silence warning assert(FALSE); } } @@ -1898,6 +1901,7 @@ execute_operator_cg(dill_stream s, operator_t op, int op_type, dill_reg result, if (cod_expr_is_string(ptr)) { size = 1; } else { + size = 1; // avoid warning assert(FALSE); } } diff --git a/thirdparty/ffs/ffs/cod/cod.h b/thirdparty/ffs/ffs/cod/cod.h index 0a866ea8e8..41a611aceb 100644 --- a/thirdparty/ffs/ffs/cod/cod.h +++ b/thirdparty/ffs/ffs/cod/cod.h @@ -6,14 +6,6 @@ extern "C" { #endif -#ifndef FFS_DECL_SPEC -#if defined(_MSC_VER) && !defined(FFS_SRC) -#define FFS_DECLSPEC __declspec(dllimport) -#else -#define FFS_DECLSPEC -#endif -#endif - #include /*! @@ -112,7 +104,7 @@ typedef cod_extern_entry *cod_extern_list; * \return Will return a new initialized cod_parse_context unless there is * no available memory. */ -extern FFS_DECLSPEC cod_parse_context new_cod_parse_context(void); +extern cod_parse_context new_cod_parse_context(void); /*! * Free a handle to an cod_parse_context. @@ -121,7 +113,7 @@ extern FFS_DECLSPEC cod_parse_context new_cod_parse_context(void); * Calling this routine frees all memory associated with the parse context, * but not that of code that has been generated from this context. */ -extern FFS_DECLSPEC void cod_free_parse_context(cod_parse_context context); +extern void cod_free_parse_context(cod_parse_context context); /*! * Associate a set of "name, external address" pairs with a parse context @@ -135,7 +127,7 @@ extern FFS_DECLSPEC void cod_free_parse_context(cod_parse_context context); * \param externs the list of "name, external address" pairs to be * associated. This list should be terminated with a {NULL, 0} pair. */ -extern FFS_DECLSPEC void cod_assoc_externs(cod_parse_context context, +extern void cod_assoc_externs(cod_parse_context context, cod_extern_list externs); /*! @@ -151,7 +143,7 @@ extern FFS_DECLSPEC void cod_assoc_externs(cod_parse_context context, * "int proc(double d, int *i)" * */ -extern FFS_DECLSPEC void +extern void cod_subroutine_declaration(const char *decl, cod_parse_context context); /*! @@ -163,7 +155,7 @@ cod_subroutine_declaration(const char *decl, cod_parse_context context); * the structure. * \param context the context in which the type is to be made available. */ -extern FFS_DECLSPEC void cod_add_simple_struct_type(const char *name, FMFieldList field_list, +extern void cod_add_simple_struct_type(const char *name, FMFieldList field_list, cod_parse_context context); /*! @@ -174,7 +166,7 @@ extern FFS_DECLSPEC void cod_add_simple_struct_type(const char *name, FMFieldLis * the structures. * \param context the context in which the type is to be made available. */ -extern FFS_DECLSPEC void cod_add_struct_type(FMStructDescList format_list, +extern void cod_add_struct_type(FMStructDescList format_list, cod_parse_context context); /*! @@ -185,7 +177,7 @@ extern FFS_DECLSPEC void cod_add_struct_type(FMStructDescList format_list, * \param param_num the numeral of the new parameter (0 is first) * \param context the context in which the subroutine is being declared. */ -extern FFS_DECLSPEC void +extern void cod_add_param(const char *id, const char *typ, int param_num, cod_parse_context context); @@ -199,7 +191,7 @@ cod_add_param(const char *id, const char *typ, int param_num, * \param context the context in which the subroutine is being declared. */ #ifdef __FM__ -extern FFS_DECLSPEC void +extern void cod_add_encoded_param(const char *id, char *data, int param_num, FMContext c, cod_parse_context context); #endif @@ -210,7 +202,7 @@ cod_add_encoded_param(const char *id, char *data, int param_num, * \param typ the data type of the return value. * \param context the context in which the subroutine is being declared. */ -extern FFS_DECLSPEC void +extern void cod_set_return_type(char *typ, cod_parse_context context); /*! @@ -263,7 +255,7 @@ int cod_code_verify(char *code, cod_parse_context context); * * \param code the handle to the resources that will be free'd. */ -extern FFS_DECLSPEC void cod_code_free(cod_code code); +extern void cod_code_free(cod_code code); /*! * create an execution context associated with a code block @@ -271,7 +263,7 @@ extern FFS_DECLSPEC void cod_code_free(cod_code code); * \param code the handle to the code bloc * \return the created execution context */ -extern FFS_DECLSPEC cod_exec_context cod_create_exec_context(cod_code code); +extern cod_exec_context cod_create_exec_context(cod_code code); /*! * Free all resources associated with the generated code associated with the @@ -279,7 +271,7 @@ extern FFS_DECLSPEC cod_exec_context cod_create_exec_context(cod_code code); * * \param code the handle to the resources that will be free'd. */ -extern FFS_DECLSPEC void cod_exec_context_free(cod_exec_context ec); +extern void cod_exec_context_free(cod_exec_context ec); /*! * Associate application-level data with an execution context. This is @@ -291,7 +283,7 @@ extern FFS_DECLSPEC void cod_exec_context_free(cod_exec_context ec); * \param key the value that will serve as a key to retrieve the data * \param value the 'long' data that will be associated with the key */ -extern FFS_DECLSPEC void cod_assoc_client_data(cod_exec_context ec, int key, intptr_t value); +extern void cod_assoc_client_data(cod_exec_context ec, int key, intptr_t value); /*! * Retrieve application-level data with an execution context. This is @@ -305,7 +297,7 @@ extern FFS_DECLSPEC void cod_assoc_client_data(cod_exec_context ec, int key, int * \param key the value that will serve as a key to retrieve the data * \return the 'long' data that was associated with the key */ -extern FFS_DECLSPEC intptr_t cod_get_client_data(cod_exec_context ec, int key); +extern intptr_t cod_get_client_data(cod_exec_context ec, int key); /*! * Extract static state from an execution context. @@ -315,7 +307,7 @@ extern FFS_DECLSPEC intptr_t cod_get_client_data(cod_exec_context ec, int key); * the length of the returned state block * \return a pointer to the extracted state */ -extern FFS_DECLSPEC void *cod_extract_state(cod_exec_context ec, int *length_p); +extern void *cod_extract_state(cod_exec_context ec, int *length_p); /*! * Install static state into an execution context. @@ -325,7 +317,7 @@ extern FFS_DECLSPEC void *cod_extract_state(cod_exec_context ec, int *length_p); * \param state_size the size of the state block * \return 1 on success, 0 on failure */ -extern FFS_DECLSPEC int cod_install_state(cod_exec_context ec, void *state, int length); +extern int cod_install_state(cod_exec_context ec, void *state, int length); /*! * \brief This parses a string to determine what external @@ -356,7 +348,7 @@ int cod_parse_for_globals(char *code, cod_parse_context context); * * \param context the cod_parse_context to be duplicated. */ -extern FFS_DECLSPEC cod_parse_context cod_copy_context(cod_parse_context context); +extern cod_parse_context cod_copy_context(cod_parse_context context); /*! * Duplicate a handle to an cod_parse_context, specifically adapting the results to @@ -365,7 +357,7 @@ extern FFS_DECLSPEC cod_parse_context cod_copy_context(cod_parse_context context * * \param context the cod_parse_context to be duplicated. */ -extern FFS_DECLSPEC cod_parse_context cod_copy_globals(cod_parse_context context); +extern cod_parse_context cod_copy_globals(cod_parse_context context); /*! * err_out_func_t is a function pointer type. Functions matching this @@ -408,7 +400,7 @@ void cod_dump(cod_code code); * \param format2 the old format. This is the format of the output message. * \param xform_code The COD code string that transforms data from format1 to format2. */ -extern FFS_DECLSPEC cod_code +extern cod_code gen_rollback_code(FMStructDescList format1, FMStructDescList format2, char *xform_code); /*! @@ -417,7 +409,7 @@ gen_rollback_code(FMStructDescList format1, FMStructDescList format2, char *xfor * \param value The value of the constant * \param context the context in which this is to be created */ -extern FFS_DECLSPEC void cod_add_int_constant_to_parse_context(const char *id, int value, +extern void cod_add_int_constant_to_parse_context(const char *id, int value, cod_parse_context context); /*! @@ -426,7 +418,7 @@ extern FFS_DECLSPEC void cod_add_int_constant_to_parse_context(const char *id, i * \param value The value to send * \param context The context in which the subroutine has been declared. */ -extern FFS_DECLSPEC void cod_set_closure(char *name, void* value, cod_parse_context context); +extern void cod_set_closure(char *name, void* value, cod_parse_context context); #if defined(__cplusplus) || defined(c_plusplus) } diff --git a/thirdparty/ffs/ffs/cod/cod.y b/thirdparty/ffs/ffs/cod/cod.y index a4d0d17515..3d99807dcb 100644 --- a/thirdparty/ffs/ffs/cod/cod.y +++ b/thirdparty/ffs/ffs/cod/cod.y @@ -6447,7 +6447,7 @@ space_to_underscore(char *str){ while(*str != '\0'){ if(isspace(*str)) *str = '_'; - str++; + str++; } } @@ -6732,7 +6732,7 @@ evaluate_constant_return_expr(cod_parse_context context, sm_ref expr, int *free_ *free_result = 1; } else { /* we get an integer result */ - intptr_t left_val = 0, right_val = 0, value; + intptr_t left_val = 0, right_val = 0, value = 0; char str_val[40]; if (expr->node.operator.left) left_val = get_constant_long_value(context, left); diff --git a/thirdparty/ffs/ffs/cod/pregen_source/Windows/cod.l b/thirdparty/ffs/ffs/cod/pregen_source/Windows/cod.l index 724916d39c..6b816a740f 100644 --- a/thirdparty/ffs/ffs/cod/pregen_source/Windows/cod.l +++ b/thirdparty/ffs/ffs/cod/pregen_source/Windows/cod.l @@ -359,70 +359,9 @@ yywrap YY_PROTO(( void )) } -#ifndef input /* flex, not lex */ void yy_delete_buffer YY_PROTO((YY_BUFFER_STATE b)); -#ifdef WINNT -/* old Windows code for MKS Toolkit version of flex */ - -static void -terminate_string_parse() -{ - yyrestart(NULL); -} - -#ifdef YY_USE_PROTOS -static void *yy_flex_alloc( yy_size_t size ) -#else -static void *yy_flex_alloc( size ) -yy_size_t size; -#endif - { - return (void *) malloc( size ); - } - -static char* current_input_string; - -int my_yy_input(buf,result,max_size) { - - if (current_input_string == NULL) - { - - result = 0; - } - else - if (max_size < strlen(current_input_string)) - { - memcpy((void*)buf, current_input_string, max_size); - current_input_string += max_size; - result = max_size; - } else { - int n = strlen(current_input_string); - memcpy((void*)buf, current_input_string, n+1); - current_input_string = NULL; - result = n; - } - -/* printf("my_yy_input buf[%s],result[%d]\n",buf,result);*/ - return result; -} - -static void -setup_for_string_parse(string, defined_types, enum_constants) -const char *string; -char **defined_types; -char **enum_constants; -{ - type_count = defined_type_count; - types = defined_types; - enums = enum_constants; - - current_input_string = string; - lex_offset = 1; - line_count = 1; -} -#else static YY_BUFFER_STATE bb = NULL; @@ -455,5 +394,3 @@ terminate_string_parse() } } -#endif -#endif diff --git a/thirdparty/ffs/ffs/cod/pregen_source/Windows/cod.tab.c b/thirdparty/ffs/ffs/cod/pregen_source/Windows/cod.tab.c index 531a170505..70198e7fba 100644 --- a/thirdparty/ffs/ffs/cod/pregen_source/Windows/cod.tab.c +++ b/thirdparty/ffs/ffs/cod/pregen_source/Windows/cod.tab.c @@ -289,138 +289,7 @@ cod_dup_list(sm_list list) # endif # endif - -/* Debug traces. */ -#ifndef YYDEBUG -# define YYDEBUG 0 -#endif -#if YYDEBUG -extern int yydebug; -#endif - -/* Token kinds. */ -#ifndef YYTOKENTYPE -# define YYTOKENTYPE - enum yytokentype - { - YYEMPTY = -2, - YYEOF = 0, /* "end of file" */ - YYerror = 256, /* error */ - YYUNDEF = 257, /* "invalid token" */ - ARROW = 258, /* ARROW */ - LPAREN = 259, /* LPAREN */ - RPAREN = 260, /* RPAREN */ - LCURLY = 261, /* LCURLY */ - RCURLY = 262, /* RCURLY */ - COLON = 263, /* COLON */ - QUESTION = 264, /* QUESTION */ - LBRACKET = 265, /* LBRACKET */ - RBRACKET = 266, /* RBRACKET */ - DOT = 267, /* DOT */ - STAR = 268, /* STAR */ - AT = 269, /* AT */ - SLASH = 270, /* SLASH */ - MODULUS = 271, /* MODULUS */ - PLUS = 272, /* PLUS */ - MINUS = 273, /* MINUS */ - TILDE = 274, /* TILDE */ - LEQ = 275, /* LEQ */ - LT = 276, /* LT */ - GEQ = 277, /* GEQ */ - GT = 278, /* GT */ - EQ = 279, /* EQ */ - NEQ = 280, /* NEQ */ - LEFT_SHIFT = 281, /* LEFT_SHIFT */ - RIGHT_SHIFT = 282, /* RIGHT_SHIFT */ - ASSIGN = 283, /* ASSIGN */ - MUL_ASSIGN = 284, /* MUL_ASSIGN */ - DIV_ASSIGN = 285, /* DIV_ASSIGN */ - MOD_ASSIGN = 286, /* MOD_ASSIGN */ - ADD_ASSIGN = 287, /* ADD_ASSIGN */ - SUB_ASSIGN = 288, /* SUB_ASSIGN */ - LEFT_ASSIGN = 289, /* LEFT_ASSIGN */ - RIGHT_ASSIGN = 290, /* RIGHT_ASSIGN */ - AND_ASSIGN = 291, /* AND_ASSIGN */ - XOR_ASSIGN = 292, /* XOR_ASSIGN */ - OR_ASSIGN = 293, /* OR_ASSIGN */ - LOG_OR = 294, /* LOG_OR */ - LOG_AND = 295, /* LOG_AND */ - ARITH_OR = 296, /* ARITH_OR */ - ARITH_AND = 297, /* ARITH_AND */ - ARITH_XOR = 298, /* ARITH_XOR */ - INC_OP = 299, /* INC_OP */ - DEC_OP = 300, /* DEC_OP */ - BANG = 301, /* BANG */ - SEMI = 302, /* SEMI */ - IF = 303, /* IF */ - ELSE = 304, /* ELSE */ - FOR = 305, /* FOR */ - DO = 306, /* DO */ - WHILE = 307, /* WHILE */ - CHAR = 308, /* CHAR */ - SHORT = 309, /* SHORT */ - INT = 310, /* INT */ - LONG = 311, /* LONG */ - UNSIGNED = 312, /* UNSIGNED */ - SIGNED = 313, /* SIGNED */ - FLOAT = 314, /* FLOAT */ - DOUBLE = 315, /* DOUBLE */ - VOID = 316, /* VOID */ - STRING = 317, /* STRING */ - STATIC = 318, /* STATIC */ - EXTERN_TOKEN = 319, /* EXTERN_TOKEN */ - STRUCT = 320, /* STRUCT */ - ENUM = 321, /* ENUM */ - UNION = 322, /* UNION */ - CONST = 323, /* CONST */ - SIZEOF = 324, /* SIZEOF */ - TYPEDEF = 325, /* TYPEDEF */ - RETURN_TOKEN = 326, /* RETURN_TOKEN */ - CONTINUE = 327, /* CONTINUE */ - BREAK = 328, /* BREAK */ - GOTO = 329, /* GOTO */ - PRINT = 330, /* PRINT */ - COMMA = 331, /* COMMA */ - DOTDOTDOT = 332, /* DOTDOTDOT */ - integer_constant = 333, /* integer_constant */ - character_constant = 334, /* character_constant */ - string_constant = 335, /* string_constant */ - floating_constant = 336, /* floating_constant */ - identifier_ref = 337, /* identifier_ref */ - type_identifier = 338, /* type_id */ - enumeration_constant = 339 /* enumeration_constant */ - }; - typedef enum yytokentype yytoken_kind_t; -#endif - -/* Value type. */ -#if ! defined YYSTYPE && ! defined YYSTYPE_IS_DECLARED -union YYSTYPE -{ -#line 201 "cod.y" - - lx_info info; - sm_ref reference; - operator_t operator; - sm_list list; - char *string; - -#line 409 "cod.tab.c" - -}; -typedef union YYSTYPE YYSTYPE; -# define YYSTYPE_IS_TRIVIAL 1 -# define YYSTYPE_IS_DECLARED 1 -#endif - - -extern YYSTYPE yylval; - - -int yyparse (void); - - - +#include "cod.tab.h" /* Symbol kind. */ enum yysymbol_kind_t { @@ -508,7 +377,7 @@ enum yysymbol_kind_t YYSYMBOL_string_constant = 80, /* string_constant */ YYSYMBOL_floating_constant = 81, /* floating_constant */ YYSYMBOL_identifier_ref = 82, /* identifier_ref */ - YYSYMBOL_type_identifier = 83, /* type_id */ + YYSYMBOL_type_identifier = 83, /* type_identifier */ YYSYMBOL_enumeration_constant = 84, /* enumeration_constant */ YYSYMBOL_YYACCEPT = 85, /* $accept */ YYSYMBOL_start = 86, /* start */ @@ -1019,18 +888,19 @@ static const char *const yytname[] = "ENUM", "UNION", "CONST", "SIZEOF", "TYPEDEF", "RETURN_TOKEN", "CONTINUE", "BREAK", "GOTO", "PRINT", "COMMA", "DOTDOTDOT", "integer_constant", "character_constant", "string_constant", - "floating_constant", "identifier_ref", "type_identifier", "enumeration_constant", - "$accept", "start", "primary_expression", "postfix_expression", - "argument_expression_list", "unary_expression", "unary_operator", - "cast_expression", "multiplicative_expression", "additive_expression", - "shift_expression", "relational_expression", "equality_expression", - "and_expression", "exclusive_or_expression", "inclusive_or_expression", - "logical_and_expression", "logical_or_expression", - "conditional_expression", "assignment_operator", "assignment_expression", - "expression", "constant_expression", "init_declarator_list", - "declaration", "$@1", "@2", "declaration_specifiers", "init_declarator", - "storage_class_specifier", "type_specifier", "struct_or_union_specifier", - "struct_or_union", "struct_declaration_list", "struct_declaration", + "floating_constant", "identifier_ref", "type_identifier", + "enumeration_constant", "$accept", "start", "primary_expression", + "postfix_expression", "argument_expression_list", "unary_expression", + "unary_operator", "cast_expression", "multiplicative_expression", + "additive_expression", "shift_expression", "relational_expression", + "equality_expression", "and_expression", "exclusive_or_expression", + "inclusive_or_expression", "logical_and_expression", + "logical_or_expression", "conditional_expression", "assignment_operator", + "assignment_expression", "expression", "constant_expression", + "init_declarator_list", "declaration", "$@1", "@2", + "declaration_specifiers", "init_declarator", "storage_class_specifier", + "type_specifier", "struct_or_union_specifier", "struct_or_union", + "struct_declaration_list", "struct_declaration", "struct_declarator_list", "struct_declarator", "specifier_qualifier_list", "enum_specifier", "enumerator_list", "enumerator", "type_qualifier", "declarator", "direct_declarator", @@ -1928,7 +1798,7 @@ yyparse (void) { yyparse_value = (sm_ref)(yyvsp[0].list); } -#line 1932 "cod.tab.c" +#line 1802 "cod.tab.c" break; case 3: /* start: compound_statement */ @@ -1936,7 +1806,7 @@ yyparse (void) { yyparse_value = (yyvsp[0].reference); } -#line 1940 "cod.tab.c" +#line 1810 "cod.tab.c" break; case 4: /* primary_expression: identifier_ref */ @@ -1946,13 +1816,13 @@ yyparse (void) (yyval.reference)->node.identifier.id = (yyvsp[0].info).string; (yyval.reference)->node.identifier.lx_srcpos = (yyvsp[0].info).lx_srcpos; } -#line 1950 "cod.tab.c" +#line 1820 "cod.tab.c" break; case 6: /* primary_expression: LPAREN expression RPAREN */ #line 377 "cod.y" { (yyval.reference) = (yyvsp[-1].reference); } -#line 1956 "cod.tab.c" +#line 1826 "cod.tab.c" break; case 8: /* postfix_expression: postfix_expression LBRACKET expression RBRACKET */ @@ -1963,7 +1833,7 @@ yyparse (void) (yyval.reference)->node.element_ref.expression = (yyvsp[-1].reference); (yyval.reference)->node.element_ref.array_ref = (yyvsp[-3].reference); } -#line 1967 "cod.tab.c" +#line 1837 "cod.tab.c" break; case 9: /* postfix_expression: postfix_expression DOT identifier_ref */ @@ -1974,7 +1844,7 @@ yyparse (void) (yyval.reference)->node.field_ref.lx_field = (yyvsp[0].info).string; (yyval.reference)->node.field_ref.struct_ref = (yyvsp[-2].reference); } -#line 1978 "cod.tab.c" +#line 1848 "cod.tab.c" break; case 10: /* postfix_expression: postfix_expression LPAREN argument_expression_list RPAREN */ @@ -1985,7 +1855,7 @@ yyparse (void) (yyval.reference)->node.subroutine_call.arguments = (yyvsp[-1].list); (yyval.reference)->node.subroutine_call.sm_func_ref = (yyvsp[-3].reference); } -#line 1989 "cod.tab.c" +#line 1859 "cod.tab.c" break; case 11: /* postfix_expression: postfix_expression LPAREN RPAREN */ @@ -1996,7 +1866,7 @@ yyparse (void) (yyval.reference)->node.subroutine_call.arguments = NULL; (yyval.reference)->node.subroutine_call.sm_func_ref = (yyvsp[-2].reference); } -#line 2000 "cod.tab.c" +#line 1870 "cod.tab.c" break; case 12: /* postfix_expression: postfix_expression ARROW identifier_ref */ @@ -2007,7 +1877,7 @@ yyparse (void) (yyval.reference)->node.field_ref.lx_field = (yyvsp[0].info).string; (yyval.reference)->node.field_ref.struct_ref = (yyvsp[-2].reference); } -#line 2011 "cod.tab.c" +#line 1881 "cod.tab.c" break; case 13: /* postfix_expression: postfix_expression INC_OP */ @@ -2019,7 +1889,7 @@ yyparse (void) (yyval.reference)->node.operator.right = NULL; (yyval.reference)->node.operator.left = (yyvsp[-1].reference); } -#line 2023 "cod.tab.c" +#line 1893 "cod.tab.c" break; case 14: /* postfix_expression: postfix_expression DEC_OP */ @@ -2031,7 +1901,7 @@ yyparse (void) (yyval.reference)->node.operator.right = NULL; (yyval.reference)->node.operator.left = (yyvsp[-1].reference); } -#line 2035 "cod.tab.c" +#line 1905 "cod.tab.c" break; case 15: /* argument_expression_list: assignment_expression */ @@ -2041,7 +1911,7 @@ yyparse (void) (yyval.list)->node = (yyvsp[0].reference); (yyval.list)->next = NULL; } -#line 2045 "cod.tab.c" +#line 1915 "cod.tab.c" break; case 16: /* argument_expression_list: argument_expression_list COMMA assignment_expression */ @@ -2056,7 +1926,7 @@ yyparse (void) tmp->next->next = NULL; (yyval.list) = (yyvsp[-2].list); } -#line 2060 "cod.tab.c" +#line 1930 "cod.tab.c" break; case 18: /* unary_expression: INC_OP unary_expression */ @@ -2068,7 +1938,7 @@ yyparse (void) (yyval.reference)->node.operator.right = (yyvsp[0].reference); (yyval.reference)->node.operator.left = NULL; } -#line 2072 "cod.tab.c" +#line 1942 "cod.tab.c" break; case 19: /* unary_expression: DEC_OP unary_expression */ @@ -2080,7 +1950,7 @@ yyparse (void) (yyval.reference)->node.operator.right = (yyvsp[0].reference); (yyval.reference)->node.operator.left = NULL; } -#line 2084 "cod.tab.c" +#line 1954 "cod.tab.c" break; case 20: /* unary_expression: unary_operator cast_expression */ @@ -2092,7 +1962,7 @@ yyparse (void) (yyval.reference)->node.operator.right = (yyvsp[0].reference); (yyval.reference)->node.operator.left = NULL; } -#line 2096 "cod.tab.c" +#line 1966 "cod.tab.c" break; case 21: /* unary_expression: SIZEOF unary_expression */ @@ -2104,7 +1974,7 @@ yyparse (void) (yyval.reference)->node.operator.right = (yyvsp[0].reference); (yyval.reference)->node.operator.left = NULL; } -#line 2108 "cod.tab.c" +#line 1978 "cod.tab.c" break; case 22: /* unary_expression: SIZEOF LPAREN type_name RPAREN */ @@ -2122,7 +1992,7 @@ yyparse (void) (yyval.reference)->node.operator.right = cast; (yyval.reference)->node.operator.left = NULL; } -#line 2126 "cod.tab.c" +#line 1996 "cod.tab.c" break; case 23: /* unary_operator: ARITH_AND */ @@ -2130,7 +2000,7 @@ yyparse (void) { (yyval.info).op = op_address; } -#line 2134 "cod.tab.c" +#line 2004 "cod.tab.c" break; case 24: /* unary_operator: STAR */ @@ -2138,7 +2008,7 @@ yyparse (void) { (yyval.info).op = op_deref; } -#line 2142 "cod.tab.c" +#line 2012 "cod.tab.c" break; case 25: /* unary_operator: PLUS */ @@ -2146,7 +2016,7 @@ yyparse (void) { (yyval.info).op = op_plus; } -#line 2150 "cod.tab.c" +#line 2020 "cod.tab.c" break; case 26: /* unary_operator: MINUS */ @@ -2154,7 +2024,7 @@ yyparse (void) { (yyval.info).op = op_minus; } -#line 2158 "cod.tab.c" +#line 2028 "cod.tab.c" break; case 27: /* unary_operator: TILDE */ @@ -2162,7 +2032,7 @@ yyparse (void) { (yyval.info).op = op_not; } -#line 2166 "cod.tab.c" +#line 2036 "cod.tab.c" break; case 28: /* unary_operator: BANG */ @@ -2170,7 +2040,7 @@ yyparse (void) { (yyval.info).op = op_log_neg; } -#line 2174 "cod.tab.c" +#line 2044 "cod.tab.c" break; case 30: /* cast_expression: LPAREN type_name RPAREN cast_expression */ @@ -2181,7 +2051,7 @@ yyparse (void) (yyval.reference)->node.cast.type_spec = (yyvsp[-2].list); (yyval.reference)->node.cast.expression = (yyvsp[0].reference); } -#line 2185 "cod.tab.c" +#line 2055 "cod.tab.c" break; case 32: /* multiplicative_expression: multiplicative_expression STAR cast_expression */ @@ -2193,7 +2063,7 @@ yyparse (void) (yyval.reference)->node.operator.right = (yyvsp[0].reference); (yyval.reference)->node.operator.left = (yyvsp[-2].reference); } -#line 2197 "cod.tab.c" +#line 2067 "cod.tab.c" break; case 33: /* multiplicative_expression: multiplicative_expression SLASH cast_expression */ @@ -2205,7 +2075,7 @@ yyparse (void) (yyval.reference)->node.operator.right = (yyvsp[0].reference); (yyval.reference)->node.operator.left = (yyvsp[-2].reference); } -#line 2209 "cod.tab.c" +#line 2079 "cod.tab.c" break; case 34: /* multiplicative_expression: multiplicative_expression MODULUS cast_expression */ @@ -2217,7 +2087,7 @@ yyparse (void) (yyval.reference)->node.operator.right = (yyvsp[0].reference); (yyval.reference)->node.operator.left = (yyvsp[-2].reference); } -#line 2221 "cod.tab.c" +#line 2091 "cod.tab.c" break; case 36: /* additive_expression: additive_expression PLUS multiplicative_expression */ @@ -2229,7 +2099,7 @@ yyparse (void) (yyval.reference)->node.operator.right = (yyvsp[0].reference); (yyval.reference)->node.operator.left = (yyvsp[-2].reference); } -#line 2233 "cod.tab.c" +#line 2103 "cod.tab.c" break; case 37: /* additive_expression: additive_expression MINUS multiplicative_expression */ @@ -2241,7 +2111,7 @@ yyparse (void) (yyval.reference)->node.operator.right = (yyvsp[0].reference); (yyval.reference)->node.operator.left = (yyvsp[-2].reference); } -#line 2245 "cod.tab.c" +#line 2115 "cod.tab.c" break; case 39: /* shift_expression: shift_expression LEFT_SHIFT additive_expression */ @@ -2253,7 +2123,7 @@ yyparse (void) (yyval.reference)->node.operator.right = (yyvsp[0].reference); (yyval.reference)->node.operator.left = (yyvsp[-2].reference); } -#line 2257 "cod.tab.c" +#line 2127 "cod.tab.c" break; case 40: /* shift_expression: shift_expression RIGHT_SHIFT additive_expression */ @@ -2265,7 +2135,7 @@ yyparse (void) (yyval.reference)->node.operator.right = (yyvsp[0].reference); (yyval.reference)->node.operator.left = (yyvsp[-2].reference); } -#line 2269 "cod.tab.c" +#line 2139 "cod.tab.c" break; case 42: /* relational_expression: relational_expression LT shift_expression */ @@ -2277,7 +2147,7 @@ yyparse (void) (yyval.reference)->node.operator.right = (yyvsp[0].reference); (yyval.reference)->node.operator.left = (yyvsp[-2].reference); } -#line 2281 "cod.tab.c" +#line 2151 "cod.tab.c" break; case 43: /* relational_expression: relational_expression GT shift_expression */ @@ -2289,7 +2159,7 @@ yyparse (void) (yyval.reference)->node.operator.right = (yyvsp[0].reference); (yyval.reference)->node.operator.left = (yyvsp[-2].reference); } -#line 2293 "cod.tab.c" +#line 2163 "cod.tab.c" break; case 44: /* relational_expression: relational_expression LEQ shift_expression */ @@ -2301,7 +2171,7 @@ yyparse (void) (yyval.reference)->node.operator.right = (yyvsp[0].reference); (yyval.reference)->node.operator.left = (yyvsp[-2].reference); } -#line 2305 "cod.tab.c" +#line 2175 "cod.tab.c" break; case 45: /* relational_expression: relational_expression GEQ shift_expression */ @@ -2313,7 +2183,7 @@ yyparse (void) (yyval.reference)->node.operator.right = (yyvsp[0].reference); (yyval.reference)->node.operator.left = (yyvsp[-2].reference); } -#line 2317 "cod.tab.c" +#line 2187 "cod.tab.c" break; case 47: /* equality_expression: equality_expression EQ relational_expression */ @@ -2325,7 +2195,7 @@ yyparse (void) (yyval.reference)->node.operator.right = (yyvsp[0].reference); (yyval.reference)->node.operator.left = (yyvsp[-2].reference); } -#line 2329 "cod.tab.c" +#line 2199 "cod.tab.c" break; case 48: /* equality_expression: equality_expression NEQ relational_expression */ @@ -2337,7 +2207,7 @@ yyparse (void) (yyval.reference)->node.operator.right = (yyvsp[0].reference); (yyval.reference)->node.operator.left = (yyvsp[-2].reference); } -#line 2341 "cod.tab.c" +#line 2211 "cod.tab.c" break; case 50: /* and_expression: and_expression ARITH_AND equality_expression */ @@ -2349,7 +2219,7 @@ yyparse (void) (yyval.reference)->node.operator.right = (yyvsp[0].reference); (yyval.reference)->node.operator.left = (yyvsp[-2].reference); } -#line 2353 "cod.tab.c" +#line 2223 "cod.tab.c" break; case 52: /* exclusive_or_expression: exclusive_or_expression ARITH_XOR and_expression */ @@ -2361,7 +2231,7 @@ yyparse (void) (yyval.reference)->node.operator.right = (yyvsp[0].reference); (yyval.reference)->node.operator.left = (yyvsp[-2].reference); } -#line 2365 "cod.tab.c" +#line 2235 "cod.tab.c" break; case 54: /* inclusive_or_expression: inclusive_or_expression ARITH_OR exclusive_or_expression */ @@ -2373,7 +2243,7 @@ yyparse (void) (yyval.reference)->node.operator.right = (yyvsp[0].reference); (yyval.reference)->node.operator.left = (yyvsp[-2].reference); } -#line 2377 "cod.tab.c" +#line 2247 "cod.tab.c" break; case 56: /* logical_and_expression: logical_and_expression LOG_AND inclusive_or_expression */ @@ -2385,7 +2255,7 @@ yyparse (void) (yyval.reference)->node.operator.right = (yyvsp[0].reference); (yyval.reference)->node.operator.left = (yyvsp[-2].reference); } -#line 2389 "cod.tab.c" +#line 2259 "cod.tab.c" break; case 58: /* logical_or_expression: logical_or_expression LOG_OR logical_and_expression */ @@ -2397,7 +2267,7 @@ yyparse (void) (yyval.reference)->node.operator.right = (yyvsp[0].reference); (yyval.reference)->node.operator.left = (yyvsp[-2].reference); } -#line 2401 "cod.tab.c" +#line 2271 "cod.tab.c" break; case 60: /* conditional_expression: logical_or_expression QUESTION expression COLON conditional_expression */ @@ -2409,79 +2279,79 @@ yyparse (void) (yyval.reference)->node.conditional_operator.e1 = (yyvsp[-2].reference); (yyval.reference)->node.conditional_operator.e2 = (yyvsp[0].reference); } -#line 2413 "cod.tab.c" +#line 2283 "cod.tab.c" break; case 61: /* assignment_operator: ASSIGN */ #line 746 "cod.y" { (yyval.info) = (yyvsp[0].info); (yyval.info).op = op_eq;} -#line 2419 "cod.tab.c" +#line 2289 "cod.tab.c" break; case 62: /* assignment_operator: MUL_ASSIGN */ #line 748 "cod.y" { (yyval.info) = (yyvsp[0].info); (yyval.info).op = op_mult;} -#line 2425 "cod.tab.c" +#line 2295 "cod.tab.c" break; case 63: /* assignment_operator: DIV_ASSIGN */ #line 750 "cod.y" { (yyval.info) = (yyvsp[0].info); (yyval.info).op = op_div;} -#line 2431 "cod.tab.c" +#line 2301 "cod.tab.c" break; case 64: /* assignment_operator: MOD_ASSIGN */ #line 752 "cod.y" { (yyval.info) = (yyvsp[0].info); (yyval.info).op = op_modulus;} -#line 2437 "cod.tab.c" +#line 2307 "cod.tab.c" break; case 65: /* assignment_operator: ADD_ASSIGN */ #line 754 "cod.y" { (yyval.info) = (yyvsp[0].info); (yyval.info).op = op_plus;} -#line 2443 "cod.tab.c" +#line 2313 "cod.tab.c" break; case 66: /* assignment_operator: SUB_ASSIGN */ #line 756 "cod.y" { (yyval.info) = (yyvsp[0].info); (yyval.info).op = op_minus;} -#line 2449 "cod.tab.c" +#line 2319 "cod.tab.c" break; case 67: /* assignment_operator: LEFT_ASSIGN */ #line 758 "cod.y" { (yyval.info) = (yyvsp[0].info); (yyval.info).op = op_left_shift;} -#line 2455 "cod.tab.c" +#line 2325 "cod.tab.c" break; case 68: /* assignment_operator: RIGHT_ASSIGN */ #line 760 "cod.y" { (yyval.info) = (yyvsp[0].info); (yyval.info).op = op_right_shift;} -#line 2461 "cod.tab.c" +#line 2331 "cod.tab.c" break; case 69: /* assignment_operator: AND_ASSIGN */ #line 762 "cod.y" { (yyval.info) = (yyvsp[0].info); (yyval.info).op = op_arith_and;} -#line 2467 "cod.tab.c" +#line 2337 "cod.tab.c" break; case 70: /* assignment_operator: XOR_ASSIGN */ #line 764 "cod.y" { (yyval.info) = (yyvsp[0].info); (yyval.info).op = op_arith_xor;} -#line 2473 "cod.tab.c" +#line 2343 "cod.tab.c" break; case 71: /* assignment_operator: OR_ASSIGN */ #line 766 "cod.y" { (yyval.info) = (yyvsp[0].info); (yyval.info).op = op_arith_or;} -#line 2479 "cod.tab.c" +#line 2349 "cod.tab.c" break; case 72: /* assignment_expression: conditional_expression */ #line 771 "cod.y" { (yyval.reference) = (yyvsp[0].reference);} -#line 2485 "cod.tab.c" +#line 2355 "cod.tab.c" break; case 73: /* assignment_expression: unary_expression assignment_operator assignment_expression */ @@ -2493,13 +2363,13 @@ yyparse (void) (yyval.reference)->node.assignment_expression.right = (yyvsp[0].reference); (yyval.reference)->node.assignment_expression.op = (yyvsp[-1].info).op; } -#line 2497 "cod.tab.c" +#line 2367 "cod.tab.c" break; case 74: /* expression: assignment_expression */ #line 785 "cod.y" {(yyval.reference) = (yyvsp[0].reference);} -#line 2503 "cod.tab.c" +#line 2373 "cod.tab.c" break; case 75: /* expression: expression COMMA assignment_expression */ @@ -2510,7 +2380,7 @@ yyparse (void) (yyval.reference)->node.comma_expression.left = (yyvsp[-2].reference); (yyval.reference)->node.comma_expression.right = (yyvsp[0].reference); } -#line 2514 "cod.tab.c" +#line 2384 "cod.tab.c" break; case 77: /* init_declarator_list: init_declarator */ @@ -2520,7 +2390,7 @@ yyparse (void) (yyval.list)->node = (yyvsp[0].reference); (yyval.list)->next = NULL; } -#line 2524 "cod.tab.c" +#line 2394 "cod.tab.c" break; case 78: /* init_declarator_list: init_declarator_list COMMA init_declarator */ @@ -2536,7 +2406,7 @@ yyparse (void) tmp->next = NULL; (yyval.list) = (yyvsp[-2].list); } -#line 2540 "cod.tab.c" +#line 2410 "cod.tab.c" break; case 79: /* $@1: %empty */ @@ -2547,7 +2417,7 @@ yyparse (void) YYACCEPT; } } -#line 2551 "cod.tab.c" +#line 2421 "cod.tab.c" break; case 80: /* @2: %empty */ @@ -2579,7 +2449,7 @@ yyparse (void) YYACCEPT; } } -#line 2583 "cod.tab.c" +#line 2453 "cod.tab.c" break; case 81: /* declaration: declaration_specifiers $@1 init_declarator_list @2 SEMI */ @@ -2639,7 +2509,7 @@ yyparse (void) } (void)(yyvsp[-1].reference); } -#line 2643 "cod.tab.c" +#line 2513 "cod.tab.c" break; case 82: /* declaration: declaration_specifiers SEMI */ @@ -2647,7 +2517,7 @@ yyparse (void) { (yyval.list) = (yyvsp[-1].list); } -#line 2651 "cod.tab.c" +#line 2521 "cod.tab.c" break; case 83: /* declaration_specifiers: storage_class_specifier */ @@ -2657,7 +2527,7 @@ yyparse (void) (yyval.list)->node = (yyvsp[0].reference); (yyval.list)->next = NULL; } -#line 2661 "cod.tab.c" +#line 2531 "cod.tab.c" break; case 84: /* declaration_specifiers: storage_class_specifier declaration_specifiers */ @@ -2668,7 +2538,7 @@ yyparse (void) tmp->next = (yyvsp[0].list); (yyval.list) = tmp; } -#line 2672 "cod.tab.c" +#line 2542 "cod.tab.c" break; case 85: /* declaration_specifiers: type_specifier */ @@ -2678,7 +2548,7 @@ yyparse (void) (yyval.list)->node = (yyvsp[0].reference); (yyval.list)->next = NULL; } -#line 2682 "cod.tab.c" +#line 2552 "cod.tab.c" break; case 86: /* declaration_specifiers: type_specifier declaration_specifiers */ @@ -2689,7 +2559,7 @@ yyparse (void) tmp->next = (yyvsp[0].list); (yyval.list) = tmp; } -#line 2693 "cod.tab.c" +#line 2563 "cod.tab.c" break; case 87: /* declaration_specifiers: type_qualifier */ @@ -2699,7 +2569,7 @@ yyparse (void) (yyval.list)->node = (yyvsp[0].reference); (yyval.list)->next = NULL; } -#line 2703 "cod.tab.c" +#line 2573 "cod.tab.c" break; case 88: /* declaration_specifiers: type_qualifier declaration_specifiers */ @@ -2710,7 +2580,7 @@ yyparse (void) tmp->next = (yyvsp[0].list); (yyval.list) = tmp; } -#line 2714 "cod.tab.c" +#line 2584 "cod.tab.c" break; case 90: /* init_declarator: declarator ASSIGN initializer */ @@ -2727,7 +2597,7 @@ yyparse (void) tmp->node.declaration.init_value = (yyvsp[0].reference); } } -#line 2731 "cod.tab.c" +#line 2601 "cod.tab.c" break; case 91: /* storage_class_specifier: TYPEDEF */ @@ -2737,7 +2607,7 @@ yyparse (void) (yyval.reference)->node.type_specifier.lx_srcpos = (yyvsp[0].info).lx_srcpos; (yyval.reference)->node.type_specifier.token = TYPEDEF; } -#line 2741 "cod.tab.c" +#line 2611 "cod.tab.c" break; case 92: /* storage_class_specifier: STATIC */ @@ -2747,7 +2617,7 @@ yyparse (void) (yyval.reference)->node.type_specifier.lx_srcpos = (yyvsp[0].info).lx_srcpos; (yyval.reference)->node.type_specifier.token = STATIC; } -#line 2751 "cod.tab.c" +#line 2621 "cod.tab.c" break; case 93: /* storage_class_specifier: EXTERN_TOKEN */ @@ -2757,7 +2627,7 @@ yyparse (void) (yyval.reference)->node.type_specifier.lx_srcpos = (yyvsp[0].info).lx_srcpos; (yyval.reference)->node.type_specifier.token = EXTERN_TOKEN; } -#line 2761 "cod.tab.c" +#line 2631 "cod.tab.c" break; case 94: /* type_specifier: CHAR */ @@ -2767,7 +2637,7 @@ yyparse (void) (yyval.reference)->node.type_specifier.lx_srcpos = (yyvsp[0].info).lx_srcpos; (yyval.reference)->node.type_specifier.token = CHAR; } -#line 2771 "cod.tab.c" +#line 2641 "cod.tab.c" break; case 95: /* type_specifier: SHORT */ @@ -2777,7 +2647,7 @@ yyparse (void) (yyval.reference)->node.type_specifier.lx_srcpos = (yyvsp[0].info).lx_srcpos; (yyval.reference)->node.type_specifier.token = SHORT; } -#line 2781 "cod.tab.c" +#line 2651 "cod.tab.c" break; case 96: /* type_specifier: INT */ @@ -2787,7 +2657,7 @@ yyparse (void) (yyval.reference)->node.type_specifier.lx_srcpos = (yyvsp[0].info).lx_srcpos; (yyval.reference)->node.type_specifier.token = INT; } -#line 2791 "cod.tab.c" +#line 2661 "cod.tab.c" break; case 97: /* type_specifier: LONG */ @@ -2797,7 +2667,7 @@ yyparse (void) (yyval.reference)->node.type_specifier.lx_srcpos = (yyvsp[0].info).lx_srcpos; (yyval.reference)->node.type_specifier.token = LONG; } -#line 2801 "cod.tab.c" +#line 2671 "cod.tab.c" break; case 98: /* type_specifier: FLOAT */ @@ -2807,7 +2677,7 @@ yyparse (void) (yyval.reference)->node.type_specifier.lx_srcpos = (yyvsp[0].info).lx_srcpos; (yyval.reference)->node.type_specifier.token = FLOAT; } -#line 2811 "cod.tab.c" +#line 2681 "cod.tab.c" break; case 99: /* type_specifier: DOUBLE */ @@ -2817,7 +2687,7 @@ yyparse (void) (yyval.reference)->node.type_specifier.lx_srcpos = (yyvsp[0].info).lx_srcpos; (yyval.reference)->node.type_specifier.token = DOUBLE; } -#line 2821 "cod.tab.c" +#line 2691 "cod.tab.c" break; case 100: /* type_specifier: VOID */ @@ -2827,7 +2697,7 @@ yyparse (void) (yyval.reference)->node.type_specifier.lx_srcpos = (yyvsp[0].info).lx_srcpos; (yyval.reference)->node.type_specifier.token = VOID; } -#line 2831 "cod.tab.c" +#line 2701 "cod.tab.c" break; case 101: /* type_specifier: SIGNED */ @@ -2837,7 +2707,7 @@ yyparse (void) (yyval.reference)->node.type_specifier.lx_srcpos = (yyvsp[0].info).lx_srcpos; (yyval.reference)->node.type_specifier.token = SIGNED; } -#line 2841 "cod.tab.c" +#line 2711 "cod.tab.c" break; case 102: /* type_specifier: UNSIGNED */ @@ -2847,7 +2717,7 @@ yyparse (void) (yyval.reference)->node.type_specifier.lx_srcpos = (yyvsp[0].info).lx_srcpos; (yyval.reference)->node.type_specifier.token = UNSIGNED; } -#line 2851 "cod.tab.c" +#line 2721 "cod.tab.c" break; case 103: /* type_specifier: STRING */ @@ -2857,17 +2727,17 @@ yyparse (void) (yyval.reference)->node.type_specifier.lx_srcpos = (yyvsp[0].info).lx_srcpos; (yyval.reference)->node.type_specifier.token = STRING; } -#line 2861 "cod.tab.c" +#line 2731 "cod.tab.c" break; case 104: /* type_specifier: type_identifier */ #line 1042 "cod.y" - { + { (yyval.reference) = cod_new_identifier(); (yyval.reference)->node.identifier.lx_srcpos = (yyvsp[0].info).lx_srcpos; (yyval.reference)->node.identifier.id = (yyvsp[0].info).string; } -#line 2871 "cod.tab.c" +#line 2741 "cod.tab.c" break; case 105: /* type_specifier: struct_or_union_specifier */ @@ -2875,7 +2745,7 @@ yyparse (void) { (yyval.reference) = (yyvsp[0].reference); } -#line 2879 "cod.tab.c" +#line 2749 "cod.tab.c" break; case 106: /* type_specifier: enum_specifier */ @@ -2883,7 +2753,7 @@ yyparse (void) { (yyval.reference) = (yyvsp[0].reference); } -#line 2887 "cod.tab.c" +#line 2757 "cod.tab.c" break; case 107: /* struct_or_union_specifier: struct_or_union identifier_ref LCURLY struct_declaration_list RCURLY */ @@ -2891,7 +2761,7 @@ yyparse (void) { (yyval.reference) = cod_build_parsed_type_node(yycontext, (yyvsp[-3].info).string, (yyvsp[-1].list)); } -#line 2895 "cod.tab.c" +#line 2765 "cod.tab.c" break; case 108: /* struct_or_union_specifier: struct_or_union LCURLY struct_declaration_list RCURLY */ @@ -2899,7 +2769,7 @@ yyparse (void) { (yyval.reference) = cod_build_parsed_type_node(yycontext, strdup("anon"), (yyvsp[-1].list)); } -#line 2903 "cod.tab.c" +#line 2773 "cod.tab.c" break; case 109: /* struct_or_union_specifier: struct_or_union identifier_ref */ @@ -2907,7 +2777,7 @@ yyparse (void) { (yyval.reference) = cod_build_parsed_type_node(yycontext, (yyvsp[0].info).string, NULL); } -#line 2911 "cod.tab.c" +#line 2781 "cod.tab.c" break; case 111: /* struct_or_union: UNION */ @@ -2915,7 +2785,7 @@ yyparse (void) { yyerror("UNIONs not supported!"); } -#line 2919 "cod.tab.c" +#line 2789 "cod.tab.c" break; case 113: /* struct_declaration_list: struct_declaration_list struct_declaration */ @@ -2928,13 +2798,13 @@ yyparse (void) tmp->next =(yyvsp[0].list); (yyval.list) = (yyvsp[-1].list); } -#line 2932 "cod.tab.c" +#line 2802 "cod.tab.c" break; case 114: /* struct_declaration: specifier_qualifier_list SEMI */ #line 1088 "cod.y" { } -#line 2938 "cod.tab.c" +#line 2808 "cod.tab.c" break; case 115: /* struct_declaration: specifier_qualifier_list struct_declarator_list SEMI */ @@ -2986,7 +2856,7 @@ yyparse (void) } } } -#line 2990 "cod.tab.c" +#line 2860 "cod.tab.c" break; case 116: /* struct_declarator_list: struct_declarator */ @@ -2996,7 +2866,7 @@ yyparse (void) (yyval.list)->node = (yyvsp[0].reference); (yyval.list)->next = NULL; } -#line 3000 "cod.tab.c" +#line 2870 "cod.tab.c" break; case 117: /* struct_declarator_list: struct_declarator_list COMMA struct_declarator */ @@ -3011,7 +2881,7 @@ yyparse (void) tmp->next->next = NULL; (yyval.list) = (yyvsp[-2].list); } -#line 3015 "cod.tab.c" +#line 2885 "cod.tab.c" break; case 119: /* specifier_qualifier_list: type_specifier specifier_qualifier_list */ @@ -3022,7 +2892,7 @@ yyparse (void) tmp->next = (yyvsp[0].list); (yyval.list) = tmp; } -#line 3026 "cod.tab.c" +#line 2896 "cod.tab.c" break; case 120: /* specifier_qualifier_list: type_specifier */ @@ -3032,7 +2902,7 @@ yyparse (void) (yyval.list)->node = (yyvsp[0].reference); (yyval.list)->next = NULL; } -#line 3036 "cod.tab.c" +#line 2906 "cod.tab.c" break; case 121: /* specifier_qualifier_list: type_qualifier specifier_qualifier_list */ @@ -3043,7 +2913,7 @@ yyparse (void) tmp->next = (yyvsp[0].list); (yyval.list) = tmp; } -#line 3047 "cod.tab.c" +#line 2917 "cod.tab.c" break; case 122: /* specifier_qualifier_list: type_qualifier */ @@ -3053,7 +2923,7 @@ yyparse (void) (yyval.list)->node = (yyvsp[0].reference); (yyval.list)->next = NULL; } -#line 3057 "cod.tab.c" +#line 2927 "cod.tab.c" break; case 123: /* enum_specifier: ENUM LCURLY enumerator_list RCURLY */ @@ -3065,7 +2935,7 @@ yyparse (void) (yyval.reference)->node.enum_type_decl.lx_srcpos = (yyvsp[-3].info).lx_srcpos; // cod_add_defined_type(decl->node.declaration.id, yycontext); } -#line 3069 "cod.tab.c" +#line 2939 "cod.tab.c" break; case 124: /* enum_specifier: ENUM LCURLY enumerator_list COMMA RCURLY */ @@ -3077,7 +2947,7 @@ yyparse (void) (yyval.reference)->node.enum_type_decl.lx_srcpos = (yyvsp[-4].info).lx_srcpos; // cod_add_defined_type(decl->node.declaration.id, yycontext); } -#line 3081 "cod.tab.c" +#line 2951 "cod.tab.c" break; case 125: /* enum_specifier: ENUM identifier_ref LCURLY enumerator_list RCURLY */ @@ -3089,7 +2959,7 @@ yyparse (void) (yyval.reference)->node.enum_type_decl.lx_srcpos = (yyvsp[-4].info).lx_srcpos; // cod_add_defined_type(decl->node.declaration.id, yycontext); } -#line 3093 "cod.tab.c" +#line 2963 "cod.tab.c" break; case 126: /* enum_specifier: ENUM identifier_ref LCURLY enumerator_list COMMA RCURLY */ @@ -3101,7 +2971,7 @@ yyparse (void) (yyval.reference)->node.enum_type_decl.lx_srcpos = (yyvsp[-5].info).lx_srcpos; // cod_add_defined_type(decl->node.declaration.id, yycontext); } -#line 3105 "cod.tab.c" +#line 2975 "cod.tab.c" break; case 127: /* enum_specifier: ENUM identifier_ref */ @@ -3113,7 +2983,7 @@ yyparse (void) (yyval.reference)->node.enum_type_decl.lx_srcpos = (yyvsp[-1].info).lx_srcpos; // cod_add_defined_type(decl->node.declaration.id, yycontext); } -#line 3117 "cod.tab.c" +#line 2987 "cod.tab.c" break; case 128: /* enumerator_list: enumerator */ @@ -3124,7 +2994,7 @@ yyparse (void) tmp->next = NULL; (yyval.list) = tmp; } -#line 3128 "cod.tab.c" +#line 2998 "cod.tab.c" break; case 129: /* enumerator_list: enumerator_list COMMA enumerator */ @@ -3135,7 +3005,7 @@ yyparse (void) tmp->next = (yyvsp[-2].list); (yyval.list) = tmp; } -#line 3139 "cod.tab.c" +#line 3009 "cod.tab.c" break; case 130: /* enumerator: identifier_ref ASSIGN constant_expression */ @@ -3145,7 +3015,7 @@ yyparse (void) (yyval.reference)->node.enumerator.id = (yyvsp[-2].info).string; (yyval.reference)->node.enumerator.const_expression = (yyvsp[0].reference); } -#line 3149 "cod.tab.c" +#line 3019 "cod.tab.c" break; case 131: /* enumerator: identifier_ref */ @@ -3155,7 +3025,7 @@ yyparse (void) (yyval.reference)->node.enumerator.id = (yyvsp[0].info).string; (yyval.reference)->node.enumerator.const_expression = NULL; } -#line 3159 "cod.tab.c" +#line 3029 "cod.tab.c" break; case 132: /* type_qualifier: CONST */ @@ -3165,7 +3035,7 @@ yyparse (void) (yyval.reference)->node.type_specifier.lx_srcpos = (yyvsp[0].info).lx_srcpos; (yyval.reference)->node.type_specifier.token = CONST; } -#line 3169 "cod.tab.c" +#line 3039 "cod.tab.c" break; case 134: /* declarator: pointer direct_declarator */ @@ -3181,7 +3051,7 @@ yyparse (void) cod_print((yyval.reference)); } } -#line 3185 "cod.tab.c" +#line 3055 "cod.tab.c" break; case 135: /* direct_declarator: identifier_ref */ @@ -3195,7 +3065,7 @@ yyparse (void) (yyval.reference)->node.declaration.is_subroutine = 0; (yyval.reference)->node.declaration.params = NULL; } -#line 3199 "cod.tab.c" +#line 3069 "cod.tab.c" break; case 136: /* direct_declarator: LPAREN declarator RPAREN */ @@ -3203,7 +3073,7 @@ yyparse (void) { (yyval.reference) = (yyvsp[-1].reference); } -#line 3207 "cod.tab.c" +#line 3077 "cod.tab.c" break; case 137: /* direct_declarator: identifier_ref LPAREN parameter_type_list RPAREN */ @@ -3217,7 +3087,7 @@ yyparse (void) (yyval.reference)->node.declaration.is_subroutine = 1; (yyval.reference)->node.declaration.params = (yyvsp[-1].list); } -#line 3221 "cod.tab.c" +#line 3091 "cod.tab.c" break; case 138: /* direct_declarator: identifier_ref LPAREN RPAREN */ @@ -3231,7 +3101,7 @@ yyparse (void) (yyval.reference)->node.declaration.is_subroutine = 1; (yyval.reference)->node.declaration.params = NULL; } -#line 3235 "cod.tab.c" +#line 3105 "cod.tab.c" break; case 139: /* direct_declarator: direct_declarator LBRACKET constant_expression RBRACKET */ @@ -3243,7 +3113,7 @@ yyparse (void) (yyval.reference)->node.array_type_decl.element_ref = (yyvsp[-3].reference); (yyval.reference)->node.array_type_decl.sm_dynamic_size = NULL; } -#line 3247 "cod.tab.c" +#line 3117 "cod.tab.c" break; case 140: /* direct_declarator: direct_declarator LBRACKET RBRACKET */ @@ -3255,7 +3125,7 @@ yyparse (void) (yyval.reference)->node.array_type_decl.element_ref = (yyvsp[-2].reference); (yyval.reference)->node.array_type_decl.sm_dynamic_size = NULL; } -#line 3259 "cod.tab.c" +#line 3129 "cod.tab.c" break; case 141: /* pointer: STAR */ @@ -3268,7 +3138,7 @@ yyparse (void) (yyval.list)->node = star; (yyval.list)->next = NULL; } -#line 3272 "cod.tab.c" +#line 3142 "cod.tab.c" break; case 142: /* pointer: STAR type_qualifier_list */ @@ -3281,7 +3151,7 @@ yyparse (void) (yyval.list)->node = star; (yyval.list)->next = (yyvsp[0].list); } -#line 3285 "cod.tab.c" +#line 3155 "cod.tab.c" break; case 143: /* pointer: STAR pointer */ @@ -3294,7 +3164,7 @@ yyparse (void) (yyval.list)->node = star; (yyval.list)->next = (yyvsp[0].list); } -#line 3298 "cod.tab.c" +#line 3168 "cod.tab.c" break; case 144: /* pointer: STAR type_qualifier_list pointer */ @@ -3313,7 +3183,7 @@ yyparse (void) (yyval.list)->node = star; (yyval.list)->next = (yyvsp[-1].list); } -#line 3317 "cod.tab.c" +#line 3187 "cod.tab.c" break; case 145: /* pointer: AT */ @@ -3329,7 +3199,7 @@ yyparse (void) (yyval.list)->node = star; (yyval.list)->next = NULL; } -#line 3333 "cod.tab.c" +#line 3203 "cod.tab.c" break; case 146: /* pointer: AT type_qualifier_list */ @@ -3345,7 +3215,7 @@ yyparse (void) (yyval.list)->node = star; (yyval.list)->next = (yyvsp[0].list); } -#line 3349 "cod.tab.c" +#line 3219 "cod.tab.c" break; case 147: /* pointer: AT pointer */ @@ -3361,7 +3231,7 @@ yyparse (void) (yyval.list)->node = star; (yyval.list)->next = (yyvsp[0].list); } -#line 3365 "cod.tab.c" +#line 3235 "cod.tab.c" break; case 148: /* pointer: AT type_qualifier_list pointer */ @@ -3383,7 +3253,7 @@ yyparse (void) (yyval.list)->node = star; (yyval.list)->next = (yyvsp[-1].list); } -#line 3387 "cod.tab.c" +#line 3257 "cod.tab.c" break; case 149: /* type_qualifier_list: type_qualifier */ @@ -3393,7 +3263,7 @@ yyparse (void) (yyval.list)->node = (yyvsp[0].reference); (yyval.list)->next = NULL; } -#line 3397 "cod.tab.c" +#line 3267 "cod.tab.c" break; case 150: /* type_qualifier_list: type_qualifier_list type_qualifier */ @@ -3408,7 +3278,7 @@ yyparse (void) tmp->next->next = NULL; (yyval.list) = (yyvsp[-1].list); } -#line 3412 "cod.tab.c" +#line 3282 "cod.tab.c" break; case 152: /* parameter_type_list: parameter_list COMMA DOTDOTDOT */ @@ -3425,7 +3295,7 @@ yyparse (void) id->node.declaration.id = strdup("..."); (yyval.list) = (yyvsp[-2].list); } -#line 3429 "cod.tab.c" +#line 3299 "cod.tab.c" break; case 153: /* parameter_list: parameter_declaration */ @@ -3435,7 +3305,7 @@ yyparse (void) (yyval.list)->node = (yyvsp[0].reference); (yyval.list)->next = NULL; } -#line 3439 "cod.tab.c" +#line 3309 "cod.tab.c" break; case 154: /* parameter_list: parameter_list COMMA parameter_declaration */ @@ -3450,7 +3320,7 @@ yyparse (void) tmp->next->next = NULL; (yyval.list) = (yyvsp[-2].list); } -#line 3454 "cod.tab.c" +#line 3324 "cod.tab.c" break; case 155: /* parameter_declaration: declaration_specifiers */ @@ -3464,7 +3334,7 @@ yyparse (void) (yyval.reference)->node.declaration.params = NULL; (yyval.reference)->node.declaration.type_spec = (yyvsp[0].list); } -#line 3468 "cod.tab.c" +#line 3338 "cod.tab.c" break; case 156: /* parameter_declaration: declaration_specifiers declarator */ @@ -3506,7 +3376,7 @@ yyparse (void) printf("unexpected node in parameter_declaration"); } } -#line 3510 "cod.tab.c" +#line 3380 "cod.tab.c" break; case 158: /* type_name: specifier_qualifier_list abstract_declarator */ @@ -3519,7 +3389,7 @@ yyparse (void) tmp->next = (yyvsp[0].list); (yyval.list) = (yyvsp[-1].list); } -#line 3523 "cod.tab.c" +#line 3393 "cod.tab.c" break; case 160: /* initializer: LCURLY initializer_list RCURLY */ @@ -3528,7 +3398,7 @@ yyparse (void) (yyval.reference) = cod_new_initializer_list(); (yyval.reference)->node.initializer_list.initializers = (yyvsp[-1].list); } -#line 3532 "cod.tab.c" +#line 3402 "cod.tab.c" break; case 161: /* initializer: LCURLY initializer_list COMMA RCURLY */ @@ -3537,13 +3407,13 @@ yyparse (void) (yyval.reference) = cod_new_initializer_list(); (yyval.reference)->node.initializer_list.initializers = (yyvsp[-2].list); } -#line 3541 "cod.tab.c" +#line 3411 "cod.tab.c" break; case 162: /* initializer: assignment_expression */ #line 1579 "cod.y" { (yyval.reference) = (yyvsp[0].reference);} -#line 3547 "cod.tab.c" +#line 3417 "cod.tab.c" break; case 163: /* initializer_list: designation initializer */ @@ -3556,7 +3426,7 @@ yyparse (void) (yyval.list)->node = initializer; (yyval.list)->next = NULL; } -#line 3560 "cod.tab.c" +#line 3430 "cod.tab.c" break; case 164: /* initializer_list: initializer */ @@ -3569,7 +3439,7 @@ yyparse (void) (yyval.list)->node = initializer; (yyval.list)->next = NULL; } -#line 3573 "cod.tab.c" +#line 3443 "cod.tab.c" break; case 165: /* initializer_list: initializer_list COMMA designation initializer */ @@ -3587,7 +3457,7 @@ yyparse (void) tmp->next->next = NULL; (yyval.list) = (yyvsp[-3].list); } -#line 3591 "cod.tab.c" +#line 3461 "cod.tab.c" break; case 166: /* initializer_list: initializer_list COMMA initializer */ @@ -3605,13 +3475,13 @@ yyparse (void) tmp->next->next = NULL; (yyval.list) = (yyvsp[-2].list); } -#line 3609 "cod.tab.c" +#line 3479 "cod.tab.c" break; case 167: /* designation: designator_list ASSIGN */ #line 1630 "cod.y" { (yyval.list) = (yyvsp[-1].list);} -#line 3615 "cod.tab.c" +#line 3485 "cod.tab.c" break; case 168: /* designator_list: designator */ @@ -3621,7 +3491,7 @@ yyparse (void) (yyval.list)->node = (yyvsp[0].reference); (yyval.list)->next = NULL; } -#line 3625 "cod.tab.c" +#line 3495 "cod.tab.c" break; case 169: /* designator_list: designator_list designator */ @@ -3636,7 +3506,7 @@ yyparse (void) tmp->next->next = NULL; (yyval.list) = (yyvsp[-1].list); } -#line 3640 "cod.tab.c" +#line 3510 "cod.tab.c" break; case 170: /* designator: LBRACKET constant_expression RBRACKET */ @@ -3646,7 +3516,7 @@ yyparse (void) (yyval.reference)->node.designator.expression = (yyvsp[-1].reference); (yyval.reference)->node.designator.id = NULL; } -#line 3650 "cod.tab.c" +#line 3520 "cod.tab.c" break; case 171: /* designator: DOT identifier_ref */ @@ -3656,7 +3526,7 @@ yyparse (void) (yyval.reference)->node.designator.expression = NULL; (yyval.reference)->node.designator.id = (yyvsp[0].info).string; } -#line 3660 "cod.tab.c" +#line 3530 "cod.tab.c" break; case 172: /* decls_stmts_list: statement */ @@ -3667,7 +3537,7 @@ yyparse (void) tmp->next = NULL; (yyval.list) = tmp; } -#line 3671 "cod.tab.c" +#line 3541 "cod.tab.c" break; case 173: /* decls_stmts_list: declaration */ @@ -3675,7 +3545,7 @@ yyparse (void) { (yyval.list) = (yyvsp[0].list); } -#line 3679 "cod.tab.c" +#line 3549 "cod.tab.c" break; case 174: /* decls_stmts_list: error SEMI */ @@ -3683,7 +3553,7 @@ yyparse (void) { (yyval.list) = NULL; } -#line 3687 "cod.tab.c" +#line 3557 "cod.tab.c" break; case 175: /* decls_stmts_list: decls_stmts_list statement */ @@ -3694,7 +3564,7 @@ yyparse (void) tmp->next = NULL; (yyval.list) = cod_append_list((yyvsp[-1].list), tmp); } -#line 3698 "cod.tab.c" +#line 3568 "cod.tab.c" break; case 176: /* decls_stmts_list: decls_stmts_list declaration */ @@ -3702,7 +3572,7 @@ yyparse (void) { (yyval.list) = cod_append_list((yyvsp[-1].list), (yyvsp[0].list)); } -#line 3706 "cod.tab.c" +#line 3576 "cod.tab.c" break; case 183: /* labeled_statement: identifier_ref COLON statement */ @@ -3712,7 +3582,7 @@ yyparse (void) (yyval.reference)->node.label_statement.name = (yyvsp[-2].info).string; (yyval.reference)->node.label_statement.statement = (yyvsp[0].reference); } -#line 3716 "cod.tab.c" +#line 3586 "cod.tab.c" break; case 184: /* compound_statement: LCURLY RCURLY */ @@ -3720,7 +3590,7 @@ yyparse (void) { (yyval.reference) = cod_new_compound_statement(); } -#line 3724 "cod.tab.c" +#line 3594 "cod.tab.c" break; case 185: /* compound_statement: LCURLY decls_stmts_list RCURLY */ @@ -3731,13 +3601,13 @@ yyparse (void) (yyval.reference)->node.compound_statement.decls = (yyvsp[-1].list); cod_remove_defined_types(yycontext, count); } -#line 3735 "cod.tab.c" +#line 3605 "cod.tab.c" break; case 186: /* declaration_list: declaration */ #line 1720 "cod.y" { (yyval.list) = (yyvsp[0].list); } -#line 3741 "cod.tab.c" +#line 3611 "cod.tab.c" break; case 187: /* declaration_list: declaration_list declaration */ @@ -3754,7 +3624,7 @@ yyparse (void) (yyval.list) = (yyvsp[-1].list); } } -#line 3758 "cod.tab.c" +#line 3628 "cod.tab.c" break; case 188: /* jump_statement: RETURN_TOKEN expression SEMI */ @@ -3764,7 +3634,7 @@ yyparse (void) (yyval.reference)->node.return_statement.expression = (yyvsp[-1].reference); (yyval.reference)->node.return_statement.lx_srcpos = (yyvsp[-2].info).lx_srcpos; } -#line 3768 "cod.tab.c" +#line 3638 "cod.tab.c" break; case 189: /* jump_statement: RETURN_TOKEN SEMI */ @@ -3774,7 +3644,7 @@ yyparse (void) (yyval.reference)->node.return_statement.expression = NULL; (yyval.reference)->node.return_statement.lx_srcpos = (yyvsp[-1].info).lx_srcpos; } -#line 3778 "cod.tab.c" +#line 3648 "cod.tab.c" break; case 190: /* jump_statement: CONTINUE SEMI */ @@ -3785,7 +3655,7 @@ yyparse (void) (yyval.reference)->node.jump_statement.goto_target = NULL; (yyval.reference)->node.jump_statement.lx_srcpos = (yyvsp[-1].info).lx_srcpos; } -#line 3789 "cod.tab.c" +#line 3659 "cod.tab.c" break; case 191: /* jump_statement: BREAK SEMI */ @@ -3796,7 +3666,7 @@ yyparse (void) (yyval.reference)->node.jump_statement.goto_target = NULL; (yyval.reference)->node.jump_statement.lx_srcpos = (yyvsp[-1].info).lx_srcpos; } -#line 3800 "cod.tab.c" +#line 3670 "cod.tab.c" break; case 192: /* jump_statement: GOTO identifier_ref SEMI */ @@ -3807,7 +3677,7 @@ yyparse (void) (yyval.reference)->node.jump_statement.goto_target = (yyvsp[-1].info).string; (yyval.reference)->node.jump_statement.lx_srcpos = (yyvsp[-2].info).lx_srcpos; } -#line 3811 "cod.tab.c" +#line 3681 "cod.tab.c" break; case 193: /* expression_statement: SEMI */ @@ -3815,7 +3685,7 @@ yyparse (void) { (yyval.reference) = NULL; } -#line 3819 "cod.tab.c" +#line 3689 "cod.tab.c" break; case 194: /* expression_statement: expression SEMI */ @@ -3824,7 +3694,7 @@ yyparse (void) (yyval.reference) = cod_new_expression_statement(); (yyval.reference)->node.expression_statement.expression = (yyvsp[-1].reference); } -#line 3828 "cod.tab.c" +#line 3698 "cod.tab.c" break; case 195: /* selection_statement: IF LPAREN expression RPAREN statement */ @@ -3836,7 +3706,7 @@ yyparse (void) (yyval.reference)->node.selection_statement.then_part = (yyvsp[0].reference); (yyval.reference)->node.selection_statement.else_part = NULL; } -#line 3840 "cod.tab.c" +#line 3710 "cod.tab.c" break; case 196: /* selection_statement: IF LPAREN expression RPAREN statement ELSE statement */ @@ -3848,7 +3718,7 @@ yyparse (void) (yyval.reference)->node.selection_statement.then_part = (yyvsp[-2].reference); (yyval.reference)->node.selection_statement.else_part = (yyvsp[0].reference); } -#line 3852 "cod.tab.c" +#line 3722 "cod.tab.c" break; case 197: /* iteration_statement: FOR LPAREN expression_opt SEMI expression_opt SEMI expression_opt RPAREN statement */ @@ -3861,7 +3731,7 @@ yyparse (void) (yyval.reference)->node.iteration_statement.iter_expr = (yyvsp[-2].reference); (yyval.reference)->node.iteration_statement.statement = (yyvsp[0].reference); } -#line 3865 "cod.tab.c" +#line 3735 "cod.tab.c" break; case 198: /* iteration_statement: WHILE LPAREN expression RPAREN statement */ @@ -3874,7 +3744,7 @@ yyparse (void) (yyval.reference)->node.iteration_statement.iter_expr = NULL; (yyval.reference)->node.iteration_statement.statement = (yyvsp[0].reference); } -#line 3878 "cod.tab.c" +#line 3748 "cod.tab.c" break; case 199: /* iteration_statement: DO statement WHILE LPAREN expression RPAREN SEMI */ @@ -3888,13 +3758,13 @@ yyparse (void) (yyval.reference)->node.iteration_statement.iter_expr = NULL; (yyval.reference)->node.iteration_statement.statement = (yyvsp[-5].reference); } -#line 3892 "cod.tab.c" +#line 3762 "cod.tab.c" break; case 200: /* expression_opt: %empty */ #line 1841 "cod.y" { (yyval.reference) = NULL; } -#line 3898 "cod.tab.c" +#line 3768 "cod.tab.c" break; case 202: /* constant: integer_constant */ @@ -3905,7 +3775,7 @@ yyparse (void) (yyval.reference)->node.constant.const_val = (yyvsp[0].info).string; (yyval.reference)->node.constant.lx_srcpos = (yyvsp[0].info).lx_srcpos; } -#line 3909 "cod.tab.c" +#line 3779 "cod.tab.c" break; case 203: /* constant: floating_constant */ @@ -3916,7 +3786,7 @@ yyparse (void) (yyval.reference)->node.constant.const_val = (yyvsp[0].info).string; (yyval.reference)->node.constant.lx_srcpos = (yyvsp[0].info).lx_srcpos; } -#line 3920 "cod.tab.c" +#line 3790 "cod.tab.c" break; case 204: /* constant: string_constant */ @@ -3927,7 +3797,7 @@ yyparse (void) (yyval.reference)->node.constant.const_val = (yyvsp[0].info).string; (yyval.reference)->node.constant.lx_srcpos = (yyvsp[0].info).lx_srcpos; } -#line 3931 "cod.tab.c" +#line 3801 "cod.tab.c" break; case 205: /* constant: character_constant */ @@ -3938,7 +3808,7 @@ yyparse (void) (yyval.reference)->node.constant.const_val = (yyvsp[0].info).string; (yyval.reference)->node.constant.lx_srcpos = (yyvsp[0].info).lx_srcpos; } -#line 3942 "cod.tab.c" +#line 3812 "cod.tab.c" break; case 206: /* constant: enumeration_constant */ @@ -3949,11 +3819,11 @@ yyparse (void) (yyval.reference)->node.constant.const_val = (yyvsp[0].info).string; (yyval.reference)->node.constant.lx_srcpos = (yyvsp[0].info).lx_srcpos; } -#line 3953 "cod.tab.c" +#line 3823 "cod.tab.c" break; -#line 3957 "cod.tab.c" +#line 3827 "cod.tab.c" default: break; } @@ -8715,7 +8585,7 @@ space_to_underscore(char *str){ while(*str != '\0'){ if(isspace(*str)) *str = '_'; - str++; + str++; } } @@ -9000,7 +8870,7 @@ evaluate_constant_return_expr(cod_parse_context context, sm_ref expr, int *free_ *free_result = 1; } else { /* we get an integer result */ - intptr_t left_val = 0, right_val = 0, value; + intptr_t left_val = 0, right_val = 0, value = 0; char str_val[40]; if (expr->node.operator.left) left_val = get_constant_long_value(context, left); @@ -9079,7 +8949,7 @@ evaluate_constant_return_expr(cod_parse_context context, sm_ref expr, int *free_ } ret = cod_new_constant(); ret->node.constant.token = integer_constant; - sprintf(str_val, "%Id", value); + sprintf(str_val, "%zd", value); ret->node.constant.const_val = strdup(str_val); *free_result = 1; } diff --git a/thirdparty/ffs/ffs/cod/pregen_source/Windows/cod.tab.h b/thirdparty/ffs/ffs/cod/pregen_source/Windows/cod.tab.h index 9dfb351295..f291f75d49 100644 --- a/thirdparty/ffs/ffs/cod/pregen_source/Windows/cod.tab.h +++ b/thirdparty/ffs/ffs/cod/pregen_source/Windows/cod.tab.h @@ -1,14 +1,14 @@ -/* A Bison parser, made by GNU Bison 2.3. */ +/* A Bison parser, made by GNU Bison 3.8.2. */ -/* Skeleton interface for Bison's Yacc-like parsers in C +/* Bison interface for Yacc-like parsers in C - Copyright (C) 1984, 1989, 1990, 2000, 2001, 2002, 2003, 2004, 2005, 2006 - Free Software Foundation, Inc. + Copyright (C) 1984, 1989-1990, 2000-2015, 2018-2021 Free Software Foundation, + Inc. - This program is free software; you can redistribute it and/or modify + This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2, or (at your option) - any later version. + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of @@ -16,9 +16,7 @@ GNU General Public License for more details. You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, - Boston, MA 02110-1301, USA. */ + along with this program. If not, see . */ /* As a special exception, you may create a larger work that contains part or all of the Bison parser skeleton and distribute that work @@ -33,200 +31,140 @@ This special exception was added by the Free Software Foundation in version 2.2 of Bison. */ -/* Tokens. */ +/* DO NOT RELY ON FEATURES THAT ARE NOT DOCUMENTED in the manual, + especially those whose name start with YY_ or yy_. They are + private implementation details that can be changed or removed. */ + +#ifndef YY_YY_COD_TAB_H_INCLUDED +# define YY_YY_COD_TAB_H_INCLUDED +/* Debug traces. */ +#ifndef YYDEBUG +# define YYDEBUG 0 +#endif +#if YYDEBUG +extern int yydebug; +#endif + +/* Token kinds. */ #ifndef YYTOKENTYPE # define YYTOKENTYPE - /* Put the tokens into the symbol table, so that GDB and other debuggers - know about them. */ - enum yytokentype { - ARROW = 258, - LPAREN = 259, - RPAREN = 260, - LCURLY = 261, - RCURLY = 262, - COLON = 263, - QUESTION = 264, - LBRACKET = 265, - RBRACKET = 266, - DOT = 267, - STAR = 268, - AT = 269, - SLASH = 270, - MODULUS = 271, - PLUS = 272, - MINUS = 273, - TILDE = 274, - LEQ = 275, - LT = 276, - GEQ = 277, - GT = 278, - EQ = 279, - NEQ = 280, - LEFT_SHIFT = 281, - RIGHT_SHIFT = 282, - ASSIGN = 283, - MUL_ASSIGN = 284, - DIV_ASSIGN = 285, - MOD_ASSIGN = 286, - ADD_ASSIGN = 287, - SUB_ASSIGN = 288, - LEFT_ASSIGN = 289, - RIGHT_ASSIGN = 290, - AND_ASSIGN = 291, - XOR_ASSIGN = 292, - OR_ASSIGN = 293, - LOG_OR = 294, - LOG_AND = 295, - ARITH_OR = 296, - ARITH_AND = 297, - ARITH_XOR = 298, - INC_OP = 299, - DEC_OP = 300, - BANG = 301, - SEMI = 302, - IF = 303, - ELSE = 304, - FOR = 305, - DO = 306, - WHILE = 307, - CHAR = 308, - SHORT = 309, - INT = 310, - LONG = 311, - UNSIGNED = 312, - SIGNED = 313, - FLOAT = 314, - DOUBLE = 315, - VOID = 316, - STRING = 317, - STATIC = 318, - EXTERN_TOKEN = 319, - STRUCT = 320, - ENUM = 321, - UNION = 322, - CONST = 323, - SIZEOF = 324, - TYPEDEF = 325, - RETURN_TOKEN = 326, - CONTINUE = 327, - BREAK = 328, - GOTO = 329, - PRINT = 330, - COMMA = 331, - DOTDOTDOT = 332, - integer_constant = 333, - character_constant = 334, - string_constant = 335, - floating_constant = 336, - identifier_ref = 337, - type_identifier = 338, - enumeration_constant = 339 - }; + enum yytokentype + { + YYEMPTY = -2, + YYEOF = 0, /* "end of file" */ + YYerror = 256, /* error */ + YYUNDEF = 257, /* "invalid token" */ + ARROW = 258, /* ARROW */ + LPAREN = 259, /* LPAREN */ + RPAREN = 260, /* RPAREN */ + LCURLY = 261, /* LCURLY */ + RCURLY = 262, /* RCURLY */ + COLON = 263, /* COLON */ + QUESTION = 264, /* QUESTION */ + LBRACKET = 265, /* LBRACKET */ + RBRACKET = 266, /* RBRACKET */ + DOT = 267, /* DOT */ + STAR = 268, /* STAR */ + AT = 269, /* AT */ + SLASH = 270, /* SLASH */ + MODULUS = 271, /* MODULUS */ + PLUS = 272, /* PLUS */ + MINUS = 273, /* MINUS */ + TILDE = 274, /* TILDE */ + LEQ = 275, /* LEQ */ + LT = 276, /* LT */ + GEQ = 277, /* GEQ */ + GT = 278, /* GT */ + EQ = 279, /* EQ */ + NEQ = 280, /* NEQ */ + LEFT_SHIFT = 281, /* LEFT_SHIFT */ + RIGHT_SHIFT = 282, /* RIGHT_SHIFT */ + ASSIGN = 283, /* ASSIGN */ + MUL_ASSIGN = 284, /* MUL_ASSIGN */ + DIV_ASSIGN = 285, /* DIV_ASSIGN */ + MOD_ASSIGN = 286, /* MOD_ASSIGN */ + ADD_ASSIGN = 287, /* ADD_ASSIGN */ + SUB_ASSIGN = 288, /* SUB_ASSIGN */ + LEFT_ASSIGN = 289, /* LEFT_ASSIGN */ + RIGHT_ASSIGN = 290, /* RIGHT_ASSIGN */ + AND_ASSIGN = 291, /* AND_ASSIGN */ + XOR_ASSIGN = 292, /* XOR_ASSIGN */ + OR_ASSIGN = 293, /* OR_ASSIGN */ + LOG_OR = 294, /* LOG_OR */ + LOG_AND = 295, /* LOG_AND */ + ARITH_OR = 296, /* ARITH_OR */ + ARITH_AND = 297, /* ARITH_AND */ + ARITH_XOR = 298, /* ARITH_XOR */ + INC_OP = 299, /* INC_OP */ + DEC_OP = 300, /* DEC_OP */ + BANG = 301, /* BANG */ + SEMI = 302, /* SEMI */ + IF = 303, /* IF */ + ELSE = 304, /* ELSE */ + FOR = 305, /* FOR */ + DO = 306, /* DO */ + WHILE = 307, /* WHILE */ + CHAR = 308, /* CHAR */ + SHORT = 309, /* SHORT */ + INT = 310, /* INT */ + LONG = 311, /* LONG */ + UNSIGNED = 312, /* UNSIGNED */ + SIGNED = 313, /* SIGNED */ + FLOAT = 314, /* FLOAT */ + DOUBLE = 315, /* DOUBLE */ + VOID = 316, /* VOID */ + STRING = 317, /* STRING */ + STATIC = 318, /* STATIC */ + EXTERN_TOKEN = 319, /* EXTERN_TOKEN */ + STRUCT = 320, /* STRUCT */ + ENUM = 321, /* ENUM */ + UNION = 322, /* UNION */ + CONST = 323, /* CONST */ + SIZEOF = 324, /* SIZEOF */ + TYPEDEF = 325, /* TYPEDEF */ + RETURN_TOKEN = 326, /* RETURN_TOKEN */ + CONTINUE = 327, /* CONTINUE */ + BREAK = 328, /* BREAK */ + GOTO = 329, /* GOTO */ + PRINT = 330, /* PRINT */ + COMMA = 331, /* COMMA */ + DOTDOTDOT = 332, /* DOTDOTDOT */ + integer_constant = 333, /* integer_constant */ + character_constant = 334, /* character_constant */ + string_constant = 335, /* string_constant */ + floating_constant = 336, /* floating_constant */ + identifier_ref = 337, /* identifier_ref */ + type_identifier = 338, /* type_identifier */ + enumeration_constant = 339 /* enumeration_constant */ + }; + typedef enum yytokentype yytoken_kind_t; #endif -/* Tokens. */ -#define ARROW 258 -#define LPAREN 259 -#define RPAREN 260 -#define LCURLY 261 -#define RCURLY 262 -#define COLON 263 -#define QUESTION 264 -#define LBRACKET 265 -#define RBRACKET 266 -#define DOT 267 -#define STAR 268 -#define AT 269 -#define SLASH 270 -#define MODULUS 271 -#define PLUS 272 -#define MINUS 273 -#define TILDE 274 -#define LEQ 275 -#define LT 276 -#define GEQ 277 -#define GT 278 -#define EQ 279 -#define NEQ 280 -#define LEFT_SHIFT 281 -#define RIGHT_SHIFT 282 -#define ASSIGN 283 -#define MUL_ASSIGN 284 -#define DIV_ASSIGN 285 -#define MOD_ASSIGN 286 -#define ADD_ASSIGN 287 -#define SUB_ASSIGN 288 -#define LEFT_ASSIGN 289 -#define RIGHT_ASSIGN 290 -#define AND_ASSIGN 291 -#define XOR_ASSIGN 292 -#define OR_ASSIGN 293 -#define LOG_OR 294 -#define LOG_AND 295 -#define ARITH_OR 296 -#define ARITH_AND 297 -#define ARITH_XOR 298 -#define INC_OP 299 -#define DEC_OP 300 -#define BANG 301 -#define SEMI 302 -#define IF 303 -#define ELSE 304 -#define FOR 305 -#define DO 306 -#define WHILE 307 -#define CHAR 308 -#define SHORT 309 -#define INT 310 -#define LONG 311 -#define UNSIGNED 312 -#define SIGNED 313 -#define FLOAT 314 -#define DOUBLE 315 -#define VOID 316 -#define STRING 317 -#define STATIC 318 -#define EXTERN_TOKEN 319 -#define STRUCT 320 -#define ENUM 321 -#define UNION 322 -#define CONST 323 -#define SIZEOF 324 -#define TYPEDEF 325 -#define RETURN_TOKEN 326 -#define CONTINUE 327 -#define BREAK 328 -#define GOTO 329 -#define PRINT 330 -#define COMMA 331 -#define DOTDOTDOT 332 -#define integer_constant 333 -#define character_constant 334 -#define string_constant 335 -#define floating_constant 336 -#define identifier_ref 337 -#define type_identifier 338 -#define enumeration_constant 339 - - - +/* Value type. */ #if ! defined YYSTYPE && ! defined YYSTYPE_IS_DECLARED -typedef union YYSTYPE -#line 187 "cod/cod.y" +union YYSTYPE { +#line 201 "cod.y" + lx_info info; sm_ref reference; operator_t operator; sm_list list; char *string; -} -/* Line 1529 of yacc.c. */ -#line 225 "/Users/eisen/prog/ffs/build/cod.tab.h" - YYSTYPE; -# define yystype YYSTYPE /* obsolescent; will be withdrawn */ -# define YYSTYPE_IS_DECLARED 1 + +#line 156 "cod.tab.h" + +}; +typedef union YYSTYPE YYSTYPE; # define YYSTYPE_IS_TRIVIAL 1 +# define YYSTYPE_IS_DECLARED 1 #endif + extern YYSTYPE yylval; + +int yyparse (void); + + +#endif /* !YY_YY_COD_TAB_H_INCLUDED */ diff --git a/thirdparty/ffs/ffs/cod/pregen_source/Windows/cod.y b/thirdparty/ffs/ffs/cod/pregen_source/Windows/cod.y index 264b112904..3d99807dcb 100644 --- a/thirdparty/ffs/ffs/cod/pregen_source/Windows/cod.y +++ b/thirdparty/ffs/ffs/cod/pregen_source/Windows/cod.y @@ -6447,7 +6447,7 @@ space_to_underscore(char *str){ while(*str != '\0'){ if(isspace(*str)) *str = '_'; - str++; + str++; } } @@ -6732,7 +6732,7 @@ evaluate_constant_return_expr(cod_parse_context context, sm_ref expr, int *free_ *free_result = 1; } else { /* we get an integer result */ - intptr_t left_val = 0, right_val = 0, value; + intptr_t left_val = 0, right_val = 0, value = 0; char str_val[40]; if (expr->node.operator.left) left_val = get_constant_long_value(context, left); @@ -6811,7 +6811,7 @@ evaluate_constant_return_expr(cod_parse_context context, sm_ref expr, int *free_ } ret = cod_new_constant(); ret->node.constant.token = integer_constant; - sprintf(str_val, "%Id", value); + sprintf(str_val, "%zd", value); ret->node.constant.const_val = strdup(str_val); *free_result = 1; } diff --git a/thirdparty/ffs/ffs/cod/pregen_source/Windows/lex.yy.c b/thirdparty/ffs/ffs/cod/pregen_source/Windows/lex.yy.c index 100cc2fc18..0daf24e1f1 100644 --- a/thirdparty/ffs/ffs/cod/pregen_source/Windows/lex.yy.c +++ b/thirdparty/ffs/ffs/cod/pregen_source/Windows/lex.yy.c @@ -2728,70 +2728,9 @@ yywrap YY_PROTO(( void )) } -#ifndef input /* flex, not lex */ void yy_delete_buffer YY_PROTO((YY_BUFFER_STATE b)); -#ifdef WINNT -/* old Windows code for MKS Toolkit version of flex */ - -static void -terminate_string_parse() -{ - yyrestart(NULL); -} - -#ifdef YY_USE_PROTOS -static void *yy_flex_alloc( yy_size_t size ) -#else -static void *yy_flex_alloc( size ) -yy_size_t size; -#endif - { - return (void *) malloc( size ); - } - -static char* current_input_string; - -int my_yy_input(buf,result,max_size) { - - if (current_input_string == NULL) - { - - result = 0; - } - else - if (max_size < strlen(current_input_string)) - { - memcpy((void*)buf, current_input_string, max_size); - current_input_string += max_size; - result = max_size; - } else { - int n = strlen(current_input_string); - memcpy((void*)buf, current_input_string, n+1); - current_input_string = NULL; - result = n; - } - -/* printf("my_yy_input buf[%s],result[%d]\n",buf,result);*/ - return result; -} - -static void -setup_for_string_parse(string, defined_types, enum_constants) -const char *string; -char **defined_types; -char **enum_constants; -{ - type_count = defined_type_count; - types = defined_types; - enums = enum_constants; - - current_input_string = string; - lex_offset = 1; - line_count = 1; -} -#else static YY_BUFFER_STATE bb = NULL; @@ -2824,6 +2763,4 @@ terminate_string_parse() } } -#endif -#endif diff --git a/thirdparty/ffs/ffs/cod/standard.c b/thirdparty/ffs/ffs/cod/standard.c index 2735741153..df9c7145dd 100644 --- a/thirdparty/ffs/ffs/cod/standard.c +++ b/thirdparty/ffs/ffs/cod/standard.c @@ -199,6 +199,10 @@ typedef struct timeval { long tv_usec; } timeval; #endif +typedef struct timezone { + int tz_minuteswest; /* minutes west of Greenwich */ + int tz_dsttime; /* type of DST correction */ +} timezone; int gettimeofday(struct timeval* tp, struct timezone* tzp) { @@ -464,7 +468,7 @@ cod_add_standard_elements(cod_parse_context context) } #endif /* LINUX_KERNEL_MODULE */ -#if NO_DYNAMIC_LINKING +#if defined(NO_DYNAMIC_LINKING) && !defined(_MSC_VER) #define sym(x) (void*)(intptr_t)x #else #define sym(x) (void*)0 diff --git a/thirdparty/ffs/ffs/cod/tests/CMakeLists.txt b/thirdparty/ffs/ffs/cod/tests/CMakeLists.txt index 7da5c38893..2ade40f9da 100644 --- a/thirdparty/ffs/ffs/cod/tests/CMakeLists.txt +++ b/thirdparty/ffs/ffs/cod/tests/CMakeLists.txt @@ -15,7 +15,10 @@ foreach (TEST ${TESTS} ) TARGET_LINK_LIBRARIES(${TEST} ${ATL_LIBRARIES} ) ADD_TEST(NAME ${TEST} COMMAND ${TEST}) endforeach() -SET_SOURCE_FILES_PROPERTIES(general.c PROPERTIES COMPILE_FLAGS -O0) + +if (NOT CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + SET_SOURCE_FILES_PROPERTIES(general.c PROPERTIES COMPILE_FLAGS -O0) +endif() set_tests_properties (t2 PROPERTIES PASS_REGULAR_EXPRESSION "Expect -> .values are is 5, 3.14159, hello!. diff --git a/thirdparty/ffs/ffs/cod/tests/general.ops b/thirdparty/ffs/ffs/cod/tests/general.ops index a3a2d588f7..5f5570fedf 100644 --- a/thirdparty/ffs/ffs/cod/tests/general.ops +++ b/thirdparty/ffs/ffs/cod/tests/general.ops @@ -906,6 +906,7 @@ EOF $cnt = $cnt + 3; } } + print COUT " (void) rand1_p;\n (void) rand2_p;\n"; foreach $dr_type (split(' ', "f d c s i l uc us u ul")) { $rand_type = $rand_types{$dr_type}; $c_type = $c_types{$dr_type}; diff --git a/thirdparty/ffs/ffs/cod/tests/t1.c b/thirdparty/ffs/ffs/cod/tests/t1.c index 194d005822..8307f6a369 100644 --- a/thirdparty/ffs/ffs/cod/tests/t1.c +++ b/thirdparty/ffs/ffs/cod/tests/t1.c @@ -223,7 +223,7 @@ main(int argc, char**argv) str.j = 4; str.k = 10; str.l = 3; - long tmp = func(EC_param1 param); + (void) func(EC_param1 param); assert(func(EC_param1 param) == 90); cod_exec_context_free(ec); cod_code_free(gen_code); diff --git a/thirdparty/ffs/ffs/ffs/ffs.c b/thirdparty/ffs/ffs/ffs/ffs.c index 2bc61274c8..4b1f0e85f6 100755 --- a/thirdparty/ffs/ffs/ffs/ffs.c +++ b/thirdparty/ffs/ffs/ffs/ffs.c @@ -1563,6 +1563,8 @@ add_to_tmp_buffer(FFSBuffer buf, size_t size) } #if SIZEOF_LONG != 8 +#ifndef WORDS_BIGENDIAN + static int words_bigendian = -1; static int @@ -1577,7 +1579,6 @@ set_bigendian () { words_bigendian = (u.c[sizeof (long) - 1] == 1); return words_bigendian; } -#ifndef WORDS_BIGENDIAN #define WORDS_BIGENDIAN ((words_bigendian == -1) ? set_bigendian() : words_bigendian) #endif #endif diff --git a/thirdparty/ffs/ffs/ffs/ffs.h.in b/thirdparty/ffs/ffs/ffs/ffs.h.in index 3708464633..21640793cc 100644 --- a/thirdparty/ffs/ffs/ffs/ffs.h.in +++ b/thirdparty/ffs/ffs/ffs/ffs.h.in @@ -51,13 +51,13 @@ typedef struct _FFSTypeHandle *FFSTypeHandle; typedef struct _FFSIndexItem *FFSIndexItem; #define create_FFSContext() create_FFSContext_FM(NULL) -extern FFS_DECLSPEC FFSContext create_FFSContext_FM(FMContext fmc); +extern FFSContext create_FFSContext_FM(FMContext fmc); -extern FFS_DECLSPEC FFSBuffer create_FFSBuffer(); -extern FFS_DECLSPEC FFSBuffer create_fixed_FFSBuffer(char *buffer, size_t size); -extern FFS_DECLSPEC void free_FFSBuffer(FFSBuffer buf); +extern FFSBuffer create_FFSBuffer(); +extern FFSBuffer create_fixed_FFSBuffer(char *buffer, size_t size); +extern void free_FFSBuffer(FFSBuffer buf); -extern FFS_DECLSPEC char * +extern char * FFSencode(FFSBuffer b, FMFormat ioformat, void *data, size_t *buf_size); typedef struct FFSEncodeVec { @@ -65,133 +65,133 @@ typedef struct FFSEncodeVec { @UIO_SIZE_T_TYPE@ iov_len; } *FFSEncodeVector; -extern FFS_DECLSPEC FFSEncodeVector +extern FFSEncodeVector FFSencode_vector(FFSBuffer b, FMFormat fmformat, void *data); -extern FFS_DECLSPEC char * +extern char * FFSencode_no_leaf_copy(FFSBuffer b, FMFormat fmformat, void *data, size_t *buf_size); -extern FFS_DECLSPEC int FFSdecode_in_place_possible(FFSTypeHandle); +extern int FFSdecode_in_place_possible(FFSTypeHandle); -extern FFS_DECLSPEC FFSTypeHandle FFSTypeHandle_from_encode(FFSContext c, char *b); +extern FFSTypeHandle FFSTypeHandle_from_encode(FFSContext c, char *b); -extern FFS_DECLSPEC FFSTypeHandle FFSTypeHandle_by_index(FFSContext c, int index); +extern FFSTypeHandle FFSTypeHandle_by_index(FFSContext c, int index); -extern FFS_DECLSPEC char * FFSTypeHandle_name(FFSTypeHandle f); +extern char * FFSTypeHandle_name(FFSTypeHandle f); -extern FFS_DECLSPEC void +extern void establish_conversion(FFSContext c, FFSTypeHandle f, FMStructDescList struct_list); -extern FFS_DECLSPEC int +extern int FFShas_conversion(FFSTypeHandle format); -extern FFS_DECLSPEC size_t +extern size_t FFS_est_decode_length(FFSContext context, char *encoded, size_t record_length); -extern FFS_DECLSPEC int +extern int FFSdecode_in_place(FFSContext context, char *encode, void **dest_ptr); -extern FFS_DECLSPEC int +extern int FFSdecode_to_buffer(FFSContext context, char *encode, void *dest); -extern FFS_DECLSPEC int +extern int FFSdecode(FFSContext context, char *encode, char *dest); -extern FFS_DECLSPEC FFSTypeHandle +extern FFSTypeHandle FFSset_fixed_target(FFSContext c, FMStructDescList struct_list); -extern FFS_DECLSPEC FFSTypeHandle +extern FFSTypeHandle FFS_target_from_encode(FFSContext c, char *data); -extern FFS_DECLSPEC FMFormat +extern FMFormat FMFormat_of_original(FFSTypeHandle h); -extern FFS_DECLSPEC FFSEncodeVector +extern FFSEncodeVector copy_all_to_FFSBuffer(FFSBuffer buf, FFSEncodeVector vec); -extern FFS_DECLSPEC FFSEncodeVector +extern FFSEncodeVector copy_vector_to_FFSBuffer(FFSBuffer buf, FFSEncodeVector vec); -extern FFS_DECLSPEC FMContext +extern FMContext FMContext_from_FFS(FFSContext c); -extern FFS_DECLSPEC void +extern void free_FFSContext(FFSContext c); -extern FFS_DECLSPEC FMStructDescList +extern FMStructDescList get_localized_formats(FMFormat f); /* file interface follows*/ -extern FFS_DECLSPEC FFSFile +extern FFSFile open_FFSfile(const char *path, const char *flags); -extern FFS_DECLSPEC FFSFile +extern FFSFile open_FFSfd(void *fd, const char *flags); -extern FFS_DECLSPEC void +extern void close_FFSfile(FFSFile file); -extern FFS_DECLSPEC void +extern void free_FFSfile(FFSFile file); -extern FFS_DECLSPEC int +extern int write_FFSfile(FFSFile f, FMFormat format, void *data); -extern FFS_DECLSPEC int +extern int write_FFSfile_attrs(FFSFile f, FMFormat format, void *data, attr_list attrs); -extern FFS_DECLSPEC int +extern int write_comment_FFSfile(FFSFile f, const char *comment); -extern FFS_DECLSPEC size_t +extern size_t FFSfile_next_decode_length(FFSFile iofile); typedef enum { FFSerror=1, FFSend=2, FFSdata=4, FFSformat=8, FFScomment=16, FFSindex=32 } FFSRecordType; -extern FFS_DECLSPEC void +extern void FFSset_visible(FFSFile ffsfile, int bitmap); -extern FFS_DECLSPEC FFSRecordType +extern FFSRecordType FFSnext_record_type(FFSFile ffsfile); -extern FFS_DECLSPEC size_t +extern size_t FFSnext_data_length(FFSFile file); -extern FFS_DECLSPEC FFSTypeHandle +extern FFSTypeHandle FFSnext_type_handle(FFSFile ffsfile); -extern FFS_DECLSPEC char * +extern char * FFSread_comment(FFSFile ffsfile); -extern FFS_DECLSPEC int +extern int FFSread(FFSFile ffsfile, void *dest); -extern FFS_DECLSPEC int +extern int FFSread_attr(FFSFile file, void *dest, attr_list *attr); -extern FFS_DECLSPEC int +extern int FFSread_to_buffer(FFSFile file, FFSBuffer b, void **dest); -extern FFS_DECLSPEC attr_list +extern attr_list FFSattrs_from_last_read(FFSFile file); -extern FFS_DECLSPEC FFSTypeHandle +extern FFSTypeHandle FFSread_format(FFSFile ffsfile); -extern FFS_DECLSPEC FFSIndexItem +extern FFSIndexItem FFSread_index(FFSFile ffsfile); -extern FFS_DECLSPEC FFSContext +extern FFSContext FFSContext_of_file(FFSFile f); -extern FFS_DECLSPEC FMContext +extern FMContext FMContext_of_file(FFSFile f); -extern FFS_DECLSPEC int +extern int FFSseek(FFSFile file, int data_item); #if defined(__cplusplus) || defined(c_plusplus) diff --git a/thirdparty/ffs/ffs/fm/fm.h b/thirdparty/ffs/ffs/fm/fm.h index cba86f288a..aa8df30fa7 100755 --- a/thirdparty/ffs/ffs/fm/fm.h +++ b/thirdparty/ffs/ffs/fm/fm.h @@ -9,18 +9,12 @@ extern "C" { typedef struct _FMContextStruct *FMContext; -#if defined(_MSC_VER) && !defined(FFS_SRC) -#define FFS_DECLSPEC __declspec(dllimport) -#else -#define FFS_DECLSPEC -#endif - -extern FFS_DECLSPEC FMContext create_FMcontext(); -extern FFS_DECLSPEC FMContext create_local_FMcontext(); -extern FFS_DECLSPEC void free_FMcontext(FMContext c); -extern FFS_DECLSPEC void add_ref_FMcontext(FMContext c); -extern FFS_DECLSPEC void FMcontext_allow_self_formats(FMContext fmc); -extern FFS_DECLSPEC int +extern FMContext create_FMcontext(); +extern FMContext create_local_FMcontext(); +extern void free_FMcontext(FMContext c); +extern void add_ref_FMcontext(FMContext c); +extern void FMcontext_allow_self_formats(FMContext fmc); +extern int FMcontext_get_format_server_identifier(FMContext fmc); @@ -85,19 +79,19 @@ typedef enum { typedef struct _FMFormatBody *FMFormatBodyPtr; typedef FMFormatBodyPtr FMFormat; -extern FFS_DECLSPEC char * +extern char * get_server_rep_FMformat(FMFormat ioformat, int *rep_length); -extern FFS_DECLSPEC char * +extern char * get_server_ID_FMformat(FMFormat ioformat, int *id_length); -extern FFS_DECLSPEC FMContext +extern FMContext FMContext_from_FMformat(FMFormat ioformat); -extern FFS_DECLSPEC int +extern int get_rep_len_format_ID(void *format_ID); -extern FFS_DECLSPEC void +extern void set_array_order_FMContext(FMContext iofile, int column_major); FMFormat @@ -109,23 +103,23 @@ FMformat_index(FMFormat f); FMFormat FMformat_by_index(FMContext c, int index); -extern FFS_DECLSPEC FMFormat +extern FMFormat load_external_format_FMcontext(FMContext iocontext, char *server_id, int id_size, char *server_rep); -extern FFS_DECLSPEC void +extern void add_opt_info_FMformat(FMFormat format, int typ, int len, void *block); -extern FFS_DECLSPEC FMFormat +extern FMFormat FMregister_simple_format(FMContext context, char *format_name, FMFieldList field_list, int struct_size); -extern FFS_DECLSPEC FMFormat +extern FMFormat register_data_format(FMContext context, FMStructDescList struct_list); -extern FFS_DECLSPEC FMFormat +extern FMFormat FMregister_data_format(FMContext context, FMStructDescList struct_list); -extern FFS_DECLSPEC void free_FMfield_list(FMFieldList list); +extern void free_FMfield_list(FMFieldList list); /*! * lookup the FMFormat associated with a particular FMStructDescList @@ -148,7 +142,7 @@ extern FFS_DECLSPEC void free_FMfield_list(FMFieldList list); * address. Normally if you use static format lists, the addresses will * be unique. */ -extern FFS_DECLSPEC FMFormat +extern FMFormat FMlookup_format(FMContext context, FMStructDescList struct_list); typedef enum {Format_Less, Format_Greater, Format_Equal, @@ -161,125 +155,125 @@ typedef struct compat_formats { char *xform_code; } *FMcompat_formats; -extern FFS_DECLSPEC FMcompat_formats +extern FMcompat_formats FMget_compat_formats(FMFormat ioformat); -extern FFS_DECLSPEC char * +extern char * name_of_FMformat(FMFormat format); -extern FFS_DECLSPEC FMStructDescList +extern FMStructDescList format_list_of_FMFormat(FMFormat format); -extern FFS_DECLSPEC void +extern void FMlocalize_structs(FMStructDescList list); -extern FFS_DECLSPEC char * +extern char * global_name_of_FMFormat(FMFormat format); -extern FFS_DECLSPEC FMFieldList +extern FMFieldList copy_field_list(FMFieldList list); -extern FFS_DECLSPEC FMStructDescList +extern FMStructDescList FMcopy_struct_list(FMStructDescList list); -extern FFS_DECLSPEC void +extern void FMfree_struct_list(FMStructDescList list); -extern FFS_DECLSPEC int +extern int FMstruct_size_field_list(FMFieldList list, int pointer_size); -extern FFS_DECLSPEC FMStructDescList +extern FMStructDescList FMlocalize_formats(FMStructDescList list); -extern FFS_DECLSPEC int +extern int count_FMfield(FMFieldList list); -extern FFS_DECLSPEC void print_server_ID(unsigned char *ID); -extern FFS_DECLSPEC void print_format_ID(FMFormat ioformat); -extern FFS_DECLSPEC void fprint_server_ID(void * file,unsigned char *ID); +extern void print_server_ID(unsigned char *ID); +extern void print_format_ID(FMFormat ioformat); +extern void fprint_server_ID(void * file,unsigned char *ID); -extern FFS_DECLSPEC int FMformatID_len(char *buffer); +extern int FMformatID_len(char *buffer); -extern FFS_DECLSPEC int +extern int FMdump_data(FMFormat format, void *data, int character_limit); -extern FFS_DECLSPEC int +extern int FMdump_encoded_data(FMFormat format, void *data, int character_limit); -extern FFS_DECLSPEC void +extern void FMdump_XML(FMFormat format, void *data, int encoded); -extern FFS_DECLSPEC void +extern void FMdump_encoded_XML(FMContext c, void *data, int character_limit); -extern FFS_DECLSPEC int +extern int FMfdump_data(void *file, FMFormat format, void *data, int character_limit); -extern FFS_DECLSPEC int +extern int FMfdump_encoded_data(void *file, FMFormat format, void *data, int character_limit); -extern FFS_DECLSPEC void +extern void FMfdump_XML(void *file, FMFormat format, void *data, int encoded); -extern FFS_DECLSPEC void +extern void FMfdump_encoded_XML(void *file, FMContext c, void *data, int character_limit); -extern FFS_DECLSPEC char* +extern char* FMunencoded_to_XML_string(FMContext fmcontext, FMFormat format, void *data); -extern FFS_DECLSPEC void +extern void FMfree_var_rec_elements(FMFormat format, void *data); -extern FFS_DECLSPEC char *FMbase_type(const char *field_type); +extern char *FMbase_type(const char *field_type); #define XML_OPT_INFO 0x584D4C20 #define COMPAT_OPT_INFO 0x45564F4C #define COMPAT_OPT_INFO_FMFILE 0x45564F4D typedef struct _FMgetFieldStruct *FMFieldPtr; -extern FFS_DECLSPEC FMFieldPtr get_FMfieldPtrFromList(FMFieldList field_list, +extern FMFieldPtr get_FMfieldPtrFromList(FMFieldList field_list, const char *fieldname); -extern FFS_DECLSPEC void * +extern void * get_FMfieldAddr_by_name(FMFieldList field_list, const char *fieldname, void *data); -extern FFS_DECLSPEC void * +extern void * get_FMPtrField_by_name(FMFieldList field_list, const char *fieldname, void *data, int encode); -extern FFS_DECLSPEC int +extern int set_FMPtrField_by_name(FMFieldList field_list, const char *fieldname, void *data, void *ptr_value); -extern FFS_DECLSPEC int +extern int get_FMfieldInt_by_name(FMFieldList field_list, const char *fieldname, void *data); -extern FFS_DECLSPEC size_t +extern size_t get_FMfieldLong_by_name(FMFieldList field_list, const char *fieldname, void *data); -extern FFS_DECLSPEC void * FMheader_skip(FMContext c, void *data); -extern FFS_DECLSPEC char *get_FMstring_base(FMFieldPtr iofield, void *data, void *string_base); -extern FFS_DECLSPEC void *get_FMFieldAddr(FMFieldPtr iofield, void *data); -extern FFS_DECLSPEC void *get_FMaddr (FMFieldPtr iofield, void *data, void *string_base, int encode); -extern FFS_DECLSPEC void *put_FMaddr (FMFieldPtr iofield, void *data); -extern FFS_DECLSPEC float get_FMfloat(FMFieldPtr iofield, void *data); -extern FFS_DECLSPEC double get_FMdouble(FMFieldPtr iofield, void *data); -extern FFS_DECLSPEC short get_FMshort(FMFieldPtr iofield, void *data); -extern FFS_DECLSPEC int get_FMint(FMFieldPtr iofield, void *data); -extern FFS_DECLSPEC size_t get_FMlong(FMFieldPtr iofield, void *data); -extern FFS_DECLSPEC void get_FMlong8(FMFieldPtr iofield, void *data, unsigned long *low_long, long *high_long); +extern void * FMheader_skip(FMContext c, void *data); +extern char *get_FMstring_base(FMFieldPtr iofield, void *data, void *string_base); +extern void *get_FMFieldAddr(FMFieldPtr iofield, void *data); +extern void *get_FMaddr (FMFieldPtr iofield, void *data, void *string_base, int encode); +extern void *put_FMaddr (FMFieldPtr iofield, void *data); +extern float get_FMfloat(FMFieldPtr iofield, void *data); +extern double get_FMdouble(FMFieldPtr iofield, void *data); +extern short get_FMshort(FMFieldPtr iofield, void *data); +extern int get_FMint(FMFieldPtr iofield, void *data); +extern size_t get_FMlong(FMFieldPtr iofield, void *data); +extern void get_FMlong8(FMFieldPtr iofield, void *data, unsigned long *low_long, long *high_long); #if defined(SIZEOF_LONG_LONG) #if SIZEOF_LONG_LONG != 0 -extern FFS_DECLSPEC long long get_FMlong_long(FMFieldPtr iofield, void *data); -extern FFS_DECLSPEC unsigned long long get_FMulong_long(FMFieldPtr iofield, void *data); +extern long long get_FMlong_long(FMFieldPtr iofield, void *data); +extern unsigned long long get_FMulong_long(FMFieldPtr iofield, void *data); #endif #endif #if defined(SIZEOF_LONG_DOUBLE) #if SIZEOF_LONG_DOUBLE != 0 -extern FFS_DECLSPEC long double get_FMlong_double(FMFieldPtr iofield, void *data); +extern long double get_FMlong_double(FMFieldPtr iofield, void *data); #endif #endif -extern FFS_DECLSPEC unsigned short get_FMushort(FMFieldPtr iofield, void *data); -extern FFS_DECLSPEC unsigned int get_FMuint(FMFieldPtr iofield, void *data); -extern FFS_DECLSPEC size_t get_FMulong(FMFieldPtr iofield, void *data); -extern FFS_DECLSPEC int get_FMulong8(FMFieldPtr iofield, void *data, unsigned long *low_long, unsigned long *high_long); -extern FFS_DECLSPEC char *get_FMstring(FMFieldPtr iofield, void *data); -extern FFS_DECLSPEC char get_FMchar(FMFieldPtr iofield, void *data); -extern FFS_DECLSPEC int get_FMenum(FMFieldPtr iofield, void *data); +extern unsigned short get_FMushort(FMFieldPtr iofield, void *data); +extern unsigned int get_FMuint(FMFieldPtr iofield, void *data); +extern size_t get_FMulong(FMFieldPtr iofield, void *data); +extern int get_FMulong8(FMFieldPtr iofield, void *data, unsigned long *low_long, unsigned long *high_long); +extern char *get_FMstring(FMFieldPtr iofield, void *data); +extern char get_FMchar(FMFieldPtr iofield, void *data); +extern int get_FMenum(FMFieldPtr iofield, void *data); #if defined(__cplusplus) || defined(c_plusplus) } diff --git a/thirdparty/ffs/ffs/fm/fm_dump.c b/thirdparty/ffs/ffs/fm/fm_dump.c index 2c77715be3..8fd7e28d46 100644 --- a/thirdparty/ffs/ffs/fm/fm_dump.c +++ b/thirdparty/ffs/ffs/fm/fm_dump.c @@ -109,6 +109,7 @@ dump_output(dstate s, size_t length_estimate, char *format, ...) } #if SIZEOF_LONG != 8 +#ifndef WORDS_BIGENDIAN static int words_bigendian = -1; static int @@ -123,7 +124,6 @@ set_bigendian () { words_bigendian = (u.c[sizeof (long) - 1] == 1); return words_bigendian; } -#ifndef WORDS_BIGENDIAN #define WORDS_BIGENDIAN ((words_bigendian == -1) ? set_bigendian() : words_bigendian) #endif #endif diff --git a/thirdparty/ffs/ffs/fm/io_interface.h b/thirdparty/ffs/ffs/fm/io_interface.h index e0f635bdec..d17200cfad 100644 --- a/thirdparty/ffs/ffs/fm/io_interface.h +++ b/thirdparty/ffs/ffs/fm/io_interface.h @@ -1,4 +1,4 @@ -#if defined(HAVE_WINDOWS_H) && !defined(NEED_IOVEC_DEFINE) && !defined(_STRUCT_IOVEC) +#if defined(_MSC_VER) && !defined(_STRUCT_IOVEC) #define _STRUCT_IOVEC struct iovec { const void *iov_base; diff --git a/thirdparty/ffs/ffs/fm/nt_io.c b/thirdparty/ffs/ffs/fm/nt_io.c index c0d047c808..d33155826e 100755 --- a/thirdparty/ffs/ffs/fm/nt_io.c +++ b/thirdparty/ffs/ffs/fm/nt_io.c @@ -82,7 +82,7 @@ char **result_p; (tmp != WSAEINTR)) { /* serious error */ fprintf(stderr, "WINSOCK ERROR during receive, %i on socket %p\n", - tmp, conn); + (int)tmp, conn); return -1; } else { if (tmp == WSAECONNRESET) @@ -110,7 +110,7 @@ char **result_p; /* serious error */ fprintf(stderr, "WINSOCK ERROR during receive2, %i on socket %p\n", - tmp, conn); + (int) tmp, conn); return (length - left); } else { if (tmp == WSAECONNRESET) @@ -141,10 +141,9 @@ char **result_p; while (left > 0) { bResult = WriteFile((HANDLE) conn, (char *) buffer + length - left, - left, &iget, NULL); + left, (unsigned long *)&iget, NULL); if (!bResult) { DWORD tmp = GetLastError(); - if (errno_p) tmp = tmp; if ((tmp != WSAEWOULDBLOCK) && (tmp != WSAEINPROGRESS) && (tmp != WSAEINTR)) { @@ -277,7 +276,7 @@ char **result_p; int i = 0; for (; i < icount; i++) { - if (nt_socket_read_func(conn, iov[i].iov_base, iov[i].iov_len, + if (nt_socket_read_func(conn, (void*)iov[i].iov_base, iov[i].iov_len, errno_p, result_p) != iov[i].iov_len) { return i; } @@ -297,7 +296,7 @@ char **result_p; int i = 0; for (; i < icount; i++) { - if (nt_file_read_func(conn, iov[i].iov_base, iov[i].iov_len, errno_p, + if (nt_file_read_func(conn, (void*)iov[i].iov_base, iov[i].iov_len, errno_p, result_p) != iov[i].iov_len) { return i; } @@ -316,7 +315,7 @@ char** result_p; int i = 0; for (; i < icount; i++) { - if (nt_file_write_func(conn, iov[i].iov_base, iov[i].iov_len, errno_p, + if (nt_file_write_func(conn, (void*)iov[i].iov_base, iov[i].iov_len, errno_p, result_p) != iov[i].iov_len) { return i; } diff --git a/thirdparty/ffs/ffs/fm/tests/scale_test.c b/thirdparty/ffs/ffs/fm/tests/scale_test.c index 5e06f7f20b..a3190d8c9c 100644 --- a/thirdparty/ffs/ffs/fm/tests/scale_test.c +++ b/thirdparty/ffs/ffs/fm/tests/scale_test.c @@ -41,7 +41,7 @@ main(int argc, char **argv) { FMStructDescRec str_list[5]; - struct timespec start, stop; + struct timespec start = {0,0}, stop = {0,0}; FMContext context; int field_count = 20000; diff --git a/thirdparty/ffs/ffs/scripts/ci/cmake/windows-common.cmake b/thirdparty/ffs/ffs/scripts/ci/cmake/windows-common.cmake index 1b227e5395..d5ab37c858 100644 --- a/thirdparty/ffs/ffs/scripts/ci/cmake/windows-common.cmake +++ b/thirdparty/ffs/ffs/scripts/ci/cmake/windows-common.cmake @@ -4,4 +4,9 @@ string(APPEND dashboard_cache " ") list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") + +# the two lines below shouldn't be necessary, but something wrong with vcpkg maybe +set(ENV{dill_DIR} "${CMAKE_CURRENT_LIST_DIR}/../../../../dill/install/lib/cmake/dill") +set(ENV{atl_DIR} "${CMAKE_CURRENT_LIST_DIR}/../../../../atl/install/lib/cmake/atl") + include(${CMAKE_CURRENT_LIST_DIR}/common.cmake) diff --git a/thirdparty/ffs/ffs/scripts/ci/cmake/windows2022-vs2022-msvc-static.cmake b/thirdparty/ffs/ffs/scripts/ci/cmake/windows2022-vs2022-msvc-static.cmake new file mode 100644 index 0000000000..cdb9e59d4e --- /dev/null +++ b/thirdparty/ffs/ffs/scripts/ci/cmake/windows2022-vs2022-msvc-static.cmake @@ -0,0 +1,8 @@ +# Client maintainer: chuck.atkins@kitware.com + +set(CTEST_CMAKE_GENERATOR "Visual Studio 17 2022") +set(CTEST_CMAKE_GENERATOR_PLATFORM x64) +set(BUILD_SHARED_LIBS off) + +list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") +include(${CMAKE_CURRENT_LIST_DIR}/windows-common.cmake) From 8337718e8a6e9c3bbac1aa3a74973af7c524195b Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Thu, 22 Feb 2024 07:44:58 -0500 Subject: [PATCH 049/124] Add arguments to IO.AvailableAttributes() and io.available_attributes() to get attributes of a specific variable. --- bindings/Python/py11IO.cpp | 5 +- bindings/Python/py11IO.h | 3 +- bindings/Python/py11glue.cpp | 5 +- python/adios2/io.py | 15 +++++- python/adios2/stream.py | 15 +++++- .../bindings/python/TestBPWriteReadTypes.py | 45 ++++++++++++++---- .../python/TestBPWriteTypesHighLevelAPI.py | 47 +++++++++++++++++-- 7 files changed, 116 insertions(+), 19 deletions(-) diff --git a/bindings/Python/py11IO.cpp b/bindings/Python/py11IO.cpp index 34466e09aa..f44e260dd9 100644 --- a/bindings/Python/py11IO.cpp +++ b/bindings/Python/py11IO.cpp @@ -344,10 +344,11 @@ std::map IO::AvailableVariables() return m_IO->GetAvailableVariables(); } -std::map IO::AvailableAttributes() +std::map IO::AvailableAttributes(const std::string &varname, + const std::string &separator) { helper::CheckForNullptr(m_IO, "in call to IO::AvailableAttributes"); - return m_IO->GetAvailableAttributes(); + return m_IO->GetAvailableAttributes(varname, separator); } std::string IO::VariableType(const std::string &name) const diff --git a/bindings/Python/py11IO.h b/bindings/Python/py11IO.h index ef290874e6..80c7c7599b 100644 --- a/bindings/Python/py11IO.h +++ b/bindings/Python/py11IO.h @@ -113,7 +113,8 @@ class IO std::map AvailableVariables(); - std::map AvailableAttributes(); + std::map AvailableAttributes(const std::string &varname = "", + const std::string &separator = "/"); std::string VariableType(const std::string &name) const; diff --git a/bindings/Python/py11glue.cpp b/bindings/Python/py11glue.cpp index 2f0c1742f0..13ca876517 100644 --- a/bindings/Python/py11glue.cpp +++ b/bindings/Python/py11glue.cpp @@ -292,8 +292,11 @@ PYBIND11_MODULE(ADIOS2_PYTHON_MODULE_NAME, m) adios2::py11::MPI4PY_Comm comm)) & adios2::py11::IO::Open) #endif + .def("AvailableAttributes", &adios2::py11::IO::AvailableAttributes, + pybind11::arg("varname") = "", pybind11::arg("separator") = "/", + pybind11::return_value_policy::move) + .def("AvailableVariables", &adios2::py11::IO::AvailableVariables) - .def("AvailableAttributes", &adios2::py11::IO::AvailableAttributes) .def("FlushAll", &adios2::py11::IO::FlushAll) .def("EngineType", &adios2::py11::IO::EngineType) .def("RemoveVariable", &adios2::py11::IO::RemoveVariable) diff --git a/python/adios2/io.py b/python/adios2/io.py index 974addee7f..ae0318d0cf 100644 --- a/python/adios2/io.py +++ b/python/adios2/io.py @@ -99,11 +99,22 @@ def inquire_attribute(self, name, variable_name="", separator="/"): attr.impl = attrimpl return attr - def available_attributes(self): + def available_attributes(self, varname="", separator="/"): """ Returns a 2-level dictionary with attribute information. Read mode only. + Parameters + variable_name + If varname is set, attributes assigned to that variable are returned. + The keys returned are attribute names with the prefix of varname + separator + removed. + + separator + concatenation string between variable_name and attribute + e.g. varname + separator + name ("var/attr") + Not used if varname is empty + Returns attributes dictionary key @@ -111,7 +122,7 @@ def available_attributes(self): value attribute information dictionary """ - return self.impl.AvailableAttributes() + return self.impl.AvailableAttributes(varname, separator) def remove_attribute(self, name): """ diff --git a/python/adios2/stream.py b/python/adios2/stream.py index 63490b4b3f..1d5ad78baa 100644 --- a/python/adios2/stream.py +++ b/python/adios2/stream.py @@ -172,11 +172,22 @@ def available_variables(self): """ return self._io.available_variables() - def available_attributes(self): + def available_attributes(self, varname="", separator="/"): """ Returns a 2-level dictionary with attribute information. Read mode only. + Parameters + variable_name + If varname is set, attributes assigned to that variable are returned. + The keys returned are attribute names with the prefix of varname + separator + removed. + + separator + concatenation string between variable_name and attribute + e.g. varname + separator + name ("var/attr") + Not used if varname is empty + Returns attributes dictionary key @@ -184,7 +195,7 @@ def available_attributes(self): value attribute information dictionary """ - return self._io.available_attributes() + return self._io.available_attributes(varname, separator) def define_variable(self, name): """ diff --git a/testing/adios2/bindings/python/TestBPWriteReadTypes.py b/testing/adios2/bindings/python/TestBPWriteReadTypes.py index 04a0a22c05..c2eda114db 100644 --- a/testing/adios2/bindings/python/TestBPWriteReadTypes.py +++ b/testing/adios2/bindings/python/TestBPWriteReadTypes.py @@ -49,8 +49,8 @@ def check_array(np1, np2, hint): "attrU16", "attrU32", "attrU64", - "attrR32", - "attrR64", + "varR32/attrR32", + "varR64::attrR64", ] var_names = [ "varStr", @@ -104,8 +104,9 @@ def check_array(np1, np2, hint): attU16 = ioWriter.DefineAttribute("attrU16", data.U16) attU32 = ioWriter.DefineAttribute("attrU32", data.U32) attU64 = ioWriter.DefineAttribute("attrU64", data.U64) -attR32 = ioWriter.DefineAttribute("attrR32", data.R32) -attR64 = ioWriter.DefineAttribute("attrR64", data.R64) +# add an attribute to a variable +attR32 = ioWriter.DefineAttribute("attrR32", data.R32, "varR32") +attR64 = ioWriter.DefineAttribute("attrR64", data.R64, "varR64", "::") ioWriter.SetEngine("BPFile") ioParams = {"Threads": "1", "InitialBufferSize": "17Kb"} @@ -166,8 +167,8 @@ def check_array(np1, np2, hint): attrU16 = ioReader.InquireAttribute("attrU16") attrU32 = ioReader.InquireAttribute("attrU32") attrU64 = ioReader.InquireAttribute("attrU64") -attrR32 = ioReader.InquireAttribute("attrR32") -attrR64 = ioReader.InquireAttribute("attrR64") +attrR32 = ioReader.InquireAttribute("attrR32", "varR32") +attrR64 = ioReader.InquireAttribute("attrR64", "varR64", "::") check_object(attrString, "attrString") check_object(attrStringArray, "attrStringArray") @@ -179,8 +180,14 @@ def check_array(np1, np2, hint): check_object(attrU16, "attrU16") check_object(attrU32, "attrU32") check_object(attrU64, "attrU64") -check_object(attrR32, "attrR32") -check_object(attrR64, "attrR64") +check_object(attrR32, "varR32/attrR32") +check_object(attrR64, "varR64::attrR64") + +# alternative inquire format +attrR32 = ioReader.InquireAttribute("varR32/attrR32") +attrR64 = ioReader.InquireAttribute("varR64::attrR64") +check_object(attrR32, "varR32/attrR32") +check_object(attrR64, "varR64::attrR64") attrStringData = attrString.DataString() print(f"attrString = {attrStringData}", flush=True) @@ -218,6 +225,7 @@ def check_array(np1, np2, hint): check_array(attrR32Data, data.R32, "R32") check_array(attrR64Data, data.R64, "R64") +print("=========== Attributes ===========") attributesInfo = ioReader.AvailableAttributes() for name, info in attributesInfo.items(): check_name(name, attr_names) @@ -227,6 +235,27 @@ def check_array(np1, np2, hint): print("\t" + key + ": " + value) print("\n") +print("=========== Available attributes of varR32 ===========") +attributesInfoR32 = ioReader.AvailableAttributes("varR32") +for name, info in attributesInfoR32.items(): + check_name(name, ["attrR32"]) + if rank == 0: + print("attribute_name: " + name) + for key, value in info.items(): + print("\t" + key + ": " + value) + print("\n") + +print("=========== Available attributes of varR64 ===========") +attributesInfoR32 = ioReader.AvailableAttributes("varR64", "::") +for name, info in attributesInfoR32.items(): + check_name(name, ["attrR64"]) + if rank == 0: + print("attribute_name: " + name) + for key, value in info.items(): + print("\t" + key + ": " + value) + print("\n") + +print("=========== Variables ===========") varStr = ioReader.InquireVariable("varStr") varI8 = ioReader.InquireVariable("varI8") varI16 = ioReader.InquireVariable("varI16") diff --git a/testing/adios2/python/TestBPWriteTypesHighLevelAPI.py b/testing/adios2/python/TestBPWriteTypesHighLevelAPI.py index 231032fde8..3b300fca55 100644 --- a/testing/adios2/python/TestBPWriteTypesHighLevelAPI.py +++ b/testing/adios2/python/TestBPWriteTypesHighLevelAPI.py @@ -56,9 +56,8 @@ s.write("a_float_list", data.float_list) s.write("a_complex_list", data.complex_list) - # single value attributes + # single value attributes with numpy variables s.write_attribute("attrStr", "Testing single string attribute") - s.write_attribute("attrNx", data.Nx) s.write_attribute("attrI8", np.array(data.i8[0])) s.write_attribute("attrI16", np.array(data.i16[0])) s.write_attribute("attrI32", np.array(data.i32[0])) @@ -71,15 +70,19 @@ s.write_attribute("attrR64", np.array(data.r64[0])) s.write_attribute("attrC64", np.array(data.c64[0])) + # single value attributes with Python variables + s.write_attribute("attrNx", data.Nx) s.write_attribute("attr_int_value", i) s.write_attribute("attr_float_value", f) s.write_attribute("attr_complex_value", c) + # array attributes with Python lists s.write_attribute("attr_int_list", data.int_list) s.write_attribute("attr_float_list", data.float_list) s.write_attribute("attr_complex_list", data.complex_list) - s.write_attribute("attrStrArray", ["string1", "string2", "string3"]) + + # array attributes with numpy arrays s.write_attribute("attrI8Array", data.i8) s.write_attribute("attrI16Array", data.i16) s.write_attribute("attrI32Array", data.i32) @@ -106,6 +109,9 @@ s.write("varC64", data.c64, shape, start, count) if rank == 0 and step.current_step() == 0: + # attribute assigned to variable + s.write_attribute("size", data.Nx, "varI8") + s.write_attribute("size", data.Nx, "varI16", "::") s.write_attribute("varattrStrArray", ["varattr1", "varattr2", "varattr3"], "steps") s.write_attribute("varattrI8Array", data.i8, "varI8") s.write_attribute("varattrI16Array", data.i16, "varI16") @@ -376,6 +382,41 @@ if not (inR64 == data.r64).all(): raise ValueError("var attrR64 array read failed") + # Attributes assigned to a variable + sizeI8 = fr_step.read_attribute("size", "varI8") + sizeI16 = fr_step.read_attribute("size", "varI16", "::") + + if sizeI8[0] != data.Nx: + raise ValueError("attribute varI8/size read failed") + + if sizeI16[0] != data.Nx: + raise ValueError("attribute varI16::size read failed") + + sizeI8 = fr_step.read_attribute("varI8/size") + sizeI16 = fr_step.read_attribute("varI16::size") + + if sizeI8[0] != data.Nx: + raise ValueError("attribute varI8/size read failed") + + if sizeI16[0] != data.Nx: + raise ValueError("attribute varI16::size read failed") + + step_attrs = fr_step.available_attributes() + # for name, info in step_attrs.items(): + # print(f"attribute {name} : {info}") + + step_attrs = fr_step.available_attributes("varI8") + # for name, info in step_attrs.items(): + # print(f"attribute {name} : {info}") + if not [*step_attrs] == ["size", "varattrI8Array"]: + raise ValueError("getting attributes of varI8 failed") + + step_attrs = fr_step.available_attributes("varI16", "::") + for name, info in step_attrs.items(): + print(f"attribute {name} : {info}") + if not [*step_attrs] == ["size"]: + raise ValueError("getting attributes of varI16 with separator :: failed") + stepStr = "Step:" + str(step) instepStr = fr_step.read("steps") From 82b9fff82d13ca6a62f2de729d4ed3fe522bf84a Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Sat, 24 Feb 2024 07:42:18 -0500 Subject: [PATCH 050/124] Return a 0-dim numpy array of size 1 when reading a scalar variable instead of a 1-dim array of size 1. The difference can be seen with gray-scott gsplot, as step is printed now 10 instead of [10]. Same when running bpReaderHeatMap2D.py. --- python/adios2/file_reader.py | 1 + python/adios2/stream.py | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/python/adios2/file_reader.py b/python/adios2/file_reader.py index ee351e7e26..74f0f640fe 100644 --- a/python/adios2/file_reader.py +++ b/python/adios2/file_reader.py @@ -2,6 +2,7 @@ Distributed under the OSI-approved Apache License, Version 2.0. See accompanying file Copyright.txt for details. """ + from functools import singledispatchmethod from adios2 import Stream, IO diff --git a/python/adios2/stream.py b/python/adios2/stream.py index 1d5ad78baa..9025073a7e 100644 --- a/python/adios2/stream.py +++ b/python/adios2/stream.py @@ -2,6 +2,7 @@ Distributed under the OSI-approved Apache License, Version 2.0. See accompanying file Copyright.txt for details. """ + from functools import singledispatchmethod from sys import maxsize import numpy as np @@ -349,7 +350,11 @@ def read(self, variable: Variable): output_shape[0] *= steps else: # scalar - output_shape = (variable.selection_size(),) + size_all_steps = variable.selection_size() + if size_all_steps > 1: + output_shape = [size_all_steps] + else: + output_shape = [] output = np.zeros(output_shape, dtype=dtype) self._engine.get(variable, output) From 4e5047614c8b46cde2bb3a3604b8232c8c1a3e0e Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Mon, 26 Feb 2024 08:14:09 -0500 Subject: [PATCH 051/124] Add timeout parameter to stream.steps(). Necessary for controlling timeout in in situ processing. Also very useful for not being stuck forever in processing files that were not closed properly. Default is still infinity, which may not be the ideal setup for python scripts in practice. Usage: for _ in f.steps(timeout=1.0): ... --- python/adios2/stream.py | 44 ++++++++++++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 12 deletions(-) diff --git a/python/adios2/stream.py b/python/adios2/stream.py index 9025073a7e..eb632f19fb 100644 --- a/python/adios2/stream.py +++ b/python/adios2/stream.py @@ -50,6 +50,9 @@ def string_to_mode(mode: str) -> [bindings.Mode, bool]: class Stream: """High level implementation of the Stream class from the core API""" + # Default timeout for stream.begin_step() + DEFAULT_TIMEOUT_SEC = -1.0 + @singledispatchmethod def __init__(self, path, mode, comm=None): # pylint: disable=R0912 # Too many branches @@ -71,6 +74,7 @@ def __init__(self, path, mode, comm=None): self.index = -1 self.max_steps = maxsize self._step_status = bindings.StepStatus.EndOfStream + self._step_timeout_sec = self.DEFAULT_TIMEOUT_SEC # e.g. Stream(io: adios2.IO, path, mode) @__init__.register(IO) @@ -82,6 +86,7 @@ def _(self, io: IO, path, mode, comm=None): self.index = -1 self.max_steps = maxsize self._step_status = bindings.StepStatus.EndOfStream + self._step_timeout_sec = self.DEFAULT_TIMEOUT_SEC @property def mode(self): @@ -123,10 +128,17 @@ def __next__(self): raise StopIteration self.index += 1 - self._step_status = self.begin_step() + self._step_status = self.begin_step(timeout=self._step_timeout_sec) if self._step_status == bindings.StepStatus.EndOfStream: raise StopIteration + if self._step_status == bindings.StepStatus.NotReady: + print( + "ERROR: Stream returned no new step within the time limit of" + f" {self._step_timeout_sec} seconds. Ending the loop" + ) + raise StopIteration + if self._step_status == bindings.StepStatus.OtherError: print("ERROR: Stream returned an error. Ending the loop") raise StopIteration @@ -241,7 +253,9 @@ def inquire_attribute(self, name, variable_name="", separator="/"): @singledispatchmethod def write(self, variable: Variable, content): """ - writes a variable + Writes a variable. + Note that the content will be available for consumption only at + the end of the for loop or in the end_step() call. Parameters variable @@ -257,7 +271,7 @@ def write(self, variable: Variable, content): @write.register(str) def _(self, name, content, shape=[], start=[], count=[], operations=None): """ - writes a variable + Writes a variable Parameters name @@ -485,25 +499,29 @@ def read_attribute_string(self, name, variable_name="", separator="/"): return attribute.data_string() - def begin_step(self): + def begin_step(self, *, timeout=DEFAULT_TIMEOUT_SEC): """ - Write mode: advances to the next step. Convenient when declaring - variable attributes as advancing to the next step is not attached - to any variable. + Write mode: declare the starting of an output step. Pass data in + stream.write() and stream.write_attribute(). All data will be published + in end_step(). Read mode: in streaming mode releases the current step (no effect in file based engines) """ + if self._read_mode: + mode = bindings.StepMode.Read + else: + mode = bindings.StepMode.Append if not self._engine.between_step_pairs(): - return self._engine.begin_step() + return self._engine.begin_step(mode=mode, timeoutSeconds=timeout) return bindings.StepStatus.OtherError def end_step(self): """ - Write mode: advances to the next step. Convenient when declaring - variable attributes as advancing to the next step is not attached - to any variable. + Write mode: declaring the end of an output step. All data passed in + stream.write() and all attributes passed in stream.write_attribute() + will be published for consumers. Read mode: in streaming mode releases the current step (no effect in file based engines) @@ -549,7 +567,7 @@ def loop_index(self): """ return self.index - def steps(self, num_steps=0): + def steps(self, num_steps=0, *, timeout=DEFAULT_TIMEOUT_SEC): """ Returns an interator that can be use to itererate throught the steps. In each iteration begin_step() and end_step() will be internally called. @@ -580,6 +598,8 @@ def steps(self, num_steps=0): else: self.max_steps = maxsize # engine steps will limit the loop + self._step_timeout_sec = timeout + # in write mode we can run yet another loop self.index = -1 From f4ac309ed9e77062a4167ace377941b5923d3be7 Mon Sep 17 00:00:00 2001 From: Vicente Adolfo Bolea Sanchez Date: Tue, 27 Feb 2024 14:56:48 -0500 Subject: [PATCH 052/124] blosc: do not export blosc lib --- CMakeLists.txt | 6 +++--- cmake/DetectOptions.cmake | 7 +++---- cmake/adios2-config-common.cmake.in | 5 ----- cmake/install/pre/CMakeLists.txt | 2 +- source/adios2/CMakeLists.txt | 2 +- 5 files changed, 8 insertions(+), 14 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1a3b2a8534..8511e152bf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -187,6 +187,9 @@ adios_option(Campaign "Enable support for Campaigns (requires SQLite3 and ZLIB adios_option(AWSSDK "Enable support for S3 compatible storage using AWS SDK's S3 module" OFF) adios_option(Derived_Variable "Enable support for derived variables" OFF) adios_option(PIP "Enable support for pip packaging" OFF) + +option(ADIOS2_Blosc2_PREFER_SHARED "Prefer shared Blosc2 libraries" ON) +mark_as_advanced(ADIOS2_Blosc2_PREFER_SHARED) mark_as_advanced(ADIOS2_USE_PIP) include(${PROJECT_SOURCE_DIR}/cmake/DetectOptions.cmake) @@ -217,9 +220,6 @@ if(ADIOS2_HAVE_MPI) add_definitions(-DOMPI_SKIP_MPICXX -DMPICH_SKIP_MPICXX) endif() -cmake_dependent_option(ADIOS2_Blosc2_PREFER_SHARED "Prefer shared Blosc2 libraries" - ON "Blosc2_shlib_available" OFF) - #------------------------------------------------------------------------------# # POSIX O_DIRECT is only working for Unix in adios for now #------------------------------------------------------------------------------# diff --git a/cmake/DetectOptions.cmake b/cmake/DetectOptions.cmake index 2b3b8cbe3d..13c98e4f5c 100644 --- a/cmake/DetectOptions.cmake +++ b/cmake/DetectOptions.cmake @@ -92,15 +92,14 @@ if(Blosc2_FOUND) set(adios2_blosc2_tgt Blosc2::Blosc2) if (Blosc2_VERSION VERSION_GREATER_EQUAL 2.10.1) - if (Blosc2_shlib_available) - set(adios2_blosc2_tgt Blosc2::blosc2_$,shared,static>) + if (Blosc2_shlib_available AND ADIOS2_Blosc2_PREFER_SHARED) + set(adios2_blosc2_tgt Blosc2::blosc2_shared) else() set(adios2_blosc2_tgt Blosc2::blosc2_static) endif() endif() - add_library(adios2_blosc2 INTERFACE) - target_link_libraries(adios2_blosc2 INTERFACE ${adios2_blosc2_tgt}) + add_library(adios2_blosc2 ALIAS ${adios2_blosc2_tgt}) endif() # BZip2 diff --git a/cmake/adios2-config-common.cmake.in b/cmake/adios2-config-common.cmake.in index 04a41531f1..4be4ff7ff0 100644 --- a/cmake/adios2-config-common.cmake.in +++ b/cmake/adios2-config-common.cmake.in @@ -81,11 +81,6 @@ else() endif() if(NOT @BUILD_SHARED_LIBS@) - set(ADIOS2_HAVE_Blosc2 @ADIOS2_HAVE_Blosc2@) - if(ADIOS2_HAVE_Blosc2) - find_dependency(Blosc2) - endif() - set(ADIOS2_HAVE_BZip2 @ADIOS2_HAVE_BZip2@) if(ADIOS2_HAVE_BZip2) find_dependency(BZip2) diff --git a/cmake/install/pre/CMakeLists.txt b/cmake/install/pre/CMakeLists.txt index a9dd447228..6f0fd32be6 100644 --- a/cmake/install/pre/CMakeLists.txt +++ b/cmake/install/pre/CMakeLists.txt @@ -4,7 +4,7 @@ install(CODE " " COMPONENT adios2_core-development ) -file(GLOB ADIOS2_MODULE_FILES +file(GLOB ADIOS2_MODULE_FILES "${ADIOS2_SOURCE_DIR}/cmake/Find*.cmake" "${ADIOS2_SOURCE_DIR}/cmake/CMake*.cmake" ) diff --git a/source/adios2/CMakeLists.txt b/source/adios2/CMakeLists.txt index ff96f8e2c1..c4f8d893d2 100644 --- a/source/adios2/CMakeLists.txt +++ b/source/adios2/CMakeLists.txt @@ -480,7 +480,7 @@ install(DIRECTORY toolkit/ ) # Library installation -install(TARGETS adios2_core ${maybe_adios2_core_mpi} ${maybe_adios2_core_cuda} ${maybe_adios2_core_kokkos} ${maybe_adios2_blosc2} ${maybe_adios2_core_derived} EXPORT adios2Exports +install(TARGETS adios2_core ${maybe_adios2_core_mpi} ${maybe_adios2_core_cuda} ${maybe_adios2_core_kokkos} ${maybe_adios2_core_derived} EXPORT adios2Exports RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT adios2_core-runtime LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT adios2_core-libraries NAMELINK_COMPONENT adios2_core-development ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT adios2_core-development From 37bd41a7250160cc74a605816d50975979cb4ad9 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Mon, 26 Feb 2024 17:21:26 -0500 Subject: [PATCH 053/124] Fix segfault when asking for BlocksInfo for a variable name that does not exist. Return value is now empty dictionary. Also fix handling 1D arrays made from LocalValues. Now this code works for examples/basics/values/valuesWrite.cpp ProcessID: with adios2.Stream("values.bp", 'rra') as f: e = f.engine bi = e.blocks_info("ProcessID", 0) print(f"Blocks info = {bi}") --- bindings/Python/py11Engine.cpp | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/bindings/Python/py11Engine.cpp b/bindings/Python/py11Engine.cpp index faa4d26716..8787bd1419 100644 --- a/bindings/Python/py11Engine.cpp +++ b/bindings/Python/py11Engine.cpp @@ -267,12 +267,18 @@ std::vector> Engine::BlocksInfo(std::string & const size_t step) const { std::vector> rv; + auto &varMap = m_Engine->m_IO.GetVariables(); + auto itVariable = varMap.find(var_name); + if (itVariable == varMap.end()) + { + return rv; + } // Grab the specified variable object and get its type string adios2::DataType var_type = m_Engine->GetIO().InquireVariableType(var_name); MinVarInfo *minBlocksInfo = nullptr; - auto itVariable = m_Engine->m_IO.GetVariables().find(var_name); + auto Variable = itVariable->second.get(); minBlocksInfo = m_Engine->MinBlocksInfo(*Variable, 0); if (minBlocksInfo) @@ -293,7 +299,8 @@ std::vector> Engine::BlocksInfo(std::string & { start_ss << ","; } - start_ss << info.Start[i]; + start_ss << (minBlocksInfo->WasLocalValue ? reinterpret_cast(info.Start) + : info.Start[i]); } } info_map["Start"] = start_ss.str(); @@ -310,7 +317,8 @@ std::vector> Engine::BlocksInfo(std::string & { count_ss << ","; } - count_ss << info.Count[i]; + count_ss << (minBlocksInfo->WasLocalValue ? reinterpret_cast(info.Count) + : info.Count[i]); } } info_map["Count"] = count_ss.str(); From 37b147db570812e453f2ee99f2426c24daff5c95 Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Mon, 26 Feb 2024 20:42:49 +0000 Subject: [PATCH 054/124] Allow plugin operators to take advantage of the estimated size API --- .../adios2/operator/plugin/PluginOperator.cpp | 22 +++++++++++++++++++ .../adios2/operator/plugin/PluginOperator.h | 3 +++ 2 files changed, 25 insertions(+) diff --git a/source/adios2/operator/plugin/PluginOperator.cpp b/source/adios2/operator/plugin/PluginOperator.cpp index d66cb5b973..9f14529f4c 100644 --- a/source/adios2/operator/plugin/PluginOperator.cpp +++ b/source/adios2/operator/plugin/PluginOperator.cpp @@ -79,6 +79,28 @@ void PluginOperator::PluginInit(const std::string &pluginName, const std::string m_Impl->m_Plugin = m_Impl->m_HandleCreate(m_Parameters); } +size_t PluginOperator::GetEstimatedSize(const size_t ElemCount, const size_t ElemSize, + const size_t ndims, const size_t *dims) const +{ + // Need to calculate the size of the header written by Operate and then add our plugin's size. + constexpr size_t commonHeaderSize = + sizeof(m_TypeEnum) + sizeof(std::uint8_t) + sizeof(std::uint16_t); + + auto &pp = m_Impl->m_PluginParams; + // Want to use std::transform_reduce but C++11 + size_t paramsSize = 1; // for the number of parameters + for (auto &&p : pp) + { + // Need length and string for key and values. + paramsSize += p.first.size() + p.second.size() + 2; + } + + // Plugin's estimate of size so it doesn't need to know about headers. + auto implSize = m_Impl->m_Plugin->GetEstimatedSize(ElemCount, ElemSize, ndims, dims); + + return commonHeaderSize + paramsSize + implSize; +} + size_t PluginOperator::Operate(const char *dataIn, const Dims &blockStart, const Dims &blockCount, const DataType type, char *bufferOut) { diff --git a/source/adios2/operator/plugin/PluginOperator.h b/source/adios2/operator/plugin/PluginOperator.h index 36e8f87bd8..3c80173298 100644 --- a/source/adios2/operator/plugin/PluginOperator.h +++ b/source/adios2/operator/plugin/PluginOperator.h @@ -37,6 +37,9 @@ class PluginOperator : public core::Operator PluginOperator(const Params ¶meters); virtual ~PluginOperator(); + size_t GetEstimatedSize(const size_t ElemCount, const size_t ElemSize, const size_t ndims, + const size_t *dims) const override; + size_t Operate(const char *dataIn, const Dims &blockStart, const Dims &blockCount, const DataType type, char *bufferOut) override; From d77e848bc052e9b4ec0fbbfab77f6dce1c47bc6b Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Fri, 1 Mar 2024 17:55:24 +0000 Subject: [PATCH 055/124] Use GetEstimatedSize in encryption operator plugin --- plugins/operators/EncryptionOperator.cpp | 10 ++++++++++ plugins/operators/EncryptionOperator.h | 3 +++ 2 files changed, 13 insertions(+) diff --git a/plugins/operators/EncryptionOperator.cpp b/plugins/operators/EncryptionOperator.cpp index 4579a4933b..3bf79b18f1 100644 --- a/plugins/operators/EncryptionOperator.cpp +++ b/plugins/operators/EncryptionOperator.cpp @@ -166,6 +166,16 @@ EncryptionOperator::InverseOperate(const char *bufferIn, const size_t sizeIn, ch bool EncryptionOperator::IsDataTypeValid(const DataType type) const { return true; } +size_t EncryptionOperator::GetEstimatedSize(const size_t ElemCount, const size_t ElemSize, + const size_t ndims, const size_t *dims) const +{ + size_t sizeIn = ElemCount * ElemSize; + return (sizeof(size_t) // Data size + + crypto_secretbox_NONCEBYTES // Nonce + + sizeIn // Data + + crypto_secretbox_MACBYTES // MAC + ); +} } // end namespace plugin } // end namespace adios2 diff --git a/plugins/operators/EncryptionOperator.h b/plugins/operators/EncryptionOperator.h index f9b5502750..42a007ffcf 100644 --- a/plugins/operators/EncryptionOperator.h +++ b/plugins/operators/EncryptionOperator.h @@ -41,6 +41,9 @@ class EncryptionOperator : public PluginOperatorInterface bool IsDataTypeValid(const DataType type) const override; + size_t GetEstimatedSize(const size_t ElemCount, const size_t ElemSize, const size_t ndims, + const size_t *dims) const override; + private: struct EncryptImpl; std::unique_ptr Impl; From 745427f06139329af10bc0ab768a9e961d33c7de Mon Sep 17 00:00:00 2001 From: dmitry-ganyushin Date: Tue, 27 Feb 2024 11:50:32 -0500 Subject: [PATCH 056/124] Added reading of configuration parametyers from ./config/adios2 --- .../adios2/engine/campaign/CampaignReader.cpp | 36 +++++++++++++++++++ .../adios2/engine/campaign/CampaignReader.h | 1 + 2 files changed, 37 insertions(+) diff --git a/source/adios2/engine/campaign/CampaignReader.cpp b/source/adios2/engine/campaign/CampaignReader.cpp index c06dad0a96..7586424480 100644 --- a/source/adios2/engine/campaign/CampaignReader.cpp +++ b/source/adios2/engine/campaign/CampaignReader.cpp @@ -116,10 +116,46 @@ void CampaignReader::EndStep() void CampaignReader::Init() { + // read config parameters from ~/.config/adios2 + std::string homePath; +#ifdef _WIN32 + homePath = getenv("HOMEPATH"); +#else + homePath = getenv("HOME"); +#endif + ReadConfig(std::string(homePath + "/.config/adios2")); InitParameters(); InitTransports(); } +void CampaignReader::ReadConfig(std::string configPath) +{ + std::ifstream fileStream(configPath); + + if (!fileStream) + { + return; + } + + std::ostringstream fileSS; + fileSS << fileStream.rdbuf(); + fileStream.close(); + size_t posEndline = 0; + size_t posSpace = 0; + std::string token; + std::string endline = "\n"; + std::string fileString = fileSS.str(); + while ((posEndline = fileString.find(endline)) != std::string::npos) + { + std::string line = fileString.substr(0, posEndline); + posSpace = fileString.find(" "); + std::string token1 = line.substr(0, posSpace); + std::string token2 = line.substr(posSpace + 1, line.size()); + // trim? + fileString.erase(0, posEndline + endline.length()); + } + return; +} void CampaignReader::InitParameters() { for (const auto &pair : m_IO.m_Parameters) diff --git a/source/adios2/engine/campaign/CampaignReader.h b/source/adios2/engine/campaign/CampaignReader.h index 711099fbd1..8e4226fc9d 100644 --- a/source/adios2/engine/campaign/CampaignReader.h +++ b/source/adios2/engine/campaign/CampaignReader.h @@ -76,6 +76,7 @@ class CampaignReader : public Engine void Init() final; ///< called from constructor, gets the selected Skeleton /// transport method from settings + void ReadConfig(std::string path); void InitParameters() final; void InitTransports() final; From 03716ff2b73e34248299f090d196a9e57999faed Mon Sep 17 00:00:00 2001 From: Dmitry Ganyushin Date: Wed, 28 Feb 2024 09:51:59 -0500 Subject: [PATCH 057/124] WIP. Changed the name of the campaign config file. --- source/adios2/engine/campaign/CampaignReader.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/source/adios2/engine/campaign/CampaignReader.cpp b/source/adios2/engine/campaign/CampaignReader.cpp index 7586424480..b6f3b38353 100644 --- a/source/adios2/engine/campaign/CampaignReader.cpp +++ b/source/adios2/engine/campaign/CampaignReader.cpp @@ -116,14 +116,15 @@ void CampaignReader::EndStep() void CampaignReader::Init() { - // read config parameters from ~/.config/adios2 + // read config parameters from config file + const std::string cfgFile = "/.config/adios2/campaign.cfg"; std::string homePath; #ifdef _WIN32 homePath = getenv("HOMEPATH"); #else homePath = getenv("HOME"); #endif - ReadConfig(std::string(homePath + "/.config/adios2")); + ReadConfig(std::string(homePath + cfgFile)); InitParameters(); InitTransports(); } From 90c243049a1aac31ac6eba9d3150b69a47f7eb49 Mon Sep 17 00:00:00 2001 From: Dmitry Ganyushin Date: Wed, 28 Feb 2024 10:34:03 -0500 Subject: [PATCH 058/124] Campaign engine is recognized by file extension. --- source/adios2/core/IO.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/adios2/core/IO.cpp b/source/adios2/core/IO.cpp index bbc797038d..83bf0f3781 100644 --- a/source/adios2/core/IO.cpp +++ b/source/adios2/core/IO.cpp @@ -556,6 +556,10 @@ Engine &IO::Open(const std::string &name, const Mode mode, helper::Comm comm) { engineTypeLC = "hdf5"; } + else if (helper::EndsWith(name, ".aca", false)) + { + engineTypeLC = "campaign"; + } else if ((mode_to_use == Mode::Read) || (mode_to_use == Mode::ReadRandomAccess)) { if (adios2sys::SystemTools::FileIsDirectory(name)) From 2096e781986c01ff057acb54b74013dd02d92791 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Thu, 29 Feb 2024 15:42:15 -0500 Subject: [PATCH 059/124] do not flush io and adios in read mode. BP5 reader does not like it. --- python/adios2/stream.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/python/adios2/stream.py b/python/adios2/stream.py index eb632f19fb..826316d9cd 100644 --- a/python/adios2/stream.py +++ b/python/adios2/stream.py @@ -536,9 +536,10 @@ def close(self): """ self._engine.close() self._engine = None - self._io.flush_all() + if not self._read_mode: + self._io.flush_all() + self._adios.flush_all() self._io = None - self._adios.flush_all() self._adios = None def current_step(self): From c2f4b4fc7505ad7aebb9f30a42dd17b6f1c90ad5 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Sat, 2 Mar 2024 07:04:40 -0500 Subject: [PATCH 060/124] Add minmax and shape functions to CampaignReader, so that per-block info is complete when listing campaign archives --- .../adios2/engine/campaign/CampaignReader.cpp | 25 +++++++++++++++++++ .../adios2/engine/campaign/CampaignReader.h | 2 ++ 2 files changed, 27 insertions(+) diff --git a/source/adios2/engine/campaign/CampaignReader.cpp b/source/adios2/engine/campaign/CampaignReader.cpp index b6f3b38353..29642d64dc 100644 --- a/source/adios2/engine/campaign/CampaignReader.cpp +++ b/source/adios2/engine/campaign/CampaignReader.cpp @@ -351,6 +351,31 @@ MinVarInfo *CampaignReader::MinBlocksInfo(const VariableBase &Var, size_t Step) return nullptr; } +bool CampaignReader::VarShape(const VariableBase &Var, const size_t Step, Dims &Shape) const +{ + auto it = m_VarInternalInfo.find(Var.m_Name); + if (it != m_VarInternalInfo.end()) + { + VariableBase *vb = reinterpret_cast(it->second.originalVar); + Engine *e = m_Engines[it->second.engineIdx]; + return e->VarShape(*vb, Step, Shape); + } + return false; +} + +bool CampaignReader::VariableMinMax(const VariableBase &Var, const size_t Step, + MinMaxStruct &MinMax) +{ + auto it = m_VarInternalInfo.find(Var.m_Name); + if (it != m_VarInternalInfo.end()) + { + VariableBase *vb = reinterpret_cast(it->second.originalVar); + Engine *e = m_Engines[it->second.engineIdx]; + return e->VariableMinMax(*vb, Step, MinMax); + } + return false; +} + #define declare_type(T) \ void CampaignReader::DoGetSync(Variable &variable, T *data) \ { \ diff --git a/source/adios2/engine/campaign/CampaignReader.h b/source/adios2/engine/campaign/CampaignReader.h index 8e4226fc9d..3678318f16 100644 --- a/source/adios2/engine/campaign/CampaignReader.h +++ b/source/adios2/engine/campaign/CampaignReader.h @@ -50,6 +50,8 @@ class CampaignReader : public Engine void EndStep() final; MinVarInfo *MinBlocksInfo(const VariableBase &, const size_t Step) const; + bool VarShape(const VariableBase &Var, const size_t Step, Dims &Shape) const; + bool VariableMinMax(const VariableBase &, const size_t Step, MinMaxStruct &MinMax); private: int m_Verbosity = 0; // runtime parameter Verbose From fc582bd716a97bb85b9a306aef68bdc95cc9aab4 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Thu, 29 Feb 2024 15:35:13 -0500 Subject: [PATCH 061/124] fixes to still be able to build with gcc 4.8.2. Needed for OLCF DTN nodes --- source/adios2/engine/bp5/BP5Reader.cpp | 4 ++-- source/adios2/toolkit/transport/file/FileHTTP.cpp | 2 +- source/adios2/toolkit/transport/file/FileHTTP.h | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/source/adios2/engine/bp5/BP5Reader.cpp b/source/adios2/engine/bp5/BP5Reader.cpp index 636f67ace4..4465380d4c 100644 --- a/source/adios2/engine/bp5/BP5Reader.cpp +++ b/source/adios2/engine/bp5/BP5Reader.cpp @@ -300,8 +300,8 @@ void BP5Reader::PerformRemoteGets() void BP5Reader::PerformLocalGets() { - auto lf_CompareReqSubfile = [&](adios2::format::BP5Deserializer::ReadRequest &r1, - adios2::format::BP5Deserializer::ReadRequest &r2) -> bool { + auto lf_CompareReqSubfile = [&](const adios2::format::BP5Deserializer::ReadRequest &r1, + const adios2::format::BP5Deserializer::ReadRequest &r2) -> bool { return (m_WriterMap[m_WriterMapIndex[r1.Timestep]].RankToSubfile[r1.WriterRank] < m_WriterMap[m_WriterMapIndex[r2.Timestep]].RankToSubfile[r2.WriterRank]); }; diff --git a/source/adios2/toolkit/transport/file/FileHTTP.cpp b/source/adios2/toolkit/transport/file/FileHTTP.cpp index 59d06d81cc..7407cb6ff7 100644 --- a/source/adios2/toolkit/transport/file/FileHTTP.cpp +++ b/source/adios2/toolkit/transport/file/FileHTTP.cpp @@ -187,7 +187,7 @@ void FileHTTP::Read(char *buffer, size_t size, size_t start) MAX_REQUEST_LEN = 1024 }; char request[MAX_REQUEST_LEN] = {'\0'}; - int request_len = snprintf(request, MAX_REQUEST_LEN, request_template, m_Name.c_str(), + int request_len = snprintf(request, MAX_REQUEST_LEN, request_template.c_str(), m_Name.c_str(), m_hostname.c_str(), start, start + size - 1); if (request_len >= MAX_REQUEST_LEN) { diff --git a/source/adios2/toolkit/transport/file/FileHTTP.h b/source/adios2/toolkit/transport/file/FileHTTP.h index cf32da3a76..443d0c8c1c 100644 --- a/source/adios2/toolkit/transport/file/FileHTTP.h +++ b/source/adios2/toolkit/transport/file/FileHTTP.h @@ -72,7 +72,7 @@ class FileHTTP : public Transport int m_Errno = 0; bool m_IsOpening = false; /* if filename is very lomg, we can get lout from array boundaries */ - char request_template[128] = "GET %s HTTP/1.1\r\nHost: %s\r\nRange: bytes=%d-%d\r\n\r\n"; + std::string request_template = "GET %s HTTP/1.1\r\nHost: %s\r\nRange: bytes=%d-%d\r\n\r\n"; std::string m_hostname = "localhost"; int m_server_port = 9999; struct sockaddr_in sockaddr_in; From 3820db52b80a197476f46f928effb286432be407 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Thu, 29 Feb 2024 15:43:33 -0500 Subject: [PATCH 062/124] format --- source/adios2/engine/bp5/BP5Reader.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/source/adios2/engine/bp5/BP5Reader.cpp b/source/adios2/engine/bp5/BP5Reader.cpp index 4465380d4c..a2854b2de2 100644 --- a/source/adios2/engine/bp5/BP5Reader.cpp +++ b/source/adios2/engine/bp5/BP5Reader.cpp @@ -300,8 +300,9 @@ void BP5Reader::PerformRemoteGets() void BP5Reader::PerformLocalGets() { - auto lf_CompareReqSubfile = [&](const adios2::format::BP5Deserializer::ReadRequest &r1, - const adios2::format::BP5Deserializer::ReadRequest &r2) -> bool { + auto lf_CompareReqSubfile = + [&](const adios2::format::BP5Deserializer::ReadRequest &r1, + const adios2::format::BP5Deserializer::ReadRequest &r2) -> bool { return (m_WriterMap[m_WriterMapIndex[r1.Timestep]].RankToSubfile[r1.WriterRank] < m_WriterMap[m_WriterMapIndex[r2.Timestep]].RankToSubfile[r2.WriterRank]); }; From 61118484441f281b1ea75d08de42260225a2e056 Mon Sep 17 00:00:00 2001 From: guj Date: Wed, 28 Feb 2024 12:57:58 -0800 Subject: [PATCH 063/124] added support to read back from H5T_STRING VARIABLES it turns out strings written out through h5py are all variable strings --- .../toolkit/interop/hdf5/HDF5Common.cpp | 71 +++++++++++++++++-- 1 file changed, 66 insertions(+), 5 deletions(-) diff --git a/source/adios2/toolkit/interop/hdf5/HDF5Common.cpp b/source/adios2/toolkit/interop/hdf5/HDF5Common.cpp index 1b8c8c409c..8efb6f6f6b 100644 --- a/source/adios2/toolkit/interop/hdf5/HDF5Common.cpp +++ b/source/adios2/toolkit/interop/hdf5/HDF5Common.cpp @@ -929,13 +929,28 @@ void HDF5Common::ReadStringScalarDataset(hid_t dataSetId, std::string &result) hid_t h5Type = H5Dget_type(dataSetId); // get actual type; size_t typesize = H5Tget_size(h5Type); - char *val = (char *)(calloc(typesize, sizeof(char))); - hid_t ret2 = H5Dread(dataSetId, h5Type, H5S_ALL, H5S_ALL, H5P_DEFAULT, val); - CHECK_H5_RETURN(ret2, "ReadStringScalarDataset"); + if (H5Tis_variable_str(h5Type)) + { + hid_t d_space = H5Dget_space(dataSetId); + std::vector vc(typesize); // byte buffer to vlen strings + auto status = H5Dread(dataSetId, h5Type, H5S_ALL, H5S_ALL, H5P_DEFAULT, vc.data()); + result.assign(*((char **)vc.data())); - result.assign(val, typesize); - free(val); + // free dynamically allocated vlen memory from H5Dread + // recent versions shift to use H5Treclaim() + H5Dvlen_reclaim(h5Type, d_space, H5P_DEFAULT, vc.data()); + + // H5Treclaim(attr_type, attr_space, H5P_DEFAULT, vc.data()); + } + else + { + char *val = (char *)(calloc(typesize, sizeof(char))); + hid_t ret2 = H5Dread(dataSetId, h5Type, H5S_ALL, H5S_ALL, H5P_DEFAULT, val); + CHECK_H5_RETURN(ret2, "ReadStringScalarDataset"); + result.assign(val, typesize); + free(val); + } H5Tclose(h5Type); } @@ -1213,6 +1228,52 @@ void HDF5Common::ReadInStringAttr(core::IO &io, const std::string &attrName, hid hsize_t typeSize = H5Tget_size(h5Type); H5S_class_t stype = H5Sget_simple_extent_type(sid); + if (H5Tis_variable_str(h5Type)) + { + hid_t attr_space = H5Aget_space(attrId); + if (H5S_SCALAR == stype) + { + std::vector vc(typeSize); // byte buffer to vlen strings + auto status = H5Aread(attrId, h5Type, vc.data()); + CHECK_H5_RETURN(status, "ReadInStringAttr_scalar") + + std::string c_str(*((char **)vc.data())); + + // free dynamically allocated vlen memory from H5Aread + // later versions use H5Treclaim() instead. + H5Dvlen_reclaim(h5Type, attr_space, H5P_DEFAULT, vc.data()); + + io.DefineAttribute(attrName, c_str); + } + else + { + hsize_t ndims = H5Sget_simple_extent_ndims(sid); + if (ndims != 1) + CHECK_H5_RETURN(-1, "Only handles 1-D string array"); + + // ndims must be 1 + hsize_t dims[1]; + hid_t ret = H5Sget_simple_extent_dims(sid, dims, NULL); + CHECK_H5_RETURN(ret, "ReadInStringAttr"); + + std::vector vc(dims[0]); + auto status = H5Aread(attrId, h5Type, vc.data()); + CHECK_H5_RETURN(status, "ReadInStringAttr"); + + std::vector stringArray; + for (auto const &val : vc) + // stringArray.push_back(auxiliary::strip(std::string(val), {'\0'})); + stringArray.push_back(std::string(val)); + + status = H5Dvlen_reclaim(h5Type, attr_space, H5P_DEFAULT, vc.data()); + io.DefineAttribute(attrName, stringArray.data(), dims[0]); + } + + return; + } + // + // regular string, not variables + // if (H5S_SCALAR == stype) { auto val = std::unique_ptr(new char[typeSize]); From adc1d00bfa864b8824b5da27db5c0f4c1f22c9c6 Mon Sep 17 00:00:00 2001 From: guj Date: Wed, 28 Feb 2024 13:35:09 -0800 Subject: [PATCH 064/124] fixed warning --- source/adios2/toolkit/interop/hdf5/HDF5Common.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/source/adios2/toolkit/interop/hdf5/HDF5Common.cpp b/source/adios2/toolkit/interop/hdf5/HDF5Common.cpp index 8efb6f6f6b..c87786996a 100644 --- a/source/adios2/toolkit/interop/hdf5/HDF5Common.cpp +++ b/source/adios2/toolkit/interop/hdf5/HDF5Common.cpp @@ -934,6 +934,7 @@ void HDF5Common::ReadStringScalarDataset(hid_t dataSetId, std::string &result) hid_t d_space = H5Dget_space(dataSetId); std::vector vc(typesize); // byte buffer to vlen strings auto status = H5Dread(dataSetId, h5Type, H5S_ALL, H5S_ALL, H5P_DEFAULT, vc.data()); + CHECK_H5_RETURN(status, "ReadStringScalar_variable_str"); result.assign(*((char **)vc.data())); // free dynamically allocated vlen memory from H5Dread From 59064b625966c331857fd67d6ec650d43b7f9853 Mon Sep 17 00:00:00 2001 From: guj Date: Wed, 28 Feb 2024 13:41:30 -0800 Subject: [PATCH 065/124] clang-format fix --- source/adios2/toolkit/interop/hdf5/HDF5Common.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/adios2/toolkit/interop/hdf5/HDF5Common.cpp b/source/adios2/toolkit/interop/hdf5/HDF5Common.cpp index c87786996a..2e97a13ad9 100644 --- a/source/adios2/toolkit/interop/hdf5/HDF5Common.cpp +++ b/source/adios2/toolkit/interop/hdf5/HDF5Common.cpp @@ -934,7 +934,7 @@ void HDF5Common::ReadStringScalarDataset(hid_t dataSetId, std::string &result) hid_t d_space = H5Dget_space(dataSetId); std::vector vc(typesize); // byte buffer to vlen strings auto status = H5Dread(dataSetId, h5Type, H5S_ALL, H5S_ALL, H5P_DEFAULT, vc.data()); - CHECK_H5_RETURN(status, "ReadStringScalar_variable_str"); + CHECK_H5_RETURN(status, "ReadStringScalar_variable_str"); result.assign(*((char **)vc.data())); // free dynamically allocated vlen memory from H5Dread From 76da99a04ef667b2c6e430d8dc147f090a9c00aa Mon Sep 17 00:00:00 2001 From: Ana Gainaru Date: Fri, 1 Mar 2024 10:54:55 -0500 Subject: [PATCH 066/124] Using the correct flag to detect the CUDA backend in ZFP --- cmake/DetectOptions.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/DetectOptions.cmake b/cmake/DetectOptions.cmake index 13c98e4f5c..2b2bd3b285 100644 --- a/cmake/DetectOptions.cmake +++ b/cmake/DetectOptions.cmake @@ -125,7 +125,7 @@ if(ADIOS2_USE_ZFP) endif() if(ZFP_FOUND) set(ADIOS2_HAVE_ZFP TRUE) - set(ADIOS2_HAVE_ZFP_CUDA ${ZFP_CUDA}) + set(ADIOS2_HAVE_ZFP_CUDA ${ZFP_WITH_CUDA}) # Older versions of ZFP if(NOT ADIOS2_HAVE_ZFP_CUDA) From 2d8a099609ad03fcd060f954a51c8c19e0c1942e Mon Sep 17 00:00:00 2001 From: Greg Eisenhauer Date: Mon, 4 Mar 2024 15:44:02 -0500 Subject: [PATCH 067/124] Fix Reord to use MinBlocksInfo where appropriate (#4071) --- source/utils/adios_reorganize/Reorganize.cpp | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/source/utils/adios_reorganize/Reorganize.cpp b/source/utils/adios_reorganize/Reorganize.cpp index dc1288fe15..c4121a5a0c 100644 --- a/source/utils/adios_reorganize/Reorganize.cpp +++ b/source/utils/adios_reorganize/Reorganize.cpp @@ -499,8 +499,16 @@ int Reorganize::ProcessMetadata(core::Engine &rStream, core::IO &io, const core: if (v->m_ShapeID == adios2::ShapeID::LocalArray) \ { \ \ - auto blocks = rStream.BlocksInfo(*v, rStream.CurrentStep()); \ - nBlocks = blocks.size(); \ + const auto minBlocks = rStream.MinBlocksInfo(*v, step); \ + if (minBlocks) \ + { \ + nBlocks = minBlocks->BlocksInfo.size(); \ + } \ + else \ + { \ + auto blocks = rStream.BlocksInfo(*v, rStream.CurrentStep()); \ + nBlocks = blocks.size(); \ + } \ } \ variable = v; \ } From e49ad90a06f363d4d6d0536ef7b87c0877048c34 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Tue, 5 Mar 2024 20:09:59 -0500 Subject: [PATCH 068/124] - Restructure python API doc in separate main topic, add working example to it. - What's new for 2.10 - Usage on DOE machines --- docs/user_guide/source/api_full/api_full.rst | 1 - docs/user_guide/source/api_full/python.rst | 44 ---- docs/user_guide/source/api_high/api_high.rst | 3 - docs/user_guide/source/api_high/python.rst | 111 ---------- .../api_python/adios2-doc-read-filereader.py | 31 +++ .../source/api_python/adios2-doc-read.py | 31 +++ .../source/api_python/adios2-doc-write.py | 34 ++++ .../source/api_python/api_python.rst | 8 + docs/user_guide/source/api_python/python.rst | 29 +++ .../source/api_python/python_bindings.rst | 61 ++++++ .../source/api_python/python_example.rst | 190 ++++++++++++++++++ .../python_transition_from_high.rst | 41 ++++ docs/user_guide/source/conf.py | 7 +- docs/user_guide/source/index.rst | 1 + .../source/introduction/whatsnew.rst | 37 ++++ .../source/setting_up/doemachines.rst | 88 ++++++++ docs/user_guide/source/setting_up/package.rst | 23 ++- .../source/setting_up/setting_up.rst | 2 + .../source/setting_up/source/cmake.rst | 85 ++++++-- 19 files changed, 636 insertions(+), 191 deletions(-) delete mode 100644 docs/user_guide/source/api_full/python.rst delete mode 100644 docs/user_guide/source/api_high/python.rst create mode 100644 docs/user_guide/source/api_python/adios2-doc-read-filereader.py create mode 100644 docs/user_guide/source/api_python/adios2-doc-read.py create mode 100644 docs/user_guide/source/api_python/adios2-doc-write.py create mode 100644 docs/user_guide/source/api_python/api_python.rst create mode 100644 docs/user_guide/source/api_python/python.rst create mode 100644 docs/user_guide/source/api_python/python_bindings.rst create mode 100644 docs/user_guide/source/api_python/python_example.rst create mode 100644 docs/user_guide/source/api_python/python_transition_from_high.rst create mode 100644 docs/user_guide/source/setting_up/doemachines.rst diff --git a/docs/user_guide/source/api_full/api_full.rst b/docs/user_guide/source/api_full/api_full.rst index b721b8b695..c09a81d598 100644 --- a/docs/user_guide/source/api_full/api_full.rst +++ b/docs/user_guide/source/api_full/api_full.rst @@ -71,4 +71,3 @@ The following sections provide a summary of the API calls on each language and l .. include:: cxx11.rst .. include:: fortran.rst .. include:: c.rst -.. include:: python.rst diff --git a/docs/user_guide/source/api_full/python.rst b/docs/user_guide/source/api_full/python.rst deleted file mode 100644 index 828017bb00..0000000000 --- a/docs/user_guide/source/api_full/python.rst +++ /dev/null @@ -1,44 +0,0 @@ -*************** -Python bindings -*************** - -.. note:: - - Product Application Developers targeting finer-level control for their IO tasks for optimization should use the current full APIs. If you want to use ADIOS2 in simple use cases (*e.g.* reading a file for analysis, interactive Python, or saving some data for a small project) please refer to the :ref:`High-Level APIs` for a flat learning curve. - -The full Python APIs follow very closely the full C++11 API interface. - -ADIOS class --------------- -.. autoclass:: adios2.bindings.adios2_bindings::ADIOS - :members: - -IO class --------------- -.. autoclass:: adios2.bindings.adios2_bindings::IO - :members: - -Variable class --------------- -.. autoclass:: adios2.bindings.adios2_bindings::Variable - :members: - -Attribute class ---------------- -.. autoclass:: adios2.bindings.adios2_bindings::Attribute - :members: - -Engine class --------------- -.. autoclass:: adios2.bindings.adios2_bindings::Engine - :members: - -Operator class --------------- -.. autoclass:: adios2.bindings.adios2_bindings::Operator - :members: - -Query class --------------- -.. autoclass:: adios2.bindings.adios2_bindings::Query - :members: diff --git a/docs/user_guide/source/api_high/api_high.rst b/docs/user_guide/source/api_high/api_high.rst index dde197447e..879aef4736 100644 --- a/docs/user_guide/source/api_high/api_high.rst +++ b/docs/user_guide/source/api_high/api_high.rst @@ -28,14 +28,11 @@ Currently ADIOS2 support bindings for the following languages and their minimum +----------+----------+-----------------------+-------------+ | C++ | 11/newer | ``#include adios2.h`` | ``fstream`` | +----------+----------+-----------------------+-------------+ -| Python | 2.7/3 | ``import adios2`` | Python IO | -+----------+----------+-----------------------+-------------+ | Matlab | | | | +----------+----------+-----------------------+-------------+ The following sections provide a summary of the API calls on each language and links to Write and Read examples to put it all together. .. include:: cxx11.rst -.. include:: python.rst .. include:: matlab.rst diff --git a/docs/user_guide/source/api_high/python.rst b/docs/user_guide/source/api_high/python.rst deleted file mode 100644 index 76f086b15b..0000000000 --- a/docs/user_guide/source/api_high/python.rst +++ /dev/null @@ -1,111 +0,0 @@ -********************* -Python High-Level API -********************* - -Python simple bindings follow closely Python style directives. Just like the full APIs, they rely on numpy and, optionally, on ``mpi4py``, if the underlying ADIOS2 library is compiled with MPI. - -For online examples on MyBinder : - -- `Python-MPI Notebooks `_ - -- `Python-noMPI Notebooks `_ - - -Python Write example --------------------- - -.. code-block:: python - - from mpi4py import MPI - import numpy as np - from adios2 import Stream - - comm = MPI.COMM_WORLD - rank = comm.Get_rank() - size = comm.Get_size() - ... - shape = [size * nx] - start = [rank * nx] - count = [nx] - - with Stream("cfd.bp", "w", comm) as s: - # NSteps from application - for _ in s.steps(NSteps): - if rank == 0 and s.current_step() == 0: - fh.write("size", np.array([size])) - - fh.write("physical_time", np.array([physical_time]) ) - # temperature and pressure are numpy arrays - fh.write("temperature", temperature, shape, start, count) - # advances to next step - fh.write("pressure", pressure, shape, start, count, end_step=True) - - -Python Read "step-by-step" example ----------------------------------- - -.. code-block:: python - - from mpi4py import MPI - import numpy as np - from adios2 import Stream - - comm = MPI.COMM_WORLD - rank = comm.Get_rank() - size = comm.Get_size() - ... - shape = [size * nx] - start = [rank * nx] - count = [nx] - - if rank == 0: - # if only one rank is active pass MPI.COMM_SELF - with Stream("cfd.bp", "r", MPI.COMM_SELF) as s: - for _ in s.steps(): - # inspect variables in current step - for name, info in s.available_variables(): - print("variable_name: " + name) - for key, value in info.items(): - print("\t" + key + ": " + value) - print("\n") - - # track current step - if s.current_step() == 0: - size_in = s.read("size") - - # read variables return a numpy array with corresponding selection - physical_time = s.read("physical_time") - temperature = s.read("temperature", start, count) - pressure = s.read("pressure", start, count) - -.. caution:: - - When reading in stepping mode with the for-in directive, as in the example above, you can ignore the variable that iterates the s.step() as it contains the same ref of the Stream instance. - - -High Level API --------------- - -.. autoclass:: adios2::Stream - :members: - -.. autoclass:: adios2::FileReader - :members: - -.. autoclass:: adios2::Adios - :members: - -.. autoclass:: adios2::IO - :members: - -.. autoclass:: adios2::Engine - :members: - -.. autoclass:: adios2::Variable - :members: - -.. autoclass:: adios2::Attribute - :members: - -.. autoclass:: adios2::Operator - :members: diff --git a/docs/user_guide/source/api_python/adios2-doc-read-filereader.py b/docs/user_guide/source/api_python/adios2-doc-read-filereader.py new file mode 100644 index 0000000000..3211e82642 --- /dev/null +++ b/docs/user_guide/source/api_python/adios2-doc-read-filereader.py @@ -0,0 +1,31 @@ +import numpy as np +from adios2 import FileReader + +with FileReader("cfd.bp") as s: + # inspect variables + vars = s.available_variables() + for name, info in vars.items(): + print("variable_name: " + name, end=" ") + for key, value in info.items(): + print("\t" + key + ": " + value, end=" ") + print() + + nproc = s.read("nproc") + print(f"nproc is {nproc} of type {type(nproc)}") + + # read variables return a numpy array with corresponding selection + steps = int(vars["physical_time"]["AvailableStepsCount"]) + physical_time = s.read("physical_time", step_selection=[0, steps]) + print(f"physical_time is {physical_time} of type {type(physical_time)}") + + steps = int(vars["temperature"]["AvailableStepsCount"]) + temperature = s.read("temperature", step_selection=[0, steps]) + temp_unit = s.read_attribute("temperature/unit") + print(f"temperature array size is {temperature.size} of shape {temperature.shape}") + print(f"temperature unit is {temp_unit} of type {type(temp_unit)}") + + steps = int(vars["pressure"]["AvailableStepsCount"]) + pressure = s.read("pressure", step_selection=[0, steps]) + press_unit = s.read_attribute("pressure/unit") + + print() diff --git a/docs/user_guide/source/api_python/adios2-doc-read.py b/docs/user_guide/source/api_python/adios2-doc-read.py new file mode 100644 index 0000000000..5f7d1ee2cf --- /dev/null +++ b/docs/user_guide/source/api_python/adios2-doc-read.py @@ -0,0 +1,31 @@ +import numpy as np +from adios2 import Stream + +with Stream("cfd.bp", "r") as s: + # steps comes from the stream + for _ in s.steps(): + # track current step + print(f"Current step is {s.current_step()}") + + # inspect variables in current step + for name, info in s.available_variables().items(): + print("variable_name: " + name, end=" ") + for key, value in info.items(): + print("\t" + key + ": " + value, end=" ") + print() + + if s.current_step() == 0: + nproc = s.read("nproc") + print(f"nproc is {nproc} of type {type(nproc)}") + + # read variables return a numpy array with corresponding selection + physical_time = s.read("physical_time") + print(f"physical_time is {physical_time} of type {type(physical_time)}") + temperature = s.read("temperature") + temp_unit = s.read_attribute("temperature/unit") + print(f"temperature array size is {temperature.size} of shape {temperature.shape}") + print(f"temperature unit is {temp_unit} of type {type(temp_unit)}") + pressure = s.read("pressure") + press_unit = s.read_attribute("pressure/unit") + print(f"pressure unit is {press_unit} of type {type(press_unit)}") + print() diff --git a/docs/user_guide/source/api_python/adios2-doc-write.py b/docs/user_guide/source/api_python/adios2-doc-write.py new file mode 100644 index 0000000000..6a470c83cd --- /dev/null +++ b/docs/user_guide/source/api_python/adios2-doc-write.py @@ -0,0 +1,34 @@ +from mpi4py import MPI +import numpy as np +from adios2 import Stream + +comm = MPI.COMM_WORLD +rank = comm.Get_rank() +size = comm.Get_size() + +nx = 10 +shape = [size * nx] +start = [rank * nx] +count = [nx] + +temperature = np.zeros(nx, dtype=np.double) +pressure = np.ones(nx, dtype=np.double) +delta_time = 0.01 +physical_time = 0.0 +nsteps = 5 + +with Stream("cfd.bp", "w", comm) as s: + # NSteps from application + for _ in s.steps(nsteps): + if rank == 0 and s.current_step() == 0: + # write a Python integer + s.write("nproc", size) + + # write a Python floating point value + s.write("physical_time", physical_time) + # temperature and pressure are numpy arrays + s.write("temperature", temperature, shape, start, count) + s.write_attribute("temperature/unit", "K") + s.write("pressure", pressure, shape, start, count) + s.write_attribute("pressure/unit", "Pa") + physical_time += delta_time diff --git a/docs/user_guide/source/api_python/api_python.rst b/docs/user_guide/source/api_python/api_python.rst new file mode 100644 index 0000000000..d995bbe034 --- /dev/null +++ b/docs/user_guide/source/api_python/api_python.rst @@ -0,0 +1,8 @@ +#################################### +Python APIs +#################################### + +.. include:: python_example.rst +.. include:: python.rst +.. include:: python_bindings.rst +.. include:: python_transition_from_high.rst diff --git a/docs/user_guide/source/api_python/python.rst b/docs/user_guide/source/api_python/python.rst new file mode 100644 index 0000000000..3253000c6d --- /dev/null +++ b/docs/user_guide/source/api_python/python.rst @@ -0,0 +1,29 @@ +************************* +adios2 classes for Python +************************* + +``Stream`` is a high-level class that can perform most of the ADIOS functionality. ``FileReader`` is just a convenience class and is the same as Stream with "rra" (ReadRandomAccess) mode. FileReaders do not work with for loops as opposed to Streams that work step-by-step, rather one can access any step of any variable at will. The other classes, ``Adios``, ``IO``, ``Engine``, ``Variable``, ``Attribute`` and ``Operator`` correspond to the C++ classes. One need to use them to extend the capabilities of the ``Stream`` class (e.g. using an external XML file for runtime configuration, changing the engine for the run, setting up a compression operator for an output variable, etc.) + +.. autoclass:: adios2::Stream + :members: + +.. autoclass:: adios2::FileReader + :members: + +.. autoclass:: adios2::Adios + :members: + +.. autoclass:: adios2::IO + :members: + +.. autoclass:: adios2::Engine + :members: + +.. autoclass:: adios2::Variable + :members: + +.. autoclass:: adios2::Attribute + :members: + +.. autoclass:: adios2::Operator + :members: diff --git a/docs/user_guide/source/api_python/python_bindings.rst b/docs/user_guide/source/api_python/python_bindings.rst new file mode 100644 index 0000000000..cd0ef4383c --- /dev/null +++ b/docs/user_guide/source/api_python/python_bindings.rst @@ -0,0 +1,61 @@ +********************** +Python bindings to C++ +********************** + +.. note:: + + The bindings to the C++ functions is the basis of the native Python API described before. It is still accessible to users who used the "Full Python API" pre-2.10. In order to make old scripts working with 2.10 and later versions, change the import line in the python script. + +.. code-block:: python + + import adios2.bindings as adios2 + +The full Python APIs follow very closely the full C++11 API interface. All of its functionality is now in the native API as well, so its use is discouraged for future scripts. + +Examples using the Python bindings in the ADIOS2 repository +----------------------------------------------------------- + +- Simple file-based examples + - examples/hello/helloWorld/hello-world-bindings.py + - examples/hello/bpReader/bpReaderHeatMap2D-bindings.py + - examples/hello/bpWriter/bpWriter-bindings.py + +- Staging examples using staging engines SST and DataMan + - examples/hello/sstWriter/sstWriter-bindings.py + - examples/hello/sstReader/sstReader-bindings.py + + +ADIOS class +-------------- +.. autoclass:: adios2.bindings::ADIOS + :members: + +IO class +-------------- +.. autoclass:: adios2.bindings::IO + :members: + +Variable class +-------------- +.. autoclass:: adios2.bindings::Variable + :members: + +Attribute class +--------------- +.. autoclass:: adios2.bindings::Attribute + :members: + +Engine class +-------------- +.. autoclass:: adios2.bindings::Engine + :members: + +Operator class +-------------- +.. autoclass:: adios2.bindings::Operator + :members: + +Query class +-------------- +.. autoclass:: adios2.bindings::Query + :members: diff --git a/docs/user_guide/source/api_python/python_example.rst b/docs/user_guide/source/api_python/python_example.rst new file mode 100644 index 0000000000..ceec6ae43d --- /dev/null +++ b/docs/user_guide/source/api_python/python_example.rst @@ -0,0 +1,190 @@ +******************* +Python Example Code +******************* + +The Python APIs follow closely Python style directives. They rely on numpy and, optionally, on ``mpi4py``, if the underlying ADIOS2 library is compiled with MPI. + +For online examples on MyBinder : + +- `Python-MPI Notebooks `_ + +- `Python-noMPI Notebooks `_ + + +Examples in the ADIOS2 repository +--------------------------------- + +- Simple file-based examples + - examples/hello/helloWorld/hello-world.py + - examples/hello/bpReader/bpReaderHeatMap2D.py + - examples/hello/bpWriter/bpWriter.py + +- Staging examples using staging engines SST and DataMan + - examples/hello/sstWriter/sstWriter.py + - examples/hello/sstReader/sstReader.py + - examples/hello/datamanWriter/dataManWriter.py + - examples/hello/datamanReader/dataManReader.py + +Python Write example +-------------------- + +.. code-block:: python + + from mpi4py import MPI + import numpy as np + from adios2 import Stream + + comm = MPI.COMM_WORLD + rank = comm.Get_rank() + size = comm.Get_size() + + nx = 10 + shape = [size * nx] + start = [rank * nx] + count = [nx] + + temperature = np.zeros(nx, dtype=np.double) + delta_time = 0.01 + physical_time = 0.0 + nsteps = 5 + + with Stream("cfd.bp", "w", comm) as s: + # NSteps from application + for _ in s.steps(nsteps): + if rank == 0 and s.current_step() == 0: + # write a Python integer + s.write("nproc", size) + + # write a Python floating point value + s.write("physical_time", physical_time) + # temperature and pressure are numpy arrays + s.write("temperature", temperature, shape, start, count) + s.write_attribute("temperature/unit", "K") + s.write("pressure", pressure, shape, start, count) + s.write_attribute("pressure/unit", "Pa") + physical_time += delta_time + +.. code-block:: bash + + $ mpirun -n 4 python3 ./adios2-doc-write.py + $ bpls -la cfd.bp + int64_t nproc scalar = 4 + double physical_time 5*scalar = 0 / 0.04 + double pressure 5*{40} = 1 / 1 + string pressure/unit attr = "Pa" + double temperature 5*{40} = 0 / 0 + string temperature/unit attr = "K" + + +Python Read "step-by-step" example +---------------------------------- + +.. code-block:: python + + import numpy as np + from adios2 import Stream + + with Stream("cfd.bp", "r") as s: + # steps comes from the stream + for _ in s.steps(): + + # track current step + print(f"Current step is {s.current_step()}") + + # inspect variables in current step + for name, info in s.available_variables().items(): + print("variable_name: " + name, end=" ") + for key, value in info.items(): + print("\t" + key + ": " + value, end=" ") + print() + + if s.current_step() == 0: + nproc = s.read("nproc") + print(f"nproc is {nproc} of type {type(nproc)}") + + # read variables return a numpy array with corresponding selection + physical_time = s.read("physical_time") + print(f"physical_time is {physical_time} of type {type(physical_time)}") + temperature = s.read("temperature") + temp_unit = s.read_attribute("temperature/unit") + print(f"temperature array size is {temperature.size} of shape {temperature.shape}") + print(f"temperature unit is {temp_unit} of type {type(temp_unit)}") + pressure = s.read("pressure") + press_unit = s.read_attribute("pressure/unit") + print(f"pressure unit is {press_unit} of type {type(press_unit)}") + print() + +.. code-block:: bash + + $ python3 adios2-doc-read.py + Current step is 0 + variable_name: nproc AvailableStepsCount: 1 Max: 4 Min: 4 Shape: SingleValue: true Type: int64_t + variable_name: physical_time AvailableStepsCount: 1 Max: 0 Min: 0 Shape: SingleValue: true Type: double + variable_name: pressure AvailableStepsCount: 1 Max: 1 Min: 1 Shape: 40 SingleValue: false Type: double + variable_name: temperature AvailableStepsCount: 1 Max: 0 Min: 0 Shape: 40 SingleValue: false Type: double + nproc is 4 of type + physical_time is 0.0 of type + temperature array size is 40 of shape (40,) + temperature unit is ['K'] of type + pressure unit is ['Pa'] of type + + Current step is 1 + variable_name: physical_time AvailableStepsCount: 1 Max: 0.01 Min: 0.01 Shape: SingleValue: true Type: double + variable_name: pressure AvailableStepsCount: 1 Max: 1 Min: 1 Shape: 40 SingleValue: false Type: double + variable_name: temperature AvailableStepsCount: 1 Max: 0 Min: 0 Shape: 40 SingleValue: false Type: double + physical_time is 0.01 of type + temperature array size is 40 of shape (40,) + temperature unit is ['K'] of type + pressure unit is ['Pa'] of type + + ... + + +Python Read Random Access example +---------------------------------- + +.. code-block:: python + + import numpy as np + from adios2 import FileReader + + with FileReader("cfd.bp") as s: + # inspect variables + vars = s.available_variables() + for name, info in vars.items(): + print("variable_name: " + name, end=" ") + for key, value in info.items(): + print("\t" + key + ": " + value, end=" ") + print() + + nproc = s.read("nproc") + print(f"nproc is {nproc} of type {type(nproc)}") + + # read variables return a numpy array with corresponding selection + steps = int(vars['physical_time']['AvailableStepsCount']) + physical_time = s.read("physical_time", step_selection=[0, steps]) + print(f"physical_time is {physical_time} of type {type(physical_time)}") + + steps = int(vars['temperature']['AvailableStepsCount']) + temperature = s.read("temperature", step_selection=[0, steps]) + temp_unit = s.read_attribute("temperature/unit") + print(f"temperature array size is {temperature.size} of shape {temperature.shape}") + print(f"temperature unit is {temp_unit} of type {type(temp_unit)}") + + steps = int(vars['pressure']['AvailableStepsCount']) + pressure = s.read("pressure", step_selection=[0, steps]) + press_unit = s.read_attribute("pressure/unit") + print() + +.. code-block:: bash + + $ python3 adios2-doc-read-filereader.py + variable_name: nproc AvailableStepsCount: 1 Max: 4 Min: 4 Shape: SingleValue: true Type: int64_t + variable_name: physical_time AvailableStepsCount: 5 Max: 0.04 Min: 0 Shape: SingleValue: true Type: double + variable_name: pressure AvailableStepsCount: 5 Max: 1 Min: 1 Shape: 40 SingleValue: false Type: double + variable_name: temperature AvailableStepsCount: 5 Max: 0 Min: 0 Shape: 40 SingleValue: false Type: double + nproc is 4 of type + physical_time is [0. 0.01 0.02 0.03 0.04] of type + temperature array size is 200 of shape (200,) + temperature unit is ['K'] of type + diff --git a/docs/user_guide/source/api_python/python_transition_from_high.rst b/docs/user_guide/source/api_python/python_transition_from_high.rst new file mode 100644 index 0000000000..2e00f294c2 --- /dev/null +++ b/docs/user_guide/source/api_python/python_transition_from_high.rst @@ -0,0 +1,41 @@ +********************************** +Transition from old API to new API +********************************** + +A python script using the high-level API of 2.9 and earlier need to be modified to work with 2.10 and later. + +- adios2.open() is replaced with adios2.Stream(), and does not have 4th and 5th optional arguments for external xml and IO name. +- the ``for in file`` is replaced with ``for _ in file.steps()`` but it works for both writing (by specifying the number of output steps) and reading (for the number of available steps in a stream/file). + +.. code-block:: python + + # OLD API + import adios2 + + # NEW API + from adios2 import Adios, Stream + + # NEW API: this still works + import adios2 + + + # OLD API + fr = adios2.open(args.instream, "r", mpi.comm_app,"adios2.xml", "SimulationOutput") + + # NEW API + adios = Adios("adios2.xml", mpi.comm_app) + io = adios.declare_io("SimulationOutput") + fr = Stream(io, args.instream, "r", mpi.comm_app) + + + # OLD API + for fr_step in fr: + fr_step.... + + # NEW API 1 + for _ in fr.steps(): + fr.... + + # NEW API 2 + for fr_step in fr.steps(): + fr_step.... diff --git a/docs/user_guide/source/conf.py b/docs/user_guide/source/conf.py index f3f49453d4..e3cf43f332 100644 --- a/docs/user_guide/source/conf.py +++ b/docs/user_guide/source/conf.py @@ -103,11 +103,11 @@ 'setting_up/source/hpc_systems.rst', 'setting_up/source/ctest.rst', 'setting_up/source/cpp_c.rst', + 'setting_up/doemachines.rst', 'setting_up/source.rst', 'setting_up/package.rst', 'setting_up/linking.rst', 'api_full/fortran.rst', - 'api_full/python.rst', 'api_full/cxx11.rst', # 'api_full/api_full.rst', 'api_full/c.rst', @@ -121,8 +121,11 @@ # 'components/components.rst', 'components/overview.rst', 'api_high/matlab.rst', - 'api_high/python.rst', 'api_high/cxx11.rst', + 'api_python/python_example.rst', + 'api_python/python.rst', + 'api_python/python_bindings.rst', + 'python_transition_from_high.rst', # 'api_high/api_high.rst', # 'ecosystem/utilities.rst', 'ecosystem/utilities/adios2-config.rst', diff --git a/docs/user_guide/source/index.rst b/docs/user_guide/source/index.rst index c976e6826d..a30daad270 100644 --- a/docs/user_guide/source/index.rst +++ b/docs/user_guide/source/index.rst @@ -30,6 +30,7 @@ Funded by the `Exascale Computing Project (ECP) `_ + + ================== What's new in 2.9? ================== diff --git a/docs/user_guide/source/setting_up/doemachines.rst b/docs/user_guide/source/setting_up/doemachines.rst new file mode 100644 index 0000000000..f6b0a13c5f --- /dev/null +++ b/docs/user_guide/source/setting_up/doemachines.rst @@ -0,0 +1,88 @@ +################### +Use on DOE machines +################### + +ADIOS2 is installed as part of the `E4S `_ software stack and access to adios2 is the same as access to the many other packages. + +***************************************** +NERSC Perlmutter +***************************************** + +To use adios2 on Perlmutter, + +- load the e4s module +- pick your compiler environment with spack +- load adios2 with spack + +.. code-block:: bash + + ~> module load e4s + _____________________________________________________________________________________ + The Extreme-Scale Scientific Software Stack (E4S) is accessible via the Spack package manager. + + In order to access the production stack, you will need to load a spack + environment. Here are some tips to get started: + + + 'spack env list' - List all Spack environments + 'spack env activate gcc' - Activate the "gcc" Spack environment + 'spack env status' - Display the active Spack environment + 'spack load amrex' - Load the "amrex" Spack package into your user environment + + For additional support, please refer to the following references: + + NERSC E4S Documentation: https://docs.nersc.gov/applications/e4s/ + E4S Documentation: https://e4s.readthedocs.io + Spack Documentation: https://spack.readthedocs.io/en/latest/ + Spack Slack: https://spackpm.slack.com + + _____________________________________________________________________________________ + + ~> spack env list + ==> 4 environments + cce cuda gcc nvhpc + ~> spack env activate gcc + ~> spack load adios2 + + ~> which bpls + /global/common/software/spackecp/perlmutter/e4s-23.08/94543/spack/opt/spack/linux-sles15-zen3/gcc-12.3.0/adios2-2.9.1-iwv5lkkc5gyagr4uqrqr4v2fds7x66pk/bin/bpls + + ~> bpls -Vv + blps: ADIOS file introspection utility + + Build configuration: + ADIOS version: 2.9.1 + C++ Compiler: GNU 12.3.0 (CrayPrgEnv) + Target OS: Linux-5.14.21-150400.24.81_12.0.87-cray_shasta_c + Target Arch: x86_64 + Available engines = 10: BP3, BP4, BP5, SST, SSC, Inline, MHS, + ParaViewADIOSInSituEngine, Null, Skeleton + Available operators = 4: BZip2, SZ, ZFP, PNG + Available features = 16: BP5, DATAMAN, MHS, SST, FORTRAN, MPI, BZIP2, PNG, + SZ, ZFP, O_DIRECT, CATALYST, SYSVSHMEM, ZEROMQ, PROFILING, ENDIAN_REVERSE + + +***************************************** +OLCF Frontier +***************************************** + +OLCF installs the E4S packages in individual modules, hence `adios2` is also available as a module. + +.. code-block:: bash + + $ module avail adios2 + ----- /sw/frontier/spack-envs/base/modules/spack/cray-sles15-x86_64/cray-mpich/8.1.23-j56azw5/cce/15.0.0 ----- + adios2/2.8.1 adios2/2.8.3 (D) + + Where: + D: Default Module + + $ module load adios2 + $ bpls -Vv + blps: ADIOS file introspection utility + + Build configuration: + ADIOS version: 2.8.3 + C++ Compiler: GNU 12.2.0 (CrayPrgEnv) + Target OS: Linux-5.14.21-150400.24.11_12.0.57-cray_shasta_c + Target Arch: x86_64 diff --git a/docs/user_guide/source/setting_up/package.rst b/docs/user_guide/source/setting_up/package.rst index 3c7c7a8b41..286d4e1c9f 100644 --- a/docs/user_guide/source/setting_up/package.rst +++ b/docs/user_guide/source/setting_up/package.rst @@ -2,26 +2,27 @@ As Package ########## -***** +***************************************** Conda -***** +***************************************** -Currently ADIOS 2 can be obtained from anaconda cloud: +ADIOS2 can be obtained from anaconda cloud: `conda-forge adios2 `_ -* x86-64 and MacOS: `williamfgc adios2-openmpi adios2-mpich adios2-nompi `_ -* Multiple archs: `conda-forge adios2 `_ +***************************************** +PyPI +***************************************** +ADIOS2 pip package can be downloaded with `pip3 install adios2` or `python3 -m pip install adios2`. This is contains the serial build only, so MPI programs cannot use it. See `adios2 on PyPi `_ -***** +***************************************** Spack -***** +***************************************** -ADIOS 2 is packaged in Spack `adios2 package `_ +ADIOS2 is packaged in Spack. See `adios2 spack package `_ - -****** +***************************************** Docker -****** +***************************************** Docker images including building and installation of dependencies and ADIOS 2 containers for Ubuntu 20 and CentOS 7 can be found in: under the directory `scripts/docker/ `_ diff --git a/docs/user_guide/source/setting_up/setting_up.rst b/docs/user_guide/source/setting_up/setting_up.rst index 980f04c7c9..63d2e4b567 100644 --- a/docs/user_guide/source/setting_up/setting_up.rst +++ b/docs/user_guide/source/setting_up/setting_up.rst @@ -1,4 +1,6 @@ + .. include:: source.rst .. include:: package.rst .. include:: linking.rst +.. include:: doemachines.rst \ No newline at end of file diff --git a/docs/user_guide/source/setting_up/source/cmake.rst b/docs/user_guide/source/setting_up/source/cmake.rst index 0fe2210f45..d268fdd764 100644 --- a/docs/user_guide/source/setting_up/source/cmake.rst +++ b/docs/user_guide/source/setting_up/source/cmake.rst @@ -8,31 +8,77 @@ To build ADIOS v2.x, clone the repository and invoke the canonical CMake build s $ git clone https://github.com/ornladios/ADIOS2.git ADIOS2 $ mkdir adios2-build && cd adios2-build - $ cmake ../ADIOS2 - -- The C compiler identification is GNU 7.3.0 - -- The CXX compiler identification is GNU 7.3.0 + $ cmake ../ADIOS2 cmake -DADIOS2_BUILD_EXAMPLES=ON + -- The C compiler identification is GNU 9.4.0 + -- The CXX compiler identification is GNU 9.4.0 ... ADIOS2 build configuration: + ADIOS Version: 2.10.0 + C++ Compiler : GNU 9.4.0 + /usr/bin/c++ + + Fortran Compiler : GNU 9.4.0 + /usr/bin/f95 + + Installation prefix: /usr/local + bin: bin + lib: lib + include: include + cmake: lib/cmake/adios2 + python: lib/python3/dist-packages + ... Features: Library Type: shared Build Type: Release - Testing: ON + Testing: OFF + Examples: ON Build Options: - BZip2 : ON - ZFP : OFF - SZ : OFF - MGARD : OFF - MPI : ON - DataMan : ON - SST : ON - ZeroMQ : ON - HDF5 : ON - Python : ON - Fortran : ON - SysVShMem: ON - Endian_Reverse: OFF + DataMan : ON + DataSpaces : OFF + HDF5 : OFF + HDF5_VOL : OFF + MHS : ON + SST : ON + Fortran : ON + MPI : ON + Python : ON + PIP : OFF + Blosc2 : OFF + BZip2 : ON + LIBPRESSIO : OFF + MGARD : OFF + MGARD_MDR : OFF + PNG : OFF + SZ : OFF + ZFP : ON + DAOS : OFF + IME : OFF + O_DIRECT : ON + Sodium : ON + Catalyst : OFF + SysVShMem : ON + UCX : OFF + ZeroMQ : ON + Profiling : ON + Endian_Reverse : OFF + Derived_Variable : OFF + AWSSDK : OFF + GPU_Support : OFF + CUDA : OFF + Kokkos : OFF + Kokkos_CUDA : OFF + Kokkos_HIP : OFF + Kokkos_SYCL : OFF + Campaign : OFF + + +If a desired feature is OFF in the report above, tell cmake where to find the required dependencies for that feature and manually turn it on. E.g.: + +.. code-block:: bash + + $ cmake ... -DADIOS2_USE_Blosc2=ON -DCMAKE_PREFIX_PATH="" Then compile using @@ -41,7 +87,7 @@ Then compile using $ make -j 16 -Optionally, run the tests: +Optionally, run the tests (need to configure with ``-DBUILD_TESTING=ON`` cmake flag) .. code-block:: bash @@ -94,7 +140,7 @@ VAR VALUE Description ``ADIOS2_USE_SZ`` **ON**/OFF `SZ `_ compression (experimental). ``ADIOS2_USE_MGARD`` **ON**/OFF `MGARD `_ compression (experimental). ``ADIOS2_USE_PNG`` **ON**/OFF `PNG `_ compression (experimental). -``ADIOS2_USE_Blosc`` **ON**/OFF `Blosc `_ compression (experimental). +``ADIOS2_USE_Blosc2`` **ON**/OFF `Blosc `_ compression (experimental). ``ADIOS2_USE_Endian_Reverse`` ON/**OFF** Enable endian conversion if a different endianness is detected between write and read. ``ADIOS2_USE_IME`` ON/**OFF** DDN IME transport. ============================= ================ ========================================================================================================================================================================================================================== @@ -109,6 +155,7 @@ In addition to the ``ADIOS2_USE_Feature`` options, the following options are als ``BUILD_TESTING`` ON/**OFF** Build test code. ``CMAKE_INSTALL_PREFIX`` /path/to/install (``/usr/local``) Installation location. ``CMAKE_BUILD_TYPE`` Debug/**Release**/RelWithDebInfo/MinSizeRel Compiler optimization levels. +``CMAKE_PREFIX_PATH`` Semi-colon separeated list of paths Location of extra dependencies ==================================== =============================================== =============================== From fea759f7bd8d5cf09e8fecf7c06d39b57413071b Mon Sep 17 00:00:00 2001 From: anagainaru Date: Wed, 6 Mar 2024 09:12:04 -0500 Subject: [PATCH 069/124] Small typo fixes --- docs/user_guide/source/api_python/python.rst | 2 +- docs/user_guide/source/api_python/python_bindings.rst | 2 +- docs/user_guide/source/api_python/python_example.rst | 2 +- .../source/api_python/python_transition_from_high.rst | 2 +- docs/user_guide/source/introduction/whatsnew.rst | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/user_guide/source/api_python/python.rst b/docs/user_guide/source/api_python/python.rst index 3253000c6d..9b615770d9 100644 --- a/docs/user_guide/source/api_python/python.rst +++ b/docs/user_guide/source/api_python/python.rst @@ -2,7 +2,7 @@ adios2 classes for Python ************************* -``Stream`` is a high-level class that can perform most of the ADIOS functionality. ``FileReader`` is just a convenience class and is the same as Stream with "rra" (ReadRandomAccess) mode. FileReaders do not work with for loops as opposed to Streams that work step-by-step, rather one can access any step of any variable at will. The other classes, ``Adios``, ``IO``, ``Engine``, ``Variable``, ``Attribute`` and ``Operator`` correspond to the C++ classes. One need to use them to extend the capabilities of the ``Stream`` class (e.g. using an external XML file for runtime configuration, changing the engine for the run, setting up a compression operator for an output variable, etc.) +``Stream`` is a high-level class that can perform most of the ADIOS functionality. ``FileReader`` is just a convenience class and is the same as Stream with "rra" (ReadRandomAccess) mode. FileReaders do not work with for loops as opposed to Streams that work step-by-step, rather one can access any step of any variable at will. The other classes, ``Adios``, ``IO``, ``Engine``, ``Variable``, ``Attribute`` and ``Operator`` correspond to the C++ classes. One needs to use them to extend the capabilities of the ``Stream`` class (e.g. using an external XML file for runtime configuration, changing the engine for the run, setting up a compression operator for an output variable, etc.) .. autoclass:: adios2::Stream :members: diff --git a/docs/user_guide/source/api_python/python_bindings.rst b/docs/user_guide/source/api_python/python_bindings.rst index cd0ef4383c..d8dba17248 100644 --- a/docs/user_guide/source/api_python/python_bindings.rst +++ b/docs/user_guide/source/api_python/python_bindings.rst @@ -10,7 +10,7 @@ Python bindings to C++ import adios2.bindings as adios2 -The full Python APIs follow very closely the full C++11 API interface. All of its functionality is now in the native API as well, so its use is discouraged for future scripts. +The full Python APIs follows very closely the full C++11 API interface. All of its functionality is now in the native API as well, so its use is discouraged for future scripts. Examples using the Python bindings in the ADIOS2 repository ----------------------------------------------------------- diff --git a/docs/user_guide/source/api_python/python_example.rst b/docs/user_guide/source/api_python/python_example.rst index ceec6ae43d..d2458a12c2 100644 --- a/docs/user_guide/source/api_python/python_example.rst +++ b/docs/user_guide/source/api_python/python_example.rst @@ -2,7 +2,7 @@ Python Example Code ******************* -The Python APIs follow closely Python style directives. They rely on numpy and, optionally, on ``mpi4py``, if the underlying ADIOS2 library is compiled with MPI. +The Python APIs follows closely Python style directives. They rely on numpy and, optionally, on ``mpi4py``, if the underlying ADIOS2 library is compiled with MPI. For online examples on MyBinder : diff --git a/docs/user_guide/source/api_python/python_transition_from_high.rst b/docs/user_guide/source/api_python/python_transition_from_high.rst index 2e00f294c2..66ef52cb58 100644 --- a/docs/user_guide/source/api_python/python_transition_from_high.rst +++ b/docs/user_guide/source/api_python/python_transition_from_high.rst @@ -2,7 +2,7 @@ Transition from old API to new API ********************************** -A python script using the high-level API of 2.9 and earlier need to be modified to work with 2.10 and later. +A python script using the high-level API of 2.9 and earlier needs to be modified to work with 2.10 and later. - adios2.open() is replaced with adios2.Stream(), and does not have 4th and 5th optional arguments for external xml and IO name. - the ``for in file`` is replaced with ``for _ in file.steps()`` but it works for both writing (by specifying the number of output steps) and reading (for the number of available steps in a stream/file). diff --git a/docs/user_guide/source/introduction/whatsnew.rst b/docs/user_guide/source/introduction/whatsnew.rst index 179010144f..92d4191992 100644 --- a/docs/user_guide/source/introduction/whatsnew.rst +++ b/docs/user_guide/source/introduction/whatsnew.rst @@ -24,7 +24,7 @@ New/updated features -------------------- - BP5 is supported on Windows now - - SST staging engine is GPU-Aware now + - SST and DataMan staging engines are GPU-Aware now - SYCL support added for Intel GPUs (besides CUDA and HIP for NVidia and AMD GPUs) - the SST/libfabric data transport now works on Frontier (besides the MPI data transport) From 826323b72165b7413c980ae9576dde96bccfd4c5 Mon Sep 17 00:00:00 2001 From: Ana Gainaru Date: Tue, 5 Mar 2024 16:45:38 -0500 Subject: [PATCH 070/124] Adding c bindings for setting and getting the memory space --- bindings/C/adios2/c/adios2_c_types.h | 9 +++ bindings/C/adios2/c/adios2_c_variable.cpp | 84 +++++++++++++++++++++++ bindings/C/adios2/c/adios2_c_variable.h | 16 +++++ bindings/CXX11/adios2/cxx11/Variable.h | 6 +- 4 files changed, 112 insertions(+), 3 deletions(-) diff --git a/bindings/C/adios2/c/adios2_c_types.h b/bindings/C/adios2/c/adios2_c_types.h index c61504d32d..e503e3da78 100644 --- a/bindings/C/adios2/c/adios2_c_types.h +++ b/bindings/C/adios2/c/adios2_c_types.h @@ -159,6 +159,15 @@ union adios2_PrimitiveStdtypeUnion char *str; }; +typedef enum +{ + adios2_memory_space_host = 1, +#ifdef ADIOS2_HAVE_GPU_SUPPORT + adios2_memory_space_detect = 0, + adios2_memory_space_gpu = 2, +#endif +} adios2_memory_space; + typedef struct { int WriterID; diff --git a/bindings/C/adios2/c/adios2_c_variable.cpp b/bindings/C/adios2/c/adios2_c_variable.cpp index 0dc04d7ef2..da0df55540 100644 --- a/bindings/C/adios2/c/adios2_c_variable.cpp +++ b/bindings/C/adios2/c/adios2_c_variable.cpp @@ -77,6 +77,90 @@ adios2_error adios2_set_shape(adios2_variable *variable, const size_t ndims, con } } +adios2::MemorySpace adios2_ToMemorySpace(const adios2_memory_space Cmem) +{ +#ifdef ADIOS2_HAVE_GPU_SUPPORT + adios2::MemorySpace mem = adios2::MemorySpace::Detect; +#else + adios2::MemorySpace mem = adios2::MemorySpace::Host; +#endif + switch (Cmem) + { + + case adios2_memory_space_host: + mem = adios2::MemorySpace::Host; + break; + +#ifdef ADIOS2_HAVE_GPU_SUPPORT + case adios2_memory_space_gpu: + mem = adios2::MemorySpace::GPU; + break; +#endif + default: + break; + } + return mem; +} + +adios2_memory_space adios2_FromMemorySpace(const adios2::MemorySpace mem) +{ +#ifdef ADIOS2_HAVE_GPU_SUPPORT + adios2_memory_space Cmem = adios2_memory_space_detect; +#else + adios2_memory_space Cmem = adios2_memory_space_host; +#endif + switch (mem) + { + + case adios2::MemorySpace::Host: + Cmem = adios2_memory_space_host; + break; + +#ifdef ADIOS2_HAVE_GPU_SUPPORT + case adios2::MemorySpace::GPU: + Cmem = adios2_memory_space_gpu; + break; +#endif + default: + break; + } + return Cmem; +} + +adios2_error adios2_set_memory_space(adios2_variable *variable, const adios2_memory_space mem) +{ + try + { + adios2::core::VariableBase *variableBase = + reinterpret_cast(variable); + variableBase->SetMemorySpace(adios2_ToMemorySpace(mem)); + + return adios2_error_none; + } + catch (...) + { + return static_cast( + adios2::helper::ExceptionToError("adios2_set_memory_space")); + } +} + +adios2_error adios2_get_memory_space(adios2_memory_space *mem, adios2_variable *variable) +{ + try + { + adios2::core::VariableBase *variableBase = + reinterpret_cast(variable); + *mem = adios2_FromMemorySpace(variableBase->m_MemSpace); + + return adios2_error_none; + } + catch (...) + { + return static_cast( + adios2::helper::ExceptionToError("adios2_set_memory_space")); + } +} + adios2_error adios2_set_block_selection(adios2_variable *variable, const size_t block_id) { try diff --git a/bindings/C/adios2/c/adios2_c_variable.h b/bindings/C/adios2/c/adios2_c_variable.h index 165e5881f2..0e027df9f1 100644 --- a/bindings/C/adios2/c/adios2_c_variable.h +++ b/bindings/C/adios2/c/adios2_c_variable.h @@ -29,6 +29,22 @@ extern "C" { */ adios2_error adios2_set_shape(adios2_variable *variable, const size_t ndims, const size_t *shape); +/** + * Sets the memory space for all following Puts/Gets + * to either host (default) or device + * @param mem memory space where Put/Get buffers are allocated + * @return adios2_error 0: success, see enum adios2_error for errors + */ +adios2_error adios2_set_memory_space(adios2_variable *variable, const adios2_memory_space mem); + +/** + * Get the memory space that was set by the application for a given variable + * @param memory space output, the variable memory space + * @param variable handler + * @return adios2_error 0: success, see enum adios2_error for errors + */ +adios2_error adios2_get_memory_space(adios2_memory_space *mem, adios2_variable *variable); + /** * Read mode only. Required for reading local variables. For Global Arrays it * will Set diff --git a/bindings/CXX11/adios2/cxx11/Variable.h b/bindings/CXX11/adios2/cxx11/Variable.h index 2dd77adf9a..b2263a5780 100644 --- a/bindings/CXX11/adios2/cxx11/Variable.h +++ b/bindings/CXX11/adios2/cxx11/Variable.h @@ -148,9 +148,9 @@ class Variable explicit operator bool() const noexcept; /** - * Sets the memory space for all following Puts - * to either host (default) or device (currently only CUDA supported) - * @param mem memory space where Put buffers are allocated + * Sets the memory space for all following Puts/Gets + * to either host (default) or device + * @param mem memory space where Put/Get buffers are allocated */ void SetMemorySpace(const MemorySpace mem); From c1dcbf618fa371abdaa4341c897da29687515890 Mon Sep 17 00:00:00 2001 From: Ana Gainaru Date: Tue, 5 Mar 2024 16:46:17 -0500 Subject: [PATCH 071/124] Adding c bindings for getting the shape of a variable based on a memory space --- bindings/C/adios2/c/adios2_c_variable.cpp | 14 ++++++++++++-- bindings/C/adios2/c/adios2_c_variable.h | 10 ++++++++++ source/adios2/core/VariableBase.cpp | 9 +++++++++ 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/bindings/C/adios2/c/adios2_c_variable.cpp b/bindings/C/adios2/c/adios2_c_variable.cpp index da0df55540..f17b8c44ae 100644 --- a/bindings/C/adios2/c/adios2_c_variable.cpp +++ b/bindings/C/adios2/c/adios2_c_variable.cpp @@ -358,7 +358,8 @@ adios2_error adios2_variable_ndims(size_t *ndims, const adios2_variable *variabl } } -adios2_error adios2_variable_shape(size_t *shape, const adios2_variable *variable) +adios2_error adios2_variable_shape_with_memory_space(size_t *shape, const adios2_variable *variable, + adios2_memory_space mem) { try { @@ -380,7 +381,11 @@ adios2_error adios2_variable_shape(size_t *shape, const adios2_variable *variabl { \ const adios2::core::Variable *variable = \ dynamic_cast *>(variableBase); \ - const adios2::Dims shapeCpp = variable->Shape(adios2::EngineCurrentStep); \ + adios2::Dims shapeCpp; \ + if (mem == adios2_memory_space_host) \ + shapeCpp = variable->Shape(adios2::EngineCurrentStep); \ + else \ + shapeCpp = variable->Shape(adios2::EngineCurrentStep, adios2_ToMemorySpace(mem)); \ std::copy(shapeCpp.begin(), shapeCpp.end(), shape); \ } ADIOS2_FOREACH_STDTYPE_1ARG(declare_template_instantiation) @@ -394,6 +399,11 @@ adios2_error adios2_variable_shape(size_t *shape, const adios2_variable *variabl } } +adios2_error adios2_variable_shape(size_t *shape, const adios2_variable *variable) +{ + return adios2_variable_shape_with_memory_space(shape, variable, adios2_memory_space_host); +} + adios2_error adios2_variable_start(size_t *start, const adios2_variable *variable) { try diff --git a/bindings/C/adios2/c/adios2_c_variable.h b/bindings/C/adios2/c/adios2_c_variable.h index 0e027df9f1..0ed2957020 100644 --- a/bindings/C/adios2/c/adios2_c_variable.h +++ b/bindings/C/adios2/c/adios2_c_variable.h @@ -156,6 +156,16 @@ adios2_error adios2_variable_ndims(size_t *ndims, const adios2_variable *variabl */ adios2_error adios2_variable_shape(size_t *shape, const adios2_variable *variable); +/** + * Retrieve current variable shape for a given memory space + * @param shape output, must be pre-allocated with ndims + * @param variable handler + * @param memory space + * @return adios2_error 0: success, see enum adios2_error for errors + */ +adios2_error adios2_variable_shape_with_memory_space(size_t *shape, const adios2_variable *variable, + const adios2_memory_space mem); + /** * Retrieve current variable start * @param start output, single value diff --git a/source/adios2/core/VariableBase.cpp b/source/adios2/core/VariableBase.cpp index bd8058cd49..c55ca7ca15 100644 --- a/source/adios2/core/VariableBase.cpp +++ b/source/adios2/core/VariableBase.cpp @@ -647,6 +647,15 @@ Dims VariableBase::Shape(const size_t step, const MemorySpace memSpace, const ArrayOrdering layout) const { auto dims = Shape(step); +#ifdef ADIOS2_HAVE_GPU_SUPPORT + if (memSpace == MemorySpace::Detect) + { + helper::Throw("Core", "Variable", "Shape", + "can't call Shape() for variable " + m_Name + + " using the Detect memory space." + "Host/GPU need to be selected"); + } +#endif #if defined(ADIOS2_HAVE_KOKKOS) || defined(ADIOS2_HAVE_GPU_SUPPORT) bool mismatchMemSpace = (memSpace != MemorySpace::Host && m_BaseLayout != ArrayOrdering::ColumnMajor); From c2293b1466d9e45ef93048aebeacded6d547dfbc Mon Sep 17 00:00:00 2001 From: Ana Gainaru Date: Tue, 5 Mar 2024 19:05:33 -0500 Subject: [PATCH 072/124] Testing code for the C bindings with memory space API --- testing/adios2/bindings/C/CMakeLists.txt | 1 + .../adios2/bindings/C/TestBPMemorySpace.cpp | 128 ++++++++++++++++++ 2 files changed, 129 insertions(+) create mode 100644 testing/adios2/bindings/C/TestBPMemorySpace.cpp diff --git a/testing/adios2/bindings/C/CMakeLists.txt b/testing/adios2/bindings/C/CMakeLists.txt index 3990e4d5ea..0a1f8d3c67 100644 --- a/testing/adios2/bindings/C/CMakeLists.txt +++ b/testing/adios2/bindings/C/CMakeLists.txt @@ -8,3 +8,4 @@ gtest_add_tests_helper(WriteReadMultiblock MPI_ALLOW BP Bindings.C. "") gtest_add_tests_helper(NullWriteRead MPI_ALLOW "" Bindings.C. "") gtest_add_tests_helper(WriteAggregateReadLocal MPI_ONLY BP Bindings.C. "") gtest_add_tests_helper(AvailableVariablesAttribites MPI_ONLY BP Bindings.C. "") +gtest_add_tests_helper(MemorySpace MPI_NONE BP Bindings.C. "") diff --git a/testing/adios2/bindings/C/TestBPMemorySpace.cpp b/testing/adios2/bindings/C/TestBPMemorySpace.cpp new file mode 100644 index 0000000000..65c2381969 --- /dev/null +++ b/testing/adios2/bindings/C/TestBPMemorySpace.cpp @@ -0,0 +1,128 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * TestBPMemorySpace.cpp : test the C bindings related to memory space + */ + +#include +#include + +class ADIOS2_C_API : public ::testing::Test +{ +public: + ADIOS2_C_API() { adiosH = adios2_init_serial(); } + + ~ADIOS2_C_API() { adios2_finalize(adiosH); } + + adios2_adios *adiosH; +}; + +#ifdef ADIOS2_HAVE_GPU_SUPPORT +TEST_F(ADIOS2_C_API, ADIOS2BPMemorySpaceGPU) +{ + adios2_io *ioH = adios2_declare_io(adiosH, "CMemSpace"); + adios2_set_engine(ioH, "BPFile"); + + size_t shape[1]; + shape[0] = 10; + + size_t start[1]; + start[0] = 0; + + size_t count[1]; + count[0] = 10; + + adios2_define_variable(ioH, "varI32", adios2_type_int32_t, 1, shape, start, count, + adios2_constant_dims_true); + adios2_variable *varI32 = adios2_inquire_variable(ioH, "varI32"); + + // test that the default memory space is Detect + adios2_memory_space mem; + adios2_error e = adios2_get_memory_space(&mem, varI32); + EXPECT_EQ(e, adios2_error_none); + EXPECT_EQ(mem, adios2_memory_space_detect); + // test that the set memory space is GPU + adios2_set_memory_space(varI32, adios2_memory_space_gpu); + e = adios2_get_memory_space(&mem, varI32); + EXPECT_EQ(e, adios2_error_none); + EXPECT_EQ(mem, adios2_memory_space_gpu); +} +#endif + +TEST_F(ADIOS2_C_API, ADIOS2BPMemorySpaceShape) +{ + const char fname[] = "ADIOS2_C_API.ADIOS2BPMemorySpace.bp"; + // write + { + adios2_io *ioH = adios2_declare_io(adiosH, "CMemSpace"); + adios2_set_engine(ioH, "BPFile"); + + size_t shape[2]; + shape[0] = 5; + shape[1] = 2; + + size_t start[2]; + start[0] = 0; + start[1] = 0; + + size_t count[2]; + count[0] = 5; + count[1] = 2; + + int32_t data_I32[10] = {131072, 131073, -131070, 131075, -131068, + 131077, -131066, 131079, -131064, 131081}; + adios2_define_variable(ioH, "varI32", adios2_type_int32_t, 2, shape, start, count, + adios2_constant_dims_true); + adios2_variable *varI32 = adios2_inquire_variable(ioH, "varI32"); + + // test that the set memory space is Host + adios2_memory_space mem; + adios2_set_memory_space(varI32, adios2_memory_space_host); + adios2_error e = adios2_get_memory_space(&mem, varI32); + EXPECT_EQ(e, adios2_error_none); + EXPECT_EQ(mem, adios2_memory_space_host); + + adios2_engine *engineH = adios2_open(ioH, fname, adios2_mode_write); + adios2_put(engineH, varI32, data_I32, adios2_mode_deferred); + adios2_close(engineH); + } + // read shape + { + adios2_io *ioH = adios2_declare_io(adiosH, "Reader"); + adios2_engine *engineH = adios2_open(ioH, fname, adios2_mode_readRandomAccess); + size_t steps; + adios2_steps(&steps, engineH); + EXPECT_EQ(steps, 1); + + adios2_variable *varI32 = adios2_inquire_variable(ioH, "varI32"); + + // test that the shape function returns the correct dimensions + size_t cpu_shape[2]; + adios2_error e = adios2_variable_shape(cpu_shape, varI32); + EXPECT_EQ(e, adios2_error_none); + EXPECT_EQ(cpu_shape[0], 5); + EXPECT_EQ(cpu_shape[1], 2); + e = adios2_variable_shape_with_memory_space(cpu_shape, varI32, adios2_memory_space_host); + EXPECT_EQ(e, adios2_error_none); + EXPECT_EQ(cpu_shape[0], 5); + EXPECT_EQ(cpu_shape[1], 2); +#ifdef ADIOS2_HAVE_GPU_SUPPORT + size_t gpu_shape[2]; + e = adios2_variable_shape_with_memory_space(gpu_shape, varI32, adios2_memory_space_gpu); + EXPECT_EQ(e, adios2_error_none); + EXPECT_EQ(gpu_shape[0], 2); + EXPECT_EQ(gpu_shape[1], 5); +#endif + adios2_close(engineH); + } +} + +int main(int argc, char **argv) +{ + int result; + ::testing::InitGoogleTest(&argc, argv); + result = RUN_ALL_TESTS(); + + return result; +} From 5142ace710ecf601e62e27a5960a6981f67357c0 Mon Sep 17 00:00:00 2001 From: anagainaru Date: Wed, 6 Mar 2024 11:34:47 -0500 Subject: [PATCH 073/124] Making the Detect memory space available regardless of the backend used --- bindings/C/adios2/c/adios2_c_types.h | 2 +- source/adios2/common/ADIOSTypes.h | 4 +--- source/adios2/core/VariableBase.cpp | 4 ++-- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/bindings/C/adios2/c/adios2_c_types.h b/bindings/C/adios2/c/adios2_c_types.h index e503e3da78..81f6ad3479 100644 --- a/bindings/C/adios2/c/adios2_c_types.h +++ b/bindings/C/adios2/c/adios2_c_types.h @@ -161,9 +161,9 @@ union adios2_PrimitiveStdtypeUnion typedef enum { + adios2_memory_space_detect = 0, adios2_memory_space_host = 1, #ifdef ADIOS2_HAVE_GPU_SUPPORT - adios2_memory_space_detect = 0, adios2_memory_space_gpu = 2, #endif } adios2_memory_space; diff --git a/source/adios2/common/ADIOSTypes.h b/source/adios2/common/ADIOSTypes.h index 5085ccf295..817e4da2b1 100644 --- a/source/adios2/common/ADIOSTypes.h +++ b/source/adios2/common/ADIOSTypes.h @@ -45,10 +45,8 @@ enum class DerivedVarType /** Memory space for the user provided buffers */ enum class MemorySpace { -#ifdef ADIOS2_HAVE_GPU_SUPPORT Detect, ///< Detect the memory space automatically -#endif - Host, ///< Host memory space + Host, ///< Host memory space #ifdef ADIOS2_HAVE_GPU_SUPPORT GPU ///< GPU memory space #endif diff --git a/source/adios2/core/VariableBase.cpp b/source/adios2/core/VariableBase.cpp index c55ca7ca15..2eb3442c0c 100644 --- a/source/adios2/core/VariableBase.cpp +++ b/source/adios2/core/VariableBase.cpp @@ -77,19 +77,19 @@ MemorySpace VariableBase::GetMemorySpace(const void *ptr) #if defined(ADIOS2_HAVE_KOKKOS) || defined(ADIOS2_HAVE_GPU_SUPPORT) ArrayOrdering layout = m_BaseLayout; #endif -#ifdef ADIOS2_HAVE_GPU_SUPPORT // first time the memory space is set if (m_MemSpace == MemorySpace::Detect) { +#ifdef ADIOS2_HAVE_GPU_SUPPORT if (helper::IsGPUbuffer(ptr)) { m_MemSpace = MemorySpace::GPU; layout = ArrayOrdering::ColumnMajor; } else +#endif m_MemSpace = MemorySpace::Host; } -#endif #if defined(ADIOS2_HAVE_KOKKOS) || defined(ADIOS2_HAVE_GPU_SUPPORT) // set the layout based on the buffer memory space // skipping throwing an exception for a mismatch From e896e7b5dd8e6c9bffcca9654d2471f4fe04b826 Mon Sep 17 00:00:00 2001 From: Greg Eisenhauer Date: Wed, 6 Mar 2024 17:13:40 -0500 Subject: [PATCH 074/124] consolidate (#4078) --- .../adios2/toolkit/remote/remote_server.cpp | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/source/adios2/toolkit/remote/remote_server.cpp b/source/adios2/toolkit/remote/remote_server.cpp index 98ff1bcdcd..e46e06bca0 100644 --- a/source/adios2/toolkit/remote/remote_server.cpp +++ b/source/adios2/toolkit/remote/remote_server.cpp @@ -304,7 +304,25 @@ static void ReadRequestHandler(CManager cm, CMConnection conn, void *vevent, voi f->m_CurrentOffset = ReadMsg->Offset; } char *tmp = (char *)malloc(ReadMsg->Size); - read(f->m_FileDescriptor, tmp, ReadMsg->Size); + size_t remaining = ReadMsg->Size; + char *pointer = tmp; + while (remaining > 0) + { + ssize_t ret = read(f->m_FileDescriptor, pointer, remaining); + if (ret <= 0) + { + // EOF or error, should send a message back, but we haven't define error handling yet + std::cout << "Read failed! BAD!" << std::endl; + // instead free tmp and return; + free(tmp); + return; + } + else + { + remaining -= ret; + pointer += ret; + } + } f->m_CurrentOffset += ReadMsg->Size; _ReadResponseMsg Response; memset(&Response, 0, sizeof(Response)); From 7572090b02ab55ee1d03a49ea3e9c15b03e35401 Mon Sep 17 00:00:00 2001 From: Ana Gainaru Date: Thu, 7 Mar 2024 08:58:57 -0500 Subject: [PATCH 075/124] Fortran bindings for memory space related functions (#4077) * Fortran bindings to set a memory space * wip * format --- bindings/Fortran/f2c/adios2_f2c_variable.cpp | 19 +++++++++ .../Fortran/modules/adios2_parameters_mod.f90 | 4 ++ .../Fortran/modules/adios2_variable_mod.f90 | 16 ++++++++ .../adios2/bindings/fortran/CMakeLists.txt | 4 ++ .../bindings/fortran/TestBPMemorySpace.F90 | 41 +++++++++++++++++++ .../bindings/fortran/TestBPMemorySpaceGPU.F90 | 41 +++++++++++++++++++ 6 files changed, 125 insertions(+) create mode 100644 testing/adios2/bindings/fortran/TestBPMemorySpace.F90 create mode 100644 testing/adios2/bindings/fortran/TestBPMemorySpaceGPU.F90 diff --git a/bindings/Fortran/f2c/adios2_f2c_variable.cpp b/bindings/Fortran/f2c/adios2_f2c_variable.cpp index b03ac1ef84..2dc811a163 100644 --- a/bindings/Fortran/f2c/adios2_f2c_variable.cpp +++ b/bindings/Fortran/f2c/adios2_f2c_variable.cpp @@ -97,6 +97,25 @@ void FC_GLOBAL(adios2_variable_steps_f2c, } } +void FC_GLOBAL(adios2_set_memory_space_f2c, ADIOS2_SET_MEMORY_SPACE_F2C)(adios2_variable **variable, + const int *mem, int *ierr) +{ + *ierr = static_cast( + adios2_set_memory_space(*variable, static_cast(*mem))); +} + +void FC_GLOBAL(adios2_get_memory_space_f2c, + ADIOS2_GET_MEMORY_SPACE_F2C)(int *mem, adios2_variable **variable, int *ierr) +{ + *mem = 0; + adios2_memory_space Cmem; + *ierr = static_cast(adios2_get_memory_space(&Cmem, *variable)); + if (*ierr == static_cast(adios2_error_none)) + { + *mem = static_cast(Cmem); + } +} + void FC_GLOBAL(adios2_set_shape_f2c, ADIOS2_SET_SHAPE_F2C)(adios2_variable **variable, const int *ndims, const int64_t *shape, int *ierr) diff --git a/bindings/Fortran/modules/adios2_parameters_mod.f90 b/bindings/Fortran/modules/adios2_parameters_mod.f90 index 6d73405849..a96fd8549a 100644 --- a/bindings/Fortran/modules/adios2_parameters_mod.f90 +++ b/bindings/Fortran/modules/adios2_parameters_mod.f90 @@ -60,6 +60,10 @@ module adios2_parameters_mod integer, parameter :: adios2_mode_deferred = 4 integer, parameter :: adios2_mode_sync = 5 + integer, parameter :: adios2_memory_space_detect = 0 + integer, parameter :: adios2_memory_space_host = 1 + integer, parameter :: adios2_memory_space_gpu = 2 + ! Step Mode integer, parameter :: adios2_step_mode_append = 0 integer, parameter :: adios2_step_mode_update = 1 diff --git a/bindings/Fortran/modules/adios2_variable_mod.f90 b/bindings/Fortran/modules/adios2_variable_mod.f90 index d88de02155..0b15b1b3c5 100644 --- a/bindings/Fortran/modules/adios2_variable_mod.f90 +++ b/bindings/Fortran/modules/adios2_variable_mod.f90 @@ -86,6 +86,22 @@ subroutine adios2_variable_steps(steps, variable, ierr) end subroutine + subroutine adios2_set_memory_space(variable, mem, ierr) + type(adios2_variable), intent(in) :: variable + integer, intent(in) :: mem + integer, intent(out) :: ierr + + call adios2_set_memory_space_f2c(variable%f2c, mem, ierr) + end subroutine + + subroutine adios2_get_memory_space(mem, variable, ierr) + integer, intent(out) :: mem + type(adios2_variable), intent(in) :: variable + integer, intent(out) :: ierr + + call adios2_get_memory_space_f2c(mem, variable%f2c, ierr) + end subroutine + subroutine adios2_set_shape(variable, ndims, shape_dims, ierr) type(adios2_variable), intent(in) :: variable integer, intent(in) :: ndims diff --git a/testing/adios2/bindings/fortran/CMakeLists.txt b/testing/adios2/bindings/fortran/CMakeLists.txt index 341fc362f4..c026628343 100644 --- a/testing/adios2/bindings/fortran/CMakeLists.txt +++ b/testing/adios2/bindings/fortran/CMakeLists.txt @@ -54,6 +54,10 @@ fortran_add_test_helper(BPReadGlobalsByName MPI_ONLY) fortran_add_test_helper(BPWriteMemorySelectionRead2D MPI_ONLY) fortran_add_test_helper(BPWriteMemorySelectionRead3D MPI_ONLY) fortran_add_test_helper(NullEngine MPI_ONLY) +fortran_add_test_helper(BPMemorySpace MPI_NONE) +if(ADIOS2_HAVE_GPU_Support) + fortran_add_test_helper(BPMemorySpaceGPU MPI_NONE) +endif() if(ADIOS2_HAVE_MPI) add_subdirectory(operation) diff --git a/testing/adios2/bindings/fortran/TestBPMemorySpace.F90 b/testing/adios2/bindings/fortran/TestBPMemorySpace.F90 new file mode 100644 index 0000000000..e22cb67438 --- /dev/null +++ b/testing/adios2/bindings/fortran/TestBPMemorySpace.F90 @@ -0,0 +1,41 @@ +program TestBPMemorySpace + use adios2 + implicit none + + integer(kind=8), dimension(1) :: shape_dims, start_dims, count_dims + integer(kind=4) :: ierr, mem + + type(adios2_adios) :: adios + type(adios2_variable) :: variable + type(adios2_io) :: ioWrite + + shape_dims(1) = 10 + start_dims(1) = 0 + count_dims(1) = 10 + + if( adios%valid .eqv. .true. ) stop 'Invalid adios default' + if( variable%valid .eqv. .true. ) stop 'Invalid variables default' + + call adios2_init(adios, ierr) + if( adios%valid .eqv. .false. ) stop 'Invalid adios2_init' + + call adios2_declare_io(ioWrite, adios, "ioWrite", ierr) + if( ioWrite%valid .eqv. .false. ) stop 'Invalid adios2_declare_io' + + call adios2_set_engine(ioWrite, 'File', ierr) + + call adios2_define_variable(variable, ioWrite, "var_I32", & + adios2_type_integer4, 1, & + shape_dims, start_dims, count_dims, & + adios2_constant_dims, ierr) + + ! check that the default execution space is Detect + call adios2_get_memory_space(mem, variable, ierr) + if (mem /= adios2_memory_space_detect) stop 'Invalid adios2_memory_space' + + ! check that the execution space is updated to Host + call adios2_set_memory_space(variable, adios2_memory_space_host, ierr) + call adios2_get_memory_space(mem, variable, ierr) + if (mem /= adios2_memory_space_host) stop 'Invalid adios2_memory_space' + +end program TestBPMemorySpace diff --git a/testing/adios2/bindings/fortran/TestBPMemorySpaceGPU.F90 b/testing/adios2/bindings/fortran/TestBPMemorySpaceGPU.F90 new file mode 100644 index 0000000000..64fb26755f --- /dev/null +++ b/testing/adios2/bindings/fortran/TestBPMemorySpaceGPU.F90 @@ -0,0 +1,41 @@ +program TestBPMemorySpace + use adios2 + implicit none + + integer(kind=8), dimension(1) :: shape_dims, start_dims, count_dims + integer(kind=4) :: ierr, mem + + type(adios2_adios) :: adios + type(adios2_variable) :: variable + type(adios2_io) :: ioWrite + + shape_dims(1) = 10 + start_dims(1) = 0 + count_dims(1) = 10 + + if( adios%valid .eqv. .true. ) stop 'Invalid adios default' + if( variable%valid .eqv. .true. ) stop 'Invalid variables default' + + call adios2_init(adios, ierr) + if( adios%valid .eqv. .false. ) stop 'Invalid adios2_init' + + call adios2_declare_io(ioWrite, adios, "ioWrite", ierr) + if( ioWrite%valid .eqv. .false. ) stop 'Invalid adios2_declare_io' + + call adios2_set_engine(ioWrite, 'File', ierr) + + call adios2_define_variable(variable, ioWrite, "var_I32", & + adios2_type_integer4, 1, & + shape_dims, start_dims, count_dims, & + adios2_constant_dims, ierr) + + ! check that the default execution space is Detect + call adios2_get_memory_space(mem, variable, ierr) + if (mem /= adios2_memory_space_detect) stop 'Invalid adios2_memory_space' + + ! check that the execution space is updated to GPU + call adios2_set_memory_space(variable, adios2_memory_space_gpu, ierr) + call adios2_get_memory_space(mem, variable, ierr) + if (mem /= adios2_memory_space_gpu) stop 'Invalid adios2_memory_space' + +end program TestBPMemorySpace From 9bbaecd37d9ff8aa71569dfb62d44d86eb0ff2d2 Mon Sep 17 00:00:00 2001 From: Greg Eisenhauer Date: Thu, 7 Mar 2024 15:10:55 -0500 Subject: [PATCH 076/124] Raise an exception if remote open fails (#4069) Raise an exception if remote open fails --- source/adios2/engine/bp5/BP5Reader.cpp | 45 +++++++++++++++++++------- source/adios2/engine/bp5/BP5Reader.h | 1 + thirdparty/EVPath/EVPath/cmsockets.c | 1 - 3 files changed, 35 insertions(+), 12 deletions(-) diff --git a/source/adios2/engine/bp5/BP5Reader.cpp b/source/adios2/engine/bp5/BP5Reader.cpp index a2854b2de2..e8fba5f3f3 100644 --- a/source/adios2/engine/bp5/BP5Reader.cpp +++ b/source/adios2/engine/bp5/BP5Reader.cpp @@ -272,6 +272,36 @@ std::pair BP5Reader::ReadData(adios2::transportman::TransportMan void BP5Reader::PerformGets() { + // if dataIsRemote is true and m_Remote is not true, this is our first time through + // PerformGets() Either we don't need a remote open (m_dataIsRemote=false), or we need to Open + // remote file (or die trying) + if (m_dataIsRemote && !m_Remote) + { + bool RowMajorOrdering = (m_IO.m_ArrayOrder == ArrayOrdering::RowMajor); + + // If nothing is pending, don't open + if (m_BP5Deserializer->PendingGetRequests.size() == 0) + return; + + if (!m_Parameters.RemoteDataPath.empty()) + { + m_Remote.Open("localhost", RemoteCommon::ServerPort, m_Parameters.RemoteDataPath, + m_OpenMode, RowMajorOrdering); + } + else if (getenv("DoRemote")) + { + m_Remote.Open("localhost", RemoteCommon::ServerPort, m_Name, m_OpenMode, + RowMajorOrdering); + } + if (!m_Remote) + { + helper::Throw( + "Engine", "BP5Reader", "OpenFiles", + "Remote file " + m_Name + + " cannot be opened. Possible server or file specification error."); + } + } + if (m_Remote) { PerformRemoteGets(); @@ -477,18 +507,11 @@ void BP5Reader::Init() OpenFiles(timeoutInstant, pollSeconds, timeoutSeconds); UpdateBuffer(timeoutInstant, pollSeconds / 10, timeoutSeconds); - // This isn't how we'll trigger remote ops in the end, but a temporary - // solution - bool RowMajorOrdering = (m_IO.m_ArrayOrder == ArrayOrdering::RowMajor); + // Don't try to open the remote file when we open local metadata. Do that on demand. if (!m_Parameters.RemoteDataPath.empty()) - { - m_Remote.Open("localhost", RemoteCommon::ServerPort, m_Parameters.RemoteDataPath, - m_OpenMode, RowMajorOrdering); - } - else if (getenv("DoRemote")) - { - m_Remote.Open("localhost", RemoteCommon::ServerPort, m_Name, m_OpenMode, RowMajorOrdering); - } + m_dataIsRemote = true; + if (getenv("DoRemote")) + m_dataIsRemote = true; } void BP5Reader::InitParameters() diff --git a/source/adios2/engine/bp5/BP5Reader.h b/source/adios2/engine/bp5/BP5Reader.h index 440158e509..5e121cfa2d 100644 --- a/source/adios2/engine/bp5/BP5Reader.h +++ b/source/adios2/engine/bp5/BP5Reader.h @@ -92,6 +92,7 @@ class BP5Reader : public BP5Engine, public Engine /* transport manager for managing the active flag file */ transportman::TransportMan m_ActiveFlagFileManager; + bool m_dataIsRemote = false; Remote m_Remote; bool m_WriterIsActive = true; adios2::profiling::JSONProfiler m_JSONProfiler; diff --git a/thirdparty/EVPath/EVPath/cmsockets.c b/thirdparty/EVPath/EVPath/cmsockets.c index 5a70aded91..a054fd452d 100644 --- a/thirdparty/EVPath/EVPath/cmsockets.c +++ b/thirdparty/EVPath/EVPath/cmsockets.c @@ -457,7 +457,6 @@ initiate_conn(CManager cm, CMtrans_services svc, transport_entry trans, attr_lis int err = WSAGetLastError(); if (err != WSAEWOULDBLOCK || err != WSAEINPROGRESS) { #endif - printf("Errno was %d\n", errno); svc->trace_out(cm, "CMSocket connect FAILURE --> Connect() to IP %s failed", ip_str); close(sock); #ifdef WSAEWOULDBLOCK From f6ee8d59bb47844bb280e9d24e4c55fe45833099 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Wed, 6 Mar 2024 08:10:17 -0500 Subject: [PATCH 077/124] Python: add the same treatment to attributes as to variables before: return scalars (0-dim ndarray) for single value attributes. --- bindings/Python/py11Attribute.cpp | 6 +++ bindings/Python/py11Attribute.h | 2 + bindings/Python/py11IO.cpp | 11 +++++- bindings/Python/py11glue.cpp | 3 +- examples/hello/bpReader/bpReaderHeatMap2D.py | 29 +++++++++----- python/adios2/attribute.py | 19 ++++++++- testing/adios2/python/TestAttribute.py | 4 +- .../python/TestBPWriteTypesHighLevelAPI.py | 39 +++++++++++-------- .../TestBPWriteTypesHighLevelAPI_HDF5.py | 32 +++++++++------ 9 files changed, 102 insertions(+), 43 deletions(-) diff --git a/bindings/Python/py11Attribute.cpp b/bindings/Python/py11Attribute.cpp index 6eb1c0e44b..e5ea93082a 100644 --- a/bindings/Python/py11Attribute.cpp +++ b/bindings/Python/py11Attribute.cpp @@ -36,6 +36,12 @@ std::string Attribute::Type() const return ToString(m_Attribute->m_Type); } +bool Attribute::SingleValue() const +{ + helper::CheckForNullptr(m_Attribute, "in call to Attribute::SingleValue"); + return m_Attribute->m_IsSingleValue; +} + std::vector Attribute::DataString() { helper::CheckForNullptr(m_Attribute, "in call to Attribute::DataStrings"); diff --git a/bindings/Python/py11Attribute.h b/bindings/Python/py11Attribute.h index 81f2723146..4366923ae6 100644 --- a/bindings/Python/py11Attribute.h +++ b/bindings/Python/py11Attribute.h @@ -37,6 +37,8 @@ class Attribute std::string Type() const; + bool SingleValue() const; + pybind11::array Data(); std::vector DataString(); diff --git a/bindings/Python/py11IO.cpp b/bindings/Python/py11IO.cpp index f44e260dd9..3d8f03782c 100644 --- a/bindings/Python/py11IO.cpp +++ b/bindings/Python/py11IO.cpp @@ -180,8 +180,15 @@ Attribute IO::DefineAttribute(const std::string &name, const pybind11::array &ar else if (pybind11::isinstance>(array)) \ { \ const T *data = reinterpret_cast(array.data()); \ - const size_t size = static_cast(array.size()); \ - attribute = &m_IO->DefineAttribute(name, data, size, variableName, separator); \ + if (array.ndim()) \ + { \ + const size_t size = static_cast(array.size()); \ + attribute = &m_IO->DefineAttribute(name, data, size, variableName, separator); \ + } \ + else \ + { \ + attribute = &m_IO->DefineAttribute(name, data[0], variableName, separator); \ + } \ } ADIOS2_FOREACH_NUMPY_ATTRIBUTE_TYPE_1ARG(declare_type) #undef declare_type diff --git a/bindings/Python/py11glue.cpp b/bindings/Python/py11glue.cpp index 13ca876517..2c40280f99 100644 --- a/bindings/Python/py11glue.cpp +++ b/bindings/Python/py11glue.cpp @@ -395,7 +395,8 @@ PYBIND11_MODULE(ADIOS2_PYTHON_MODULE_NAME, m) .def("Name", &adios2::py11::Attribute::Name) .def("Type", &adios2::py11::Attribute::Type) .def("DataString", &adios2::py11::Attribute::DataString) - .def("Data", &adios2::py11::Attribute::Data); + .def("Data", &adios2::py11::Attribute::Data) + .def("SingleValue", &adios2::py11::Attribute::SingleValue); pybind11::class_(m, "Engine") // Python 2 diff --git a/examples/hello/bpReader/bpReaderHeatMap2D.py b/examples/hello/bpReader/bpReaderHeatMap2D.py index 0f7dfe70ea..0c7c870f63 100644 --- a/examples/hello/bpReader/bpReaderHeatMap2D.py +++ b/examples/hello/bpReader/bpReaderHeatMap2D.py @@ -35,7 +35,6 @@ for j in range(0, Ny): value = iGlobal * shape[1] + j temperatures[i * Nx + j] = value -# print(str(i) + "," + str(j) + " " + str(value)) with Stream("HeatMap2D_py.bp", "w", comm) as obpStream: obpStream.write("temperature2D", temperatures, shape, start, count) @@ -43,19 +42,31 @@ obpStream.write("N", [size, Nx, Ny]) # will be an array in output obpStream.write("Nx", numpy.array(Nx)) # will be a scalar in output obpStream.write("Ny", Ny) # will be a scalar in output + obpStream.write_attribute("nproc", size) # will be a single value attribute in output obpStream.write_attribute("dimensions", [size * Nx, Ny], "temperature2D") if not rank: with FileReader("HeatMap2D_py.bp", MPI.COMM_SELF) as ibpFile: + # scalar variables are read as a numpy array with 0 dimension + in_nx = ibpFile.read("Nx") + in_ny = ibpFile.read("Ny") + print(f"Incoming nx, ny = {in_nx}, {in_ny}") + + # single value attribute is read as a numpy array with 0 dimension + in_nproc = ibpFile.read_attribute("nproc") + print(f"Incoming nproc = {in_nproc}") + # array attribute is read as a numpy array or string list + in_dims = ibpFile.read_attribute("temperature2D/dimensions") + print(f"Incoming diumensions = {in_dims}") + + # On option is to inquire a variable to know its type, shape + # directly, not as strings, and then we can use the variable + # object to set selection and/or set steps to read var_inTemperature = ibpFile.inquire_variable("temperature2D") if var_inTemperature is not None: var_inTemperature.set_selection([[2, 2], [4, 4]]) inTemperatures = ibpFile.read(var_inTemperature) - - in_nx = ibpFile.read("Nx") # scalar is read as a numpy array with 1 element - in_ny = ibpFile.read("Ny") # scalar is read as a numpy array with 1 element - print(f"Incoming nx, ny = {in_nx}, {in_ny}") - - print("Incoming temperature map") - for i in range(0, inTemperatures.shape[1]): - print(str(inTemperatures[i])) + print(f"Incoming temperature map with selection " + f"start = {var_inTemperature.start()}, count = {var_inTemperature.count()}") + for i in range(0, inTemperatures.shape[1]): + print(str(inTemperatures[i])) diff --git a/python/adios2/attribute.py b/python/adios2/attribute.py index 8740a23ae3..ec4dd7618e 100644 --- a/python/adios2/attribute.py +++ b/python/adios2/attribute.py @@ -41,6 +41,15 @@ def type(self): Type of the Attribute as a str. """ return self.impl.Type() + + def single_value(self): + """ + True if the attribute is a single value, False if it is an array + + Returns: + True or False. + """ + return self.impl.SingleValue() def data(self): """ @@ -49,7 +58,10 @@ def data(self): Returns: Content of the Attribute as a non string. """ - return self.impl.Data() + if self.single_value(): + return self.impl.Data()[0] + else: + return self.impl.Data() def data_string(self): """ @@ -58,4 +70,7 @@ def data_string(self): Returns: Content of the Attribute as a str. """ - return self.impl.DataString() + if self.single_value(): + return self.impl.DataString()[0] + else: + return self.impl.DataString() diff --git a/testing/adios2/python/TestAttribute.py b/testing/adios2/python/TestAttribute.py index 24fd79ee0d..b547c71715 100644 --- a/testing/adios2/python/TestAttribute.py +++ b/testing/adios2/python/TestAttribute.py @@ -12,14 +12,14 @@ def test_create_write(self): with adios.declare_io("BPWriter") as io: ts = io.define_attribute("timestamp", "20231122") self.assertEqual(ts.name(), "timestamp") - self.assertEqual(ts.data_string(), ["20231122"]) + self.assertEqual(ts.data_string(), "20231122") def test_create_reader(self): adios = Adios() with adios.declare_io("BPWriter") as io: ts = io.define_attribute("timestamp", "20231122") self.assertEqual(ts.name(), "timestamp") - self.assertEqual(ts.data_string(), ["20231122"]) + self.assertEqual(ts.data_string(), "20231122") def test_create_write_ndarray(self): adios = Adios() diff --git a/testing/adios2/python/TestBPWriteTypesHighLevelAPI.py b/testing/adios2/python/TestBPWriteTypesHighLevelAPI.py index 3b300fca55..cc031e2667 100644 --- a/testing/adios2/python/TestBPWriteTypesHighLevelAPI.py +++ b/testing/adios2/python/TestBPWriteTypesHighLevelAPI.py @@ -58,6 +58,8 @@ # single value attributes with numpy variables s.write_attribute("attrStr", "Testing single string attribute") + print(f"---- type of np.array(data.i8[0]) is {type(np.array(data.i8[0]))}" + f" shape = {np.array(data.i8[0]).shape}") s.write_attribute("attrI8", np.array(data.i8[0])) s.write_attribute("attrI16", np.array(data.i16[0])) s.write_attribute("attrI32", np.array(data.i32[0])) @@ -148,6 +150,7 @@ if rank == 0 and step == 0: inTag = fr_step.read("tag") + print(f"tag = {inTag}") nx = fr_step.read("nx") print(f"nx = {nx}") assert nx == data.Nx @@ -198,9 +201,11 @@ # attributes inTag = fr_step.read_attribute("attrStr") + print(f"attrStr = {inTag}") inNx = fr_step.read_attribute("attrNx") inI8 = fr_step.read_attribute("attrI8") inI16 = fr_step.read_attribute("attrI16") + print(f"attrI16 = {inI16}") inI32 = fr_step.read_attribute("attrI32") inI64 = fr_step.read_attribute("attrI64") inU8 = fr_step.read_attribute("attrU8") @@ -210,40 +215,40 @@ inR32 = fr_step.read_attribute("attrR32") inR64 = fr_step.read_attribute("attrR64") - if inTag[0] != "Testing single string attribute": + if inTag != "Testing single string attribute": raise ValueError("attr string read failed") if inNx != data.Nx: raise ValueError("attrNx read failed") - if inI8[0] != data.i8[0]: + if inI8 != data.i8[0]: raise ValueError("attrI8 read failed") - if inI16[0] != data.i16[0]: + if inI16 != data.i16[0]: raise ValueError("attrI16 read failed") - if inI32[0] != data.i32[0]: + if inI32 != data.i32[0]: raise ValueError("attrI32 read failed") - if inI64[0] != data.i64[0]: + if inI64 != data.i64[0]: raise ValueError("attrI64 read failed") - if inU8[0] != data.u8[0]: + if inU8 != data.u8[0]: raise ValueError("attrU8 read failed") - if inU16[0] != data.u16[0]: + if inU16 != data.u16[0]: raise ValueError("attrU16 read failed") - if inU32[0] != data.u32[0]: + if inU32 != data.u32[0]: raise ValueError("attrU32 read failed") - if inU64[0] != data.u64[0]: + if inU64 != data.u64[0]: raise ValueError("attrU64 read failed") - if inR32[0] != data.r32[0]: + if inR32 != data.r32[0]: raise ValueError("attrR32 read failed") - if inR64[0] != data.r64[0]: + if inR64 != data.r64[0]: raise ValueError("attrR64 read failed") in_an_int_value = fr_step.read("an_int_value") @@ -278,9 +283,11 @@ print(f"a_complex_list = {a_complex_list} of type {type(a_complex_list)}") # Array attribute - inTag = fr_step.read_attribute_string("attrStrArray") + inTag = fr_step.read_attribute("attrStrArray") + print(f"attrStrArray = {inTag}") inI8 = fr_step.read_attribute("attrI8Array") inI16 = fr_step.read_attribute("attrI16Array") + print(f"attrI16Array = {inI16}") inI32 = fr_step.read_attribute("attrI32Array") inI64 = fr_step.read_attribute("attrI64Array") inU8 = fr_step.read_attribute("attrU8Array") @@ -386,19 +393,19 @@ sizeI8 = fr_step.read_attribute("size", "varI8") sizeI16 = fr_step.read_attribute("size", "varI16", "::") - if sizeI8[0] != data.Nx: + if sizeI8 != data.Nx: raise ValueError("attribute varI8/size read failed") - if sizeI16[0] != data.Nx: + if sizeI16 != data.Nx: raise ValueError("attribute varI16::size read failed") sizeI8 = fr_step.read_attribute("varI8/size") sizeI16 = fr_step.read_attribute("varI16::size") - if sizeI8[0] != data.Nx: + if sizeI8 != data.Nx: raise ValueError("attribute varI8/size read failed") - if sizeI16[0] != data.Nx: + if sizeI16 != data.Nx: raise ValueError("attribute varI16::size read failed") step_attrs = fr_step.available_attributes() diff --git a/testing/adios2/python/TestBPWriteTypesHighLevelAPI_HDF5.py b/testing/adios2/python/TestBPWriteTypesHighLevelAPI_HDF5.py index d95c160bc9..e7848e900f 100644 --- a/testing/adios2/python/TestBPWriteTypesHighLevelAPI_HDF5.py +++ b/testing/adios2/python/TestBPWriteTypesHighLevelAPI_HDF5.py @@ -35,6 +35,7 @@ if rank == 0 and i == 0: fw.write("tag", "Testing ADIOS2 high-level API") + fw.write("nx", data.Nx) fw.write("gvarI8", np.array(data.i8[0])) fw.write("gvarI16", np.array(data.i16[0])) fw.write("gvarI32", np.array(data.i32[0])) @@ -48,6 +49,7 @@ # single value attributes fw.write_attribute("attrStr", "Testing single string attribute") + fw.write_attribute("attrNx", data.Nx) fw.write_attribute("attrI8", np.array(data.i8[0])) fw.write_attribute("attrI16", np.array(data.i16[0])) fw.write_attribute("attrI32", np.array(data.i32[0])) @@ -120,6 +122,7 @@ if rank == 0 and step == 0: inTag = fr_step.read("tag") + inNx = fr_step.read("nx") inI8 = fr_step.read("gvarI8") inI16 = fr_step.read("gvarI16") inI32 = fr_step.read("gvarI32") @@ -135,6 +138,9 @@ print("InTag: " + str(inTag)) raise ValueError("tag variable read failed") + if inNx != nx: + raise ValueError("tag variable read failed") + if inI8 != data.i8[0]: raise ValueError("gvarI8 read failed") @@ -167,6 +173,7 @@ # attributes inTag = fr_step.read_attribute("attrStr") + inNx = fr_step.read_attribute("attrNx") inI8 = fr_step.read_attribute("attrI8") inI16 = fr_step.read_attribute("attrI16") inI32 = fr_step.read_attribute("attrI32") @@ -178,37 +185,40 @@ inR32 = fr_step.read_attribute("attrR32") inR64 = fr_step.read_attribute("attrR64") - if inTag[0] != "Testing single string attribute": + if inTag != "Testing single string attribute": raise ValueError("attr string read failed") - if inI8[0] != data.i8[0]: + if inNx != nx: raise ValueError("attrI8 read failed") - if inI16[0] != data.i16[0]: + if inI8 != data.i8[0]: + raise ValueError("attrI8 read failed") + + if inI16 != data.i16[0]: raise ValueError("attrI16 read failed") - if inI32[0] != data.i32[0]: + if inI32 != data.i32[0]: raise ValueError("attrI32 read failed") - if inI64[0] != data.i64[0]: + if inI64 != data.i64[0]: raise ValueError("attrI64 read failed") - if inU8[0] != data.u8[0]: + if inU8 != data.u8[0]: raise ValueError("attrU8 read failed") - if inU16[0] != data.u16[0]: + if inU16 != data.u16[0]: raise ValueError("attrU16 read failed") - if inU32[0] != data.u32[0]: + if inU32 != data.u32[0]: raise ValueError("attrU32 read failed") - if inU64[0] != data.u64[0]: + if inU64 != data.u64[0]: raise ValueError("attrU64 read failed") - if inR32[0] != data.r32[0]: + if inR32 != data.r32[0]: raise ValueError("attrR32 read failed") - if inR64[0] != data.r64[0]: + if inR64 != data.r64[0]: raise ValueError("attrR64 read failed") # Array attribute From 7d9632e5bae162882e7dc2d3b2808824bd5874b3 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Wed, 6 Mar 2024 11:38:01 -0500 Subject: [PATCH 078/124] format --- examples/hello/bpReader/bpReaderHeatMap2D.py | 10 ++++++---- python/adios2/attribute.py | 8 +++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/examples/hello/bpReader/bpReaderHeatMap2D.py b/examples/hello/bpReader/bpReaderHeatMap2D.py index 0c7c870f63..ce31e418d9 100644 --- a/examples/hello/bpReader/bpReaderHeatMap2D.py +++ b/examples/hello/bpReader/bpReaderHeatMap2D.py @@ -42,13 +42,13 @@ obpStream.write("N", [size, Nx, Ny]) # will be an array in output obpStream.write("Nx", numpy.array(Nx)) # will be a scalar in output obpStream.write("Ny", Ny) # will be a scalar in output - obpStream.write_attribute("nproc", size) # will be a single value attribute in output + obpStream.write_attribute("nproc", size) # will be a single value attribute in output obpStream.write_attribute("dimensions", [size * Nx, Ny], "temperature2D") if not rank: with FileReader("HeatMap2D_py.bp", MPI.COMM_SELF) as ibpFile: # scalar variables are read as a numpy array with 0 dimension - in_nx = ibpFile.read("Nx") + in_nx = ibpFile.read("Nx") in_ny = ibpFile.read("Ny") print(f"Incoming nx, ny = {in_nx}, {in_ny}") @@ -66,7 +66,9 @@ if var_inTemperature is not None: var_inTemperature.set_selection([[2, 2], [4, 4]]) inTemperatures = ibpFile.read(var_inTemperature) - print(f"Incoming temperature map with selection " - f"start = {var_inTemperature.start()}, count = {var_inTemperature.count()}") + print( + f"Incoming temperature map with selection " + f"start = {var_inTemperature.start()}, count = {var_inTemperature.count()}" + ) for i in range(0, inTemperatures.shape[1]): print(str(inTemperatures[i])) diff --git a/python/adios2/attribute.py b/python/adios2/attribute.py index ec4dd7618e..ddbaa2eb06 100644 --- a/python/adios2/attribute.py +++ b/python/adios2/attribute.py @@ -41,7 +41,7 @@ def type(self): Type of the Attribute as a str. """ return self.impl.Type() - + def single_value(self): """ True if the attribute is a single value, False if it is an array @@ -60,8 +60,7 @@ def data(self): """ if self.single_value(): return self.impl.Data()[0] - else: - return self.impl.Data() + return self.impl.Data() def data_string(self): """ @@ -72,5 +71,4 @@ def data_string(self): """ if self.single_value(): return self.impl.DataString()[0] - else: - return self.impl.DataString() + return self.impl.DataString() From 5d0660851095281b17221b6e2b8b976b7422a5c1 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Wed, 6 Mar 2024 12:08:59 -0500 Subject: [PATCH 079/124] format more --- testing/adios2/python/TestBPWriteTypesHighLevelAPI.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/testing/adios2/python/TestBPWriteTypesHighLevelAPI.py b/testing/adios2/python/TestBPWriteTypesHighLevelAPI.py index cc031e2667..dbd9cf6635 100644 --- a/testing/adios2/python/TestBPWriteTypesHighLevelAPI.py +++ b/testing/adios2/python/TestBPWriteTypesHighLevelAPI.py @@ -58,8 +58,10 @@ # single value attributes with numpy variables s.write_attribute("attrStr", "Testing single string attribute") - print(f"---- type of np.array(data.i8[0]) is {type(np.array(data.i8[0]))}" - f" shape = {np.array(data.i8[0]).shape}") + print( + f"---- type of np.array(data.i8[0]) is {type(np.array(data.i8[0]))}" + f" shape = {np.array(data.i8[0]).shape}" + ) s.write_attribute("attrI8", np.array(data.i8[0])) s.write_attribute("attrI16", np.array(data.i16[0])) s.write_attribute("attrI32", np.array(data.i32[0])) From 624592ba88a7f859eddd89e24667157504d9e1b5 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Thu, 7 Mar 2024 14:38:36 -0500 Subject: [PATCH 080/124] - Python: fix for scalar reading. If a global value has 1 step (i.e. always in streaming), read returns a 0-dim numpy array (single value). If the variable has multiple steps (only in ReadRandomAccess mode), read returns a 1-dim numpy array even if the step selection is a single step. This way, read of a certain variable always results in the same type of array no matter the number of steps selected. - Python: fix for string attributes: return a string, not a list of one element which is a string, to be consistent with string global values and with other APIs. --- .../api_python/adios2-doc-read-filereader.py | 8 ++- ...-doc-read.py => adios2-doc-read-stream.py} | 4 +- .../source/api_python/python_example.rst | 24 ++++--- python/adios2/file_reader.py | 4 -- python/adios2/stream.py | 62 ++++++++++++++----- testing/adios2/python/TestFileReader.py | 58 ++++++++++++----- 6 files changed, 113 insertions(+), 47 deletions(-) rename docs/user_guide/source/api_python/{adios2-doc-read.py => adios2-doc-read-stream.py} (89%) diff --git a/docs/user_guide/source/api_python/adios2-doc-read-filereader.py b/docs/user_guide/source/api_python/adios2-doc-read-filereader.py index 3211e82642..73fa255005 100644 --- a/docs/user_guide/source/api_python/adios2-doc-read-filereader.py +++ b/docs/user_guide/source/api_python/adios2-doc-read-filereader.py @@ -9,14 +9,18 @@ for key, value in info.items(): print("\t" + key + ": " + value, end=" ") print() + print() nproc = s.read("nproc") - print(f"nproc is {nproc} of type {type(nproc)}") + print(f"nproc is {nproc} of type {type(nproc)} with ndim {nproc.ndim}") # read variables return a numpy array with corresponding selection steps = int(vars["physical_time"]["AvailableStepsCount"]) physical_time = s.read("physical_time", step_selection=[0, steps]) - print(f"physical_time is {physical_time} of type {type(physical_time)}") + print( + f"physical_time is {physical_time} of type {type(physical_time)} with " + f"ndim {physical_time.ndim} shape = {physical_time.shape}" + ) steps = int(vars["temperature"]["AvailableStepsCount"]) temperature = s.read("temperature", step_selection=[0, steps]) diff --git a/docs/user_guide/source/api_python/adios2-doc-read.py b/docs/user_guide/source/api_python/adios2-doc-read-stream.py similarity index 89% rename from docs/user_guide/source/api_python/adios2-doc-read.py rename to docs/user_guide/source/api_python/adios2-doc-read-stream.py index 5f7d1ee2cf..0d24844ed6 100644 --- a/docs/user_guide/source/api_python/adios2-doc-read.py +++ b/docs/user_guide/source/api_python/adios2-doc-read-stream.py @@ -16,7 +16,7 @@ if s.current_step() == 0: nproc = s.read("nproc") - print(f"nproc is {nproc} of type {type(nproc)}") + print(f"nproc is {nproc} of type {type(nproc)} with ndim {nproc.ndim}") # read variables return a numpy array with corresponding selection physical_time = s.read("physical_time") @@ -26,6 +26,6 @@ print(f"temperature array size is {temperature.size} of shape {temperature.shape}") print(f"temperature unit is {temp_unit} of type {type(temp_unit)}") pressure = s.read("pressure") - press_unit = s.read_attribute("pressure/unit") + press_unit = s.read_attribute("unit", "pressure") print(f"pressure unit is {press_unit} of type {type(press_unit)}") print() diff --git a/docs/user_guide/source/api_python/python_example.rst b/docs/user_guide/source/api_python/python_example.rst index d2458a12c2..58ac0b9375 100644 --- a/docs/user_guide/source/api_python/python_example.rst +++ b/docs/user_guide/source/api_python/python_example.rst @@ -44,6 +44,7 @@ Python Write example count = [nx] temperature = np.zeros(nx, dtype=np.double) + pressure = np.ones(nx, dtype=np.double) delta_time = 0.01 physical_time = 0.0 nsteps = 5 @@ -125,8 +126,8 @@ Python Read "step-by-step" example nproc is 4 of type physical_time is 0.0 of type temperature array size is 40 of shape (40,) - temperature unit is ['K'] of type - pressure unit is ['Pa'] of type + temperature unit is K of type + pressure unit is Pa of type Current step is 1 variable_name: physical_time AvailableStepsCount: 1 Max: 0.01 Min: 0.01 Shape: SingleValue: true Type: double @@ -134,8 +135,8 @@ Python Read "step-by-step" example variable_name: temperature AvailableStepsCount: 1 Max: 0 Min: 0 Shape: 40 SingleValue: false Type: double physical_time is 0.01 of type temperature array size is 40 of shape (40,) - temperature unit is ['K'] of type - pressure unit is ['Pa'] of type + temperature unit is K of type + pressure unit is Pa of type ... @@ -156,14 +157,18 @@ Python Read Random Access example for key, value in info.items(): print("\t" + key + ": " + value, end=" ") print() + print() nproc = s.read("nproc") - print(f"nproc is {nproc} of type {type(nproc)}") + print(f"nproc is {nproc} of type {type(nproc)} with ndim {nproc.ndim}") # read variables return a numpy array with corresponding selection steps = int(vars['physical_time']['AvailableStepsCount']) physical_time = s.read("physical_time", step_selection=[0, steps]) - print(f"physical_time is {physical_time} of type {type(physical_time)}") + print( + f"physical_time is {physical_time} of type {type(physical_time)} with " + f"ndim {physical_time.ndim} shape = {physical_time.shape}" + ) steps = int(vars['temperature']['AvailableStepsCount']) temperature = s.read("temperature", step_selection=[0, steps]) @@ -183,8 +188,9 @@ Python Read Random Access example variable_name: physical_time AvailableStepsCount: 5 Max: 0.04 Min: 0 Shape: SingleValue: true Type: double variable_name: pressure AvailableStepsCount: 5 Max: 1 Min: 1 Shape: 40 SingleValue: false Type: double variable_name: temperature AvailableStepsCount: 5 Max: 0 Min: 0 Shape: 40 SingleValue: false Type: double - nproc is 4 of type - physical_time is [0. 0.01 0.02 0.03 0.04] of type + + nproc is 4 of type with ndim 0 + physical_time is [0. 0.01 0.02 0.03 0.04] of type with ndim 1 shape = (5,) temperature array size is 200 of shape (200,) - temperature unit is ['K'] of type + temperature unit is K of type diff --git a/python/adios2/file_reader.py b/python/adios2/file_reader.py index 74f0f640fe..bc0eb44452 100644 --- a/python/adios2/file_reader.py +++ b/python/adios2/file_reader.py @@ -25,7 +25,3 @@ def _(self, io: IO, path, comm=None): super().__init__(io, path, "rra", comm) # pylint: enable=E1121 - - def variables(self): - """Returns the list of variables contained in the opened file""" - return [self._io.inquire_variable(var) for var in self.available_variables()] diff --git a/python/adios2/stream.py b/python/adios2/stream.py index 826316d9cd..0285507cc9 100644 --- a/python/adios2/stream.py +++ b/python/adios2/stream.py @@ -325,11 +325,9 @@ def _(self, name, content, shape=[], start=[], count=[], operations=None): self.write(variable, content) - @singledispatchmethod - def read(self, variable: Variable): + def _read_var(self, variable: Variable): """ - Random access read allowed to select steps, - only valid with Stream Engines + Internal common function to read. Settings must be done to Variable before the call. Parameters variable @@ -342,6 +340,7 @@ def read(self, variable: Variable): """ dtype = type_adios_to_numpy(variable.type()) count = variable.count() + if count != []: # array # steps = variable.get_steps_from_step_selection() @@ -365,7 +364,8 @@ def read(self, variable: Variable): else: # scalar size_all_steps = variable.selection_size() - if size_all_steps > 1: + # if size_all_steps > 1: + if self._mode == bindings.Mode.ReadRandomAccess and variable.steps() > 1: output_shape = [size_all_steps] else: output_shape = [] @@ -374,15 +374,17 @@ def read(self, variable: Variable): self._engine.get(variable, output) return output - @read.register(str) - def _(self, name: str, start=[], count=[], block_id=None, step_selection=None): + @singledispatchmethod + def read(self, variable: Variable, start=[], count=[], block_id=None, step_selection=None): """ - Random access read allowed to select steps, - only valid with Stream Engines + Read a variable. + Random access read allowed to select steps. Parameters - name - variable to be read + variable + adios2.Variable object to be read + Use variable.set_selection(), set_block_selection(), set_step_selection() + to prepare a read start variable offset dimensions @@ -400,10 +402,6 @@ def _(self, name: str, start=[], count=[], block_id=None, step_selection=None): array resulting array from selection """ - variable = self._io.inquire_variable(name) - if not variable: - raise ValueError() - if step_selection is not None and not self._mode == bindings.Mode.ReadRandomAccess: raise RuntimeError("step_selection parameter requires 'rra' mode") @@ -419,7 +417,39 @@ def _(self, name: str, start=[], count=[], block_id=None, step_selection=None): if start != [] and count != []: variable.set_selection([start, count]) - return self.read(variable) + return self._read_var(variable) + + @read.register(str) + def _(self, name: str, start=[], count=[], block_id=None, step_selection=None): + """ + Read a variable. + Random access read allowed to select steps. + + Parameters + name + variable to be read + + start + variable offset dimensions + + count + variable local dimensions from offset + + block_id + (int) Required for reading local variables, local array, and local + value. + + step_selection + (list): On the form of [start, count]. + Returns + array + resulting array from selection + """ + variable = self._io.inquire_variable(name) + if not variable: + raise ValueError() + + return self.read(variable, start, count, block_id, step_selection) def write_attribute(self, name, content, variable_name="", separator="/"): """ diff --git a/testing/adios2/python/TestFileReader.py b/testing/adios2/python/TestFileReader.py index cda2dc6533..fbd712e681 100644 --- a/testing/adios2/python/TestFileReader.py +++ b/testing/adios2/python/TestFileReader.py @@ -1,45 +1,75 @@ -from adios2 import FileReader, Stream +from adios2 import FileReader, Stream, LocalValueDim from random import randint import unittest -class TestStream(unittest.TestCase): +class TestFileReader(unittest.TestCase): def test_basic(self): with Stream("pythonfiletest.bp", "w") as s: for _ in s.steps(10): if s.current_step() == 0: + # String s.write("Outlook", "Good") + # Global Value single step + s.write("LocationID", 42) + # Global Array s.write( - "temp", + "Temp", content=[randint(15, 35), randint(15, 35), randint(15, 35)], shape=[3], start=[0], count=[3], ) + # Local Value + s.write("Wind", [5], shape=[LocalValueDim]) + # Local Array + s.write("Coords", [38, -46], [], [], [2]) + # Global Value every step + s.write("Hour", 8 + s.current_step()) with FileReader("pythonfiletest.bp") as f: self.assertEqual(len(f.all_blocks_info("temp")), 10) - for var in f.variables(): + outlook_var = f.inquire_variable("Outlook") + outlook = f.read(outlook_var) + self.assertEqual(outlook, "Good") + + for name in f.available_variables(): + var = f.inquire_variable(name) if not var.type() == "string": - self.assertEqual(var.steps(), 10) + if var.steps() > 1: + self.assertEqual(var.steps(), 10) var.set_step_selection([0, var.steps()]) - - output = f.read(var.name()) - print(f"var:{var.name()} output:{output}") + output = f.read(var) + print(f"var:{var.name()} output:{output}") with FileReader("pythonfiletest.bp") as f: - output = f.read("temp", step_selection=[0, 10]) + output = f.read("LocationID") + self.assertEqual(output.ndim, 0) + self.assertEqual(output.size, 1) + self.assertEqual(output, 42) + + output = f.read("Hour", step_selection=[0, 10]) + self.assertEqual(output.ndim, 1) + self.assertEqual(output.size, 10) + self.assertTrue((output == [8, 9, 10, 11, 12, 13, 14, 15, 16, 17]).all()) + + output = f.read("Hour", step_selection=[5, 1]) + self.assertEqual(output.ndim, 1) + self.assertEqual(output.size, 1) + self.assertEqual(output, [13]) + + output = f.read("Temp", step_selection=[0, 10]) self.assertEqual(len(output), 30) - print(f"var:temp output:{output}") + print(f"var:Temp output:{output}") - output = f.read("temp", step_selection=[0, 5]) + output = f.read("Temp", step_selection=[0, 5]) self.assertEqual(len(output), 15) - print(f"var:temp output:{output}") + print(f"var:Temp output:{output}") - output = f.read("temp", start=[0], count=[2], step_selection=[0, 10]) + output = f.read("Temp", start=[0], count=[2], step_selection=[0, 10]) self.assertEqual(len(output), 20) - print(f"var:temp output:{output}") + print(f"var:Temp output:{output}") if __name__ == "__main__": From d5007e158cdcacb7de33ea9ad91efbed4626238f Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Fri, 8 Mar 2024 12:08:46 -0500 Subject: [PATCH 081/124] Add test for single string attribute vs string array attribute with a single element --- .../python/TestBPWriteTypesHighLevelAPI.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/testing/adios2/python/TestBPWriteTypesHighLevelAPI.py b/testing/adios2/python/TestBPWriteTypesHighLevelAPI.py index dbd9cf6635..f77aac8599 100644 --- a/testing/adios2/python/TestBPWriteTypesHighLevelAPI.py +++ b/testing/adios2/python/TestBPWriteTypesHighLevelAPI.py @@ -84,7 +84,8 @@ s.write_attribute("attr_int_list", data.int_list) s.write_attribute("attr_float_list", data.float_list) s.write_attribute("attr_complex_list", data.complex_list) - s.write_attribute("attrStrArray", ["string1", "string2", "string3"]) + s.write_attribute("attrStrArray1", ["string1"]) + s.write_attribute("attrStrArray3", ["string1", "string2", "string3"]) # array attributes with numpy arrays s.write_attribute("attrI8Array", data.i8) @@ -285,8 +286,10 @@ print(f"a_complex_list = {a_complex_list} of type {type(a_complex_list)}") # Array attribute - inTag = fr_step.read_attribute("attrStrArray") - print(f"attrStrArray = {inTag}") + inStr1 = fr_step.read_attribute("attrStrArray1") + print(f"attrStrArray1 = {inStr1}") + inStr3 = fr_step.read_attribute("attrStrArray3") + print(f"attrStrArray3 = {inStr3}") inI8 = fr_step.read_attribute("attrI8Array") inI16 = fr_step.read_attribute("attrI16Array") print(f"attrI16Array = {inI16}") @@ -299,8 +302,11 @@ inR32 = fr_step.read_attribute("attrR32Array") inR64 = fr_step.read_attribute("attrR64Array") - if inTag != ["string1", "string2", "string3"]: - raise ValueError("attrStrArray read failed") + if inStr1 != ["string1"]: + raise ValueError("attrStrArray1 read failed") + + if inStr3 != ["string1", "string2", "string3"]: + raise ValueError("attrStrArray3 read failed") if not (inI8 == data.i8).all(): raise ValueError("attrI8 array read failed") From 70e645d049da364d313a9a5096e7a03475bd5fe3 Mon Sep 17 00:00:00 2001 From: Ana Gainaru Date: Mon, 11 Mar 2024 07:34:16 -0400 Subject: [PATCH 082/124] Update documentation for 2.10 changes to the GPU-backend (#4083) * Update the GPU-backend code for the 2.10 release changes on the write side * Documentation for the new Shape calls for different memory spaces --- docs/user_guide/source/advanced/gpu_aware.rst | 66 +++++++++++++++---- 1 file changed, 55 insertions(+), 11 deletions(-) diff --git a/docs/user_guide/source/advanced/gpu_aware.rst b/docs/user_guide/source/advanced/gpu_aware.rst index 0cf34a893d..40df01cb50 100644 --- a/docs/user_guide/source/advanced/gpu_aware.rst +++ b/docs/user_guide/source/advanced/gpu_aware.rst @@ -2,12 +2,13 @@ GPU-aware I/O ################# -The ``Put`` and ``Get`` functions in the BP4 and BP5 engines can receive user buffers allocated on the host or the device in both Sync and Deferred modes. +The ``Put`` and ``Get`` functions in the default file engine (BP5) and some streaming engines (SST, DataMan) can receive user buffers allocated on the host or the device in both Sync and Deferred modes. .. note:: - CUDA, HIP and SYCL allocated buffers are supported for device data. + Buffers allocated on the device with CUDA, HIP and SYCL are supported. -If ADIOS2 is built without GPU support, only buffers allocated on the host are supported. If ADIOS2 is built with any GPU support, by default, the library will automatically detect where does the buffer memory physically resides. +If ADIOS2 is built without GPU support, only buffers allocated on the host are supported. +When GPU support is enabled, the default behavior is for ADIOS2 to automatically detect where the buffer memory physically resides. Users can also provide information about where the buffer was allocated by using the ``SetMemorySpace`` function within each variable. @@ -20,17 +21,20 @@ Users can also provide information about where the buffer was allocated by using GPU ///< GPU memory spaces }; +If ADIOS2 is built without GPU support, the available MemorySpace values are only ``Detect`` and ``Host``. + ADIOS2 can use a CUDA or Kokkos backend for enabling GPU support. Only one backend can be active at a given time based on how ADIOS2 is build. ********************************** Building ADIOS2 with a GPU backend ********************************** +By default both backends are ``OFF`` even if CUDA or Kokkos are installed and available to avoid a possible conflict between if both backends are enabled at the same time. Building with CUDA enabled -------------------------- -If there is no CUDA toolkit installed, cmake will turn CUDA off automatically. ADIOS2 default behavior for ``ADIOS2_USE_CUDA`` is to enable CUDA if it can find a CUDA toolkit on the system. In case the system has a CUDA toolkit installed, but it is desired to build ADIOS2 without CUDA enabled ``-DADIOS2_USE_CUDA=OFF`` must be used. +The ADIOS2 default behavior is to turn ``OFF`` the CUDA backend. Building with the CUDA backend requires ``-DADIOS2_USE_Kokkos=ON`` and an available CUDA toolkit on the system. When building ADIOS2 with CUDA enabled, the user is responsible with setting the correct ``CMAKE_CUDA_ARCHITECTURES`` (e.g. for Summit the ``CMAKE_CUDA_ARCHITECTURES`` needs to be set to 70 to match the NVIDIA Volta V100). @@ -43,11 +47,23 @@ The Kokkos library can be used to enable GPU within ADIOS2. Based on how Kokkos Kokkos version >= 3.7 is required to enable the GPU backend in ADIOS2 -**************** -Writing GPU code -**************** +******************* +Writing GPU buffers +******************* + +The ADIOS2 API for Device pointers is identical to using Host buffers for both the read and write logic. +Internally each ADIOS2 variable holds a memory space for the data it receives. Once the memory space is set (eithr directly by the user through calls to ``SetMemorySpace`` or after detecting the buffer memory space the first ``Put`` or ``Get`` call) to either Host or Device, it cannot be changed. + +The ``examples/hello`` folder contains several codes that use Device buffers: + - `bpStepsWriteRead{Cuda|Hip}` show CUDA and HIP codes using BP5 with GPU pointers + - `bpStepsWriteReadKokkos contains` Fortran and C++ codes using ``Kokkos::View`` with different memory spaces and a Kokkos code using different layouts on Host buffers + - `datamanKokkos` shows an example of streaming a ``Kokkos::View`` with DataMan using different memory spaces + - `sstKokkos` shows an example of streaming a ``Kokkos::View`` with SST using different memory spaces + +Example using a Device buffer +----------------------------- -The following is a simple example of writing data to storage directly from a GPU buffer allocated with CUDA relying on the automatic detection of device pointers in ADIOS2. The ADIOS2 API is identical to codes using Host buffers for both the read and write logic. +The following is a simple example of writing data to storage directly from a GPU buffer allocated with CUDA relying on the automatic detection of device pointers in ADIOS2. .. code-block:: c++ @@ -56,7 +72,7 @@ The following is a simple example of writing data to storage directly from a GPU cudaMemset(gpuSimData, 0, N); auto data = io.DefineVariable("data", shape, start, count); - io.SetEngine("BP5"); // or BPFile + io.SetEngine("BP5"); adios2::Engine bpWriter = io.Open(fname, adios2::Mode::Write); // Simulation steps for (size_t step = 0; step < nSteps; ++step) @@ -82,13 +98,13 @@ If the ``SetMemorySpace`` function is used, the ADIOS2 library will not detect a Underneath, ADIOS2 relies on the backend used at build time to transfer the data. If ADIOS2 was build with CUDA, only CUDA buffers can be provided. If ADIOS2 was build with Kokkos (with CUDA enabled) only CUDA buffers can be provided. If ADIOS2 was build with Kokkos (with HIP enabled) only HIP buffers can be provided. .. note:: - The SYCL backend in Kokkos can be used to run on Nvida, AMD and Intel GPUs + The SYCL backend in Kokkos can be used to run on Nvida, AMD and Intel GPUs, but we recommand using SYCL for Intel, HIP for AMD and CUDA for Nvidia. Kokkos applications -------------------- -ADIOS2 supports GPU buffers provided in the form of ``Kokkos::View`` directly in the Put/Get calls. The memory space can be automatically detected or provided by the user, in the same way as in the CUDA example. +ADIOS2 supports GPU buffers provided in the form of ``Kokkos::View`` directly in the Put/Get calls. The memory space is automatically detected from the View information. In addition to the memory space, for ``Kokkos::View`` ADIOS2 also extracts the layout of the array and adjust the variable dimensions to be able to build the global shape (across ranks) of the array. .. code-block:: c++ @@ -97,6 +113,34 @@ ADIOS2 supports GPU buffers provided in the form of ``Kokkos::View`` directly in If the CUDA backend is being used (and not Kokkos) to enable GPU support in ADIOS2, Kokkos applications can still directly pass ``Kokkos::View`` as long as the correct external header is included: ``#include ``. +******************* +Reading GPU buffers +******************* + +The GPU-aware backend allows different layouts for global arrays without requiring the user to update the code for each case. The user defines the shape of the global array and ADIOS2 adjusts the dimensions for each rank according to the buffer layout and memory space. + +The following example shows a global array of shape (4, 3) when running with 2 ranks, each contributing half of it. + +.. code-block:: text + + Write on LayoutRight, read on LayoutRight + 1 1 1 // rank 0 + 2 2 2 + 3 3 3 // rank 1 + 4 4 4 + Write on LayoutRight, read on LayoutLeft + 1 2 3 4 + 1 2 3 4 + 1 2 3 4 + +On the read side, the Shape function can take a memory space or a layout to return the correct dimensions of the variable. +For the previous example, if a C++ code using two ranks wants to read the data into a GPU buffer, the Shape of the local array should be (3, 2). If the same data will be read on CPU buffers, the shape should be (2, 3). Both of the following code would give acceptable answers: + +.. code-block:: c++ + + auto dims_host = data.Shape(adios2::MemorySpace::Host); + auto dims_device = data.Shape(adios2::ArrayOrdering::ColumnMajor); + *************** Build scripts *************** From 8a57203b16a379999b753f86ad3415069bb79d02 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Mon, 11 Mar 2024 09:12:05 -0400 Subject: [PATCH 083/124] BlockIndex.Evaluate() made branch based on BP5 vs BP4. To support CampaignReader engine, decision is made based on whether MinBlocksInfo is supported by engine. --- source/adios2/toolkit/query/BlockIndex.h | 34 +++++++++++------------- 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/source/adios2/toolkit/query/BlockIndex.h b/source/adios2/toolkit/query/BlockIndex.h index b3a52afcb0..6008683d9d 100644 --- a/source/adios2/toolkit/query/BlockIndex.h +++ b/source/adios2/toolkit/query/BlockIndex.h @@ -28,24 +28,27 @@ class BlockIndex throw std::runtime_error("Unable to evaluate query! Invalid Variable detected"); } - if (m_IdxReader.m_EngineType.find("5") != std::string::npos) // a bp5 reader - RunBP5Stat(query, resultBlockIDs); - else - RunBP4Stat(query, resultBlockIDs); - } - - void RunBP5Stat(const QueryVar &query, std::vector &hitBlocks) - { size_t currStep = m_IdxReader.CurrentStep(); adios2::Dims currShape = m_VarPtr->Shape(); if (!query.IsSelectionValid(currShape)) return; auto MinBlocksInfo = m_IdxReader.MinBlocksInfo(*m_VarPtr, currStep); - if (!MinBlocksInfo) - { // no info, can't do anything - return; + + if (MinBlocksInfo != nullptr) + { + RunStatMinBlocksInfo(query, MinBlocksInfo, resultBlockIDs); + delete MinBlocksInfo; + } + else + { + RunStatBlocksInfo(query, currStep, resultBlockIDs); } + } + + void RunStatMinBlocksInfo(const QueryVar &query, const adios2::MinVarInfo *MinBlocksInfo, + std::vector &hitBlocks) + { for (auto &blockInfo : MinBlocksInfo->BlocksInfo) { T bmin = *(T *)&blockInfo.MinMax.MinUnion; @@ -78,16 +81,11 @@ class BlockIndex hitBlocks.push_back(BlockHit(blockInfo.BlockID)); } } - delete MinBlocksInfo; } - void RunBP4Stat(const QueryVar &query, std::vector &hitBlocks) + void RunStatBlocksInfo(const QueryVar &query, const size_t currStep, + std::vector &hitBlocks) { - size_t currStep = m_IdxReader.CurrentStep(); - adios2::Dims currShape = m_VarPtr->Shape(); - if (!query.IsSelectionValid(currShape)) - return; - std::vector::BPInfo> varBlocksInfo = m_IdxReader.BlocksInfo(*m_VarPtr, currStep); From e492a855e1ceb67fddcd9ef1b4c941fe6805284a Mon Sep 17 00:00:00 2001 From: Ana Gainaru Date: Tue, 12 Mar 2024 12:56:30 -0400 Subject: [PATCH 084/124] Fix links to tutorial materials (#4086) --- docs/user_guide/source/tutorials/overview.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/user_guide/source/tutorials/overview.rst b/docs/user_guide/source/tutorials/overview.rst index 4130a8bfff..97317bf82d 100644 --- a/docs/user_guide/source/tutorials/overview.rst +++ b/docs/user_guide/source/tutorials/overview.rst @@ -16,5 +16,5 @@ More specifically, we will go through the following examples: More advance tutorials that cover more information related to: -- Running ADIOS2 at scale (through files or streaming) with hands-on excercises: `Exascale I/O tutorial ` -- Using ADIOS2 with Paraview, TAU, Catalyst, FIDES, VTK-M: `ADIOS2 tutorial at SC23 ` +- Running ADIOS2 at scale (through files or streaming) with hands-on excercises: `Exascale I/O tutorial `_ +- Using ADIOS2 with Paraview, TAU, Catalyst, FIDES, VTK-M: `ADIOS2 tutorial at SC23 `_ From fa34d766f83fb491e2b4744778c150ac848fd744 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Tue, 12 Mar 2024 12:44:41 -0400 Subject: [PATCH 085/124] adios2_define_derived_variable C/Fortran API. C is compiled conditionally, like the C++ API. The Fortran function is always available, it will print a WARNING if derived variable is not supported. Added Fortran test for magnitude(). --- bindings/C/adios2/c/adios2_c_io.cpp | 41 + bindings/C/adios2/c/adios2_c_io.h | 20 + bindings/C/adios2/c/adios2_c_types.h | 11 + bindings/Fortran/CMakeLists.txt | 2 + bindings/Fortran/f2c/adios2_f2c_io.cpp | 14 + .../adios2_io_define_derived_variable_mod.f90 | 40 + bindings/Fortran/modules/adios2_io_mod.f90 | 783 +++++++++--------- .../Fortran/modules/adios2_parameters_mod.f90 | 264 +++--- .../bindings/fortran/TestBPWriteTypes.F90 | 537 ++++++------ 9 files changed, 929 insertions(+), 783 deletions(-) create mode 100644 bindings/Fortran/modules/adios2_io_define_derived_variable_mod.f90 diff --git a/bindings/C/adios2/c/adios2_c_io.cpp b/bindings/C/adios2/c/adios2_c_io.cpp index a4b8921224..a0de5ec75c 100644 --- a/bindings/C/adios2/c/adios2_c_io.cpp +++ b/bindings/C/adios2/c/adios2_c_io.cpp @@ -150,6 +150,47 @@ adios2_error adios2_set_transport_parameter(adios2_io *io, const size_t transpor } } +adios2_derived_variable *adios2_define_derived_variable(adios2_io *io, const char *name, + const char *expression, + const adios2_derived_var_type type) +{ + adios2_derived_variable *variable = nullptr; +#ifdef ADIOS2_HAVE_DERIVED_VARIABLE + try + { + adios2::helper::CheckForNullptr(io, "for adios2_io, in call to adios2_define_variable"); + adios2::core::IO &ioCpp = *reinterpret_cast(io); + adios2::DerivedVarType typeCpp = adios2::DerivedVarType::MetadataOnly; + switch (type) + { + case adios2_derived_var_type_metadata_only: + typeCpp = adios2::DerivedVarType::MetadataOnly; + break; + case adios2_derived_var_type_expression_string: + typeCpp = adios2::DerivedVarType::ExpressionString; + break; + case adios2_derived_var_type_store_data: + typeCpp = adios2::DerivedVarType::StoreData; + break; + } + adios2::core::VariableDerived *variableCpp = nullptr; + variableCpp = &ioCpp.DefineDerivedVariable(name, expression, typeCpp); + variable = reinterpret_cast(variableCpp); + } + catch (...) + { + + adios2::helper::ExceptionToError("adios2_define_variable"); + } +#else + std::cout << "ADIOS2 Warning: adios2_define_derived_variable() is not supported in the " + "current ADIOS2 build. The expression " + << expression << " will be ignored and the variable " << name + << " will not be produced." << std::endl; +#endif + return variable; +} + adios2_variable *adios2_define_variable(adios2_io *io, const char *name, const adios2_type type, const size_t ndims, const size_t *shape, const size_t *start, const size_t *count, diff --git a/bindings/C/adios2/c/adios2_c_io.h b/bindings/C/adios2/c/adios2_c_io.h index 8a8272c5a7..09fa10e256 100644 --- a/bindings/C/adios2/c/adios2_c_io.h +++ b/bindings/C/adios2/c/adios2_c_io.h @@ -122,6 +122,26 @@ adios2_variable *adios2_define_variable(adios2_io *io, const char *name, const a const size_t *start, const size_t *count, const adios2_constant_dims constant_dims); +#ifdef ADIOS2_HAVE_DERIVED_VARIABLE +/** + * @brief Define a derived variable within io + * @param io handler that owns the variable + * @param name unique variable identifier + * @param type primitive type from enum adios2_type in adios2_c_types.h + * @param ndims number of dimensions + * @param shape global dimension + * @param start local offset + * @param count local dimension + * @param constant_dims adios2_constant_dims_true:: shape, start, count + * won't change; adios2_constant_dims_false: shape, start, count will change + * after definition + * @return success: handler, failure: NULL + */ +adios2_derived_variable *adios2_define_derived_variable(adios2_io *io, const char *name, + const char *expression, + const adios2_derived_var_type type); +#endif + /** * @brief Retrieve a variable handler within current io handler * @param io handler to variable io owner diff --git a/bindings/C/adios2/c/adios2_c_types.h b/bindings/C/adios2/c/adios2_c_types.h index 81f6ad3479..fb102df4cb 100644 --- a/bindings/C/adios2/c/adios2_c_types.h +++ b/bindings/C/adios2/c/adios2_c_types.h @@ -23,6 +23,7 @@ extern "C" { typedef struct adios2_adios adios2_adios; typedef struct adios2_io adios2_io; typedef struct adios2_variable adios2_variable; +typedef struct adios2_derived_variable adios2_derived_variable; typedef struct adios2_attribute adios2_attribute; typedef struct adios2_engine adios2_engine; typedef struct adios2_operator adios2_operator; @@ -139,6 +140,16 @@ typedef enum adios2_arrayordering_auto } adios2_arrayordering; +#ifdef ADIOS2_HAVE_DERIVED_VARIABLE +/** Type of derived variables */ +typedef enum +{ + adios2_derived_var_type_metadata_only = 0, + adios2_derived_var_type_expression_string = 1, + adios2_derived_var_type_store_data = 2 +} adios2_derived_var_type; +#endif + static const size_t adios2_string_array_element_max_size = 4096; static const size_t adios2_local_value_dim = SIZE_MAX - 2; diff --git a/bindings/Fortran/CMakeLists.txt b/bindings/Fortran/CMakeLists.txt index 1a998bbe95..20c8407c5e 100644 --- a/bindings/Fortran/CMakeLists.txt +++ b/bindings/Fortran/CMakeLists.txt @@ -44,6 +44,7 @@ add_library(adios2_fortran modules/adios2_io_open_mod.F90 modules/adios2_io_open_serial_smod.F90 modules/adios2_io_define_variable_mod.f90 + modules/adios2_io_define_derived_variable_mod.f90 modules/adios2_io_define_attribute_mod.f90 modules/adios2_engine_mod.f90 modules/adios2_engine_begin_step_mod.f90 @@ -54,6 +55,7 @@ add_library(adios2_fortran modules/adios2_variable_max_mod.f90 modules/adios2_operator_mod.f90 ) + set_property(TARGET adios2_fortran PROPERTY EXPORT_NAME fortran) set_property(TARGET adios2_fortran PROPERTY OUTPUT_NAME adios2${ADIOS2_LIBRARY_SUFFIX}_fortran) diff --git a/bindings/Fortran/f2c/adios2_f2c_io.cpp b/bindings/Fortran/f2c/adios2_f2c_io.cpp index ef3899d5a0..f7ae560a78 100644 --- a/bindings/Fortran/f2c/adios2_f2c_io.cpp +++ b/bindings/Fortran/f2c/adios2_f2c_io.cpp @@ -201,6 +201,20 @@ void FC_GLOBAL(adios2_define_variable_f2c, } } +#ifdef ADIOS2_HAVE_DERIVED_VARIABLE +void FC_GLOBAL(adios2_define_derived_variable_f2c, + ADIOS2_DEFINE_DERIVED_VARIABLE_F2C)(adios2_derived_variable **variable, + adios2_io **io, const char *name, + const char *expression, const int *type, + int *ierr) +{ + *variable = adios2_define_derived_variable(*io, name, expression, + static_cast(*type)); + *ierr = (*variable == NULL) ? static_cast(adios2_error_exception) + : static_cast(adios2_error_none); +} +#endif + struct cnamelist { char **names; diff --git a/bindings/Fortran/modules/adios2_io_define_derived_variable_mod.f90 b/bindings/Fortran/modules/adios2_io_define_derived_variable_mod.f90 new file mode 100644 index 0000000000..fc6feced5b --- /dev/null +++ b/bindings/Fortran/modules/adios2_io_define_derived_variable_mod.f90 @@ -0,0 +1,40 @@ +! +! Distributed under the OSI-approved Apache License, Version 2.0. See +! accompanying file Copyright.txt for details. +! +! adios2_io_define_variable_mod.f90 : ADIOS2 Fortran bindings for IO class +! overloaded (C++ template) function adios2_define_variable +! +! Created on: Mar 13, 2017 +! Author: William F Godoy godoywf@ornl.gov +! + +module adios2_io_define_derived_variable_mod + use adios2_parameters_mod + implicit none + + external adios2_define_derived_variable_f2c + +contains + + subroutine adios2_define_derived_variable(variable, io, name, expression, adios2_derived_var_type, & + ierr) + type(adios2_derived_variable), intent(out) :: variable + type(adios2_io), intent(in) :: io + character*(*), intent(in) :: name + character*(*), intent(in) :: expression + integer, intent(in):: adios2_derived_var_type + integer, intent(out) :: ierr + + call adios2_define_derived_variable_f2c(variable%f2c, io%f2c, & + TRIM(ADJUSTL(name))//char(0), TRIM(ADJUSTL(expression))//char(0), & + adios2_derived_var_type, ierr) + if( ierr == 0 ) then + variable%valid = .true. + variable%name = name + variable%type = adios2_derived_var_type + end if + + end subroutine + +end module diff --git a/bindings/Fortran/modules/adios2_io_mod.f90 b/bindings/Fortran/modules/adios2_io_mod.f90 index 6011df1978..4786097e78 100644 --- a/bindings/Fortran/modules/adios2_io_mod.f90 +++ b/bindings/Fortran/modules/adios2_io_mod.f90 @@ -9,402 +9,403 @@ ! module adios2_io_mod - use adios2_io_open_mod - use adios2_io_define_variable_mod - use adios2_io_define_attribute_mod - use adios2_variable_mod - implicit none - - external adios2_add_transport_f2c - external adios2_attribute_is_value_f2c - external adios2_attribute_length_f2c - external adios2_attribute_type_f2c - external adios2_clear_parameters_f2c - external adios2_flush_all_engines_f2c - external adios2_get_parameter_f2c - external adios2_get_parameter_length_f2c - external adios2_in_config_file_f2c - external adios2_available_variables_f2c - external adios2_available_attributes_f2c - external adios2_retrieve_namelist_f2c - external adios2_inquire_attribute_f2c - external adios2_inquire_variable_attribute_f2c - external adios2_inquire_variable_f2c - external adios2_io_engine_type_f2c - external adios2_io_engine_type_length_f2c - external adios2_remove_all_attributes_f2c - external adios2_remove_all_variables_f2c - external adios2_remove_attribute_f2c - external adios2_remove_variable_f2c - external adios2_set_engine_f2c - external adios2_set_parameter_f2c - external adios2_set_parameters_f2c - external adios2_set_transport_parameter_f2c + use adios2_io_open_mod + use adios2_io_define_variable_mod + use adios2_io_define_derived_variable_mod + use adios2_io_define_attribute_mod + use adios2_variable_mod + implicit none + + external adios2_add_transport_f2c + external adios2_attribute_is_value_f2c + external adios2_attribute_length_f2c + external adios2_attribute_type_f2c + external adios2_clear_parameters_f2c + external adios2_flush_all_engines_f2c + external adios2_get_parameter_f2c + external adios2_get_parameter_length_f2c + external adios2_in_config_file_f2c + external adios2_available_variables_f2c + external adios2_available_attributes_f2c + external adios2_retrieve_namelist_f2c + external adios2_inquire_attribute_f2c + external adios2_inquire_variable_attribute_f2c + external adios2_inquire_variable_f2c + external adios2_io_engine_type_f2c + external adios2_io_engine_type_length_f2c + external adios2_remove_all_attributes_f2c + external adios2_remove_all_variables_f2c + external adios2_remove_attribute_f2c + external adios2_remove_variable_f2c + external adios2_set_engine_f2c + external adios2_set_parameter_f2c + external adios2_set_parameters_f2c + external adios2_set_transport_parameter_f2c contains - subroutine adios2_in_config_file(result, io, ierr) - logical, intent(out):: result - type(adios2_io), intent(in):: io - integer, intent(out):: ierr + subroutine adios2_in_config_file(result, io, ierr) + logical, intent(out):: result + type(adios2_io), intent(in):: io + integer, intent(out):: ierr - ! local - integer resultInt - - call adios2_in_config_file_f2c(resultInt, io, ierr) - if(resultInt == 0) then - result = .false. - else - result = .true. - end if - - end subroutine - - subroutine adios2_io_engine_type(type, io, ierr) - character(len=:), allocatable, intent(out) :: type - type(adios2_io), intent(in) :: io - integer, intent(out) :: ierr - - !local - integer :: length - - if (allocated(type)) deallocate (type) - - call adios2_io_engine_type_length_f2c(length, io%f2c, ierr) - if (ierr == 0) then - allocate (character(length) :: type) - call adios2_io_engine_type_f2c(type, io%f2c, ierr) - end if - - end subroutine - - subroutine adios2_set_engine(io, engine_type, ierr) - type(adios2_io), intent(inout) :: io - character*(*), intent(in) :: engine_type - integer, intent(out) :: ierr - - call adios2_set_engine_f2c(io%f2c, & - TRIM(ADJUSTL(engine_type))//char(0), ierr) - - if( ierr == 0 ) io%engine_type = engine_type - - end subroutine - - subroutine adios2_set_parameters(io, parameters, ierr) - type(adios2_io), intent(in) :: io - character*(*), intent(in) :: parameters - integer, intent(out) :: ierr - - call adios2_set_parameters_f2c(io%f2c, & - TRIM(ADJUSTL(parameters))//char(0), & - ierr) - end subroutine - - subroutine adios2_set_parameter(io, key, value, ierr) - type(adios2_io), intent(in) :: io - character*(*), intent(in) :: key - character*(*), intent(in) :: value - integer, intent(out) :: ierr - - call adios2_set_parameter_f2c(io%f2c, TRIM(ADJUSTL(key))//char(0), & - TRIM(ADJUSTL(value))//char(0), ierr) - end subroutine - - subroutine adios2_get_parameter(value, io, key, ierr) - character(len=:), allocatable, intent(out) :: value - type(adios2_io), intent(in) :: io - character*(*), intent(in) :: key - integer, intent(out) :: ierr - - !local - integer :: length - - if (allocated(value)) deallocate (value) - - call adios2_get_parameter_length_f2c(length, io%f2c, & - TRIM(ADJUSTL(key))//char(0), ierr) - if (ierr == 0) then - allocate (character(length) :: value) - call adios2_get_parameter_f2c(value, io%f2c, & - TRIM(ADJUSTL(key))//char(0), ierr) - end if - end subroutine - - subroutine adios2_clear_parameters(io, ierr) - type(adios2_io), intent(in) :: io - integer, intent(out) :: ierr - call adios2_clear_parameters_f2c(io%f2c, ierr) - end subroutine - - subroutine adios2_add_transport(transport_index, io, type, ierr) - integer, intent(out):: transport_index - type(adios2_io), intent(in) :: io - character*(*), intent(in) :: type - integer, intent(out) :: ierr - - call adios2_add_transport_f2c(transport_index, io%f2c, & - TRIM(ADJUSTL(type))//char(0), ierr) - - end subroutine - - subroutine adios2_set_transport_parameter(io, transport_index, key, value, & - ierr) - type(adios2_io), intent(in):: io - integer, intent(in):: transport_index - character*(*), intent(in) :: key - character*(*), intent(in) :: value - integer, intent(out):: ierr - - call adios2_set_transport_parameter_f2c(io%f2c, transport_index, & - TRIM(ADJUSTL(key))//char(0), & - TRIM(ADJUSTL(value))//char(0), & - ierr) - end subroutine - - - subroutine adios2_available_variables(io, namestruct, ierr) - type(adios2_io), intent(in) :: io - type(adios2_namestruct), intent(out) :: namestruct - integer, intent(out) :: ierr - - call adios2_available_variables_f2c(io%f2c, namestruct%f2c, & - namestruct%count, namestruct%max_name_len, ierr) - if (ierr == 0) then - namestruct%valid = .true. - endif - end subroutine - - subroutine adios2_retrieve_names(namestruct, namelist, ierr) - type(adios2_namestruct), intent(inout) :: namestruct - character(*), dimension(*), intent(inout) :: namelist - integer, intent(out) :: ierr - - if (namestruct%valid .and. namestruct%f2c > 0_8) then - call adios2_retrieve_namelist_f2c(namestruct%f2c, namelist, ierr) - else - write(*,*) "ADIOS2 Fortran ERROR: invalid namestruct when calling adios2_retrieve_names()" - endif - namestruct%valid = .false. - end subroutine - - ! - ! F2008 implementation that allows for allocating a character array inside - ! - ! subroutine adios2_available_variables(io, nvars, varnamelist, ierr) - ! type(adios2_io), intent(in) :: io - ! integer, intent(out) :: nvars - ! character(len=:), dimension(:), allocatable, intent(out) :: varnamelist - ! integer, intent(out) :: ierr - - ! integer(kind=8):: namestruct - ! integer :: count, max_name_len - - ! call adios2_available_variables_f2c(io%f2c, namestruct, count, & - ! max_name_len, ierr) - ! if (ierr == 0) then - ! allocate(character(len=max_name_len) :: varnamelist(count)) - ! endif - - ! call adios2_retrieve_variable_names_f2c(namestruct, varnamelist, ierr) - ! nvars = count - ! end subroutine - - - subroutine adios2_inquire_variable(variable, io, name, ierr) - type(adios2_variable), intent(out) :: variable - type(adios2_io), intent(in) :: io - character*(*), intent(in) :: name - integer, intent(out) :: ierr - - call adios2_inquire_variable_f2c(variable%f2c, io%f2c, & - TRIM(ADJUSTL(name))//char(0), ierr) - - if(variable%f2c > 0_8) then - variable%valid = .true. - variable%name = name - call adios2_variable_type(variable%type, variable, ierr) - call adios2_variable_ndims(variable%ndims, variable, ierr) - else - variable%valid = .false. - variable%name = '' - variable%type = adios2_type_unknown - variable%ndims = -1 - end if - - end subroutine - - subroutine adios2_remove_variable(result, io, name, ierr) - type(adios2_io), intent(in) :: io - character*(*), intent(in) :: name - logical, intent(out) :: result - integer, intent(out) :: ierr - ! Local - type(adios2_variable):: variable - integer:: resultInt - - call adios2_inquire_variable(variable, io, name, ierr) - if( variable%valid ) then - call adios2_remove_variable_f2c(resultInt, io%f2c, & - TRIM(ADJUSTL(name))//char(0), ierr) - if( resultInt == 1) then - result = .true. - else - result = .false. - end if - end if - - end subroutine - - - subroutine adios2_remove_all_variables(io, ierr) - type(adios2_io), intent(in) :: io - integer, intent(out) :: ierr - - call adios2_remove_all_variables_f2c(io%f2c, ierr) - - end subroutine - - subroutine adios2_available_attributes(io, namestruct, ierr) - type(adios2_io), intent(in) :: io - type(adios2_namestruct), intent(out) :: namestruct - integer, intent(out) :: ierr - - call adios2_available_attributes_f2c(io%f2c, namestruct%f2c, & - namestruct%count, namestruct%max_name_len, ierr) - if (ierr == 0) then - namestruct%valid = .true. - endif - end subroutine - - ! subroutine adios2_available_attributes(io, nattrs, attrnamelist, ierr) - ! type(adios2_io), intent(in) :: io - ! integer, intent(out) :: nattrs - ! character(len=:), dimension(:), allocatable, intent(out) :: attrnamelist - ! integer, intent(out) :: ierr - - ! integer(kind=8):: namestruct - ! integer :: count, max_name_len - - ! call adios2_available_attributes_f2c(io%f2c, namestruct, count, & - ! max_name_len, ierr) - ! if (ierr == 0) then - ! allocate(character(len=max_name_len) :: attrnamelist(count)) - ! endif - - ! call adios2_retrieve_attribute_names_f2c(namestruct, count, & - ! max_name_len, attrnamelist, ierr) - ! nattrs = count - ! end subroutine - - subroutine adios2_inquire_attribute(attribute, io, name, ierr) - type(adios2_attribute), intent(out) :: attribute - type(adios2_io), intent(in) :: io - character*(*), intent(in) :: name - integer, intent(out) :: ierr - !local - integer:: is_valueInt - - call adios2_inquire_attribute_f2c(attribute%f2c, io%f2c, & - TRIM(ADJUSTL(name))//char(0), ierr) - - if(attribute%f2c > 0_8) then - attribute%valid = .true. - attribute%name = name - call adios2_attribute_type_f2c(attribute%type, attribute%f2c, ierr) - call adios2_attribute_length_f2c(attribute%length, attribute%f2c, & - ierr) - call adios2_attribute_is_value_f2c(is_valueInt, attribute%f2c, ierr) - - if(is_valueInt == 0) then - attribute%is_value = .false. - else - attribute%is_value = .true. - end if - - else - attribute%valid = .false. - attribute%name = '' - attribute%type = adios2_type_unknown - attribute%length = 0 - end if - - end subroutine - - subroutine adios2_inquire_variable_attribute(attribute, io, attribute_name, variable_name, separator, ierr) - type(adios2_attribute), intent(out) :: attribute - type(adios2_io), intent(in) :: io - character*(*), intent(in) :: attribute_name - character*(*), intent(in) :: variable_name - character*(*), intent(in) :: separator - integer, intent(out) :: ierr - !local - integer:: is_valueInt - - call adios2_inquire_variable_attribute_f2c(attribute%f2c, io%f2c, & - TRIM(ADJUSTL(attribute_name))//char(0), & - TRIM(ADJUSTL(variable_name))//char(0), & - TRIM(ADJUSTL(separator))//char(0), & - ierr) - - if(attribute%f2c > 0_8) then - attribute%valid = .true. - attribute%name = TRIM(variable_name)//TRIM(separator)//TRIM(attribute_name) - call adios2_attribute_type_f2c(attribute%type, attribute%f2c, ierr) - call adios2_attribute_length_f2c(attribute%length, attribute%f2c, & - ierr) - call adios2_attribute_is_value_f2c(is_valueInt, attribute%f2c, ierr) - - if(is_valueInt == 0) then - attribute%is_value = .false. - else - attribute%is_value = .true. - end if - - else - attribute%valid = .false. - attribute%name = '' - attribute%type = adios2_type_unknown - attribute%length = 0 - end if - - end subroutine - - - subroutine adios2_remove_attribute(result, io, name, ierr) - type(adios2_io), intent(in) :: io - character*(*), intent(in) :: name - logical, intent(out) :: result - integer, intent(out) :: ierr - - ! Local - integer :: resultInt - - call adios2_remove_attribute_f2c(resultInt, io%f2c, & - TRIM(ADJUSTL(name))//char(0), ierr) - if( resultInt == 1) then + ! local + integer resultInt + + call adios2_in_config_file_f2c(resultInt, io, ierr) + if(resultInt == 0) then + result = .false. + else + result = .true. + end if + + end subroutine + + subroutine adios2_io_engine_type(type, io, ierr) + character(len=:), allocatable, intent(out) :: type + type(adios2_io), intent(in) :: io + integer, intent(out) :: ierr + + !local + integer :: length + + if (allocated(type)) deallocate (type) + + call adios2_io_engine_type_length_f2c(length, io%f2c, ierr) + if (ierr == 0) then + allocate (character(length) :: type) + call adios2_io_engine_type_f2c(type, io%f2c, ierr) + end if + + end subroutine + + subroutine adios2_set_engine(io, engine_type, ierr) + type(adios2_io), intent(inout) :: io + character*(*), intent(in) :: engine_type + integer, intent(out) :: ierr + + call adios2_set_engine_f2c(io%f2c, & + TRIM(ADJUSTL(engine_type))//char(0), ierr) + + if( ierr == 0 ) io%engine_type = engine_type + + end subroutine + + subroutine adios2_set_parameters(io, parameters, ierr) + type(adios2_io), intent(in) :: io + character*(*), intent(in) :: parameters + integer, intent(out) :: ierr + + call adios2_set_parameters_f2c(io%f2c, & + TRIM(ADJUSTL(parameters))//char(0), & + ierr) + end subroutine + + subroutine adios2_set_parameter(io, key, value, ierr) + type(adios2_io), intent(in) :: io + character*(*), intent(in) :: key + character*(*), intent(in) :: value + integer, intent(out) :: ierr + + call adios2_set_parameter_f2c(io%f2c, TRIM(ADJUSTL(key))//char(0), & + TRIM(ADJUSTL(value))//char(0), ierr) + end subroutine + + subroutine adios2_get_parameter(value, io, key, ierr) + character(len=:), allocatable, intent(out) :: value + type(adios2_io), intent(in) :: io + character*(*), intent(in) :: key + integer, intent(out) :: ierr + + !local + integer :: length + + if (allocated(value)) deallocate (value) + + call adios2_get_parameter_length_f2c(length, io%f2c, & + TRIM(ADJUSTL(key))//char(0), ierr) + if (ierr == 0) then + allocate (character(length) :: value) + call adios2_get_parameter_f2c(value, io%f2c, & + TRIM(ADJUSTL(key))//char(0), ierr) + end if + end subroutine + + subroutine adios2_clear_parameters(io, ierr) + type(adios2_io), intent(in) :: io + integer, intent(out) :: ierr + call adios2_clear_parameters_f2c(io%f2c, ierr) + end subroutine + + subroutine adios2_add_transport(transport_index, io, type, ierr) + integer, intent(out):: transport_index + type(adios2_io), intent(in) :: io + character*(*), intent(in) :: type + integer, intent(out) :: ierr + + call adios2_add_transport_f2c(transport_index, io%f2c, & + TRIM(ADJUSTL(type))//char(0), ierr) + + end subroutine + + subroutine adios2_set_transport_parameter(io, transport_index, key, value, & + ierr) + type(adios2_io), intent(in):: io + integer, intent(in):: transport_index + character*(*), intent(in) :: key + character*(*), intent(in) :: value + integer, intent(out):: ierr + + call adios2_set_transport_parameter_f2c(io%f2c, transport_index, & + TRIM(ADJUSTL(key))//char(0), & + TRIM(ADJUSTL(value))//char(0), & + ierr) + end subroutine + + + subroutine adios2_available_variables(io, namestruct, ierr) + type(adios2_io), intent(in) :: io + type(adios2_namestruct), intent(out) :: namestruct + integer, intent(out) :: ierr + + call adios2_available_variables_f2c(io%f2c, namestruct%f2c, & + namestruct%count, namestruct%max_name_len, ierr) + if (ierr == 0) then + namestruct%valid = .true. + endif + end subroutine + + subroutine adios2_retrieve_names(namestruct, namelist, ierr) + type(adios2_namestruct), intent(inout) :: namestruct + character(*), dimension(*), intent(inout) :: namelist + integer, intent(out) :: ierr + + if (namestruct%valid .and. namestruct%f2c > 0_8) then + call adios2_retrieve_namelist_f2c(namestruct%f2c, namelist, ierr) + else + write(*,*) "ADIOS2 Fortran ERROR: invalid namestruct when calling adios2_retrieve_names()" + endif + namestruct%valid = .false. + end subroutine + + ! + ! F2008 implementation that allows for allocating a character array inside + ! + ! subroutine adios2_available_variables(io, nvars, varnamelist, ierr) + ! type(adios2_io), intent(in) :: io + ! integer, intent(out) :: nvars + ! character(len=:), dimension(:), allocatable, intent(out) :: varnamelist + ! integer, intent(out) :: ierr + + ! integer(kind=8):: namestruct + ! integer :: count, max_name_len + + ! call adios2_available_variables_f2c(io%f2c, namestruct, count, & + ! max_name_len, ierr) + ! if (ierr == 0) then + ! allocate(character(len=max_name_len) :: varnamelist(count)) + ! endif + + ! call adios2_retrieve_variable_names_f2c(namestruct, varnamelist, ierr) + ! nvars = count + ! end subroutine + + + subroutine adios2_inquire_variable(variable, io, name, ierr) + type(adios2_variable), intent(out) :: variable + type(adios2_io), intent(in) :: io + character*(*), intent(in) :: name + integer, intent(out) :: ierr + + call adios2_inquire_variable_f2c(variable%f2c, io%f2c, & + TRIM(ADJUSTL(name))//char(0), ierr) + + if(variable%f2c > 0_8) then + variable%valid = .true. + variable%name = name + call adios2_variable_type(variable%type, variable, ierr) + call adios2_variable_ndims(variable%ndims, variable, ierr) + else + variable%valid = .false. + variable%name = '' + variable%type = adios2_type_unknown + variable%ndims = -1 + end if + + end subroutine + + subroutine adios2_remove_variable(result, io, name, ierr) + type(adios2_io), intent(in) :: io + character*(*), intent(in) :: name + logical, intent(out) :: result + integer, intent(out) :: ierr + ! Local + type(adios2_variable):: variable + integer:: resultInt + + call adios2_inquire_variable(variable, io, name, ierr) + if( variable%valid ) then + call adios2_remove_variable_f2c(resultInt, io%f2c, & + TRIM(ADJUSTL(name))//char(0), ierr) + if( resultInt == 1) then result = .true. - else + else result = .false. - end if - - end subroutine - - - subroutine adios2_remove_all_attributes(io, ierr) - type(adios2_io), intent(in) :: io - integer, intent(out) :: ierr - - call adios2_remove_all_attributes_f2c(io%f2c, ierr) - - end subroutine - - - subroutine adios2_flush_all_engines(io, ierr) - type(adios2_io), intent(in) :: io - integer, intent(out) :: ierr - - call adios2_flush_all_engines_f2c(io%f2c, ierr) - - end subroutine + end if + end if + + end subroutine + + + subroutine adios2_remove_all_variables(io, ierr) + type(adios2_io), intent(in) :: io + integer, intent(out) :: ierr + + call adios2_remove_all_variables_f2c(io%f2c, ierr) + + end subroutine + + subroutine adios2_available_attributes(io, namestruct, ierr) + type(adios2_io), intent(in) :: io + type(adios2_namestruct), intent(out) :: namestruct + integer, intent(out) :: ierr + + call adios2_available_attributes_f2c(io%f2c, namestruct%f2c, & + namestruct%count, namestruct%max_name_len, ierr) + if (ierr == 0) then + namestruct%valid = .true. + endif + end subroutine + + ! subroutine adios2_available_attributes(io, nattrs, attrnamelist, ierr) + ! type(adios2_io), intent(in) :: io + ! integer, intent(out) :: nattrs + ! character(len=:), dimension(:), allocatable, intent(out) :: attrnamelist + ! integer, intent(out) :: ierr + + ! integer(kind=8):: namestruct + ! integer :: count, max_name_len + + ! call adios2_available_attributes_f2c(io%f2c, namestruct, count, & + ! max_name_len, ierr) + ! if (ierr == 0) then + ! allocate(character(len=max_name_len) :: attrnamelist(count)) + ! endif + + ! call adios2_retrieve_attribute_names_f2c(namestruct, count, & + ! max_name_len, attrnamelist, ierr) + ! nattrs = count + ! end subroutine + + subroutine adios2_inquire_attribute(attribute, io, name, ierr) + type(adios2_attribute), intent(out) :: attribute + type(adios2_io), intent(in) :: io + character*(*), intent(in) :: name + integer, intent(out) :: ierr + !local + integer:: is_valueInt + + call adios2_inquire_attribute_f2c(attribute%f2c, io%f2c, & + TRIM(ADJUSTL(name))//char(0), ierr) + + if(attribute%f2c > 0_8) then + attribute%valid = .true. + attribute%name = name + call adios2_attribute_type_f2c(attribute%type, attribute%f2c, ierr) + call adios2_attribute_length_f2c(attribute%length, attribute%f2c, & + ierr) + call adios2_attribute_is_value_f2c(is_valueInt, attribute%f2c, ierr) + + if(is_valueInt == 0) then + attribute%is_value = .false. + else + attribute%is_value = .true. + end if + + else + attribute%valid = .false. + attribute%name = '' + attribute%type = adios2_type_unknown + attribute%length = 0 + end if + + end subroutine + + subroutine adios2_inquire_variable_attribute(attribute, io, attribute_name, variable_name, separator, ierr) + type(adios2_attribute), intent(out) :: attribute + type(adios2_io), intent(in) :: io + character*(*), intent(in) :: attribute_name + character*(*), intent(in) :: variable_name + character*(*), intent(in) :: separator + integer, intent(out) :: ierr + !local + integer:: is_valueInt + + call adios2_inquire_variable_attribute_f2c(attribute%f2c, io%f2c, & + TRIM(ADJUSTL(attribute_name))//char(0), & + TRIM(ADJUSTL(variable_name))//char(0), & + TRIM(ADJUSTL(separator))//char(0), & + ierr) + + if(attribute%f2c > 0_8) then + attribute%valid = .true. + attribute%name = TRIM(variable_name)//TRIM(separator)//TRIM(attribute_name) + call adios2_attribute_type_f2c(attribute%type, attribute%f2c, ierr) + call adios2_attribute_length_f2c(attribute%length, attribute%f2c, & + ierr) + call adios2_attribute_is_value_f2c(is_valueInt, attribute%f2c, ierr) + + if(is_valueInt == 0) then + attribute%is_value = .false. + else + attribute%is_value = .true. + end if + + else + attribute%valid = .false. + attribute%name = '' + attribute%type = adios2_type_unknown + attribute%length = 0 + end if + + end subroutine + + + subroutine adios2_remove_attribute(result, io, name, ierr) + type(adios2_io), intent(in) :: io + character*(*), intent(in) :: name + logical, intent(out) :: result + integer, intent(out) :: ierr + + ! Local + integer :: resultInt + + call adios2_remove_attribute_f2c(resultInt, io%f2c, & + TRIM(ADJUSTL(name))//char(0), ierr) + if( resultInt == 1) then + result = .true. + else + result = .false. + end if + + end subroutine + + + subroutine adios2_remove_all_attributes(io, ierr) + type(adios2_io), intent(in) :: io + integer, intent(out) :: ierr + + call adios2_remove_all_attributes_f2c(io%f2c, ierr) + + end subroutine + + + subroutine adios2_flush_all_engines(io, ierr) + type(adios2_io), intent(in) :: io + integer, intent(out) :: ierr + + call adios2_flush_all_engines_f2c(io%f2c, ierr) + + end subroutine end module diff --git a/bindings/Fortran/modules/adios2_parameters_mod.f90 b/bindings/Fortran/modules/adios2_parameters_mod.f90 index a96fd8549a..bf0dd18c9b 100644 --- a/bindings/Fortran/modules/adios2_parameters_mod.f90 +++ b/bindings/Fortran/modules/adios2_parameters_mod.f90 @@ -9,131 +9,143 @@ ! module adios2_parameters_mod - implicit none - - ! Types - integer, parameter :: adios2_type_unknown = -1 - - integer, parameter :: adios2_type_character = 0 - integer, parameter :: adios2_type_string = 0 - - integer, parameter :: adios2_type_real4 = 1 - integer, parameter :: adios2_type_real = 1 - - integer, parameter :: adios2_type_real8 = 2 - integer, parameter :: adios2_type_dp = 2 - integer, parameter :: adios2_type_double_precision = 2 - - integer, parameter :: adios2_type_complex4 = 3 - integer, parameter :: adios2_type_complex = 3 - - integer, parameter :: adios2_type_complex8 = 4 - integer, parameter :: adios2_type_complex_dp = 4 - - integer, parameter :: adios2_type_integer1 = 5 - integer, parameter :: adios2_type_integer2 = 6 - integer, parameter :: adios2_type_integer4 = 7 - integer, parameter :: adios2_type_integer8 = 8 - - ! is_constant_dims - logical, parameter :: adios2_constant_dims = .true. - logical, parameter :: adios2_variable_dims = .false. - - ! Variable Found or not found, ierr value - integer, parameter :: adios2_not_found = -1 - integer, parameter :: adios2_found = 0 - - ! error - integer, parameter :: adios2_error_none = 0 - integer, parameter :: adios2_error_invalid_argument = 1 - integer, parameter :: adios2_error_system_error = 2 - integer, parameter :: adios2_error_runtime_error = 3 - integer, parameter :: adios2_error_exception = 4 - - ! Mode - integer, parameter :: adios2_mode_undefined = 0 - integer, parameter :: adios2_mode_write = 1 - integer, parameter :: adios2_mode_read = 2 - integer, parameter :: adios2_mode_append = 3 - integer, parameter :: adios2_mode_readRandomAccess = 6 - - integer, parameter :: adios2_mode_deferred = 4 - integer, parameter :: adios2_mode_sync = 5 - - integer, parameter :: adios2_memory_space_detect = 0 - integer, parameter :: adios2_memory_space_host = 1 - integer, parameter :: adios2_memory_space_gpu = 2 - - ! Step Mode - integer, parameter :: adios2_step_mode_append = 0 - integer, parameter :: adios2_step_mode_update = 1 - integer, parameter :: adios2_step_mode_read = 2 - - ! Step Status - integer, parameter :: adios2_step_status_other_error = -1 - integer, parameter :: adios2_step_status_ok = 0 - integer, parameter :: adios2_step_status_not_ready = 1 - integer, parameter :: adios2_step_status_end_of_stream = 2 - - !> Fixed size for string array, used in variables and attributes, - !! must be less or equal than C equivalent in adios2_c_types.h - integer, parameter :: adios2_string_array_element_max_size = 4096 - - integer(kind=8), parameter, dimension(1) :: adios2_null_dims = (/-1/) - integer(kind=8), parameter :: adios2_local_value_dim = -2 - - logical, parameter :: adios2_advance_yes = .true. - logical, parameter :: adios2_advance_no = .false. - - ! Low level API handlers - type adios2_adios - integer(kind=8):: f2c = 0_8 - logical :: valid = .false. - end type - - type adios2_io - integer(kind=8):: f2c = 0_8 - logical :: valid = .false. - character(len=15):: engine_type = 'BPFile' - end type - - type adios2_variable - integer(kind=8):: f2c = 0_8 - logical :: valid = .false. - character(len=4096):: name = '' - integer :: type = -1 - integer :: ndims = -1 - end type - - type adios2_attribute - integer(kind=8):: f2c = 0_8 - logical :: valid = .false. - logical :: is_value = .false. - character(len=4096):: name = '' - integer :: type = -1 - integer :: length = -1 - end type - - type adios2_engine - integer(kind=8):: f2c = 0_8 - logical :: valid = .false. - character(len=64):: name = '' - character(len=15):: type = '' - integer :: mode = adios2_mode_undefined - end type - - type adios2_operator - integer(kind=8):: f2c = 0_8 - logical :: valid = .false. - character(len=64):: name = '' - character(len=64):: type = '' - end type - - type adios2_namestruct - integer(kind=8):: f2c = 0_8 - logical :: valid = .false. - integer :: count - integer :: max_name_len - end type + implicit none + + ! Types + integer, parameter :: adios2_type_unknown = -1 + + integer, parameter :: adios2_type_character = 0 + integer, parameter :: adios2_type_string = 0 + + integer, parameter :: adios2_type_real4 = 1 + integer, parameter :: adios2_type_real = 1 + + integer, parameter :: adios2_type_real8 = 2 + integer, parameter :: adios2_type_dp = 2 + integer, parameter :: adios2_type_double_precision = 2 + + integer, parameter :: adios2_type_complex4 = 3 + integer, parameter :: adios2_type_complex = 3 + + integer, parameter :: adios2_type_complex8 = 4 + integer, parameter :: adios2_type_complex_dp = 4 + + integer, parameter :: adios2_type_integer1 = 5 + integer, parameter :: adios2_type_integer2 = 6 + integer, parameter :: adios2_type_integer4 = 7 + integer, parameter :: adios2_type_integer8 = 8 + + ! is_constant_dims + logical, parameter :: adios2_constant_dims = .true. + logical, parameter :: adios2_variable_dims = .false. + + ! Variable Found or not found, ierr value + integer, parameter :: adios2_not_found = -1 + integer, parameter :: adios2_found = 0 + + ! error + integer, parameter :: adios2_error_none = 0 + integer, parameter :: adios2_error_invalid_argument = 1 + integer, parameter :: adios2_error_system_error = 2 + integer, parameter :: adios2_error_runtime_error = 3 + integer, parameter :: adios2_error_exception = 4 + + ! Mode + integer, parameter :: adios2_mode_undefined = 0 + integer, parameter :: adios2_mode_write = 1 + integer, parameter :: adios2_mode_read = 2 + integer, parameter :: adios2_mode_append = 3 + integer, parameter :: adios2_mode_readRandomAccess = 6 + + integer, parameter :: adios2_mode_deferred = 4 + integer, parameter :: adios2_mode_sync = 5 + + integer, parameter :: adios2_memory_space_detect = 0 + integer, parameter :: adios2_memory_space_host = 1 + integer, parameter :: adios2_memory_space_gpu = 2 + + ! Step Mode + integer, parameter :: adios2_step_mode_append = 0 + integer, parameter :: adios2_step_mode_update = 1 + integer, parameter :: adios2_step_mode_read = 2 + + ! Step Status + integer, parameter :: adios2_step_status_other_error = -1 + integer, parameter :: adios2_step_status_ok = 0 + integer, parameter :: adios2_step_status_not_ready = 1 + integer, parameter :: adios2_step_status_end_of_stream = 2 + + ! Derived variable type + integer, parameter :: adios2_derived_var_type_metadata_only = 0 + integer, parameter :: adios2_derived_var_type_expression_string = 1 + integer, parameter :: adios2_derived_var_type_store_data = 2 + + !> Fixed size for string array, used in variables and attributes, + !! must be less or equal than C equivalent in adios2_c_types.h + integer, parameter :: adios2_string_array_element_max_size = 4096 + + integer(kind=8), parameter, dimension(1) :: adios2_null_dims = (/-1/) + integer(kind=8), parameter :: adios2_local_value_dim = -2 + + logical, parameter :: adios2_advance_yes = .true. + logical, parameter :: adios2_advance_no = .false. + + ! Low level API handlers + type adios2_adios + integer(kind=8):: f2c = 0_8 + logical :: valid = .false. + end type + + type adios2_io + integer(kind=8):: f2c = 0_8 + logical :: valid = .false. + character(len=15):: engine_type = 'BPFile' + end type + + type adios2_variable + integer(kind=8):: f2c = 0_8 + logical :: valid = .false. + character(len=4096):: name = '' + integer :: type = -1 + integer :: ndims = -1 + end type + + type adios2_derived_variable + integer(kind=8):: f2c = 0_8 + logical :: valid = .false. + character(len=4096):: name = '' + integer :: type = -1 + end type + + type adios2_attribute + integer(kind=8):: f2c = 0_8 + logical :: valid = .false. + logical :: is_value = .false. + character(len=4096):: name = '' + integer :: type = -1 + integer :: length = -1 + end type + + type adios2_engine + integer(kind=8):: f2c = 0_8 + logical :: valid = .false. + character(len=64):: name = '' + character(len=15):: type = '' + integer :: mode = adios2_mode_undefined + end type + + type adios2_operator + integer(kind=8):: f2c = 0_8 + logical :: valid = .false. + character(len=64):: name = '' + character(len=64):: type = '' + end type + + type adios2_namestruct + integer(kind=8):: f2c = 0_8 + logical :: valid = .false. + integer :: count + integer :: max_name_len + end type end module diff --git a/testing/adios2/bindings/fortran/TestBPWriteTypes.F90 b/testing/adios2/bindings/fortran/TestBPWriteTypes.F90 index 7a8f685181..f0e7d7b2b5 100644 --- a/testing/adios2/bindings/fortran/TestBPWriteTypes.F90 +++ b/testing/adios2/bindings/fortran/TestBPWriteTypes.F90 @@ -1,325 +1,330 @@ program TestBPWriteTypes - use small_test_data + use small_test_data #if ADIOS2_USE_MPI - use mpi + use mpi #endif - use adios2 - implicit none + use adios2 + implicit none - integer(kind=8), dimension(1) :: shape_dims, start_dims, count_dims - integer(kind=4) :: inx, irank, isize, ierr, i, step_status - integer(kind=8) :: nsteps + integer(kind=8), dimension(1) :: shape_dims, start_dims, count_dims + integer(kind=4) :: inx, irank, isize, ierr, i, step_status + integer(kind=8) :: nsteps - type(adios2_adios) :: adios - type(adios2_io) :: ioWrite, ioRead - type(adios2_variable), dimension(14) :: variables - type(adios2_engine) :: bpWriter, bpReader - character(len=15) :: inString - character(len=:), allocatable :: varName, param_value - character(len=:), allocatable :: engineType - logical :: result + type(adios2_adios) :: adios + type(adios2_io) :: ioWrite, ioRead + type(adios2_variable), dimension(14) :: variables + type(adios2_derived_variable) :: derived_variable + type(adios2_engine) :: bpWriter, bpReader + character(len=15) :: inString + character(len=:), allocatable :: varName, param_value + character(len=:), allocatable :: engineType + logical :: result - ! read local value as global array - integer(kind=4), dimension(:), allocatable :: inRanks + ! read local value as global array + integer(kind=4), dimension(:), allocatable :: inRanks - ! read handlers - integer(kind=4) :: ndims - integer(kind=8), dimension(:), allocatable :: shape_in + ! read handlers + integer(kind=4) :: ndims + integer(kind=8), dimension(:), allocatable :: shape_in - character(len=4096), dimension(:), allocatable :: varnamelist - type(adios2_namestruct) :: namestruct + character(len=4096), dimension(:), allocatable :: varnamelist + type(adios2_namestruct) :: namestruct #if ADIOS2_USE_MPI - ! Launch MPI - INTEGER provided + ! Launch MPI + INTEGER provided - ! MPI_THREAD_MULTIPLE is only required if you enable the SST MPI_DP - call MPI_Init_thread(MPI_THREAD_MULTIPLE, provided, ierr) - call MPI_Comm_rank(MPI_COMM_WORLD, irank, ierr) - call MPI_Comm_size(MPI_COMM_WORLD, isize, ierr) + ! MPI_THREAD_MULTIPLE is only required if you enable the SST MPI_DP + call MPI_Init_thread(MPI_THREAD_MULTIPLE, provided, ierr) + call MPI_Comm_rank(MPI_COMM_WORLD, irank, ierr) + call MPI_Comm_size(MPI_COMM_WORLD, isize, ierr) #else - irank = 0 - isize = 1 + irank = 0 + isize = 1 #endif - ! Application variables - inx = 10 + ! Application variables + inx = 10 - ! Variable dimensions - shape_dims(1) = isize*inx - start_dims(1) = irank*inx - count_dims(1) = inx + ! Variable dimensions + shape_dims(1) = isize*inx + start_dims(1) = irank*inx + count_dims(1) = inx - if( adios%valid .eqv. .true. ) stop 'Invalid adios default' - if( ioWrite%valid .eqv. .true. ) stop 'Invalid io default' + if( adios%valid .eqv. .true. ) stop 'Invalid adios default' + if( ioWrite%valid .eqv. .true. ) stop 'Invalid io default' - do i=1,12 - if( variables(i)%valid .eqv. .true. ) stop 'Invalid variables default' - end do + do i=1,12 + if( variables(i)%valid .eqv. .true. ) stop 'Invalid variables default' + end do - if( bpWriter%valid .eqv. .true. ) stop 'Invalid engine default' + if( bpWriter%valid .eqv. .true. ) stop 'Invalid engine default' - ! Create adios handler passing the communicator and error flag + ! Create adios handler passing the communicator and error flag #if ADIOS2_USE_MPI - call adios2_init(adios, MPI_COMM_WORLD, ierr) + call adios2_init(adios, MPI_COMM_WORLD, ierr) #else - call adios2_init(adios, ierr) + call adios2_init(adios, ierr) #endif - if( adios%valid .eqv. .false. ) stop 'Invalid adios2_init' + if( adios%valid .eqv. .false. ) stop 'Invalid adios2_init' - !!!!!!!!!!!!!!!!!!!!!!!! WRITER !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - ! Declare an IO process configuration inside adios - call adios2_declare_io(ioWrite, adios, "ioWrite", ierr) - if( ioWrite%valid .eqv. .false. ) stop 'Invalid adios2_declare_io' + !!!!!!!!!!!!!!!!!!!!!!!! WRITER !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! Declare an IO process configuration inside adios + call adios2_declare_io(ioWrite, adios, "ioWrite", ierr) + if( ioWrite%valid .eqv. .false. ) stop 'Invalid adios2_declare_io' - call adios2_at_io(ioWrite, adios, "ioWrite", ierr) - if( ioWrite%valid .eqv. .false. ) stop 'Invalid adios2_at_io' + call adios2_at_io(ioWrite, adios, "ioWrite", ierr) + if( ioWrite%valid .eqv. .false. ) stop 'Invalid adios2_at_io' - call adios2_in_config_file(result, ioWrite, ierr) - if( result .eqv. .true. ) stop 'Invalid ioWrite adios2_in_config_file' + call adios2_in_config_file(result, ioWrite, ierr) + if( result .eqv. .true. ) stop 'Invalid ioWrite adios2_in_config_file' - call adios2_set_engine(ioWrite, 'File', ierr) + call adios2_set_engine(ioWrite, 'File', ierr) - call adios2_set_parameter(ioWrite, 'ProfileUnits', 'Microseconds', ierr) + call adios2_set_parameter(ioWrite, 'ProfileUnits', 'Microseconds', ierr) - call adios2_get_parameter(param_value, ioWrite, 'ProfileUnits', ierr) - if( param_value /= "Microseconds") stop 'Failed adios2_get_parameter ProfileUnits' + call adios2_get_parameter(param_value, ioWrite, 'ProfileUnits', ierr) + if( param_value /= "Microseconds") stop 'Failed adios2_get_parameter ProfileUnits' - call adios2_set_parameters(ioWrite, 'Threads=2, CollectiveMetadata = OFF', ierr) + call adios2_set_parameters(ioWrite, 'Threads=2, CollectiveMetadata = OFF', ierr) - call adios2_get_parameter(param_value, ioWrite, 'Threads', ierr) - if( param_value /= "2") stop 'Failed adios2_get_parameter Threads' + call adios2_get_parameter(param_value, ioWrite, 'Threads', ierr) + if( param_value /= "2") stop 'Failed adios2_get_parameter Threads' - call adios2_get_parameter(param_value, ioWrite, 'CollectiveMetadata', ierr) - if( param_value /= "OFF") stop 'Failed adios2_get_parameter CollectiveMetadata' + call adios2_get_parameter(param_value, ioWrite, 'CollectiveMetadata', ierr) + if( param_value /= "OFF") stop 'Failed adios2_get_parameter CollectiveMetadata' - ! set back the default to make sure writing/reading test works - call adios2_clear_parameters(ioWrite, ierr) - call adios2_get_parameter(param_value, ioWrite, 'CollectiveMetadata', ierr) - if( param_value /= "") stop 'Still Could retrieve parameter CollectiveMetadata after clearing all parameters' + ! set back the default to make sure writing/reading test works + call adios2_clear_parameters(ioWrite, ierr) + call adios2_get_parameter(param_value, ioWrite, 'CollectiveMetadata', ierr) + if( param_value /= "") stop 'Still Could retrieve parameter CollectiveMetadata after clearing all parameters' - deallocate(param_value) + deallocate(param_value) - ! Defines a variable to be written in bp format - call adios2_define_variable(variables(1), ioWrite, "var_I8", & - adios2_type_integer1, 1, & - shape_dims, start_dims, count_dims, & - adios2_constant_dims, ierr) + ! Defines a variable to be written in bp format + call adios2_define_variable(variables(1), ioWrite, "var_I8", & + adios2_type_integer1, 1, & + shape_dims, start_dims, count_dims, & + adios2_constant_dims, ierr) - call adios2_define_variable(variables(2), ioWrite, "var_I16", & - adios2_type_integer2, 1, & - shape_dims, start_dims, count_dims, & - adios2_constant_dims, ierr) + call adios2_define_variable(variables(2), ioWrite, "var_I16", & + adios2_type_integer2, 1, & + shape_dims, start_dims, count_dims, & + adios2_constant_dims, ierr) - call adios2_define_variable(variables(3), ioWrite, "var_I32", & - adios2_type_integer4, 1, & - shape_dims, start_dims, count_dims, & - adios2_constant_dims, ierr) + call adios2_define_variable(variables(3), ioWrite, "var_I32", & + adios2_type_integer4, 1, & + shape_dims, start_dims, count_dims, & + adios2_constant_dims, ierr) - call adios2_define_variable(variables(4), ioWrite, "var_I64", & - adios2_type_integer8, 1, & - shape_dims, start_dims, count_dims, & - adios2_constant_dims, ierr) + call adios2_define_variable(variables(4), ioWrite, "var_I64", & + adios2_type_integer8, 1, & + shape_dims, start_dims, count_dims, & + adios2_constant_dims, ierr) - call adios2_define_variable(variables(5), ioWrite, "var_R32", & - adios2_type_real, 1, & - shape_dims, start_dims, count_dims, & - adios2_constant_dims, ierr) + call adios2_define_variable(variables(5), ioWrite, "var_R32", & + adios2_type_real, 1, & + shape_dims, start_dims, count_dims, & + adios2_constant_dims, ierr) - call adios2_define_variable(variables(6), ioWrite, "var_R64", & - adios2_type_dp, 1, & - shape_dims, start_dims, count_dims, & - adios2_constant_dims, ierr) + call adios2_define_variable(variables(6), ioWrite, "var_R64", & + adios2_type_dp, 1, & + shape_dims, start_dims, count_dims, & + adios2_constant_dims, ierr) - ! Global variables - call adios2_define_variable(variables(7), ioWrite, "gvar_I8", & - adios2_type_integer1, ierr) + ! Global variables + call adios2_define_variable(variables(7), ioWrite, "gvar_I8", & + adios2_type_integer1, ierr) - call adios2_define_variable(variables(8), ioWrite, "gvar_I16", & - adios2_type_integer2, ierr) + call adios2_define_variable(variables(8), ioWrite, "gvar_I16", & + adios2_type_integer2, ierr) - call adios2_define_variable(variables(9), ioWrite, "gvar_I32", & - adios2_type_integer4, ierr) + call adios2_define_variable(variables(9), ioWrite, "gvar_I32", & + adios2_type_integer4, ierr) - call adios2_define_variable(variables(10), ioWrite, "gvar_I64", & - adios2_type_integer8, ierr) + call adios2_define_variable(variables(10), ioWrite, "gvar_I64", & + adios2_type_integer8, ierr) - call adios2_define_variable(variables(11), ioWrite, "gvar_R32", & - adios2_type_real, ierr) + call adios2_define_variable(variables(11), ioWrite, "gvar_R32", & + adios2_type_real, ierr) - call adios2_define_variable(variables(12), ioWrite, "gvar_R64", & - adios2_type_dp, ierr) + call adios2_define_variable(variables(12), ioWrite, "gvar_R64", & + adios2_type_dp, ierr) - call adios2_define_variable(variables(13), ioWrite, "gvar_Str", & - adios2_type_string, ierr) + call adios2_define_variable(variables(13), ioWrite, "gvar_Str", & + adios2_type_string, ierr) - ! local value - call adios2_define_variable(variables(14), ioWrite, "lvar_i32", & - adios2_type_integer4, & - 1, (/ adios2_local_value_dim /), & - adios2_null_dims, & - adios2_null_dims, & - adios2_constant_dims, ierr) + ! local value + call adios2_define_variable(variables(14), ioWrite, "lvar_i32", & + adios2_type_integer4, & + 1, (/ adios2_local_value_dim /), & + adios2_null_dims, & + adios2_null_dims, & + adios2_constant_dims, ierr) - do i=1,13 - if( variables(i)%valid .eqv. .false. ) stop 'Invalid adios2_define_variable' - end do + ! derived variable + call adios2_define_derived_variable(derived_variable, ioWrite, "derive/sum", & + "x:var_R64 y:var_R64 z:var_R64 magnitude(x,y,z)", adios2_derived_var_type_metadata_only, ierr) - ! Testing adios2_variable_name for just two cases - call adios2_variable_name(varName, variables(1), ierr) - if (varName /= 'var_I8') stop 'Invalid adios2_variable_name' + do i=1,13 + if( variables(i)%valid .eqv. .false. ) stop 'Invalid adios2_define_variable' + end do - call adios2_variable_name(varName, variables(2), ierr) - if (varName /= 'var_I16') stop 'Invalid adios2_variable_name' + ! Testing adios2_variable_name for just two cases + call adios2_variable_name(varName, variables(1), ierr) + if (varName /= 'var_I8') stop 'Invalid adios2_variable_name' - deallocate(varName) + call adios2_variable_name(varName, variables(2), ierr) + if (varName /= 'var_I16') stop 'Invalid adios2_variable_name' - ! Open myVector_f.bp in write mode, this launches an engine - if( ioWrite%valid .eqv. .false. ) stop 'Invalid adios2_io' - if( bpWriter%valid .eqv. .true. ) stop 'Invalid adios2_engine pre-open' - - call adios2_open(bpWriter, ioWrite, "ftypes.bp", adios2_mode_write, ierr) - - if( bpWriter%valid .eqv. .false. ) stop 'Invalid adios2_engine post-open' - if( TRIM(bpWriter%name) /= "ftypes.bp") stop 'Invalid adios2_engine name' - - if( TRIM(bpWriter%type) /= 'BP5Writer') then - write(*,*) 'Engine Type ', TRIM(bpWriter%type) - stop 'Invalid adios2_engine type' - end if - call adios2_io_engine_type(engineType, ioWrite, ierr) - if( engineType /= 'File') then ! FIXME, different from the above! - write(*,*) 'Engine Type ', engineType - stop 'Invalid type from adios2_engine_type' - end if - - if( bpWriter%mode /= adios2_mode_write) stop 'Invalid adios2_engine mode' - - ! Put array contents to bp buffer, based on var1 metadata - do i = 1, 3 - call adios2_begin_step(bpWriter, adios2_step_mode_append, -1.0, & - step_status, ierr) - - if (irank == 0 .and. i == 1) then - call adios2_put(bpWriter, variables(7), data_I8(1), ierr) - call adios2_put(bpWriter, variables(8), data_I16(1), ierr) - call adios2_put(bpWriter, variables(9), data_I32(1), ierr) - call adios2_put(bpWriter, variables(10), data_I64(1), ierr) - call adios2_put(bpWriter, variables(11), data_R32(1), ierr) - call adios2_put(bpWriter, variables(12), data_R64(1), ierr) - call adios2_put(bpWriter, variables(13), data_Strings(1), ierr) - end if - - call adios2_put(bpWriter, variables(1), data_I8, ierr) - call adios2_put(bpWriter, variables(2), data_I16, ierr) - call adios2_put(bpWriter, variables(3), data_I32, ierr) - call adios2_put(bpWriter, variables(4), data_I64, ierr) - call adios2_put(bpWriter, variables(5), data_R32, ierr) - call adios2_put(bpWriter, variables(6), data_R64, ierr) - - call adios2_put(bpWriter, variables(14), irank, ierr) - - call adios2_end_step(bpWriter, ierr) - end do - - ! Closes engine1 and deallocates it, becomes unreachable - call adios2_close(bpWriter, ierr) - - if( bpWriter%valid .eqv. .true. ) stop 'Invalid adios2_close' - - !!!!!!!!!!!!!!!!!!!!!!!! READER !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - ! Declare io reader - call adios2_declare_io(ioRead, adios, "ioRead", ierr) - ! Open bpReader engine - call adios2_open(bpReader, ioRead, "ftypes.bp", adios2_mode_readRandomAccess, ierr) - - call adios2_steps(nsteps, bpReader, ierr) - if(nsteps /= 3) stop 'ftypes.bp must have 3 steps' - - call adios2_available_variables(ioRead, namestruct, ierr) - if (ierr /= 0) stop 'adios2_available_variables returned with error' - if (.not.namestruct%valid) stop 'adios2_available_variables returned invalid struct' - write(*,*) 'Number of variables = ', namestruct%count - write(*,*) 'Max name length = ', namestruct%max_name_len - if (namestruct%count /= 14) stop 'adios2_available_variables returned not the expected 14' - - allocate(varnamelist(namestruct%count)) - - call adios2_retrieve_names(namestruct, varnamelist, ierr) - if (ierr /= 0) stop 'adios2_retrieve_names returned with error' - do i=1,namestruct%count - write(*,'("Var[",i2,"] = ",a12)') i, varnamelist(i) - end do - deallocate(varnamelist) - - if (namestruct%f2c /= 0_8) stop 'namestruct f2c pointer is not null after adios2_retrieve_names()' - if (namestruct%valid) stop 'namestruct is not invalidated after adios2_retrieve_names()' - - - call adios2_inquire_variable(variables(1), ioRead, "var_I8", ierr) - if (variables(1)%name /= 'var_I8') stop 'var_I8 not recognized' - if (variables(1)%type /= adios2_type_integer1) stop 'var_I8 type not recognized' - call adios2_variable_shape(shape_in, ndims, variables(1), ierr) - if (ndims /= 1) stop 'var_I8 ndims is not 1' - if (shape_in(1) /= isize*inx) stop 'var_I8 shape_in read failed' - - call adios2_inquire_variable(variables(2), ioRead, "var_I16", ierr) - if (variables(2)%name /= 'var_I16') stop 'var_I16 not recognized' - if (variables(2)%type /= adios2_type_integer2) stop 'var_I16 type not recognized' - call adios2_variable_shape(shape_in, ndims, variables(2), ierr) - if (ndims /= 1) stop 'var_I16 ndims is not 1' - if (shape_in(1) /= isize*inx) stop 'var_I16 shape_in read failed' - - call adios2_inquire_variable(variables(3), ioRead, "var_I32", ierr) - if (variables(3)%name /= 'var_I32') stop 'var_I32 not recognized' - if (variables(3)%type /= adios2_type_integer4) stop 'var_I32 type not recognized' - call adios2_variable_shape(shape_in, ndims, variables(3), ierr) - if (ndims /= 1) stop 'var_I32 ndims is not 1' - if (shape_in(1) /= isize*inx) stop 'var_I32 shape_in read failed' - - call adios2_inquire_variable(variables(4), ioRead, "var_I64", ierr) - if (variables(4)%name /= 'var_I64') stop 'var_I64 not recognized' - if (variables(4)%type /= adios2_type_integer8) stop 'var_I64 type not recognized' - call adios2_variable_shape(shape_in, ndims, variables(4), ierr) - if (ndims /= 1) stop 'var_I64 ndims is not 1' - if (shape_in(1) /= isize*inx) stop 'var_I64 shape_in read failed' - - call adios2_inquire_variable(variables(5), ioRead, "var_R32", ierr) - if (variables(5)%name /= 'var_R32') stop 'var_R32 not recognized' - if (variables(5)%type /= adios2_type_real) stop 'var_R32 type not recognized' - call adios2_variable_shape(shape_in, ndims, variables(5), ierr) - if (ndims /= 1) stop 'var_R32 ndims is not 1' - if (shape_in(1) /= isize*inx) stop 'var_R32 shape_in read failed' - - call adios2_inquire_variable(variables(6), ioRead, "var_R64", ierr) - if (variables(6)%name /= 'var_R64') stop 'var_R64 not recognized' - if (variables(6)%type /= adios2_type_dp) stop 'var_R64 type not recognized' - call adios2_variable_shape(shape_in, ndims, variables(6), ierr) - if (ndims /= 1) stop 'var_R64 ndims is not 1' - if (shape_in(1) /= isize*inx) stop 'var_R64 shape_in read failed' - - call adios2_inquire_variable(variables(13), ioRead, "gvar_Str", ierr) - call adios2_get(bpReader, variables(13), inString, ierr) - call adios2_perform_gets(bpReader, ierr) - if( inString /= data_Strings(1) ) stop 'gvar_Str read failed' - - call adios2_inquire_variable(variables(14), ioRead, "lvar_i32", ierr) - allocate(inRanks(isize)) - call adios2_get(bpReader, variables(14), inRanks, ierr) - call adios2_perform_gets(bpReader, ierr) - if( inRanks(irank+1) /= irank ) stop 'lvar_i32 read failed' - deallocate(inRanks) - - call adios2_close(bpReader, ierr) - - ! Deallocates adios and calls its destructor - call adios2_finalize(adios, ierr) - if( adios%valid .eqv. .true. ) stop 'Invalid adios2_finalize' + deallocate(varName) + + ! Open myVector_f.bp in write mode, this launches an engine + if( ioWrite%valid .eqv. .false. ) stop 'Invalid adios2_io' + if( bpWriter%valid .eqv. .true. ) stop 'Invalid adios2_engine pre-open' + + call adios2_open(bpWriter, ioWrite, "ftypes.bp", adios2_mode_write, ierr) + + if( bpWriter%valid .eqv. .false. ) stop 'Invalid adios2_engine post-open' + if( TRIM(bpWriter%name) /= "ftypes.bp") stop 'Invalid adios2_engine name' + + if( TRIM(bpWriter%type) /= 'BP5Writer') then + write(*,*) 'Engine Type ', TRIM(bpWriter%type) + stop 'Invalid adios2_engine type' + end if + call adios2_io_engine_type(engineType, ioWrite, ierr) + if( engineType /= 'File') then ! FIXME, different from the above! + write(*,*) 'Engine Type ', engineType + stop 'Invalid type from adios2_engine_type' + end if + + if( bpWriter%mode /= adios2_mode_write) stop 'Invalid adios2_engine mode' + + ! Put array contents to bp buffer, based on var1 metadata + do i = 1, 3 + call adios2_begin_step(bpWriter, adios2_step_mode_append, -1.0, & + step_status, ierr) + + if (irank == 0 .and. i == 1) then + call adios2_put(bpWriter, variables(7), data_I8(1), ierr) + call adios2_put(bpWriter, variables(8), data_I16(1), ierr) + call adios2_put(bpWriter, variables(9), data_I32(1), ierr) + call adios2_put(bpWriter, variables(10), data_I64(1), ierr) + call adios2_put(bpWriter, variables(11), data_R32(1), ierr) + call adios2_put(bpWriter, variables(12), data_R64(1), ierr) + call adios2_put(bpWriter, variables(13), data_Strings(1), ierr) + end if + + call adios2_put(bpWriter, variables(1), data_I8, ierr) + call adios2_put(bpWriter, variables(2), data_I16, ierr) + call adios2_put(bpWriter, variables(3), data_I32, ierr) + call adios2_put(bpWriter, variables(4), data_I64, ierr) + call adios2_put(bpWriter, variables(5), data_R32, ierr) + call adios2_put(bpWriter, variables(6), data_R64, ierr) + + call adios2_put(bpWriter, variables(14), irank, ierr) + + call adios2_end_step(bpWriter, ierr) + end do + + ! Closes engine1 and deallocates it, becomes unreachable + call adios2_close(bpWriter, ierr) + + if( bpWriter%valid .eqv. .true. ) stop 'Invalid adios2_close' + + !!!!!!!!!!!!!!!!!!!!!!!! READER !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! Declare io reader + call adios2_declare_io(ioRead, adios, "ioRead", ierr) + ! Open bpReader engine + call adios2_open(bpReader, ioRead, "ftypes.bp", adios2_mode_readRandomAccess, ierr) + + call adios2_steps(nsteps, bpReader, ierr) + if(nsteps /= 3) stop 'ftypes.bp must have 3 steps' + + call adios2_available_variables(ioRead, namestruct, ierr) + if (ierr /= 0) stop 'adios2_available_variables returned with error' + if (.not.namestruct%valid) stop 'adios2_available_variables returned invalid struct' + write(*,*) 'Number of variables = ', namestruct%count + write(*,*) 'Max name length = ', namestruct%max_name_len + if (namestruct%count /= 14) stop 'adios2_available_variables returned not the expected 14' + + allocate(varnamelist(namestruct%count)) + + call adios2_retrieve_names(namestruct, varnamelist, ierr) + if (ierr /= 0) stop 'adios2_retrieve_names returned with error' + do i=1,namestruct%count + write(*,'("Var[",i2,"] = ",a12)') i, varnamelist(i) + end do + deallocate(varnamelist) + + if (namestruct%f2c /= 0_8) stop 'namestruct f2c pointer is not null after adios2_retrieve_names()' + if (namestruct%valid) stop 'namestruct is not invalidated after adios2_retrieve_names()' + + + call adios2_inquire_variable(variables(1), ioRead, "var_I8", ierr) + if (variables(1)%name /= 'var_I8') stop 'var_I8 not recognized' + if (variables(1)%type /= adios2_type_integer1) stop 'var_I8 type not recognized' + call adios2_variable_shape(shape_in, ndims, variables(1), ierr) + if (ndims /= 1) stop 'var_I8 ndims is not 1' + if (shape_in(1) /= isize*inx) stop 'var_I8 shape_in read failed' + + call adios2_inquire_variable(variables(2), ioRead, "var_I16", ierr) + if (variables(2)%name /= 'var_I16') stop 'var_I16 not recognized' + if (variables(2)%type /= adios2_type_integer2) stop 'var_I16 type not recognized' + call adios2_variable_shape(shape_in, ndims, variables(2), ierr) + if (ndims /= 1) stop 'var_I16 ndims is not 1' + if (shape_in(1) /= isize*inx) stop 'var_I16 shape_in read failed' + + call adios2_inquire_variable(variables(3), ioRead, "var_I32", ierr) + if (variables(3)%name /= 'var_I32') stop 'var_I32 not recognized' + if (variables(3)%type /= adios2_type_integer4) stop 'var_I32 type not recognized' + call adios2_variable_shape(shape_in, ndims, variables(3), ierr) + if (ndims /= 1) stop 'var_I32 ndims is not 1' + if (shape_in(1) /= isize*inx) stop 'var_I32 shape_in read failed' + + call adios2_inquire_variable(variables(4), ioRead, "var_I64", ierr) + if (variables(4)%name /= 'var_I64') stop 'var_I64 not recognized' + if (variables(4)%type /= adios2_type_integer8) stop 'var_I64 type not recognized' + call adios2_variable_shape(shape_in, ndims, variables(4), ierr) + if (ndims /= 1) stop 'var_I64 ndims is not 1' + if (shape_in(1) /= isize*inx) stop 'var_I64 shape_in read failed' + + call adios2_inquire_variable(variables(5), ioRead, "var_R32", ierr) + if (variables(5)%name /= 'var_R32') stop 'var_R32 not recognized' + if (variables(5)%type /= adios2_type_real) stop 'var_R32 type not recognized' + call adios2_variable_shape(shape_in, ndims, variables(5), ierr) + if (ndims /= 1) stop 'var_R32 ndims is not 1' + if (shape_in(1) /= isize*inx) stop 'var_R32 shape_in read failed' + + call adios2_inquire_variable(variables(6), ioRead, "var_R64", ierr) + if (variables(6)%name /= 'var_R64') stop 'var_R64 not recognized' + if (variables(6)%type /= adios2_type_dp) stop 'var_R64 type not recognized' + call adios2_variable_shape(shape_in, ndims, variables(6), ierr) + if (ndims /= 1) stop 'var_R64 ndims is not 1' + if (shape_in(1) /= isize*inx) stop 'var_R64 shape_in read failed' + + call adios2_inquire_variable(variables(13), ioRead, "gvar_Str", ierr) + call adios2_get(bpReader, variables(13), inString, ierr) + call adios2_perform_gets(bpReader, ierr) + if( inString /= data_Strings(1) ) stop 'gvar_Str read failed' + + call adios2_inquire_variable(variables(14), ioRead, "lvar_i32", ierr) + allocate(inRanks(isize)) + call adios2_get(bpReader, variables(14), inRanks, ierr) + call adios2_perform_gets(bpReader, ierr) + if( inRanks(irank+1) /= irank ) stop 'lvar_i32 read failed' + deallocate(inRanks) + + call adios2_close(bpReader, ierr) + + ! Deallocates adios and calls its destructor + call adios2_finalize(adios, ierr) + if( adios%valid .eqv. .true. ) stop 'Invalid adios2_finalize' #if ADIOS2_USE_MPI - call MPI_Finalize(ierr) + call MPI_Finalize(ierr) #endif end program TestBPWriteTypes From 131eb3ce5c14714dad805486a255382158233181 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Tue, 12 Mar 2024 15:03:10 -0400 Subject: [PATCH 086/124] Completely hide derived variables in C API if not enabled. Print warning inside Fortran F2C function. --- bindings/C/adios2/c/adios2_c_io.cpp | 10 +++------- bindings/Fortran/f2c/adios2_f2c_io.cpp | 11 +++++++++-- testing/adios2/bindings/fortran/TestBPWriteTypes.F90 | 2 +- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/bindings/C/adios2/c/adios2_c_io.cpp b/bindings/C/adios2/c/adios2_c_io.cpp index a0de5ec75c..d3842d6033 100644 --- a/bindings/C/adios2/c/adios2_c_io.cpp +++ b/bindings/C/adios2/c/adios2_c_io.cpp @@ -150,12 +150,13 @@ adios2_error adios2_set_transport_parameter(adios2_io *io, const size_t transpor } } +#ifdef ADIOS2_HAVE_DERIVED_VARIABLE adios2_derived_variable *adios2_define_derived_variable(adios2_io *io, const char *name, const char *expression, const adios2_derived_var_type type) { adios2_derived_variable *variable = nullptr; -#ifdef ADIOS2_HAVE_DERIVED_VARIABLE + try { adios2::helper::CheckForNullptr(io, "for adios2_io, in call to adios2_define_variable"); @@ -182,14 +183,9 @@ adios2_derived_variable *adios2_define_derived_variable(adios2_io *io, const cha adios2::helper::ExceptionToError("adios2_define_variable"); } -#else - std::cout << "ADIOS2 Warning: adios2_define_derived_variable() is not supported in the " - "current ADIOS2 build. The expression " - << expression << " will be ignored and the variable " << name - << " will not be produced." << std::endl; -#endif return variable; } +#endif adios2_variable *adios2_define_variable(adios2_io *io, const char *name, const adios2_type type, const size_t ndims, const size_t *shape, diff --git a/bindings/Fortran/f2c/adios2_f2c_io.cpp b/bindings/Fortran/f2c/adios2_f2c_io.cpp index f7ae560a78..da477de384 100644 --- a/bindings/Fortran/f2c/adios2_f2c_io.cpp +++ b/bindings/Fortran/f2c/adios2_f2c_io.cpp @@ -201,19 +201,26 @@ void FC_GLOBAL(adios2_define_variable_f2c, } } -#ifdef ADIOS2_HAVE_DERIVED_VARIABLE void FC_GLOBAL(adios2_define_derived_variable_f2c, ADIOS2_DEFINE_DERIVED_VARIABLE_F2C)(adios2_derived_variable **variable, adios2_io **io, const char *name, const char *expression, const int *type, int *ierr) { +#ifdef ADIOS2_HAVE_DERIVED_VARIABLE *variable = adios2_define_derived_variable(*io, name, expression, static_cast(*type)); *ierr = (*variable == NULL) ? static_cast(adios2_error_exception) : static_cast(adios2_error_none); -} +#else + std::cout << "ADIOS2 Warning: adios2_define_derived_variable() is not supported in the " + "current ADIOS2 build. The expression " + << expression << " will be ignored and the variable " << name + << " will not be produced." << std::endl; + *variable = nullptr; + *ierr = static_cast(adios2_error_exception); #endif +} struct cnamelist { diff --git a/testing/adios2/bindings/fortran/TestBPWriteTypes.F90 b/testing/adios2/bindings/fortran/TestBPWriteTypes.F90 index f0e7d7b2b5..7cb02991e0 100644 --- a/testing/adios2/bindings/fortran/TestBPWriteTypes.F90 +++ b/testing/adios2/bindings/fortran/TestBPWriteTypes.F90 @@ -165,7 +165,7 @@ program TestBPWriteTypes adios2_constant_dims, ierr) ! derived variable - call adios2_define_derived_variable(derived_variable, ioWrite, "derive/sum", & + call adios2_define_derived_variable(derived_variable, ioWrite, "derived/magnitude_of_var_R64", & "x:var_R64 y:var_R64 z:var_R64 magnitude(x,y,z)", adios2_derived_var_type_metadata_only, ierr) do i=1,13 From 20222e5688e9817c40c9f6dc1359489e905bbbcd Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Tue, 12 Mar 2024 13:48:51 -0400 Subject: [PATCH 087/124] Add setup for Aurora (load adios2 as e4s package) --- .../source/setting_up/doemachines.rst | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/docs/user_guide/source/setting_up/doemachines.rst b/docs/user_guide/source/setting_up/doemachines.rst index f6b0a13c5f..8357bf94cd 100644 --- a/docs/user_guide/source/setting_up/doemachines.rst +++ b/docs/user_guide/source/setting_up/doemachines.rst @@ -86,3 +86,24 @@ OLCF installs the E4S packages in individual modules, hence `adios2` is also ava C++ Compiler: GNU 12.2.0 (CrayPrgEnv) Target OS: Linux-5.14.21-150400.24.11_12.0.57-cray_shasta_c Target Arch: x86_64 + +***************************************** +ALCF Aurora +***************************************** + +To use adios2 on Aurora, + +- Load the default oneAPI (loaded automatically on login) +- module use /soft/modulefiles +- module load spack-pe-oneapi/0.5-rc1 + +This is a "metamoduile" that makes many software packages from E4S loadable as modules. + +.. code-block:: bash + + $ module use /soft/modulefiles + $ module load spack-pe-oneapi/0.5-rc1 + $ module avail adios2 + + ---------- /soft/packaging/spack/oneapi/0.5-rc1/modulefiles/Core ----------- + adios2/2.9.0-oneapi-mpich-testing From d4a80eae45a4b975965439ad50bb5a6b1c8e2855 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Wed, 13 Mar 2024 09:31:22 -0400 Subject: [PATCH 088/124] Do not create adios-campaign/ unless there is something to record --- source/adios2/engine/campaign/CampaignManager.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/adios2/engine/campaign/CampaignManager.cpp b/source/adios2/engine/campaign/CampaignManager.cpp index 2b963f5907..bd511ce687 100644 --- a/source/adios2/engine/campaign/CampaignManager.cpp +++ b/source/adios2/engine/campaign/CampaignManager.cpp @@ -81,7 +81,6 @@ CampaignManager::CampaignManager(adios2::helper::Comm &comm) { std::cout << "Campaign Manager " << m_WriterRank << " constructor called" << std::endl; } - helper::CreateDirectory(m_CampaignDir); } CampaignManager::~CampaignManager() @@ -103,6 +102,7 @@ void CampaignManager::Open(const std::string &name) { std::cout << "Campaign Manager " << m_WriterRank << " Open(" << m_Name << ")\n"; } + m_Opened = true; } void CampaignManager::Record(const std::string &name, const size_t step, const double time) @@ -151,8 +151,10 @@ void CampaignManager::Close() { if (!cmap.empty()) { + helper::CreateDirectory(m_CampaignDir); CMapToSqlite(cmap, m_WriterRank, m_Name); } + m_Opened = false; } } // end namespace engine From e923816339ee5e77787c1eb7e79da3c68656688f Mon Sep 17 00:00:00 2001 From: Liz Dulac <47396187+lizdulac@users.noreply.github.com> Date: Tue, 19 Mar 2024 14:05:44 -0400 Subject: [PATCH 089/124] Bison 3.8 Parser (#4062) * Derived Expressions Curl function with correctness test --- cmake/ADIOSBisonFlexSub.cmake | 15 + scripts/ci/scripts/run-clang-format.sh | 2 +- source/adios2/CMakeLists.txt | 43 +- source/adios2/core/VariableDerived.cpp | 20 +- source/adios2/core/VariableDerived.h | 2 +- source/adios2/engine/bp5/BP5Writer.cpp | 4 +- source/adios2/toolkit/derived/ExprHelper.h | 2 +- source/adios2/toolkit/derived/Expression.cpp | 42 +- source/adios2/toolkit/derived/Expression.h | 2 +- source/adios2/toolkit/derived/Function.cpp | 2 +- source/adios2/toolkit/derived/Function.tcc | 2 +- .../toolkit/derived/parser/ASTDriver.cpp | 101 + .../adios2/toolkit/derived/parser/ASTDriver.h | 65 + .../adios2/toolkit/derived/parser/ASTNode.cpp | 157 +- .../adios2/toolkit/derived/parser/ASTNode.h | 62 +- .../adios2/toolkit/derived/parser/lexer.cpp | 1884 ---------------- source/adios2/toolkit/derived/parser/lexer.l | 172 +- .../adios2/toolkit/derived/parser/parser.cpp | 1666 -------------- source/adios2/toolkit/derived/parser/parser.h | 110 - .../toolkit/derived/parser/parser.output | 350 --- source/adios2/toolkit/derived/parser/parser.y | 225 +- .../derived/parser/pregen-source/lexer.cpp | 1906 +++++++++++++++++ .../parser/{ => pregen-source}/lexer.h | 182 +- .../derived/parser/pregen-source/location.hh | 306 +++ .../derived/parser/pregen-source/parser.cpp | 1414 ++++++++++++ .../derived/parser/pregen-source/parser.h | 1609 ++++++++++++++ .../toolkit/format/bp5/BP5Deserializer.cpp | 2 + .../adios2/bindings/fortran/CMakeLists.txt | 6 + .../bindings/fortran/TestBPWriteTypes.F90 | 284 ++- .../derived/TestBPDerivedCorrectness.cpp | 43 +- 30 files changed, 6174 insertions(+), 4506 deletions(-) create mode 100644 cmake/ADIOSBisonFlexSub.cmake create mode 100644 source/adios2/toolkit/derived/parser/ASTDriver.cpp create mode 100644 source/adios2/toolkit/derived/parser/ASTDriver.h delete mode 100644 source/adios2/toolkit/derived/parser/lexer.cpp delete mode 100644 source/adios2/toolkit/derived/parser/parser.cpp delete mode 100644 source/adios2/toolkit/derived/parser/parser.h delete mode 100644 source/adios2/toolkit/derived/parser/parser.output create mode 100644 source/adios2/toolkit/derived/parser/pregen-source/lexer.cpp rename source/adios2/toolkit/derived/parser/{ => pregen-source}/lexer.h (71%) create mode 100644 source/adios2/toolkit/derived/parser/pregen-source/location.hh create mode 100644 source/adios2/toolkit/derived/parser/pregen-source/parser.cpp create mode 100644 source/adios2/toolkit/derived/parser/pregen-source/parser.h diff --git a/cmake/ADIOSBisonFlexSub.cmake b/cmake/ADIOSBisonFlexSub.cmake new file mode 100644 index 0000000000..2a95e1bbcd --- /dev/null +++ b/cmake/ADIOSBisonFlexSub.cmake @@ -0,0 +1,15 @@ +FUNCTION (SETUP_ADIOS_BISON_FLEX_SUB) + +set (BISON_FLEX_PRECOMPILE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/toolkit/derived/parser/pregen-source") + +ADD_CUSTOM_COMMAND(OUTPUT parser.cpp + COMMAND ${CMAKE_COMMAND} -E copy ${BISON_FLEX_PRECOMPILE_DIR}/parser.cpp ${CMAKE_CURRENT_BINARY_DIR} + COMMAND ${CMAKE_COMMAND} -E copy ${BISON_FLEX_PRECOMPILE_DIR}/parser.h ${CMAKE_CURRENT_BINARY_DIR} + COMMAND ${CMAKE_COMMAND} -E copy ${BISON_FLEX_PRECOMPILE_DIR}/location.hh ${CMAKE_CURRENT_BINARY_DIR} + COMMENT "Using pre-generated Bison Output from ${BISON_FLEX_PRECOMPILE_DIR}") +ADD_CUSTOM_COMMAND(OUTPUT lexer.cpp + COMMAND ${CMAKE_COMMAND} -E copy ${BISON_FLEX_PRECOMPILE_DIR}/lexer.cpp ${CMAKE_CURRENT_BINARY_DIR} + COMMENT "Using pre-generated Flex Output from ${BISON_FLEX_PRECOMPILE_DIR}") + +set (BISON_Parser_OUTPUT_SOURCE parser.cpp PARENT_SCOPE) +ENDFUNCTION() diff --git a/scripts/ci/scripts/run-clang-format.sh b/scripts/ci/scripts/run-clang-format.sh index c37ab4444d..b85f7fa636 100755 --- a/scripts/ci/scripts/run-clang-format.sh +++ b/scripts/ci/scripts/run-clang-format.sh @@ -12,7 +12,7 @@ then fi # Check C and C++ code with clang-format -find source plugins testing examples bindings -regextype posix-extended -iregex '.*\.(h|c|cpp|tcc|cu)' | xargs clang-format -i +find source plugins testing examples bindings -regextype posix-extended -iregex '.*\.(h|c|cpp|tcc|cu)' ! -path "source/adios2/toolkit/derived/parser/pregen-source/*" | xargs clang-format -i DIFF="$(git diff)" if [ -n "${DIFF}" ] then diff --git a/source/adios2/CMakeLists.txt b/source/adios2/CMakeLists.txt index c4f8d893d2..ff1645b4bb 100644 --- a/source/adios2/CMakeLists.txt +++ b/source/adios2/CMakeLists.txt @@ -135,12 +135,51 @@ if (ADIOS2_HAVE_Derived_Variable) toolkit/derived/Expression.cpp toolkit/derived/Function.cpp toolkit/derived/Function.tcc toolkit/derived/ExprHelper.h) + set_target_properties(adios2_core PROPERTIES + INCLUDE_DIRECTORIES "$;$") + find_package(BISON "3.8.2") + find_package(FLEX) + + if(NOT BISON_FOUND OR NOT FLEX_FOUND) + include(ADIOSBisonFlexSub) + SETUP_ADIOS_BISON_FLEX_SUB() + else() + BISON_TARGET(MyParser + toolkit/derived/parser/parser.y + ${CMAKE_CURRENT_BINARY_DIR}/parser.cpp + COMPILE_FLAGS "-o parser.cpp --header=parser.h" + DEFINES_FILE ${CMAKE_CURRENT_BINARY_DIR}/parser.h) + FLEX_TARGET(MyScanner + toolkit/derived/parser/lexer.l + COMPILE_FLAGS "-o lexer.cpp --header-file=lexer.h" + ${CMAKE_CURRENT_BINARY_DIR}/lexer.cpp + DEFINES_FILE ${CMAKE_CURRENT_BINARY_DIR}/lexer.h) + ADD_FLEX_BISON_DEPENDENCY(MyScanner MyParser) + endif() + if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + SET_SOURCE_FILES_PROPERTIES(${CMAKE_CURRENT_BINARY_DIR}/lexer.cpp PROPERTIES COMPILE_FLAGS -Wno-sign-compare) + elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Intel") + SET_SOURCE_FILES_PROPERTIES(${CMAKE_CURRENT_BINARY_DIR}/parser.cpp PROPERTIES COMPILE_FLAGS -Wno-unused-but-set-variable) + elseif(CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM") + SET_SOURCE_FILES_PROPERTIES(${CMAKE_CURRENT_BINARY_DIR}/parser.cpp PROPERTIES COMPILE_FLAGS -Wno-unused-but-set-variable) + elseif(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") + SET_SOURCE_FILES_PROPERTIES(toolkit/derived/Expression.cpp toolkit/derived/Function.cpp PROPERTIES COMPILE_FLAGS "/wd4005 /wd4065 /wd4267 -DYY_NO_UNISTD_H") + endif() add_library(adios2_core_derived - toolkit/derived/parser/lexer.cpp - toolkit/derived/parser/parser.cpp + ${CMAKE_CURRENT_BINARY_DIR}/lexer.cpp + ${CMAKE_CURRENT_BINARY_DIR}/parser.cpp + toolkit/derived/parser/ASTDriver.cpp toolkit/derived/parser/ASTNode.cpp) + set_target_properties(adios2_core_derived PROPERTIES + VISIBILITY_INLINES_HIDDEN ON + INCLUDE_DIRECTORIES "$;$" + EXPORT_NAME core_derived + OUTPUT_NAME adios2${ADIOS2_LIBRARY_SUFFIX}_core_derived) target_link_libraries(adios2_core PRIVATE adios2_core_derived) set(maybe_adios2_core_derived adios2_core_derived) + if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") + set_target_properties(adios2_core_derived PROPERTIES COMPILE_FLAGS "/wd4005 /wd4267 /wd4065 -DYY_NO_UNISTD_H") + endif() endif() set(maybe_adios2_core_cuda) diff --git a/source/adios2/core/VariableDerived.cpp b/source/adios2/core/VariableDerived.cpp index 670ad0bc35..07819bf00e 100644 --- a/source/adios2/core/VariableDerived.cpp +++ b/source/adios2/core/VariableDerived.cpp @@ -11,7 +11,7 @@ VariableDerived::VariableDerived(const std::string &name, adios2::derived::Expre const DerivedVarType varType) : VariableBase(name, exprType, helper::GetDataTypeSize(exprType), expr.GetShape(), expr.GetStart(), expr.GetCount(), isConstant), - m_Expr(expr), m_DerivedType(varType) + m_DerivedType(varType), m_Expr(expr) { } @@ -27,15 +27,15 @@ void VariableDerived::UpdateExprDim(std::map> -VariableDerived::ApplyExpression(std::map NameToMVI) +VariableDerived::ApplyExpression(std::map> &NameToMVI) { size_t numBlocks = 0; // check that all variables have the same number of blocks - for (auto variable : NameToMVI) + for (const auto &variable : NameToMVI) { if (numBlocks == 0) - numBlocks = variable.second.BlocksInfo.size(); - if (numBlocks != variable.second.BlocksInfo.size()) + numBlocks = variable.second->BlocksInfo.size(); + if (numBlocks != variable.second->BlocksInfo.size()) helper::Throw("Core", "VariableDerived", "ApplyExpression", " variables do not have the same number of blocks " " in computing the derived variable " + @@ -44,7 +44,7 @@ VariableDerived::ApplyExpression(std::map NameToMVI) std::map> inputData; // create the map between variable name and DerivedData object - for (auto variable : NameToMVI) + for (const auto &variable : NameToMVI) { // add the dimensions of all blocks into a vector std::vector varData; @@ -52,13 +52,13 @@ VariableDerived::ApplyExpression(std::map NameToMVI) { Dims start; Dims count; - for (size_t d = 0; d < variable.second.Dims; d++) + for (int d = 0; d < variable.second->Dims; d++) { - start.push_back(variable.second.BlocksInfo[i].Start[d]); - count.push_back(variable.second.BlocksInfo[i].Count[d]); + start.push_back(variable.second->BlocksInfo[i].Start[d]); + count.push_back(variable.second->BlocksInfo[i].Count[d]); } varData.push_back(adios2::derived::DerivedData( - {variable.second.BlocksInfo[i].BufferP, start, count})); + {variable.second->BlocksInfo[i].BufferP, start, count})); } inputData.insert({variable.first, varData}); } diff --git a/source/adios2/core/VariableDerived.h b/source/adios2/core/VariableDerived.h index 2ff9eb3903..cc7bf8cbf2 100644 --- a/source/adios2/core/VariableDerived.h +++ b/source/adios2/core/VariableDerived.h @@ -32,7 +32,7 @@ class VariableDerived : public VariableBase ApplyExpression(std::map> NameToData, std::map> NameToDims); std::vector> - ApplyExpression(std::map mvi); + ApplyExpression(std::map> &mvi); }; } // end namespace core diff --git a/source/adios2/engine/bp5/BP5Writer.cpp b/source/adios2/engine/bp5/BP5Writer.cpp index 78c0646e6b..ff1889577e 100644 --- a/source/adios2/engine/bp5/BP5Writer.cpp +++ b/source/adios2/engine/bp5/BP5Writer.cpp @@ -516,7 +516,7 @@ void BP5Writer::ComputeDerivedVariables() auto derivedVar = dynamic_cast((*it).second.get()); std::vector varList = derivedVar->VariableNameList(); // to create a mapping between variable name and the varInfo (dim and data pointer) - std::map nameToVarInfo; + std::map> nameToVarInfo; bool computeDerived = true; for (auto varName : varList) { @@ -536,7 +536,7 @@ void BP5Writer::ComputeDerivedVariables() std::cout << " .. skip derived variable " << (*it).second->m_Name << std::endl; break; } - nameToVarInfo.insert({varName, *mvi}); + nameToVarInfo.insert({varName, std::unique_ptr(mvi)}); } // skip computing derived variables if it contains variables that are not written this step if (!computeDerived) diff --git a/source/adios2/toolkit/derived/ExprHelper.h b/source/adios2/toolkit/derived/ExprHelper.h index c72888f95f..e1d31b8bff 100644 --- a/source/adios2/toolkit/derived/ExprHelper.h +++ b/source/adios2/toolkit/derived/ExprHelper.h @@ -57,4 +57,4 @@ inline ExpressionOperator get_op(std::string op) { return string_to_op.at(op); } } } -#endif \ No newline at end of file +#endif diff --git a/source/adios2/toolkit/derived/Expression.cpp b/source/adios2/toolkit/derived/Expression.cpp index d773236526..81db7ccd15 100644 --- a/source/adios2/toolkit/derived/Expression.cpp +++ b/source/adios2/toolkit/derived/Expression.cpp @@ -2,19 +2,35 @@ #define ADIOS2_DERIVED_Expression_CPP_ #include "Expression.h" -#include "parser/ASTNode.h" -#include "parser/parser.h" +#include "parser/ASTDriver.h" namespace adios2 { namespace detail { +// helper function +adios2::detail::ExpressionOperator convert_op(std::string opname) +{ + adios2::detail::ExpressionOperator op; + try + { + op = adios2::detail::get_op(opname); + } + catch (std::out_of_range &e) + { + (void)e; // use e + helper::Throw("Derived", "ExprHelper", "get_op", + "Parser cannot recognize operator '" + opname + "'."); + } + return op; +}; + adios2::derived::ExpressionTree ASTNode_to_ExpressionTree(adios2::detail::ASTNode *node) { - adios2::derived::ExpressionTree exprTree_node(node->operation); - for (adios2::detail::ASTNode *e : node->sub_expr) + adios2::derived::ExpressionTree exprTree_node(convert_op(node->get_opname())); + for (adios2::detail::ASTNode *e : node->get_subexprs()) { - switch (e->operation) + switch (convert_op(e->get_opname())) { case adios2::detail::ExpressionOperator::OP_ALIAS: // add variable given by alias // add an index operation in the chain if the variable contains indeces @@ -25,19 +41,19 @@ adios2::derived::ExpressionTree ASTNode_to_ExpressionTree(adios2::detail::ASTNod index_expr.add_child(e->lookup_var_path(e->alias)); expTree_node->add_child(expr); }*/ - exprTree_node.add_child(e->lookup_var_path(e->alias)); + exprTree_node.add_child(e->get_varname()); break; case adios2::detail::ExpressionOperator::OP_PATH: // add variable name - exprTree_node.add_child(e->alias); + exprTree_node.add_child(e->get_varname()); break; case adios2::detail::ExpressionOperator::OP_NUM: // set the base value for the operation - exprTree_node.set_base(e->value); + exprTree_node.set_base(e->get_value()); break; default: // if the children nodes are other expressions, convert them to expressions auto temp_node = ASTNode_to_ExpressionTree(e); // move from a binary to a multinary tree if the child has the same operation - if (e->operation == node->operation && - adios2::detail::op_property.at(e->operation).is_associative) + if (convert_op(e->get_opname()) == convert_op(node->get_opname()) && + adios2::detail::op_property.at(convert_op(e->get_opname())).is_associative) { // concatenate exprTree with temp_node for (std::tuple childTree : @@ -63,10 +79,10 @@ namespace derived { Expression::Expression(std::string string_exp) -: ExprString(string_exp), m_Shape({0}), m_Start({0}), m_Count({0}) +: m_Shape({0}), m_Start({0}), m_Count({0}), ExprString(string_exp) { - adios2::detail::ASTNode *root_node = adios2::detail::parse_expression(string_exp); - m_Expr = adios2::detail::ASTNode_to_ExpressionTree(root_node); + adios2::detail::ASTDriver drv(string_exp); + m_Expr = adios2::detail::ASTNode_to_ExpressionTree(drv.getAST()); } std::vector Expression::VariableNameList() { return m_Expr.VariableNameList(); } diff --git a/source/adios2/toolkit/derived/Expression.h b/source/adios2/toolkit/derived/Expression.h index 6f60f1c262..2b6f8912a3 100644 --- a/source/adios2/toolkit/derived/Expression.h +++ b/source/adios2/toolkit/derived/Expression.h @@ -34,7 +34,7 @@ class ExpressionTree std::vector> sub_exprs; OpInfo detail; - ExpressionTree(){}; + ExpressionTree() : detail({adios2::detail::ExpressionOperator::OP_NULL, {}, 0}) {} ExpressionTree(adios2::detail::ExpressionOperator o) : detail({o, {}, 0}) {} ExpressionTree(adios2::detail::ExpressionOperator o, double c) : detail({o, {}, 0}) {} ExpressionTree(std::vector> indices) diff --git a/source/adios2/toolkit/derived/Function.cpp b/source/adios2/toolkit/derived/Function.cpp index c524ac8c15..cb145d8927 100644 --- a/source/adios2/toolkit/derived/Function.cpp +++ b/source/adios2/toolkit/derived/Function.cpp @@ -39,7 +39,7 @@ DerivedData MagnitudeFunc(std::vector inputData, DataType type) T *magValues = ApplyOneToOne(inputData, dataSize, [](T a, T b) { return a + b * b; }); \ for (size_t i = 0; i < dataSize; i++) \ { \ - magValues[i] = std::sqrt(magValues[i]); \ + magValues[i] = (T)std::sqrt(magValues[i]); \ } \ return DerivedData({(void *)magValues, inputData[0].Start, inputData[0].Count}); \ } diff --git a/source/adios2/toolkit/derived/Function.tcc b/source/adios2/toolkit/derived/Function.tcc index 1c47c1665b..077ad1e933 100644 --- a/source/adios2/toolkit/derived/Function.tcc +++ b/source/adios2/toolkit/derived/Function.tcc @@ -22,7 +22,7 @@ T *ApplyOneToOne(std::vector inputData, size_t dataSize, std::cout << "Allocation failed for the derived data" << std::endl; // TODO - throw an exception } - memset(outValues, 0, dataSize * sizeof(T)); + memset((void *)outValues, 0, dataSize * sizeof(T)); for (auto &variable : inputData) { for (size_t i = 0; i < dataSize; i++) diff --git a/source/adios2/toolkit/derived/parser/ASTDriver.cpp b/source/adios2/toolkit/derived/parser/ASTDriver.cpp new file mode 100644 index 0000000000..53871e0db8 --- /dev/null +++ b/source/adios2/toolkit/derived/parser/ASTDriver.cpp @@ -0,0 +1,101 @@ +#include "ASTDriver.h" + +namespace adios2 +{ +namespace detail +{ + +using indx_type = std::vector>; + +ASTDriver::ASTDriver() {} + +ASTDriver::ASTDriver(const std::string input) { ASTDriver::parse(input); } + +ASTDriver::~ASTDriver() +{ + while (holding.size() > 0) + { + delete holding.top(); + holding.pop(); + } +} + +ASTNode *ASTDriver::getAST() +{ + // TODO: check only one ASTNode remains in holding + // else throw error that parsing failed + resolve(holding.top()); + return holding.top(); +} + +void ASTDriver::resolve(ASTNode *node) +{ + if (!node->get_alias().empty()) + { + std::tuple var_info; + var_info = lookup_var(node->get_alias()); + node->set_varname(std::get<0>(var_info)); + node->set_indices(std::get<1>(var_info)); + } + for (ASTNode *subexpr : node->get_subexprs()) + { + resolve(subexpr); + } +} + +std::tuple ASTDriver::lookup_var(const std::string alias) +{ + return aliases[alias]; +} + +std::string ASTDriver::lookup_var_name(const std::string alias) +{ + std::tuple var = aliases[alias]; + return std::get<0>(var); +} + +indx_type ASTDriver::lookup_var_indices(const std::string alias) +{ + std::tuple var = aliases[alias]; + return std::get<1>(var); +} + +void ASTDriver::add_lookup_entry(std::string alias, std::string var_name, indx_type indices) +{ + aliases.insert({alias, {var_name, indices}}); +} + +void ASTDriver::add_lookup_entry(std::string alias, std::string var_name) +{ + aliases.insert({alias, {var_name, {}}}); +} + +void ASTDriver::createNode(std::string op_name, size_t numsubexprs) +{ + ASTNode *node = new ASTNode(op_name, numsubexprs); + for (size_t i = 1; i <= numsubexprs; ++i) + { + // TODO: check that holding contains ASTNode(s) + // else throw error that parsing failed + ASTNode *subexpr = holding.top(); + node->insert_subexpr_n(subexpr, numsubexprs - i); + holding.pop(); + } + holding.push(node); +} + +void ASTDriver::createNode(std::string alias) +{ + ASTNode *node = new ASTNode("ALIAS", alias); + holding.push(node); +} + +void ASTDriver::createNode(std::string alias, indx_type indices) +{ + ASTNode *node = new ASTNode("INDEX", indices); + node->pushback_subexpr(new ASTNode("ALIAS", alias)); + holding.push(node); +} + +} +} diff --git a/source/adios2/toolkit/derived/parser/ASTDriver.h b/source/adios2/toolkit/derived/parser/ASTDriver.h new file mode 100644 index 0000000000..c1f2855800 --- /dev/null +++ b/source/adios2/toolkit/derived/parser/ASTDriver.h @@ -0,0 +1,65 @@ +#ifndef ASTDRIVER_HH_ +#define ASTDRIVER_HH_ + +#include "ASTNode.h" +#include "parser.h" +#include +#include +#include +#include + +#define YY_DECL adios2::detail::parser::symbol_type yylex(adios2::detail::ASTDriver &drv) +YY_DECL; + +namespace adios2 +{ +namespace detail +{ + +using indx_type = std::vector>; + +class ASTDriver +{ +public: + ASTDriver(); + ASTDriver(const std::string input); + ~ASTDriver(); + + // Defined in lexer.l + void parse(const std::string input); + + ASTNode *getAST(); + + void resolve(ASTNode *node); + + std::tuple lookup_var(const std::string alias); + std::string lookup_var_name(const std::string alias); + indx_type lookup_var_indices(const std::string alias); + + void add_lookup_entry(std::string alias, std::string var_name, indx_type indices); + void add_lookup_entry(std::string alias, std::string var_name); + + void createNode(std::string, size_t); + void createNode(std::string); + void createNode(std::string, indx_type); + + // Whether to generate parser debug traces. + bool trace_parsing = false; + // Whether to generate scanner debug traces. + bool trace_scanning = false; + // The token's location used by the scanner. + adios2::detail::location location; + +private: + // While parsing, holds ASTNodes until parent node is created + // (since root node is created last from bottom up design) + std::stack holding; + + // Variable lookup table: maps alias names + // to variable names and indices from alias definition + std::map> aliases; +}; + +} +} +#endif // ! ASTDRIVER_HH_ diff --git a/source/adios2/toolkit/derived/parser/ASTNode.cpp b/source/adios2/toolkit/derived/parser/ASTNode.cpp index 5cbc07322d..3563a4c78f 100644 --- a/source/adios2/toolkit/derived/parser/ASTNode.cpp +++ b/source/adios2/toolkit/derived/parser/ASTNode.cpp @@ -1,6 +1,3 @@ -#ifndef ADIOS2_DERIVED_PARSER_ASTNODE_CPP_ -#define ADIOS2_DERIVED_PARSER_ASTNODE_CPP_ - #include "ASTNode.h" namespace adios2 @@ -8,136 +5,92 @@ namespace adios2 namespace detail { -/*****************************************/ -// alias maps to pair of path and indices (indices may be empty string) -std::map> ASTNode::var_lookup; - ASTNode::ASTNode() {} -ASTNode::ASTNode(ExpressionOperator op) : operation(op) {} - -ASTNode::ASTNode(ExpressionOperator op, const char *str) : operation(op) -{ - switch (operation) - { - case ExpressionOperator::OP_ALIAS: - alias = str; - break; - case ExpressionOperator::OP_PATH: - alias = str; - break; - case ExpressionOperator::OP_INDEX: - indices = str; - break; - default: - // TODO: Make some error - // std::cout << "***That's a problem... ASTNode constructed with string should be alias - // type, path type, or index type\n"; - break; - } -} - -ASTNode::ASTNode(ExpressionOperator op, double val) : operation(op), value(val) {} +ASTNode::ASTNode(std::string op) { opname = op; } -ASTNode::ASTNode(ExpressionOperator op, ASTNode *e) : operation(op) { sub_expr.push_back(e); } - -// for index -ASTNode::ASTNode(ExpressionOperator op, ASTNode *e, const char *str) : operation(op), indices(str) +ASTNode::ASTNode(std::string op, size_t numsubexprs) { - sub_expr.push_back(e); + opname = op; + sub_exprs.resize(numsubexprs); } -ASTNode::ASTNode(ExpressionOperator op, ASTNode *e1, ASTNode *e2) : operation(op) +ASTNode::ASTNode(std::string op, std::string a) { - sub_expr.push_back(e1); - sub_expr.push_back(e2); + opname = op; + alias = a; } -// Copy constructor -ASTNode::ASTNode(const ASTNode &e) -: operation(e.operation), alias(e.alias), value(e.value), sub_expr(e.sub_expr) +ASTNode::ASTNode(std::string op, std::vector> i) { + opname = op; + indices = i; } ASTNode::~ASTNode() { - for (ASTNode *e : sub_expr) + for (ASTNode *sub_expr : sub_exprs) { - delete e; + delete sub_expr; } + sub_exprs.clear(); } -std::pair ASTNode::lookup_var(const std::string var_alias) -{ - return var_lookup[var_alias]; -} - -std::string ASTNode::lookup_var_path(const std::string var_alias) -{ - return var_lookup[var_alias].first; -} - -std::string ASTNode::lookup_var_indices(const std::string var_alias) -{ - return var_lookup[var_alias].second; -} - -void ASTNode::add_lookup_entry(const std::string alias, const std::string var_name, - const std::string indices) -{ - // std::cout << "Adding alias to lookup table:\n\talias: " << alias << "\n\tvar_name: " << - // var_name << "\n\tindices: " << indices << std::endl; - var_lookup[alias] = std::make_pair(var_name, indices); -} - -void ASTNode::add_subexpr(ASTNode *e) { sub_expr.push_back(e); } +void ASTNode::set_num_subexprs(size_t n) { sub_exprs.resize(n); } -void ASTNode::add_back_subexpr(ASTNode *e, size_t n) -{ - size_t index = sub_expr.size() - n; - // std::cout << "ASTNode add_back_subexpr index: " << index << std::endl; - // if (index > 0 && sub_expr[index] == nullptr) - sub_expr[index] = e; -} +void ASTNode::pushback_subexpr(ASTNode *subexpr) { sub_exprs.push_back(subexpr); } -void ASTNode::extend_subexprs(size_t n) -{ - // std::cout << "ASTNode extending subexprs from size " << sub_expr.size() << " to " << - // (sub_expr.size() + n) << std::endl; - sub_expr.resize(sub_expr.size() + n); -} +void ASTNode::insert_subexpr_n(ASTNode *subexpr, size_t index) { sub_exprs[index] = subexpr; } -void ASTNode::printpretty(std::string indent) +std::string ASTNode::printpretty(std::string indent) { - std::cout << indent << get_op_name(operation) << ":"; - if (operation == ExpressionOperator::OP_ALIAS) + std::string result = indent + "Node: " + opname + "\n"; + if (!alias.empty()) { - std::cout << " (alias " << alias << " maps to Variable '"; - std::cout << lookup_var_path(alias) << "'"; - if (lookup_var_indices(alias) != "") - { - std::cout << " [" << lookup_var_indices(alias) << "]"; - } - std::cout << ")"; + result += indent + " (alias: \"" + alias + "\")\n"; } - else if (operation == ExpressionOperator::OP_PATH) + if (!varname.empty()) { - std::cout << " (" << alias << ")"; + result += indent + " (varname: \"" + varname + "\")\n"; } - else if (operation == ExpressionOperator::OP_INDEX) + else if (!alias.empty()) { - std::cout << " [" << indices << "]"; + result += indent + " (varname not found)\n"; + } + if (!indices.empty()) + { + result += indent + " (indices: [ "; + for (std::tuple idx : indices) + { + result += std::to_string(std::get<0>(idx)) + ":"; + result += std::to_string(std::get<1>(idx)) + ":"; + result += std::to_string(std::get<2>(idx)) + " "; + } + result += "] )\n"; } - std::cout << std::endl; - for (ASTNode *e : sub_expr) + for (ASTNode *node : sub_exprs) { - if (e != nullptr) - e->printpretty(indent + " "); - else - std::cout << "sub_expr is nullptr" << std::endl; + result += node->printpretty(indent + " "); } + + return result; } +std::vector ASTNode::get_subexprs() { return sub_exprs; } + +std::string ASTNode::get_opname() { return opname; } + +std::string ASTNode::get_alias() { return alias; } + +std::string ASTNode::get_varname() { return varname; } + +std::vector> ASTNode::get_indices() { return indices; } + +double ASTNode::get_value() { return value; } + +void ASTNode::set_varname(const std::string s) { varname = s; } + +void ASTNode::set_indices(const std::vector> idx) { indices = idx; } + } } -#endif diff --git a/source/adios2/toolkit/derived/parser/ASTNode.h b/source/adios2/toolkit/derived/parser/ASTNode.h index 72cec1a812..1d83ab5855 100644 --- a/source/adios2/toolkit/derived/parser/ASTNode.h +++ b/source/adios2/toolkit/derived/parser/ASTNode.h @@ -1,15 +1,9 @@ -#ifndef ADIOS2_DERIVED_PARSER_ASTNODE_H_ -#define ADIOS2_DERIVED_PARSER_ASTNODE_H_ - -#include -#include +#ifndef ASTNODE_HH +#define ASTNODE_HH #include +#include #include -#include "../ExprHelper.h" - -/*****************************************/ - namespace adios2 { namespace detail @@ -19,40 +13,36 @@ class ASTNode { public: ASTNode(); - ASTNode(ExpressionOperator); - ASTNode(ExpressionOperator, const char *a); - ASTNode(ExpressionOperator, double val); - ASTNode(ExpressionOperator, ASTNode *e); - ASTNode(ExpressionOperator, ASTNode *e, const char *i); - ASTNode(ExpressionOperator, ASTNode *e1, ASTNode *e2); - - // Copy constructor - ASTNode(const ASTNode &e); - + ASTNode(std::string); + ASTNode(std::string, size_t); + ASTNode(std::string, std::string); + ASTNode(std::string, std::vector>); ~ASTNode(); - static std::pair lookup_var(const std::string var_alias); - static std::string lookup_var_path(const std::string var_alias); - static std::string lookup_var_indices(const std::string var_alias); - static void add_lookup_entry(const std::string alias, const std::string var_name, - const std::string indices); + void set_num_subexprs(size_t); + void pushback_subexpr(ASTNode *); + void insert_subexpr_n(ASTNode *, size_t); + std::string printpretty(std::string = ""); + + std::vector get_subexprs(); + std::string get_opname(); + std::string get_alias(); + std::string get_varname(); + std::vector> get_indices(); + double get_value(); - void add_subexpr(ASTNode *e); - void add_back_subexpr(ASTNode *e, size_t i); - void extend_subexprs(size_t n); - void infer_type(); - void printpretty(std::string indent = ""); + void set_varname(const std::string); + void set_indices(const std::vector>); - // private: - ExpressionOperator operation; +private: + std::vector sub_exprs; + std::string opname; std::string alias; - std::string indices; + std::string varname; + std::vector> indices; double value; - std::vector sub_expr; - - static std::map> var_lookup; }; } } -#endif \ No newline at end of file +#endif // ! ASTNODE_HH diff --git a/source/adios2/toolkit/derived/parser/lexer.cpp b/source/adios2/toolkit/derived/parser/lexer.cpp deleted file mode 100644 index 03b101c191..0000000000 --- a/source/adios2/toolkit/derived/parser/lexer.cpp +++ /dev/null @@ -1,1884 +0,0 @@ -#line 1 "lexer.cpp" - -#line 3 "lexer.cpp" - -#define YY_INT_ALIGNED short int - -/* A lexical scanner generated by flex */ - -#define FLEX_SCANNER -#define YY_FLEX_MAJOR_VERSION 2 -#define YY_FLEX_MINOR_VERSION 6 -#define YY_FLEX_SUBMINOR_VERSION 4 -#if YY_FLEX_SUBMINOR_VERSION > 0 -#define FLEX_BETA -#endif - -/* First, we deal with platform-specific or compiler-specific issues. */ - -/* begin standard C headers. */ -#include -#include -#include -#include - -/* end standard C headers. */ - -/* flex integer type definitions */ - -#ifndef FLEXINT_H -#define FLEXINT_H - -/* C99 systems have . Non-C99 systems may or may not. */ - -#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L - -/* C99 says to define __STDC_LIMIT_MACROS before including stdint.h, - * if you want the limit (max/min) macros for int types. - */ -#ifndef __STDC_LIMIT_MACROS -#define __STDC_LIMIT_MACROS 1 -#endif - -#include -typedef int8_t flex_int8_t; -typedef uint8_t flex_uint8_t; -typedef int16_t flex_int16_t; -typedef uint16_t flex_uint16_t; -typedef int32_t flex_int32_t; -typedef uint32_t flex_uint32_t; -typedef uint64_t flex_uint64_t; -#else -typedef signed char flex_int8_t; -typedef short int flex_int16_t; -typedef int flex_int32_t; -typedef unsigned char flex_uint8_t; -typedef unsigned short int flex_uint16_t; -typedef unsigned int flex_uint32_t; - -/* Limits of integral types. */ -#ifndef INT8_MIN -#define INT8_MIN (-128) -#endif -#ifndef INT16_MIN -#define INT16_MIN (-32767 - 1) -#endif -#ifndef INT32_MIN -#define INT32_MIN (-2147483647 - 1) -#endif -#ifndef INT8_MAX -#define INT8_MAX (127) -#endif -#ifndef INT16_MAX -#define INT16_MAX (32767) -#endif -#ifndef INT32_MAX -#define INT32_MAX (2147483647) -#endif -#ifndef UINT8_MAX -#define UINT8_MAX (255U) -#endif -#ifndef UINT16_MAX -#define UINT16_MAX (65535U) -#endif -#ifndef UINT32_MAX -#define UINT32_MAX (4294967295U) -#endif - -#ifndef SIZE_MAX -#define SIZE_MAX (~(size_t)0) -#endif - -#endif /* ! C99 */ - -#endif /* ! FLEXINT_H */ - -/* begin standard C++ headers. */ - -/* TODO: this is always defined, so inline it */ -#define yyconst const - -#if defined(__GNUC__) && __GNUC__ >= 3 -#define yynoreturn __attribute__((__noreturn__)) -#else -#define yynoreturn -#endif - -/* Returned upon end-of-file. */ -#define YY_NULL 0 - -/* Promotes a possibly negative, possibly signed char to an - * integer in range [0..255] for use as an array index. - */ -#define YY_SC_TO_UI(c) ((YY_CHAR)(c)) - -/* Enter a start condition. This macro really ought to take a parameter, - * but we do it the disgusting crufty way forced on us by the ()-less - * definition of BEGIN. - */ -#define BEGIN (yy_start) = 1 + 2 * -/* Translate the current start state into a value that can be later handed - * to BEGIN to return to the state. The YYSTATE alias is for lex - * compatibility. - */ -#define YY_START (((yy_start)-1) / 2) -#define YYSTATE YY_START -/* Action number for EOF rule of a given start state. */ -#define YY_STATE_EOF(state) (YY_END_OF_BUFFER + state + 1) -/* Special action meaning "start processing a new file". */ -#define YY_NEW_FILE yyrestart(yyin) -#define YY_END_OF_BUFFER_CHAR 0 - -/* Size of default input buffer. */ -#ifndef YY_BUF_SIZE -#ifdef __ia64__ -/* On IA-64, the buffer size is 16k, not 8k. - * Moreover, YY_BUF_SIZE is 2*YY_READ_BUF_SIZE in the general case. - * Ditto for the __ia64__ case accordingly. - */ -#define YY_BUF_SIZE 32768 -#else -#define YY_BUF_SIZE 16384 -#endif /* __ia64__ */ -#endif - -/* The state buf must be large enough to hold one state per character in the main buffer. - */ -#define YY_STATE_BUF_SIZE ((YY_BUF_SIZE + 2) * sizeof(yy_state_type)) - -#ifndef YY_TYPEDEF_YY_BUFFER_STATE -#define YY_TYPEDEF_YY_BUFFER_STATE -typedef struct yy_buffer_state *YY_BUFFER_STATE; -#endif - -#ifndef YY_TYPEDEF_YY_SIZE_T -#define YY_TYPEDEF_YY_SIZE_T -typedef size_t yy_size_t; -#endif - -extern yy_size_t yyleng; - -extern FILE *yyin, *yyout; - -#define EOB_ACT_CONTINUE_SCAN 0 -#define EOB_ACT_END_OF_FILE 1 -#define EOB_ACT_LAST_MATCH 2 - -#define YY_LESS_LINENO(n) -#define YY_LINENO_REWIND_TO(ptr) - -/* Return all but the first "n" matched characters back to the input stream. */ -#define yyless(n) \ - do \ - { \ - /* Undo effects of setting up yytext. */ \ - int yyless_macro_arg = (n); \ - YY_LESS_LINENO(yyless_macro_arg); \ - *yy_cp = (yy_hold_char); \ - YY_RESTORE_YY_MORE_OFFSET(yy_c_buf_p) = yy_cp = yy_bp + yyless_macro_arg - YY_MORE_ADJ; \ - YY_DO_BEFORE_ACTION; /* set up yytext again */ \ - } while (0) -#define unput(c) yyunput(c, (yytext_ptr)) - -#ifndef YY_STRUCT_YY_BUFFER_STATE -#define YY_STRUCT_YY_BUFFER_STATE -struct yy_buffer_state -{ - FILE *yy_input_file; - - char *yy_ch_buf; /* input buffer */ - char *yy_buf_pos; /* current position in input buffer */ - - /* Size of input buffer in bytes, not including room for EOB - * characters. - */ - int yy_buf_size; - - /* Number of characters read into yy_ch_buf, not including EOB - * characters. - */ - yy_size_t yy_n_chars; - - /* Whether we "own" the buffer - i.e., we know we created it, - * and can realloc() it to grow it, and should free() it to - * delete it. - */ - int yy_is_our_buffer; - - /* Whether this is an "interactive" input source; if so, and - * if we're using stdio for input, then we want to use getc() - * instead of fread(), to make sure we stop fetching input after - * each newline. - */ - int yy_is_interactive; - - /* Whether we're considered to be at the beginning of a line. - * If so, '^' rules will be active on the next match, otherwise - * not. - */ - int yy_at_bol; - - int yy_bs_lineno; /**< The line count. */ - int yy_bs_column; /**< The column count. */ - - /* Whether to try to fill the input buffer when we reach the - * end of it. - */ - int yy_fill_buffer; - - int yy_buffer_status; - -#define YY_BUFFER_NEW 0 -#define YY_BUFFER_NORMAL 1 - /* When an EOF's been seen but there's still some text to process - * then we mark the buffer as YY_EOF_PENDING, to indicate that we - * shouldn't try reading from the input source any more. We might - * still have a bunch of tokens to match, though, because of - * possible backing-up. - * - * When we actually see the EOF, we change the status to "new" - * (via yyrestart()), so that the user can continue scanning by - * just pointing yyin at a new input file. - */ -#define YY_BUFFER_EOF_PENDING 2 -}; -#endif /* !YY_STRUCT_YY_BUFFER_STATE */ - -/* Stack of input buffers. */ -static size_t yy_buffer_stack_top = 0; /**< index of top of stack. */ -static size_t yy_buffer_stack_max = 0; /**< capacity of stack. */ -static YY_BUFFER_STATE *yy_buffer_stack = NULL; /**< Stack as an array. */ - -/* We provide macros for accessing buffer states in case in the - * future we want to put the buffer states in a more general - * "scanner state". - * - * Returns the top of the stack, or NULL. - */ -#define YY_CURRENT_BUFFER ((yy_buffer_stack) ? (yy_buffer_stack)[(yy_buffer_stack_top)] : NULL) -/* Same as previous macro, but useful when we know that the buffer stack is not - * NULL or when we need an lvalue. For internal use only. - */ -#define YY_CURRENT_BUFFER_LVALUE (yy_buffer_stack)[(yy_buffer_stack_top)] - -/* yy_hold_char holds the character lost when yytext is formed. */ -static char yy_hold_char; -static yy_size_t yy_n_chars; /* number of characters read into yy_ch_buf */ -yy_size_t yyleng; - -/* Points to current character in buffer. */ -static char *yy_c_buf_p = NULL; -static int yy_init = 0; /* whether we need to initialize */ -static int yy_start = 0; /* start state number */ - -/* Flag which is used to allow yywrap()'s to do buffer switches - * instead of setting up a fresh yyin. A bit of a hack ... - */ -static int yy_did_buffer_switch_on_eof; - -void yyrestart(FILE *input_file); -void yy_switch_to_buffer(YY_BUFFER_STATE new_buffer); -YY_BUFFER_STATE yy_create_buffer(FILE *file, int size); -void yy_delete_buffer(YY_BUFFER_STATE b); -void yy_flush_buffer(YY_BUFFER_STATE b); -void yypush_buffer_state(YY_BUFFER_STATE new_buffer); -void yypop_buffer_state(void); - -static void yyensure_buffer_stack(void); -static void yy_load_buffer_state(void); -static void yy_init_buffer(YY_BUFFER_STATE b, FILE *file); -#define YY_FLUSH_BUFFER yy_flush_buffer(YY_CURRENT_BUFFER) - -YY_BUFFER_STATE yy_scan_buffer(char *base, yy_size_t size); -YY_BUFFER_STATE yy_scan_string(const char *yy_str); -YY_BUFFER_STATE yy_scan_bytes(const char *bytes, yy_size_t len); - -void *yyalloc(yy_size_t); -void *yyrealloc(void *, yy_size_t); -void yyfree(void *); - -#define yy_new_buffer yy_create_buffer -#define yy_set_interactive(is_interactive) \ - { \ - if (!YY_CURRENT_BUFFER) \ - { \ - yyensure_buffer_stack(); \ - YY_CURRENT_BUFFER_LVALUE = yy_create_buffer(yyin, YY_BUF_SIZE); \ - } \ - YY_CURRENT_BUFFER_LVALUE->yy_is_interactive = is_interactive; \ - } -#define yy_set_bol(at_bol) \ - { \ - if (!YY_CURRENT_BUFFER) \ - { \ - yyensure_buffer_stack(); \ - YY_CURRENT_BUFFER_LVALUE = yy_create_buffer(yyin, YY_BUF_SIZE); \ - } \ - YY_CURRENT_BUFFER_LVALUE->yy_at_bol = at_bol; \ - } -#define YY_AT_BOL() (YY_CURRENT_BUFFER_LVALUE->yy_at_bol) - -/* Begin user sect3 */ - -#define yywrap() (/*CONSTCOND*/ 1) -#define YY_SKIP_YYWRAP -typedef flex_uint8_t YY_CHAR; - -FILE *yyin = NULL, *yyout = NULL; - -typedef int yy_state_type; - -extern int yylineno; -int yylineno = 1; - -extern char *yytext; -#ifdef yytext_ptr -#undef yytext_ptr -#endif -#define yytext_ptr yytext - -static yy_state_type yy_get_previous_state(void); -static yy_state_type yy_try_NUL_trans(yy_state_type current_state); -static int yy_get_next_buffer(void); -static void yynoreturn yy_fatal_error(const char *msg); - -/* Done after the current pattern has been matched and before the - * corresponding action - sets up yytext. - */ -#define YY_DO_BEFORE_ACTION \ - (yytext_ptr) = yy_bp; \ - yyleng = (yy_size_t)(yy_cp - yy_bp); \ - (yy_hold_char) = *yy_cp; \ - *yy_cp = '\0'; \ - (yy_c_buf_p) = yy_cp; -#define YY_NUM_RULES 24 -#define YY_END_OF_BUFFER 25 -/* This struct is not used in this scanner, - but its presence is necessary. */ -struct yy_trans_info -{ - flex_int32_t yy_verify; - flex_int32_t yy_nxt; -}; -static const flex_int16_t yy_accept[91] = { - 0, 0, 0, 25, 23, 17, 18, 23, 23, 6, 7, 3, 1, 8, 2, 23, 4, 16, 23, 19, 19, 23, 5, - 19, 19, 19, 19, 19, 18, 17, 0, 0, 0, 16, 16, 16, 19, 19, 0, 20, 19, 0, 0, 19, 19, 19, - 19, 19, 19, 19, 0, 21, 0, 0, 16, 0, 0, 16, 20, 20, 18, 0, 22, 0, 0, 9, 12, 19, 19, - 11, 19, 13, 18, 0, 16, 20, 0, 0, 0, 15, 19, 10, 0, 0, 19, 0, 19, 19, 19, 14, 0}; - -static const YY_CHAR yy_ec[256] = { - 0, 1, 1, 1, 1, 1, 1, 1, 1, 2, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 4, 1, 1, 5, 6, 7, - 8, 9, 10, 11, 12, 13, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 15, 1, 1, 1, 1, - 1, 1, 16, 16, 16, 16, 17, 18, 16, 16, 16, 16, 16, 16, 16, 16, 19, 16, 16, 16, 16, - 16, 16, 16, 16, 16, 16, 16, 20, 21, 22, 23, 24, 1, 25, 26, 27, 28, - - 29, 26, 30, 26, 31, 26, 26, 32, 33, 34, 35, 26, 36, 37, 38, 39, 40, 26, 26, 26, 26, - 26, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}; - -static const YY_CHAR yy_meta[42] = {0, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 3, 4, 5, 6, 1, 6, 6, 6, 6, 1, - 5, 1, 1, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 1}; - -static const flex_int16_t yy_base[98] = { - 0, 0, 0, 221, 254, 218, 254, 190, 29, 254, 254, 254, 254, 254, 254, 203, 254, - 32, 34, 204, 33, 42, 254, 39, 42, 47, 48, 50, 254, 213, 180, 50, 75, 199, - 52, 56, 200, 81, 84, 89, 89, 101, 103, 66, 53, 75, 92, 94, 97, 102, 183, - 254, 116, 128, 118, 134, 196, 112, 140, 144, 196, 105, 254, 145, 152, 195, 193, 138, - 144, 157, 132, 126, 254, 162, 117, 166, 174, 167, 181, 113, 173, 71, 178, 184, 160, - 187, 188, 190, 134, 64, 254, 224, 227, 229, 234, 238, 243, 247 - -}; - -static const flex_int16_t yy_def[98] = { - 0, 90, 1, 90, 90, 90, 90, 90, 91, 90, 90, 90, 90, 90, 90, 90, 90, 92, 93, 92, - 92, 90, 90, 92, 92, 92, 92, 92, 90, 90, 90, 91, 94, 90, 90, 92, 92, 92, 93, 95, - 92, 90, 90, 92, 92, 92, 92, 92, 92, 92, 90, 90, 96, 94, 90, 90, 90, 92, 97, 95, - 92, 90, 90, 90, 90, 92, 92, 92, 92, 92, 92, 92, 90, 96, 90, 97, 90, 90, 90, 92, - 92, 92, 90, 90, 92, 90, 92, 92, 92, 92, 0, 90, 90, 90, 90, 90, 90, 90 - -}; - -static const flex_int16_t yy_nxt[296] = { - 0, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 19, 19, 21, 4, 4, 22, - 19, 23, 19, 24, 19, 19, 19, 19, 19, 25, 19, 19, 19, 19, 26, 27, 19, 28, 31, 31, 34, 90, 35, 38, - 38, 37, 31, 90, 40, 31, 90, 38, 41, 42, 38, 90, 90, 37, 90, 31, 31, 90, 54, 43, 34, 55, 35, 31, - 46, 37, 31, 49, 90, 44, 90, 47, 51, 55, 45, 90, 48, 37, 52, 90, 52, 52, 56, 66, 56, 90, 65, 57, - 52, 38, 38, 52, 58, - - 90, 58, 58, 90, 38, 90, 60, 38, 90, 58, 61, 67, 58, 90, 41, 42, 63, 64, 76, 77, 51, 68, 62, 90, - 90, 57, 52, 69, 52, 52, 74, 54, 51, 70, 55, 71, 52, 90, 52, 52, 52, 52, 56, 90, 56, 90, 55, 74, - 52, 90, 58, 52, 58, 58, 58, 90, 58, 58, 63, 64, 58, 61, 89, 58, 58, 78, 51, 58, 90, 79, 81, 90, - 52, 62, 52, 52, 58, 80, 58, 58, 82, 83, 52, 61, 90, 52, 58, 76, 77, 58, 61, 82, 83, 61, 78, 62, - 61, 85, 86, 90, - - 85, 90, 62, 84, 90, 62, 90, 90, 62, 74, 72, 90, 33, 50, 29, 90, 33, 88, 30, 29, 90, 90, 90, 90, - 90, 90, 90, 87, 32, 32, 36, 90, 36, 39, 39, 53, 53, 90, 53, 53, 59, 90, 59, 59, 73, 73, 90, 73, - 73, 75, 90, 75, 75, 3, 90, 90, 90, 90, 90, 90, 90, 90, 90, 90, 90, 90, 90, 90, 90, 90, 90, 90, - 90, 90, 90, 90, 90, 90, 90, 90, 90, 90, 90, 90, 90, 90, 90, 90, 90, 90, 90, 90, 90, 90, 90 - -}; - -static const flex_int16_t yy_chk[296] = { - 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 8, 8, 17, 20, 17, 18, - 18, 17, 8, 23, 20, 8, 24, 18, 21, 21, 18, 25, 26, 17, 27, 31, 31, 44, 34, 23, 35, 34, 35, 31, - 25, 35, 31, 27, 89, 24, 43, 26, 32, 34, 24, 81, 26, 35, 32, 45, 32, 32, 37, 44, 37, 37, 43, 37, - 32, 38, 38, 32, 39, - - 40, 39, 39, 46, 38, 47, 40, 38, 48, 39, 41, 45, 39, 49, 41, 41, 42, 42, 61, 61, 52, 46, 41, 57, - 79, 57, 52, 47, 52, 52, 74, 54, 53, 48, 54, 49, 52, 71, 53, 52, 53, 53, 55, 70, 55, 88, 54, 55, - 53, 67, 58, 53, 58, 58, 59, 68, 59, 59, 63, 63, 58, 64, 88, 58, 59, 64, 73, 59, 69, 67, 70, 84, - 73, 64, 73, 73, 75, 68, 75, 75, 77, 77, 73, 76, 80, 73, 75, 76, 76, 75, 78, 82, 82, 83, 78, 76, - 85, 83, 84, 86, - - 85, 87, 78, 80, 66, 83, 65, 60, 85, 56, 50, 36, 33, 30, 29, 19, 15, 87, 7, 5, 3, 0, 0, 0, - 0, 0, 0, 86, 91, 91, 92, 0, 92, 93, 93, 94, 94, 0, 94, 94, 95, 0, 95, 95, 96, 96, 0, 96, - 96, 97, 0, 97, 97, 90, 90, 90, 90, 90, 90, 90, 90, 90, 90, 90, 90, 90, 90, 90, 90, 90, 90, 90, - 90, 90, 90, 90, 90, 90, 90, 90, 90, 90, 90, 90, 90, 90, 90, 90, 90, 90, 90, 90, 90, 90, 90 - -}; - -static yy_state_type yy_last_accepting_state; -static char *yy_last_accepting_cpos; - -extern int yy_flex_debug; -int yy_flex_debug = 0; - -/* The intent behind this definition is that it'll catch - * any uses of REJECT which flex missed. - */ -#define REJECT reject_used_but_not_detected -#define yymore() yymore_used_but_not_detected -#define YY_MORE_ADJ 0 -#define YY_RESTORE_YY_MORE_OFFSET -char *yytext; -#line 1 "lexer.l" -#line 2 "lexer.l" -#include "parser.h" -#define YY_USER_ACTION yylloc.first_line = yylloc.last_line = yylineno; -#line 544 "lexer.cpp" -#line 545 "lexer.cpp" - -#define INITIAL 0 - -#ifndef YY_NO_UNISTD_H -/* Special case for "unistd.h", since it is non-ANSI. We include it way - * down here because we want the user's section 1 to have been scanned first. - * The user has a chance to override it with an option. - */ -#include -#endif - -#ifndef YY_EXTRA_TYPE -#define YY_EXTRA_TYPE void * -#endif - -static int yy_init_globals(void); - -/* Accessor methods to globals. - These are made visible to non-reentrant scanners for convenience. */ - -int yylex_destroy(void); - -int yyget_debug(void); - -void yyset_debug(int debug_flag); - -YY_EXTRA_TYPE yyget_extra(void); - -void yyset_extra(YY_EXTRA_TYPE user_defined); - -FILE *yyget_in(void); - -void yyset_in(FILE *_in_str); - -FILE *yyget_out(void); - -void yyset_out(FILE *_out_str); - -yy_size_t yyget_leng(void); - -char *yyget_text(void); - -int yyget_lineno(void); - -void yyset_lineno(int _line_number); - -/* Macros after this point can all be overridden by user definitions in - * section 1. - */ - -#ifndef YY_SKIP_YYWRAP -#ifdef __cplusplus -extern "C" int yywrap(void); -#else -extern int yywrap(void); -#endif -#endif - -#ifndef YY_NO_UNPUT - -static void yyunput(int c, char *buf_ptr); - -#endif - -#ifndef yytext_ptr -static void yy_flex_strncpy(char *, const char *, int); -#endif - -#ifdef YY_NEED_STRLEN -static int yy_flex_strlen(const char *); -#endif - -#ifndef YY_NO_INPUT -#ifdef __cplusplus -static int yyinput(void); -#else -static int input(void); -#endif - -#endif - -/* Amount of stuff to slurp up with each read. */ -#ifndef YY_READ_BUF_SIZE -#ifdef __ia64__ -/* On IA-64, the buffer size is 16k, not 8k */ -#define YY_READ_BUF_SIZE 16384 -#else -#define YY_READ_BUF_SIZE 8192 -#endif /* __ia64__ */ -#endif - -/* Copy whatever the last rule matched to the standard output. */ -#ifndef ECHO -/* This used to be an fputs(), but since the string might contain NUL's, - * we now use fwrite(). - */ -#define ECHO \ - do \ - { \ - if (fwrite(yytext, (size_t)yyleng, 1, yyout)) \ - { \ - } \ - } while (0) -#endif - -/* Gets input and stuffs it into "buf". number of characters read, or YY_NULL, - * is returned in "result". - */ -#ifndef YY_INPUT -#define YY_INPUT(buf, result, max_size) \ - if (YY_CURRENT_BUFFER_LVALUE->yy_is_interactive) \ - { \ - int c = '*'; \ - yy_size_t n; \ - for (n = 0; n < max_size && (c = getc(yyin)) != EOF && c != '\n'; ++n) \ - buf[n] = (char)c; \ - if (c == '\n') \ - buf[n++] = (char)c; \ - if (c == EOF && ferror(yyin)) \ - YY_FATAL_ERROR("input in flex scanner failed"); \ - result = n; \ - } \ - else \ - { \ - errno = 0; \ - while ((result = (int)fread(buf, 1, (yy_size_t)max_size, yyin)) == 0 && ferror(yyin)) \ - { \ - if (errno != EINTR) \ - { \ - YY_FATAL_ERROR("input in flex scanner failed"); \ - break; \ - } \ - errno = 0; \ - clearerr(yyin); \ - } \ - } - -#endif - -/* No semi-colon after return; correct usage is to write "yyterminate();" - - * we don't want an extra ';' after the "return" because that will cause - * some compilers to complain about unreachable statements. - */ -#ifndef yyterminate -#define yyterminate() return YY_NULL -#endif - -/* Number of entries by which start-condition stack grows. */ -#ifndef YY_START_STACK_INCR -#define YY_START_STACK_INCR 25 -#endif - -/* Report a fatal error. */ -#ifndef YY_FATAL_ERROR -#define YY_FATAL_ERROR(msg) yy_fatal_error(msg) -#endif - -/* end tables serialization structures and prototypes */ - -/* Default declaration of generated scanner - a define so the user can - * easily add parameters. - */ -#ifndef YY_DECL -#define YY_DECL_IS_OURS 1 - -extern int yylex(void); - -#define YY_DECL int yylex(void) -#endif /* !YY_DECL */ - -/* Code executed at the beginning of each rule, after yytext and yyleng - * have been set up. - */ -#ifndef YY_USER_ACTION -#define YY_USER_ACTION -#endif - -/* Code executed at the end of each rule. */ -#ifndef YY_BREAK -#define YY_BREAK /*LINTED*/ break; -#endif - -#define YY_RULE_SETUP YY_USER_ACTION - -/** The main scanner function which does all the work. - */ -YY_DECL -{ - yy_state_type yy_current_state; - char *yy_cp, *yy_bp; - int yy_act; - - if (!(yy_init)) - { - (yy_init) = 1; - -#ifdef YY_USER_INIT - YY_USER_INIT; -#endif - - if (!(yy_start)) - (yy_start) = 1; /* first start state */ - - if (!yyin) - yyin = stdin; - - if (!yyout) - yyout = stdout; - - if (!YY_CURRENT_BUFFER) - { - yyensure_buffer_stack(); - YY_CURRENT_BUFFER_LVALUE = yy_create_buffer(yyin, YY_BUF_SIZE); - } - - yy_load_buffer_state(); - } - - { -#line 12 "lexer.l" - -#line 765 "lexer.cpp" - - while (/*CONSTCOND*/ 1) /* loops until end-of-file is reached */ - { - yy_cp = (yy_c_buf_p); - - /* Support of yytext. */ - *yy_cp = (yy_hold_char); - - /* yy_bp points to the position in yy_ch_buf of the start of - * the current run. - */ - yy_bp = yy_cp; - - yy_current_state = (yy_start); - yy_match: - do - { - YY_CHAR yy_c = yy_ec[YY_SC_TO_UI(*yy_cp)]; - if (yy_accept[yy_current_state]) - { - (yy_last_accepting_state) = yy_current_state; - (yy_last_accepting_cpos) = yy_cp; - } - while (yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state) - { - yy_current_state = (int)yy_def[yy_current_state]; - if (yy_current_state >= 91) - yy_c = yy_meta[yy_c]; - } - yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c]; - ++yy_cp; - } while (yy_base[yy_current_state] != 254); - - yy_find_action: - yy_act = yy_accept[yy_current_state]; - if (yy_act == 0) - { /* have to back up */ - yy_cp = (yy_last_accepting_cpos); - yy_current_state = (yy_last_accepting_state); - yy_act = yy_accept[yy_current_state]; - } - - YY_DO_BEFORE_ACTION; - - do_action: /* This label is used only to access EOF actions. */ - - switch (yy_act) - { /* beginning of action switch */ - case 0: /* must back up */ - /* undo the effects of YY_DO_BEFORE_ACTION */ - *yy_cp = (yy_hold_char); - yy_cp = (yy_last_accepting_cpos); - yy_current_state = (yy_last_accepting_state); - goto yy_find_action; - - case 1: - YY_RULE_SETUP -#line 14 "lexer.l" - { - yylval.sval = strdup(yytext); - return OPERATOR; - } - YY_BREAK - case 2: - YY_RULE_SETUP -#line 15 "lexer.l" - { - yylval.sval = strdup(yytext); - return OPERATOR; - } - YY_BREAK - case 3: - YY_RULE_SETUP -#line 16 "lexer.l" - { - yylval.sval = strdup(yytext); - return OPERATOR; - } - YY_BREAK - case 4: - YY_RULE_SETUP -#line 17 "lexer.l" - { - yylval.sval = strdup(yytext); - return OPERATOR; - } - YY_BREAK - case 5: - YY_RULE_SETUP -#line 18 "lexer.l" - { - yylval.sval = strdup(yytext); - return OPERATOR; - } - YY_BREAK - case 6: - YY_RULE_SETUP -#line 19 "lexer.l" - { - return L_PAREN; - } - YY_BREAK - case 7: - YY_RULE_SETUP -#line 20 "lexer.l" - { - return R_PAREN; - } - YY_BREAK - case 8: - YY_RULE_SETUP -#line 21 "lexer.l" - { - return COMMA; - } - YY_BREAK - case 9: - YY_RULE_SETUP -#line 22 "lexer.l" - { - yylval.sval = strdup(yytext); - return FUNCTION; - } - YY_BREAK - case 10: - YY_RULE_SETUP -#line 23 "lexer.l" - { - yylval.sval = strdup(yytext); - return FUNCTION; - } - YY_BREAK - case 11: - YY_RULE_SETUP -#line 24 "lexer.l" - { - yylval.sval = strdup(yytext); - return FUNCTION; - } - YY_BREAK - case 12: - YY_RULE_SETUP -#line 25 "lexer.l" - { - yylval.sval = strdup(yytext); - return FUNCTION; - } - YY_BREAK - case 13: - YY_RULE_SETUP -#line 26 "lexer.l" - { - yylval.sval = strdup(yytext); - return FUNCTION; - } - YY_BREAK - case 14: - YY_RULE_SETUP -#line 27 "lexer.l" - { - yylval.sval = strdup(yytext); - return FUNCTION; - } - YY_BREAK - case 15: - YY_RULE_SETUP -#line 28 "lexer.l" - { - yylval.sval = strdup(yytext); - return FUNCTION; - } - YY_BREAK - case 16: - YY_RULE_SETUP -#line 31 "lexer.l" - { - yylval.dval = atof(yytext); - return NUMBER; - } - YY_BREAK - case 17: - YY_RULE_SETUP -#line 33 "lexer.l" - { /* ignore spaces */ - } - YY_BREAK - case 18: - /* rule 18 can match eol */ - YY_RULE_SETUP -#line 35 "lexer.l" - { - return ENDL; - } - YY_BREAK - case 19: - YY_RULE_SETUP -#line 37 "lexer.l" - { - yylval.sval = strdup(yytext); - return ALIAS; - } - YY_BREAK - case 20: - YY_RULE_SETUP -#line 39 "lexer.l" - { - yylval.sval = strndup(yytext + 1, strlen(yytext) - 1); - return PATH; - } - YY_BREAK - case 21: - YY_RULE_SETUP -#line 41 "lexer.l" - { - yylval.sval = strndup(yytext + 1, strlen(yytext) - 2); - return PATH; - } - YY_BREAK - case 22: - YY_RULE_SETUP -#line 43 "lexer.l" - { - yylval.sval = strndup(yytext + 1, strlen(yytext) - 2); - return INDICES; - } - YY_BREAK - case 23: - YY_RULE_SETUP -#line 45 "lexer.l" - { - printf("Error at line %d: unrecognized symbol \"%s\"\n", yylloc.first_line, - yytext); - exit(0); - } - YY_BREAK - case 24: - YY_RULE_SETUP -#line 47 "lexer.l" - ECHO; - YY_BREAK -#line 943 "lexer.cpp" - case YY_STATE_EOF(INITIAL): - yyterminate(); - - case YY_END_OF_BUFFER: { - /* Amount of text matched not including the EOB char. */ - int yy_amount_of_matched_text = (int)(yy_cp - (yytext_ptr)) - 1; - - /* Undo the effects of YY_DO_BEFORE_ACTION. */ - *yy_cp = (yy_hold_char); - YY_RESTORE_YY_MORE_OFFSET - - if (YY_CURRENT_BUFFER_LVALUE->yy_buffer_status == YY_BUFFER_NEW) - { - /* We're scanning a new file or input source. It's - * possible that this happened because the user - * just pointed yyin at a new source and called - * yylex(). If so, then we have to assure - * consistency between YY_CURRENT_BUFFER and our - * globals. Here is the right place to do so, because - * this is the first action (other than possibly a - * back-up) that will match for the new input source. - */ - (yy_n_chars) = YY_CURRENT_BUFFER_LVALUE->yy_n_chars; - YY_CURRENT_BUFFER_LVALUE->yy_input_file = yyin; - YY_CURRENT_BUFFER_LVALUE->yy_buffer_status = YY_BUFFER_NORMAL; - } - - /* Note that here we test for yy_c_buf_p "<=" to the position - * of the first EOB in the buffer, since yy_c_buf_p will - * already have been incremented past the NUL character - * (since all states make transitions on EOB to the - * end-of-buffer state). Contrast this with the test - * in input(). - */ - if ((yy_c_buf_p) <= &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[(yy_n_chars)]) - { /* This was really a NUL. */ - yy_state_type yy_next_state; - - (yy_c_buf_p) = (yytext_ptr) + yy_amount_of_matched_text; - - yy_current_state = yy_get_previous_state(); - - /* Okay, we're now positioned to make the NUL - * transition. We couldn't have - * yy_get_previous_state() go ahead and do it - * for us because it doesn't know how to deal - * with the possibility of jamming (and we don't - * want to build jamming into it because then it - * will run more slowly). - */ - - yy_next_state = yy_try_NUL_trans(yy_current_state); - - yy_bp = (yytext_ptr) + YY_MORE_ADJ; - - if (yy_next_state) - { - /* Consume the NUL. */ - yy_cp = ++(yy_c_buf_p); - yy_current_state = yy_next_state; - goto yy_match; - } - - else - { - yy_cp = (yy_c_buf_p); - goto yy_find_action; - } - } - - else - switch (yy_get_next_buffer()) - { - case EOB_ACT_END_OF_FILE: { - (yy_did_buffer_switch_on_eof) = 0; - - if (yywrap()) - { - /* Note: because we've taken care in - * yy_get_next_buffer() to have set up - * yytext, we can now set up - * yy_c_buf_p so that if some total - * hoser (like flex itself) wants to - * call the scanner after we return the - * YY_NULL, it'll still work - another - * YY_NULL will get returned. - */ - (yy_c_buf_p) = (yytext_ptr) + YY_MORE_ADJ; - - yy_act = YY_STATE_EOF(YY_START); - goto do_action; - } - - else - { - if (!(yy_did_buffer_switch_on_eof)) - YY_NEW_FILE; - } - break; - } - - case EOB_ACT_CONTINUE_SCAN: - (yy_c_buf_p) = (yytext_ptr) + yy_amount_of_matched_text; - - yy_current_state = yy_get_previous_state(); - - yy_cp = (yy_c_buf_p); - yy_bp = (yytext_ptr) + YY_MORE_ADJ; - goto yy_match; - - case EOB_ACT_LAST_MATCH: - (yy_c_buf_p) = &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[(yy_n_chars)]; - - yy_current_state = yy_get_previous_state(); - - yy_cp = (yy_c_buf_p); - yy_bp = (yytext_ptr) + YY_MORE_ADJ; - goto yy_find_action; - } - break; - } - - default: - YY_FATAL_ERROR("fatal flex scanner internal error--no action found"); - } /* end of action switch */ - } /* end of scanning one token */ - } /* end of user's declarations */ -} /* end of yylex */ - -/* yy_get_next_buffer - try to read in a new buffer - * - * Returns a code representing an action: - * EOB_ACT_LAST_MATCH - - * EOB_ACT_CONTINUE_SCAN - continue scanning from current position - * EOB_ACT_END_OF_FILE - end of file - */ -static int yy_get_next_buffer(void) -{ - char *dest = YY_CURRENT_BUFFER_LVALUE->yy_ch_buf; - char *source = (yytext_ptr); - int number_to_move, i; - int ret_val; - - if ((yy_c_buf_p) > &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[(yy_n_chars) + 1]) - YY_FATAL_ERROR("fatal flex scanner internal error--end of buffer missed"); - - if (YY_CURRENT_BUFFER_LVALUE->yy_fill_buffer == 0) - { /* Don't try to fill the buffer, so this is an EOF. */ - if ((yy_c_buf_p) - (yytext_ptr)-YY_MORE_ADJ == 1) - { - /* We matched a single character, the EOB, so - * treat this as a final EOF. - */ - return EOB_ACT_END_OF_FILE; - } - - else - { - /* We matched some text prior to the EOB, first - * process it. - */ - return EOB_ACT_LAST_MATCH; - } - } - - /* Try to read more data. */ - - /* First move last chars to start of buffer. */ - number_to_move = (int)((yy_c_buf_p) - (yytext_ptr)-1); - - for (i = 0; i < number_to_move; ++i) - *(dest++) = *(source++); - - if (YY_CURRENT_BUFFER_LVALUE->yy_buffer_status == YY_BUFFER_EOF_PENDING) - /* don't do the read, it's not guaranteed to return an EOF, - * just force an EOF - */ - YY_CURRENT_BUFFER_LVALUE->yy_n_chars = (yy_n_chars) = 0; - - else - { - yy_size_t num_to_read = YY_CURRENT_BUFFER_LVALUE->yy_buf_size - number_to_move - 1; - - while (num_to_read <= 0) - { /* Not enough room in the buffer - grow it. */ - - /* just a shorter name for the current buffer */ - YY_BUFFER_STATE b = YY_CURRENT_BUFFER_LVALUE; - - int yy_c_buf_p_offset = (int)((yy_c_buf_p)-b->yy_ch_buf); - - if (b->yy_is_our_buffer) - { - yy_size_t new_size = b->yy_buf_size * 2; - - if (new_size <= 0) - b->yy_buf_size += b->yy_buf_size / 8; - else - b->yy_buf_size *= 2; - - b->yy_ch_buf = (char *) - /* Include room in for 2 EOB chars. */ - yyrealloc((void *)b->yy_ch_buf, (yy_size_t)(b->yy_buf_size + 2)); - } - else - /* Can't grow it, we don't own it. */ - b->yy_ch_buf = NULL; - - if (!b->yy_ch_buf) - YY_FATAL_ERROR("fatal error - scanner input buffer overflow"); - - (yy_c_buf_p) = &b->yy_ch_buf[yy_c_buf_p_offset]; - - num_to_read = YY_CURRENT_BUFFER_LVALUE->yy_buf_size - number_to_move - 1; - } - - if (num_to_read > YY_READ_BUF_SIZE) - num_to_read = YY_READ_BUF_SIZE; - - /* Read in more data. */ - YY_INPUT((&YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[number_to_move]), (yy_n_chars), num_to_read); - - YY_CURRENT_BUFFER_LVALUE->yy_n_chars = (yy_n_chars); - } - - if ((yy_n_chars) == 0) - { - if (number_to_move == YY_MORE_ADJ) - { - ret_val = EOB_ACT_END_OF_FILE; - yyrestart(yyin); - } - - else - { - ret_val = EOB_ACT_LAST_MATCH; - YY_CURRENT_BUFFER_LVALUE->yy_buffer_status = YY_BUFFER_EOF_PENDING; - } - } - - else - ret_val = EOB_ACT_CONTINUE_SCAN; - - if (((yy_n_chars) + number_to_move) > YY_CURRENT_BUFFER_LVALUE->yy_buf_size) - { - /* Extend the array by 50%, plus the number we really need. */ - yy_size_t new_size = (yy_n_chars) + number_to_move + ((yy_n_chars) >> 1); - YY_CURRENT_BUFFER_LVALUE->yy_ch_buf = - (char *)yyrealloc((void *)YY_CURRENT_BUFFER_LVALUE->yy_ch_buf, (yy_size_t)new_size); - if (!YY_CURRENT_BUFFER_LVALUE->yy_ch_buf) - YY_FATAL_ERROR("out of dynamic memory in yy_get_next_buffer()"); - /* "- 2" to take care of EOB's */ - YY_CURRENT_BUFFER_LVALUE->yy_buf_size = (int)(new_size - 2); - } - - (yy_n_chars) += number_to_move; - YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[(yy_n_chars)] = YY_END_OF_BUFFER_CHAR; - YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[(yy_n_chars) + 1] = YY_END_OF_BUFFER_CHAR; - - (yytext_ptr) = &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[0]; - - return ret_val; -} - -/* yy_get_previous_state - get the state just before the EOB char was reached */ - -static yy_state_type yy_get_previous_state(void) -{ - yy_state_type yy_current_state; - char *yy_cp; - - yy_current_state = (yy_start); - - for (yy_cp = (yytext_ptr) + YY_MORE_ADJ; yy_cp < (yy_c_buf_p); ++yy_cp) - { - YY_CHAR yy_c = (*yy_cp ? yy_ec[YY_SC_TO_UI(*yy_cp)] : 41); - if (yy_accept[yy_current_state]) - { - (yy_last_accepting_state) = yy_current_state; - (yy_last_accepting_cpos) = yy_cp; - } - while (yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state) - { - yy_current_state = (int)yy_def[yy_current_state]; - if (yy_current_state >= 91) - yy_c = yy_meta[yy_c]; - } - yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c]; - } - - return yy_current_state; -} - -/* yy_try_NUL_trans - try to make a transition on the NUL character - * - * synopsis - * next_state = yy_try_NUL_trans( current_state ); - */ -static yy_state_type yy_try_NUL_trans(yy_state_type yy_current_state) -{ - int yy_is_jam; - char *yy_cp = (yy_c_buf_p); - - YY_CHAR yy_c = 41; - if (yy_accept[yy_current_state]) - { - (yy_last_accepting_state) = yy_current_state; - (yy_last_accepting_cpos) = yy_cp; - } - while (yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state) - { - yy_current_state = (int)yy_def[yy_current_state]; - if (yy_current_state >= 91) - yy_c = yy_meta[yy_c]; - } - yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c]; - yy_is_jam = (yy_current_state == 90); - - return yy_is_jam ? 0 : yy_current_state; -} - -#ifndef YY_NO_UNPUT - -static void yyunput(int c, char *yy_bp) -{ - char *yy_cp; - - yy_cp = (yy_c_buf_p); - - /* undo effects of setting up yytext */ - *yy_cp = (yy_hold_char); - - if (yy_cp < YY_CURRENT_BUFFER_LVALUE->yy_ch_buf + 2) - { /* need to shift things up to make room */ - /* +2 for EOB chars. */ - yy_size_t number_to_move = (yy_n_chars) + 2; - char *dest = - &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[YY_CURRENT_BUFFER_LVALUE->yy_buf_size + 2]; - char *source = &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[number_to_move]; - - while (source > YY_CURRENT_BUFFER_LVALUE->yy_ch_buf) - *--dest = *--source; - - yy_cp += (int)(dest - source); - yy_bp += (int)(dest - source); - YY_CURRENT_BUFFER_LVALUE->yy_n_chars = (yy_n_chars) = - (int)YY_CURRENT_BUFFER_LVALUE->yy_buf_size; - - if (yy_cp < YY_CURRENT_BUFFER_LVALUE->yy_ch_buf + 2) - YY_FATAL_ERROR("flex scanner push-back overflow"); - } - - *--yy_cp = (char)c; - - (yytext_ptr) = yy_bp; - (yy_hold_char) = *yy_cp; - (yy_c_buf_p) = yy_cp; -} - -#endif - -#ifndef YY_NO_INPUT -#ifdef __cplusplus -static int yyinput(void) -#else -static int input(void) -#endif - -{ - int c; - - *(yy_c_buf_p) = (yy_hold_char); - - if (*(yy_c_buf_p) == YY_END_OF_BUFFER_CHAR) - { - /* yy_c_buf_p now points to the character we want to return. - * If this occurs *before* the EOB characters, then it's a - * valid NUL; if not, then we've hit the end of the buffer. - */ - if ((yy_c_buf_p) < &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[(yy_n_chars)]) - /* This was really a NUL. */ - *(yy_c_buf_p) = '\0'; - - else - { /* need more input */ - yy_size_t offset = (yy_c_buf_p) - (yytext_ptr); - ++(yy_c_buf_p); - - switch (yy_get_next_buffer()) - { - case EOB_ACT_LAST_MATCH: - /* This happens because yy_g_n_b() - * sees that we've accumulated a - * token and flags that we need to - * try matching the token before - * proceeding. But for input(), - * there's no matching to consider. - * So convert the EOB_ACT_LAST_MATCH - * to EOB_ACT_END_OF_FILE. - */ - - /* Reset buffer status. */ - yyrestart(yyin); - - /*FALLTHROUGH*/ - - case EOB_ACT_END_OF_FILE: { - if (yywrap()) - return 0; - - if (!(yy_did_buffer_switch_on_eof)) - YY_NEW_FILE; -#ifdef __cplusplus - return yyinput(); -#else - return input(); -#endif - } - - case EOB_ACT_CONTINUE_SCAN: - (yy_c_buf_p) = (yytext_ptr) + offset; - break; - } - } - } - - c = *(unsigned char *)(yy_c_buf_p); /* cast for 8-bit char's */ - *(yy_c_buf_p) = '\0'; /* preserve yytext */ - (yy_hold_char) = *++(yy_c_buf_p); - - return c; -} -#endif /* ifndef YY_NO_INPUT */ - -/** Immediately switch to a different input stream. - * @param input_file A readable stream. - * - * @note This function does not reset the start condition to @c INITIAL . - */ -void yyrestart(FILE *input_file) -{ - - if (!YY_CURRENT_BUFFER) - { - yyensure_buffer_stack(); - YY_CURRENT_BUFFER_LVALUE = yy_create_buffer(yyin, YY_BUF_SIZE); - } - - yy_init_buffer(YY_CURRENT_BUFFER, input_file); - yy_load_buffer_state(); -} - -/** Switch to a different input buffer. - * @param new_buffer The new input buffer. - * - */ -void yy_switch_to_buffer(YY_BUFFER_STATE new_buffer) -{ - - /* TODO. We should be able to replace this entire function body - * with - * yypop_buffer_state(); - * yypush_buffer_state(new_buffer); - */ - yyensure_buffer_stack(); - if (YY_CURRENT_BUFFER == new_buffer) - return; - - if (YY_CURRENT_BUFFER) - { - /* Flush out information for old buffer. */ - *(yy_c_buf_p) = (yy_hold_char); - YY_CURRENT_BUFFER_LVALUE->yy_buf_pos = (yy_c_buf_p); - YY_CURRENT_BUFFER_LVALUE->yy_n_chars = (yy_n_chars); - } - - YY_CURRENT_BUFFER_LVALUE = new_buffer; - yy_load_buffer_state(); - - /* We don't actually know whether we did this switch during - * EOF (yywrap()) processing, but the only time this flag - * is looked at is after yywrap() is called, so it's safe - * to go ahead and always set it. - */ - (yy_did_buffer_switch_on_eof) = 1; -} - -static void yy_load_buffer_state(void) -{ - (yy_n_chars) = YY_CURRENT_BUFFER_LVALUE->yy_n_chars; - (yytext_ptr) = (yy_c_buf_p) = YY_CURRENT_BUFFER_LVALUE->yy_buf_pos; - yyin = YY_CURRENT_BUFFER_LVALUE->yy_input_file; - (yy_hold_char) = *(yy_c_buf_p); -} - -/** Allocate and initialize an input buffer state. - * @param file A readable stream. - * @param size The character buffer size in bytes. When in doubt, use @c YY_BUF_SIZE. - * - * @return the allocated buffer state. - */ -YY_BUFFER_STATE yy_create_buffer(FILE *file, int size) -{ - YY_BUFFER_STATE b; - - b = (YY_BUFFER_STATE)yyalloc(sizeof(struct yy_buffer_state)); - if (!b) - YY_FATAL_ERROR("out of dynamic memory in yy_create_buffer()"); - - b->yy_buf_size = size; - - /* yy_ch_buf has to be 2 characters longer than the size given because - * we need to put in 2 end-of-buffer characters. - */ - b->yy_ch_buf = (char *)yyalloc((yy_size_t)(b->yy_buf_size + 2)); - if (!b->yy_ch_buf) - YY_FATAL_ERROR("out of dynamic memory in yy_create_buffer()"); - - b->yy_is_our_buffer = 1; - - yy_init_buffer(b, file); - - return b; -} - -/** Destroy the buffer. - * @param b a buffer created with yy_create_buffer() - * - */ -void yy_delete_buffer(YY_BUFFER_STATE b) -{ - - if (!b) - return; - - if (b == YY_CURRENT_BUFFER) /* Not sure if we should pop here. */ - YY_CURRENT_BUFFER_LVALUE = (YY_BUFFER_STATE)0; - - if (b->yy_is_our_buffer) - yyfree((void *)b->yy_ch_buf); - - yyfree((void *)b); -} - -/* Initializes or reinitializes a buffer. - * This function is sometimes called more than once on the same buffer, - * such as during a yyrestart() or at EOF. - */ -static void yy_init_buffer(YY_BUFFER_STATE b, FILE *file) - -{ - int oerrno = errno; - - yy_flush_buffer(b); - - b->yy_input_file = file; - b->yy_fill_buffer = 1; - - /* If b is the current buffer, then yy_init_buffer was _probably_ - * called from yyrestart() or through yy_get_next_buffer. - * In that case, we don't want to reset the lineno or column. - */ - if (b != YY_CURRENT_BUFFER) - { - b->yy_bs_lineno = 1; - b->yy_bs_column = 0; - } - - b->yy_is_interactive = file ? (isatty(fileno(file)) > 0) : 0; - - errno = oerrno; -} - -/** Discard all buffered characters. On the next scan, YY_INPUT will be called. - * @param b the buffer state to be flushed, usually @c YY_CURRENT_BUFFER. - * - */ -void yy_flush_buffer(YY_BUFFER_STATE b) -{ - if (!b) - return; - - b->yy_n_chars = 0; - - /* We always need two end-of-buffer characters. The first causes - * a transition to the end-of-buffer state. The second causes - * a jam in that state. - */ - b->yy_ch_buf[0] = YY_END_OF_BUFFER_CHAR; - b->yy_ch_buf[1] = YY_END_OF_BUFFER_CHAR; - - b->yy_buf_pos = &b->yy_ch_buf[0]; - - b->yy_at_bol = 1; - b->yy_buffer_status = YY_BUFFER_NEW; - - if (b == YY_CURRENT_BUFFER) - yy_load_buffer_state(); -} - -/** Pushes the new state onto the stack. The new state becomes - * the current state. This function will allocate the stack - * if necessary. - * @param new_buffer The new state. - * - */ -void yypush_buffer_state(YY_BUFFER_STATE new_buffer) -{ - if (new_buffer == NULL) - return; - - yyensure_buffer_stack(); - - /* This block is copied from yy_switch_to_buffer. */ - if (YY_CURRENT_BUFFER) - { - /* Flush out information for old buffer. */ - *(yy_c_buf_p) = (yy_hold_char); - YY_CURRENT_BUFFER_LVALUE->yy_buf_pos = (yy_c_buf_p); - YY_CURRENT_BUFFER_LVALUE->yy_n_chars = (yy_n_chars); - } - - /* Only push if top exists. Otherwise, replace top. */ - if (YY_CURRENT_BUFFER) - (yy_buffer_stack_top)++; - YY_CURRENT_BUFFER_LVALUE = new_buffer; - - /* copied from yy_switch_to_buffer. */ - yy_load_buffer_state(); - (yy_did_buffer_switch_on_eof) = 1; -} - -/** Removes and deletes the top of the stack, if present. - * The next element becomes the new top. - * - */ -void yypop_buffer_state(void) -{ - if (!YY_CURRENT_BUFFER) - return; - - yy_delete_buffer(YY_CURRENT_BUFFER); - YY_CURRENT_BUFFER_LVALUE = NULL; - if ((yy_buffer_stack_top) > 0) - --(yy_buffer_stack_top); - - if (YY_CURRENT_BUFFER) - { - yy_load_buffer_state(); - (yy_did_buffer_switch_on_eof) = 1; - } -} - -/* Allocates the stack if it does not exist. - * Guarantees space for at least one push. - */ -static void yyensure_buffer_stack(void) -{ - yy_size_t num_to_alloc; - - if (!(yy_buffer_stack)) - { - - /* First allocation is just for 2 elements, since we don't know if this - * scanner will even need a stack. We use 2 instead of 1 to avoid an - * immediate realloc on the next call. - */ - num_to_alloc = 1; /* After all that talk, this was set to 1 anyways... */ - (yy_buffer_stack) = - (struct yy_buffer_state **)yyalloc(num_to_alloc * sizeof(struct yy_buffer_state *)); - if (!(yy_buffer_stack)) - YY_FATAL_ERROR("out of dynamic memory in yyensure_buffer_stack()"); - - memset((yy_buffer_stack), 0, num_to_alloc * sizeof(struct yy_buffer_state *)); - - (yy_buffer_stack_max) = num_to_alloc; - (yy_buffer_stack_top) = 0; - return; - } - - if ((yy_buffer_stack_top) >= ((yy_buffer_stack_max)) - 1) - { - - /* Increase the buffer to prepare for a possible push. */ - yy_size_t grow_size = 8 /* arbitrary grow size */; - - num_to_alloc = (yy_buffer_stack_max) + grow_size; - (yy_buffer_stack) = (struct yy_buffer_state **)yyrealloc( - (yy_buffer_stack), num_to_alloc * sizeof(struct yy_buffer_state *)); - if (!(yy_buffer_stack)) - YY_FATAL_ERROR("out of dynamic memory in yyensure_buffer_stack()"); - - /* zero only the new slots.*/ - memset((yy_buffer_stack) + (yy_buffer_stack_max), 0, - grow_size * sizeof(struct yy_buffer_state *)); - (yy_buffer_stack_max) = num_to_alloc; - } -} - -/** Setup the input buffer state to scan directly from a user-specified character buffer. - * @param base the character buffer - * @param size the size in bytes of the character buffer - * - * @return the newly allocated buffer state object. - */ -YY_BUFFER_STATE yy_scan_buffer(char *base, yy_size_t size) -{ - YY_BUFFER_STATE b; - - if (size < 2 || base[size - 2] != YY_END_OF_BUFFER_CHAR || - base[size - 1] != YY_END_OF_BUFFER_CHAR) - /* They forgot to leave room for the EOB's. */ - return NULL; - - b = (YY_BUFFER_STATE)yyalloc(sizeof(struct yy_buffer_state)); - if (!b) - YY_FATAL_ERROR("out of dynamic memory in yy_scan_buffer()"); - - b->yy_buf_size = (int)(size - 2); /* "- 2" to take care of EOB's */ - b->yy_buf_pos = b->yy_ch_buf = base; - b->yy_is_our_buffer = 0; - b->yy_input_file = NULL; - b->yy_n_chars = b->yy_buf_size; - b->yy_is_interactive = 0; - b->yy_at_bol = 1; - b->yy_fill_buffer = 0; - b->yy_buffer_status = YY_BUFFER_NEW; - - yy_switch_to_buffer(b); - - return b; -} - -/** Setup the input buffer state to scan a string. The next call to yylex() will - * scan from a @e copy of @a str. - * @param yystr a NUL-terminated string to scan - * - * @return the newly allocated buffer state object. - * @note If you want to scan bytes that may contain NUL values, then use - * yy_scan_bytes() instead. - */ -YY_BUFFER_STATE yy_scan_string(const char *yystr) -{ - - return yy_scan_bytes(yystr, (int)strlen(yystr)); -} - -/** Setup the input buffer state to scan the given bytes. The next call to yylex() will - * scan from a @e copy of @a bytes. - * @param yybytes the byte buffer to scan - * @param _yybytes_len the number of bytes in the buffer pointed to by @a bytes. - * - * @return the newly allocated buffer state object. - */ -YY_BUFFER_STATE yy_scan_bytes(const char *yybytes, yy_size_t _yybytes_len) -{ - YY_BUFFER_STATE b; - char *buf; - yy_size_t n; - yy_size_t i; - - /* Get memory for full buffer, including space for trailing EOB's. */ - n = (yy_size_t)(_yybytes_len + 2); - buf = (char *)yyalloc(n); - if (!buf) - YY_FATAL_ERROR("out of dynamic memory in yy_scan_bytes()"); - - for (i = 0; i < _yybytes_len; ++i) - buf[i] = yybytes[i]; - - buf[_yybytes_len] = buf[_yybytes_len + 1] = YY_END_OF_BUFFER_CHAR; - - b = yy_scan_buffer(buf, n); - if (!b) - YY_FATAL_ERROR("bad buffer in yy_scan_bytes()"); - - /* It's okay to grow etc. this buffer, and we should throw it - * away when we're done. - */ - b->yy_is_our_buffer = 1; - - return b; -} - -#ifndef YY_EXIT_FAILURE -#define YY_EXIT_FAILURE 2 -#endif - -static void yynoreturn yy_fatal_error(const char *msg) -{ - fprintf(stderr, "%s\n", msg); - exit(YY_EXIT_FAILURE); -} - -/* Redefine yyless() so it works in section 3 code. */ - -#undef yyless -#define yyless(n) \ - do \ - { \ - /* Undo effects of setting up yytext. */ \ - yy_size_t yyless_macro_arg = (n); \ - YY_LESS_LINENO(yyless_macro_arg); \ - yytext[yyleng] = (yy_hold_char); \ - (yy_c_buf_p) = yytext + yyless_macro_arg; \ - (yy_hold_char) = *(yy_c_buf_p); \ - *(yy_c_buf_p) = '\0'; \ - yyleng = yyless_macro_arg; \ - } while (0) - -/* Accessor methods (get/set functions) to struct members. */ - -/** Get the current line number. - * - */ -int yyget_lineno(void) { return yylineno; } - -/** Get the input stream. - * - */ -FILE *yyget_in(void) { return yyin; } - -/** Get the output stream. - * - */ -FILE *yyget_out(void) { return yyout; } - -/** Get the length of the current token. - * - */ -yy_size_t yyget_leng(void) { return yyleng; } - -/** Get the current token. - * - */ - -char *yyget_text(void) { return yytext; } - -/** Set the current line number. - * @param _line_number line number - * - */ -void yyset_lineno(int _line_number) { yylineno = _line_number; } - -/** Set the input stream. This does not discard the current - * input buffer. - * @param _in_str A readable stream. - * - * @see yy_switch_to_buffer - */ -void yyset_in(FILE *_in_str) { yyin = _in_str; } - -void yyset_out(FILE *_out_str) { yyout = _out_str; } - -int yyget_debug(void) { return yy_flex_debug; } - -void yyset_debug(int _bdebug) { yy_flex_debug = _bdebug; } - -static int yy_init_globals(void) -{ - /* Initialization is the same as for the non-reentrant scanner. - * This function is called from yylex_destroy(), so don't allocate here. - */ - - (yy_buffer_stack) = NULL; - (yy_buffer_stack_top) = 0; - (yy_buffer_stack_max) = 0; - (yy_c_buf_p) = NULL; - (yy_init) = 0; - (yy_start) = 0; - -/* Defined in main.c */ -#ifdef YY_STDINIT - yyin = stdin; - yyout = stdout; -#else - yyin = NULL; - yyout = NULL; -#endif - - /* For future reference: Set errno on error, since we are called by - * yylex_init() - */ - return 0; -} - -/* yylex_destroy is for both reentrant and non-reentrant scanners. */ -int yylex_destroy(void) -{ - - /* Pop the buffer stack, destroying each element. */ - while (YY_CURRENT_BUFFER) - { - yy_delete_buffer(YY_CURRENT_BUFFER); - YY_CURRENT_BUFFER_LVALUE = NULL; - yypop_buffer_state(); - } - - /* Destroy the stack itself. */ - yyfree((yy_buffer_stack)); - (yy_buffer_stack) = NULL; - - /* Reset the globals. This is important in a non-reentrant scanner so the next time - * yylex() is called, initialization will occur. */ - yy_init_globals(); - - return 0; -} - -/* - * Internal utility routines. - */ - -#ifndef yytext_ptr -static void yy_flex_strncpy(char *s1, const char *s2, int n) -{ - - int i; - for (i = 0; i < n; ++i) - s1[i] = s2[i]; -} -#endif - -#ifdef YY_NEED_STRLEN -static int yy_flex_strlen(const char *s) -{ - int n; - for (n = 0; s[n]; ++n) - ; - - return n; -} -#endif - -void *yyalloc(yy_size_t size) { return malloc(size); } - -void *yyrealloc(void *ptr, yy_size_t size) -{ - - /* The cast to (char *) in the following accommodates both - * implementations that use char* generic pointers, and those - * that use void* generic pointers. It works with the latter - * because both ANSI C and C++ allow castless assignment from - * any pointer type to void*, and deal with argument conversions - * as though doing an assignment. - */ - return realloc(ptr, size); -} - -void yyfree(void *ptr) { free((char *)ptr); /* see yyrealloc() for (char *) cast */ } - -#define YYTABLES_NAME "yytables" - -#line 47 "lexer.l" diff --git a/source/adios2/toolkit/derived/parser/lexer.l b/source/adios2/toolkit/derived/parser/lexer.l index f81a98bcf9..566e4a41c7 100644 --- a/source/adios2/toolkit/derived/parser/lexer.l +++ b/source/adios2/toolkit/derived/parser/lexer.l @@ -1,47 +1,155 @@ -%{ -#include "parser.hpp" -#define YY_USER_ACTION yylloc.first_line = yylloc.last_line = yylineno; +%{ +#include +#include +#include +#include // strerror +#include +#include "ASTDriver.h" +#include "parser.h" +#if defined(_MSC_VER) +#include +#define YY_NO_UNISTD_H +#define strdup _strdup +#define isatty _isatty +#define fileno _fileno +#include +#include +#endif %} -%option noyywrap +%{ +#if defined __clang__ +# define CLANG_VERSION (__clang_major__ * 100 + __clang_minor__) +#endif +// Clang and ICC like to pretend they are GCC. +#if defined __GNUC__ && !defined __clang__ && !defined __ICC +# define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) +#endif -DIGIT [0-9] -CHAR ([a-z]|[A-Z]) - -%% +// Pacify warnings in yy_init_buffer (observed with Flex 2.6.4) +// and GCC 6.4.0, 7.3.0 with -O3. +#if defined GCC_VERSION && 600 <= GCC_VERSION +# pragma GCC diagnostic ignored "-Wnull-dereference" +#endif + +// This example uses Flex's C back end, yet compiles it as C++. +// So expect warnings about C style casts and NULL. +#if defined CLANG_VERSION && 500 <= CLANG_VERSION +# pragma clang diagnostic ignored "-Wold-style-cast" +# pragma clang diagnostic ignored "-Wzero-as-null-pointer-constant" +#elif defined GCC_VERSION && 407 <= GCC_VERSION +# pragma GCC diagnostic ignored "-Wold-style-cast" +# pragma GCC diagnostic ignored "-Wzero-as-null-pointer-constant" +#endif + +#define FLEX_VERSION (YY_FLEX_MAJOR_VERSION * 100 + YY_FLEX_MINOR_VERSION) -"+" {yylval.sval=strdup(yytext); return OPERATOR;} -"-" {yylval.sval=strdup(yytext); return OPERATOR;} -"*" {yylval.sval=strdup(yytext); return OPERATOR;} -"/" {yylval.sval=strdup(yytext); return OPERATOR;} -"^" {yylval.sval=strdup(yytext); return OPERATOR;} -"(" {return L_PAREN;} -")" {return R_PAREN;} -"," {return COMMA;} -"add" {yylval.sval=strdup(yytext); return FUNCTION;} -"sqrt" {yylval.sval=strdup(yytext); return FUNCTION;} -"sin" {yylval.sval=strdup(yytext); return FUNCTION;} -"cos" {yylval.sval=strdup(yytext); return FUNCTION;} -"tan" {yylval.sval=strdup(yytext); return FUNCTION;} -"magnitude" {yylval.sval=strdup(yytext); return FUNCTION;} -"curl" {yylval.sval=strdup(yytext); return FUNCTION;} +// Old versions of Flex (2.5.35) generate an incomplete documentation comment. +// +// In file included from src/scan-code-c.c:3: +// src/scan-code.c:2198:21: error: empty paragraph passed to '@param' command +// [-Werror,-Wdocumentation] +// * @param line_number +// ~~~~~~~~~~~~~~~~~^ +// 1 error generated. +#if FLEX_VERSION < 206 && defined CLANG_VERSION +# pragma clang diagnostic ignored "-Wdocumentation" +#endif +// Old versions of Flex (2.5.35) use 'register'. Warnings introduced in +// GCC 7 and Clang 6. +#if FLEX_VERSION < 206 +# if defined CLANG_VERSION && 600 <= CLANG_VERSION +# pragma clang diagnostic ignored "-Wdeprecated-register" +# elif defined GCC_VERSION && 700 <= GCC_VERSION +# pragma GCC diagnostic ignored "-Wregister" +# endif +#endif -(\.{DIGIT}+)|({DIGIT}+(\.{DIGIT}*)?([eE][+-]?[0-9]+)?) {yylval.dval = atof(yytext); return NUMBER;} +#if FLEX_VERSION < 206 +# if defined CLANG_VERSION +# pragma clang diagnostic ignored "-Wconversion" +# pragma clang diagnostic ignored "-Wdocumentation" +# pragma clang diagnostic ignored "-Wshorten-64-to-32" +# pragma clang diagnostic ignored "-Wsign-conversion" +# elif defined GCC_VERSION +# pragma GCC diagnostic ignored "-Wconversion" +# pragma GCC diagnostic ignored "-Wsign-conversion" +# endif +#endif + +// Flex 2.6.4, GCC 9 +// warning: useless cast to type 'int' [-Wuseless-cast] +// 1361 | YY_CURRENT_BUFFER_LVALUE->yy_buf_size = (int) (new_size - 2); +// | ^ +#if defined GCC_VERSION && 900 <= GCC_VERSION +# pragma GCC diagnostic ignored "-Wuseless-cast" +#endif +%} -[ \t]+ {/* ignore spaces */} +%option noyywrap nounput noinput batch -(\n|\0|EOF|$end) {return ENDL;} +%{ + // A number symbol corresponding to the value in S. + adios2::detail::parser::symbol_type + make_INT (const std::string &s, const adios2::detail::parser::location_type& loc); +%} -({CHAR}|{DIGIT}|_)+ {yylval.sval=strdup(yytext); return ALIAS;} +op [-+*/^] +id [a-zA-Z][a-zA-Z_0-9_]* +path [a-zA-Z][a-zA-Z0-9_\\./-]* +int [0-9]+ +blank [ \t\r] -:(\\|\/|_|{DIGIT})*{CHAR}+(\\|\/|-|_|{DIGIT}|{CHAR})* {yylval.sval=strndup(yytext + 1,strlen(yytext)-1); return PATH;} +%{ + // Code run each time a pattern is matched. + # define YY_USER_ACTION loc.columns (yyleng); +%} +%% +%{ + // A handy shortcut to the location held by the adios2::detail::ASTDriver. + adios2::detail::location& loc = drv.location; + // Code run each time yylex is called. + loc.step (); +%} +{blank}+ loc.step (); +\n+ loc.lines (yyleng); loc.step (); -'(\\|\/|_|{DIGIT})*{CHAR}+(\\|\/|-|_|{DIGIT}|{CHAR})*' {yylval.sval=strndup(yytext + 1,strlen(yytext)-2); return PATH;} +"=" return adios2::detail::parser::make_ASSIGN (loc); +"," return adios2::detail::parser::make_COMMA (loc); +":" return adios2::detail::parser::make_COLON (loc); +"(" return adios2::detail::parser::make_L_PAREN (loc); +")" return adios2::detail::parser::make_R_PAREN (loc); +"[" return adios2::detail::parser::make_L_BRACE (loc); +"]" return adios2::detail::parser::make_R_BRACE (loc); -"["({DIGIT}+|{DIGIT}*":"{DIGIT}*":"{DIGIT}*)(,({DIGIT}+|{DIGIT}*":"{DIGIT}*":"{DIGIT}*))*"]" {yylval.sval=strndup(yytext + 1,strlen(yytext)-2); return INDICES;} +{int} return make_INT (yytext, loc); +{op} return adios2::detail::parser::make_OPERATOR (yytext, loc); +{id} return adios2::detail::parser::make_IDENTIFIER (yytext, loc); +{path} return adios2::detail::parser::make_VARNAME (yytext, loc); +. { + throw adios2::detail::parser::syntax_error + (loc, "invalid character: " + std::string(yytext)); +} +<> return adios2::detail::parser::make_YYEOF (loc); +%% -. {printf("Error at line %d: unrecognized symbol \"%s\"\n", yylloc.first_line, yytext); exit(0);} +adios2::detail::parser::symbol_type +make_INT (const std::string &s, const adios2::detail::parser::location_type& loc) +{ + errno = 0; + long n = strtol (s.c_str(), NULL, 10); + if (! (INT_MIN <= n && n <= INT_MAX && errno != ERANGE)) + throw adios2::detail::parser::syntax_error (loc, "integer is out of range: " + s); + return adios2::detail::parser::make_INT ((int) n, loc); +} -%% \ No newline at end of file +void +adios2::detail::ASTDriver::parse (const std::string input) +{ + adios2::detail::parser parse (*this); + yy_scan_string(input.c_str()); + parse.set_debug_level (trace_parsing); + parse (); +} diff --git a/source/adios2/toolkit/derived/parser/parser.cpp b/source/adios2/toolkit/derived/parser/parser.cpp deleted file mode 100644 index 03938fc8d6..0000000000 --- a/source/adios2/toolkit/derived/parser/parser.cpp +++ /dev/null @@ -1,1666 +0,0 @@ -/* A Bison parser, made by GNU Bison 2.3. */ - -/* Skeleton implementation for Bison's Yacc-like parsers in C - - Copyright (C) 1984, 1989, 1990, 2000, 2001, 2002, 2003, 2004, 2005, 2006 - Free Software Foundation, Inc. - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2, or (at your option) - any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, - Boston, MA 02110-1301, USA. */ - -/* As a special exception, you may create a larger work that contains - part or all of the Bison parser skeleton and distribute that work - under terms of your choice, so long as that work isn't itself a - parser generator using the skeleton or a modified version thereof - as a parser skeleton. Alternatively, if you modify or redistribute - the parser skeleton itself, you may (at your option) remove this - special exception, which will cause the skeleton and the resulting - Bison output files to be licensed under the GNU General Public - License without this special exception. - - This special exception was added by the Free Software Foundation in - version 2.2 of Bison. */ - -/* C LALR(1) parser skeleton written by Richard Stallman, by - simplifying the original so-called "semantic" parser. */ - -/* All symbols defined below should begin with yy or YY, to avoid - infringing on user name space. This should be done even for local - variables, as they might otherwise be expanded by user macros. - There are some unavoidable exceptions within include files to - define necessary library symbols; they are noted "INFRINGES ON - USER NAME SPACE" below. */ - -/* Identify Bison output. */ -#define YYBISON 1 - -/* Bison version. */ -#define YYBISON_VERSION "2.3" - -/* Skeleton name. */ -#define YYSKELETON_NAME "yacc.c" - -/* Pure parsers. */ -#define YYPURE 0 - -/* Using locations. */ -#define YYLSP_NEEDED 1 - -/* Tokens. */ -#ifndef YYTOKENTYPE -#define YYTOKENTYPE -/* Put the tokens into the symbol table, so that GDB and other debuggers - know about them. */ -enum yytokentype -{ - COMMA = 258, - L_PAREN = 259, - R_PAREN = 260, - ENDL = 261, - FUNCTION = 262, - OPERATOR = 263, - INDICES = 264, - NUMBER = 265, - ALIAS = 266, - PATH = 267 -}; -#endif -/* Tokens. */ -#define COMMA 258 -#define L_PAREN 259 -#define R_PAREN 260 -#define ENDL 261 -#define FUNCTION 262 -#define OPERATOR 263 -#define INDICES 264 -#define NUMBER 265 -#define ALIAS 266 -#define PATH 267 - -/* Copy the first part of user declarations. */ -#line 2 "parser.y" - -#include "parser.h" -#include "lexer.h" -#include -#include -#include -#include -#include -#include - -extern int yyparse(std::stack *expr_stack); - -static void *yyparse_value; - -void yyerror(std::stack *expr_stack, const char *msg); - -namespace adios2 -{ -namespace detail -{ -void *createExpr(std::stack *, std::string, const char *, double, size_t); -} -} - -/* Enabling traces. */ -#ifndef YYDEBUG -#define YYDEBUG 0 -#endif - -/* Enabling verbose error messages. */ -#ifdef YYERROR_VERBOSE -#undef YYERROR_VERBOSE -#define YYERROR_VERBOSE 1 -#else -#define YYERROR_VERBOSE 1 -#endif - -/* Enabling the token table. */ -#ifndef YYTOKEN_TABLE -#define YYTOKEN_TABLE 0 -#endif - -#if !defined YYSTYPE && !defined YYSTYPE_IS_DECLARED -typedef union YYSTYPE -#line 25 "parser.y" -{ - double dval; - int ival; - char *sval; - void *expr_ptr; -} -/* Line 193 of yacc.c. */ -#line 148 "parser.cpp" -YYSTYPE; -#define yystype YYSTYPE /* obsolescent; will be withdrawn */ -#define YYSTYPE_IS_DECLARED 1 -#define YYSTYPE_IS_TRIVIAL 1 -#endif - -#if !defined YYLTYPE && !defined YYLTYPE_IS_DECLARED -typedef struct YYLTYPE -{ - int first_line; - int first_column; - int last_line; - int last_column; -} YYLTYPE; -#define yyltype YYLTYPE /* obsolescent; will be withdrawn */ -#define YYLTYPE_IS_DECLARED 1 -#define YYLTYPE_IS_TRIVIAL 1 -#endif - -/* Copy the second part of user declarations. */ - -/* Line 216 of yacc.c. */ -#line 173 "parser.cpp" - -#ifdef short -#undef short -#endif - -#ifdef YYTYPE_UINT8 -typedef YYTYPE_UINT8 yytype_uint8; -#else -typedef unsigned char yytype_uint8; -#endif - -#ifdef YYTYPE_INT8 -typedef YYTYPE_INT8 yytype_int8; -#elif (defined __STDC__ || defined __C99__FUNC__ || defined __cplusplus || defined _MSC_VER) -typedef signed char yytype_int8; -#else -typedef short int yytype_int8; -#endif - -#ifdef YYTYPE_UINT16 -typedef YYTYPE_UINT16 yytype_uint16; -#else -typedef unsigned short int yytype_uint16; -#endif - -#ifdef YYTYPE_INT16 -typedef YYTYPE_INT16 yytype_int16; -#else -typedef short int yytype_int16; -#endif - -#ifndef YYSIZE_T -#ifdef __SIZE_TYPE__ -#define YYSIZE_T __SIZE_TYPE__ -#elif defined size_t -#define YYSIZE_T size_t -#elif !defined YYSIZE_T && \ - (defined __STDC__ || defined __C99__FUNC__ || defined __cplusplus || defined _MSC_VER) -#include /* INFRINGES ON USER NAME SPACE */ -#define YYSIZE_T size_t -#else -#define YYSIZE_T unsigned int -#endif -#endif - -#define YYSIZE_MAXIMUM ((YYSIZE_T)-1) - -#ifndef YY_ -#if defined YYENABLE_NLS && YYENABLE_NLS -#if ENABLE_NLS -#include /* INFRINGES ON USER NAME SPACE */ -#define YY_(msgid) dgettext("bison-runtime", msgid) -#endif -#endif -#ifndef YY_ -#define YY_(msgid) msgid -#endif -#endif - -/* Suppress unused-variable warnings by "using" E. */ -#if !defined lint || defined __GNUC__ -#define YYUSE(e) ((void)(e)) -#else -#define YYUSE(e) /* empty */ -#endif - -/* Identity function, used to suppress warnings about constant conditions. */ -#ifndef lint -#define YYID(n) (n) -#else -#if (defined __STDC__ || defined __C99__FUNC__ || defined __cplusplus || defined _MSC_VER) -static int YYID(int i) -#else -static int YYID(i) -int i; -#endif -{ - return i; -} -#endif - -#if !defined yyoverflow || YYERROR_VERBOSE - -/* The parser invokes alloca or malloc; define the necessary symbols. */ - -#ifdef YYSTACK_USE_ALLOCA -#if YYSTACK_USE_ALLOCA -#ifdef __GNUC__ -#define YYSTACK_ALLOC __builtin_alloca -#elif defined __BUILTIN_VA_ARG_INCR -#include /* INFRINGES ON USER NAME SPACE */ -#elif defined _AIX -#define YYSTACK_ALLOC __alloca -#elif defined _MSC_VER -#include /* INFRINGES ON USER NAME SPACE */ -#define alloca _alloca -#else -#define YYSTACK_ALLOC alloca -#if !defined _ALLOCA_H && !defined _STDLIB_H && \ - (defined __STDC__ || defined __C99__FUNC__ || defined __cplusplus || defined _MSC_VER) -#include /* INFRINGES ON USER NAME SPACE */ -#ifndef _STDLIB_H -#define _STDLIB_H 1 -#endif -#endif -#endif -#endif -#endif - -#ifdef YYSTACK_ALLOC -/* Pacify GCC's `empty if-body' warning. */ -#define YYSTACK_FREE(Ptr) \ - do \ - { /* empty */ \ - ; \ - } while (YYID(0)) -#ifndef YYSTACK_ALLOC_MAXIMUM -/* The OS might guarantee only one guard page at the bottom of the stack, - and a page size can be as small as 4096 bytes. So we cannot safely - invoke alloca (N) if N exceeds 4096. Use a slightly smaller number - to allow for a few compiler-allocated temporary stack slots. */ -#define YYSTACK_ALLOC_MAXIMUM 4032 /* reasonable circa 2006 */ -#endif -#else -#define YYSTACK_ALLOC YYMALLOC -#define YYSTACK_FREE YYFREE -#ifndef YYSTACK_ALLOC_MAXIMUM -#define YYSTACK_ALLOC_MAXIMUM YYSIZE_MAXIMUM -#endif -#if (defined __cplusplus && !defined _STDLIB_H && \ - !((defined YYMALLOC || defined malloc) && (defined YYFREE || defined free))) -#include /* INFRINGES ON USER NAME SPACE */ -#ifndef _STDLIB_H -#define _STDLIB_H 1 -#endif -#endif -#ifndef YYMALLOC -#define YYMALLOC malloc -#if !defined malloc && !defined _STDLIB_H && \ - (defined __STDC__ || defined __C99__FUNC__ || defined __cplusplus || defined _MSC_VER) -void *malloc(YYSIZE_T); /* INFRINGES ON USER NAME SPACE */ -#endif -#endif -#ifndef YYFREE -#define YYFREE free -#if !defined free && !defined _STDLIB_H && \ - (defined __STDC__ || defined __C99__FUNC__ || defined __cplusplus || defined _MSC_VER) -void free(void *); /* INFRINGES ON USER NAME SPACE */ -#endif -#endif -#endif -#endif /* ! defined yyoverflow || YYERROR_VERBOSE */ - -#if (!defined yyoverflow && \ - (!defined __cplusplus || (defined YYLTYPE_IS_TRIVIAL && YYLTYPE_IS_TRIVIAL && \ - defined YYSTYPE_IS_TRIVIAL && YYSTYPE_IS_TRIVIAL))) - -/* A type that is properly aligned for any stack member. */ -union yyalloc -{ - yytype_int16 yyss; - YYSTYPE yyvs; - YYLTYPE yyls; -}; - -/* The size of the maximum gap between one aligned stack and the next. */ -#define YYSTACK_GAP_MAXIMUM (sizeof(union yyalloc) - 1) - -/* The size of an array large to enough to hold all stacks, each with - N elements. */ -#define YYSTACK_BYTES(N) \ - ((N) * (sizeof(yytype_int16) + sizeof(YYSTYPE) + sizeof(YYLTYPE)) + 2 * YYSTACK_GAP_MAXIMUM) - -/* Copy COUNT objects from FROM to TO. The source and destination do - not overlap. */ -#ifndef YYCOPY -#if defined __GNUC__ && 1 < __GNUC__ -#define YYCOPY(To, From, Count) __builtin_memcpy(To, From, (Count) * sizeof(*(From))) -#else -#define YYCOPY(To, From, Count) \ - do \ - { \ - YYSIZE_T yyi; \ - for (yyi = 0; yyi < (Count); yyi++) \ - (To)[yyi] = (From)[yyi]; \ - } while (YYID(0)) -#endif -#endif - -/* Relocate STACK from its old location to the new one. The - local variables YYSIZE and YYSTACKSIZE give the old and new number of - elements in the stack, and YYPTR gives the new location of the - stack. Advance YYPTR to a properly aligned location for the next - stack. */ -#define YYSTACK_RELOCATE(Stack) \ - do \ - { \ - YYSIZE_T yynewbytes; \ - YYCOPY(&yyptr->Stack, Stack, yysize); \ - Stack = &yyptr->Stack; \ - yynewbytes = yystacksize * sizeof(*Stack) + YYSTACK_GAP_MAXIMUM; \ - yyptr += yynewbytes / sizeof(*yyptr); \ - } while (YYID(0)) - -#endif - -/* YYFINAL -- State number of the termination state. */ -#define YYFINAL 16 -/* YYLAST -- Last index in YYTABLE. */ -#define YYLAST 37 - -/* YYNTOKENS -- Number of terminals. */ -#define YYNTOKENS 13 -/* YYNNTS -- Number of nonterminals. */ -#define YYNNTS 5 -/* YYNRULES -- Number of rules. */ -#define YYNRULES 16 -/* YYNRULES -- Number of states. */ -#define YYNSTATES 28 - -/* YYTRANSLATE(YYLEX) -- Bison symbol number corresponding to YYLEX. */ -#define YYUNDEFTOK 2 -#define YYMAXUTOK 267 - -#define YYTRANSLATE(YYX) ((unsigned int)(YYX) <= YYMAXUTOK ? yytranslate[YYX] : YYUNDEFTOK) - -/* YYTRANSLATE[YYLEX] -- Bison symbol number corresponding to YYLEX. */ -static const yytype_uint8 yytranslate[] = { - 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12}; - -#if YYDEBUG -/* YYPRHS[YYN] -- Index of the first RHS symbol of rule number YYN in - YYRHS. */ -static const yytype_uint8 yyprhs[] = {0, 0, 3, 4, 7, 10, 13, 16, 20, - 24, 26, 28, 31, 33, 35, 39, 43}; - -/* YYRHS -- A `-1'-separated list of the rules' RHS. */ -static const yytype_int8 yyrhs[] = {14, 0, -1, -1, 6, 14, -1, 15, 14, -1, 17, 14, -1, 11, 12, -1, - 11, 12, 9, -1, 16, 3, 17, -1, 17, -1, 11, -1, 11, 9, -1, 12, - -1, 10, -1, 4, 17, 5, -1, 17, 8, 17, -1, 7, 4, 16, 5, -1}; - -/* YYRLINE[YYN] -- source line where rule number YYN was defined. */ -static const yytype_uint8 yyrline[] = {0, 51, 51, 52, 53, 54, 57, 58, 65, - 66, 69, 70, 71, 72, 73, 74, 75}; -#endif - -#if YYDEBUG || YYERROR_VERBOSE || YYTOKEN_TABLE -/* YYTNAME[SYMBOL-NUM] -- String name of the symbol SYMBOL-NUM. - First, the terminals, then, starting at YYNTOKENS, nonterminals. */ -static const char *const yytname[] = {"$end", "error", "$undefined", "COMMA", "L_PAREN", - "R_PAREN", "ENDL", "FUNCTION", "OPERATOR", "INDICES", - "NUMBER", "ALIAS", "PATH", "$accept", "input", - "decl", "list", "exp", 0}; -#endif - -#ifdef YYPRINT -/* YYTOKNUM[YYLEX-NUM] -- Internal token number corresponding to - token YYLEX-NUM. */ -static const yytype_uint16 yytoknum[] = {0, 256, 257, 258, 259, 260, 261, - 262, 263, 264, 265, 266, 267}; -#endif - -/* YYR1[YYN] -- Symbol number of symbol that rule YYN derives. */ -static const yytype_uint8 yyr1[] = {0, 13, 14, 14, 14, 14, 15, 15, 16, - 16, 17, 17, 17, 17, 17, 17, 17}; - -/* YYR2[YYN] -- Number of symbols composing right hand side of rule YYN. */ -static const yytype_uint8 yyr2[] = {0, 2, 0, 2, 2, 2, 2, 3, 3, 1, 1, 2, 1, 1, 3, 3, 4}; - -/* YYDEFACT[STATE-NAME] -- Default rule to reduce with in state - STATE-NUM when YYTABLE doesn't specify something else to do. Zero - means the default is an error. */ -static const yytype_uint8 yydefact[] = {2, 0, 2, 0, 13, 10, 12, 0, 2, 2, 10, 0, 3, 0, - 11, 6, 1, 4, 0, 5, 14, 0, 9, 7, 15, 0, 16, 8}; - -/* YYDEFGOTO[NTERM-NUM]. */ -static const yytype_int8 yydefgoto[] = {-1, 7, 8, 21, 9}; - -/* YYPACT[STATE-NUM] -- Index in YYTABLE of the portion describing - STATE-NUM. */ -#define YYPACT_NINF -4 -static const yytype_int8 yypact[] = {9, 18, 9, -2, -4, 2, -4, 6, 9, -3, 1, 26, -4, 18, - -4, 14, -4, -4, 18, -4, -4, 32, 10, -4, 10, 18, -4, 10}; - -/* YYPGOTO[NTERM-NUM]. */ -static const yytype_int8 yypgoto[] = {-4, 24, -4, -4, -1}; - -/* YYTABLE[YYPACT[STATE-NUM]]. What to do in state STATE-NUM. If - positive, shift that token. If negative, reduce the rule which - number is the opposite. If zero, do what YYDEFACT says. - If YYTABLE_NINF, syntax error. */ -#define YYTABLE_NINF -1 -static const yytype_uint8 yytable[] = {11, 1, 13, 2, 3, 18, 16, 4, 5, 6, 14, 14, 22, - 1, 15, 2, 3, 24, 18, 4, 5, 6, 1, 23, 27, 3, - 12, 0, 4, 10, 6, 20, 17, 19, 18, 25, 0, 26}; - -static const yytype_int8 yycheck[] = {1, 4, 4, 6, 7, 8, 0, 10, 11, 12, 9, 9, 13, - 4, 12, 6, 7, 18, 8, 10, 11, 12, 4, 9, 25, 7, - 2, -1, 10, 11, 12, 5, 8, 9, 8, 3, -1, 5}; - -/* YYSTOS[STATE-NUM] -- The (internal number of the) accessing - symbol of state STATE-NUM. */ -static const yytype_uint8 yystos[] = {0, 4, 6, 7, 10, 11, 12, 14, 15, 17, 11, 17, 14, 4, - 9, 12, 0, 14, 8, 14, 5, 16, 17, 9, 17, 3, 5, 17}; - -#define yyerrok (yyerrstatus = 0) -#define yyclearin (yychar = YYEMPTY) -#define YYEMPTY (-2) -#define YYEOF 0 - -#define YYACCEPT goto yyacceptlab -#define YYABORT goto yyabortlab -#define YYERROR goto yyerrorlab - -/* Like YYERROR except do call yyerror. This remains here temporarily - to ease the transition to the new meaning of YYERROR, for GCC. - Once GCC version 2 has supplanted version 1, this can go. */ - -#define YYFAIL goto yyerrlab - -#define YYRECOVERING() (!!yyerrstatus) - -#define YYBACKUP(Token, Value) \ - do \ - if (yychar == YYEMPTY && yylen == 1) \ - { \ - yychar = (Token); \ - yylval = (Value); \ - yytoken = YYTRANSLATE(yychar); \ - YYPOPSTACK(1); \ - goto yybackup; \ - } \ - else \ - { \ - yyerror(expr_stack, YY_("syntax error: cannot back up")); \ - YYERROR; \ - } \ - while (YYID(0)) - -#define YYTERROR 1 -#define YYERRCODE 256 - -/* YYLLOC_DEFAULT -- Set CURRENT to span from RHS[1] to RHS[N]. - If N is 0, then set CURRENT to the empty location which ends - the previous symbol: RHS[0] (always defined). */ - -#define YYRHSLOC(Rhs, K) ((Rhs)[K]) -#ifndef YYLLOC_DEFAULT -#define YYLLOC_DEFAULT(Current, Rhs, N) \ - do \ - if (YYID(N)) \ - { \ - (Current).first_line = YYRHSLOC(Rhs, 1).first_line; \ - (Current).first_column = YYRHSLOC(Rhs, 1).first_column; \ - (Current).last_line = YYRHSLOC(Rhs, N).last_line; \ - (Current).last_column = YYRHSLOC(Rhs, N).last_column; \ - } \ - else \ - { \ - (Current).first_line = (Current).last_line = YYRHSLOC(Rhs, 0).last_line; \ - (Current).first_column = (Current).last_column = YYRHSLOC(Rhs, 0).last_column; \ - } \ - while (YYID(0)) -#endif - -/* YY_LOCATION_PRINT -- Print the location on the stream. - This macro was not mandated originally: define only if we know - we won't break user code: when these are the locations we know. */ - -#ifndef YY_LOCATION_PRINT -#if defined YYLTYPE_IS_TRIVIAL && YYLTYPE_IS_TRIVIAL -#define YY_LOCATION_PRINT(File, Loc) \ - fprintf(File, "%d.%d-%d.%d", (Loc).first_line, (Loc).first_column, (Loc).last_line, \ - (Loc).last_column) -#else -#define YY_LOCATION_PRINT(File, Loc) ((void)0) -#endif -#endif - -/* YYLEX -- calling `yylex' with the right arguments. */ - -#ifdef YYLEX_PARAM -#define YYLEX yylex(YYLEX_PARAM) -#else -#define YYLEX yylex() -#endif - -/* Enable debugging if requested. */ -#if YYDEBUG - -#ifndef YYFPRINTF -#include /* INFRINGES ON USER NAME SPACE */ -#define YYFPRINTF fprintf -#endif - -#define YYDPRINTF(Args) \ - do \ - { \ - if (yydebug) \ - YYFPRINTF Args; \ - } while (YYID(0)) - -#define YY_SYMBOL_PRINT(Title, Type, Value, Location) \ - do \ - { \ - if (yydebug) \ - { \ - YYFPRINTF(stderr, "%s ", Title); \ - yy_symbol_print(stderr, Type, Value, Location, expr_stack); \ - YYFPRINTF(stderr, "\n"); \ - } \ - } while (YYID(0)) - -/*--------------------------------. -| Print this symbol on YYOUTPUT. | -`--------------------------------*/ - -/*ARGSUSED*/ -#if (defined __STDC__ || defined __C99__FUNC__ || defined __cplusplus || defined _MSC_VER) -static void yy_symbol_value_print(FILE *yyoutput, int yytype, YYSTYPE const *const yyvaluep, - YYLTYPE const *const yylocationp, - std::stack *expr_stack) -#else -static void yy_symbol_value_print(yyoutput, yytype, yyvaluep, yylocationp, - expr_stack) FILE *yyoutput; -int yytype; -YYSTYPE const *const yyvaluep; -YYLTYPE const *const yylocationp; -std::stack *expr_stack; -#endif -{ - if (!yyvaluep) - return; - YYUSE(yylocationp); - YYUSE(expr_stack); -#ifdef YYPRINT - if (yytype < YYNTOKENS) - YYPRINT(yyoutput, yytoknum[yytype], *yyvaluep); -#else - YYUSE(yyoutput); -#endif - switch (yytype) - { - default: - break; - } -} - -/*--------------------------------. -| Print this symbol on YYOUTPUT. | -`--------------------------------*/ - -#if (defined __STDC__ || defined __C99__FUNC__ || defined __cplusplus || defined _MSC_VER) -static void yy_symbol_print(FILE *yyoutput, int yytype, YYSTYPE const *const yyvaluep, - YYLTYPE const *const yylocationp, - std::stack *expr_stack) -#else -static void yy_symbol_print(yyoutput, yytype, yyvaluep, yylocationp, expr_stack) FILE *yyoutput; -int yytype; -YYSTYPE const *const yyvaluep; -YYLTYPE const *const yylocationp; -std::stack *expr_stack; -#endif -{ - if (yytype < YYNTOKENS) - YYFPRINTF(yyoutput, "token %s (", yytname[yytype]); - else - YYFPRINTF(yyoutput, "nterm %s (", yytname[yytype]); - - YY_LOCATION_PRINT(yyoutput, *yylocationp); - YYFPRINTF(yyoutput, ": "); - yy_symbol_value_print(yyoutput, yytype, yyvaluep, yylocationp, expr_stack); - YYFPRINTF(yyoutput, ")"); -} - -/*------------------------------------------------------------------. -| yy_stack_print -- Print the state stack from its BOTTOM up to its | -| TOP (included). | -`------------------------------------------------------------------*/ - -#if (defined __STDC__ || defined __C99__FUNC__ || defined __cplusplus || defined _MSC_VER) -static void yy_stack_print(yytype_int16 *bottom, yytype_int16 *top) -#else -static void yy_stack_print(bottom, top) yytype_int16 *bottom; -yytype_int16 *top; -#endif -{ - YYFPRINTF(stderr, "Stack now"); - for (; bottom <= top; ++bottom) - YYFPRINTF(stderr, " %d", *bottom); - YYFPRINTF(stderr, "\n"); -} - -#define YY_STACK_PRINT(Bottom, Top) \ - do \ - { \ - if (yydebug) \ - yy_stack_print((Bottom), (Top)); \ - } while (YYID(0)) - -/*------------------------------------------------. -| Report that the YYRULE is going to be reduced. | -`------------------------------------------------*/ - -#if (defined __STDC__ || defined __C99__FUNC__ || defined __cplusplus || defined _MSC_VER) -static void yy_reduce_print(YYSTYPE *yyvsp, YYLTYPE *yylsp, int yyrule, - std::stack *expr_stack) -#else -static void yy_reduce_print(yyvsp, yylsp, yyrule, expr_stack) YYSTYPE *yyvsp; -YYLTYPE *yylsp; -int yyrule; -std::stack *expr_stack; -#endif -{ - int yynrhs = yyr2[yyrule]; - int yyi; - unsigned long int yylno = yyrline[yyrule]; - YYFPRINTF(stderr, "Reducing stack by rule %d (line %lu):\n", yyrule - 1, yylno); - /* The symbols being reduced. */ - for (yyi = 0; yyi < yynrhs; yyi++) - { - fprintf(stderr, " $%d = ", yyi + 1); - yy_symbol_print(stderr, yyrhs[yyprhs[yyrule] + yyi], &(yyvsp[(yyi + 1) - (yynrhs)]), - &(yylsp[(yyi + 1) - (yynrhs)]), expr_stack); - fprintf(stderr, "\n"); - } -} - -#define YY_REDUCE_PRINT(Rule) \ - do \ - { \ - if (yydebug) \ - yy_reduce_print(yyvsp, yylsp, Rule, expr_stack); \ - } while (YYID(0)) - -/* Nonzero means print parse trace. It is left uninitialized so that - multiple parsers can coexist. */ -int yydebug; -#else /* !YYDEBUG */ -#define YYDPRINTF(Args) -#define YY_SYMBOL_PRINT(Title, Type, Value, Location) -#define YY_STACK_PRINT(Bottom, Top) -#define YY_REDUCE_PRINT(Rule) -#endif /* !YYDEBUG */ - -/* YYINITDEPTH -- initial size of the parser's stacks. */ -#ifndef YYINITDEPTH -#define YYINITDEPTH 200 -#endif - -/* YYMAXDEPTH -- maximum size the stacks can grow to (effective only - if the built-in stack extension method is used). - - Do not make this value too large; the results are undefined if - YYSTACK_ALLOC_MAXIMUM < YYSTACK_BYTES (YYMAXDEPTH) - evaluated with infinite-precision integer arithmetic. */ - -#ifndef YYMAXDEPTH -#define YYMAXDEPTH 10000 -#endif - -#if YYERROR_VERBOSE - -#ifndef yystrlen -#if defined __GLIBC__ && defined _STRING_H -#define yystrlen strlen -#else -/* Return the length of YYSTR. */ -#if (defined __STDC__ || defined __C99__FUNC__ || defined __cplusplus || defined _MSC_VER) -static YYSIZE_T yystrlen(const char *yystr) -#else -static YYSIZE_T yystrlen(yystr) const char *yystr; -#endif -{ - YYSIZE_T yylen; - for (yylen = 0; yystr[yylen]; yylen++) - continue; - return yylen; -} -#endif -#endif - -#ifndef yystpcpy -#if defined __GLIBC__ && defined _STRING_H && defined _GNU_SOURCE -#define yystpcpy stpcpy -#else -/* Copy YYSRC to YYDEST, returning the address of the terminating '\0' in - YYDEST. */ -#if (defined __STDC__ || defined __C99__FUNC__ || defined __cplusplus || defined _MSC_VER) -static char *yystpcpy(char *yydest, const char *yysrc) -#else -static char *yystpcpy(yydest, yysrc) -char *yydest; -const char *yysrc; -#endif -{ - char *yyd = yydest; - const char *yys = yysrc; - - while ((*yyd++ = *yys++) != '\0') - continue; - - return yyd - 1; -} -#endif -#endif - -#ifndef yytnamerr -/* Copy to YYRES the contents of YYSTR after stripping away unnecessary - quotes and backslashes, so that it's suitable for yyerror. The - heuristic is that double-quoting is unnecessary unless the string - contains an apostrophe, a comma, or backslash (other than - backslash-backslash). YYSTR is taken from yytname. If YYRES is - null, do not copy; instead, return the length of what the result - would have been. */ -static YYSIZE_T yytnamerr(char *yyres, const char *yystr) -{ - if (*yystr == '"') - { - YYSIZE_T yyn = 0; - char const *yyp = yystr; - - for (;;) - switch (*++yyp) - { - case '\'': - case ',': - goto do_not_strip_quotes; - - case '\\': - if (*++yyp != '\\') - goto do_not_strip_quotes; - /* Fall through. */ - default: - if (yyres) - yyres[yyn] = *yyp; - yyn++; - break; - - case '"': - if (yyres) - yyres[yyn] = '\0'; - return yyn; - } - do_not_strip_quotes:; - } - - if (!yyres) - return yystrlen(yystr); - - return yystpcpy(yyres, yystr) - yyres; -} -#endif - -/* Copy into YYRESULT an error message about the unexpected token - YYCHAR while in state YYSTATE. Return the number of bytes copied, - including the terminating null byte. If YYRESULT is null, do not - copy anything; just return the number of bytes that would be - copied. As a special case, return 0 if an ordinary "syntax error" - message will do. Return YYSIZE_MAXIMUM if overflow occurs during - size calculation. */ -static YYSIZE_T yysyntax_error(char *yyresult, int yystate, int yychar) -{ - int yyn = yypact[yystate]; - - if (!(YYPACT_NINF < yyn && yyn <= YYLAST)) - return 0; - else - { - int yytype = YYTRANSLATE(yychar); - YYSIZE_T yysize0 = yytnamerr(0, yytname[yytype]); - YYSIZE_T yysize = yysize0; - YYSIZE_T yysize1; - int yysize_overflow = 0; - enum - { - YYERROR_VERBOSE_ARGS_MAXIMUM = 5 - }; - char const *yyarg[YYERROR_VERBOSE_ARGS_MAXIMUM]; - int yyx; - -#if 0 - /* This is so xgettext sees the translatable formats that are - constructed on the fly. */ - YY_("syntax error, unexpected %s"); - YY_("syntax error, unexpected %s, expecting %s"); - YY_("syntax error, unexpected %s, expecting %s or %s"); - YY_("syntax error, unexpected %s, expecting %s or %s or %s"); - YY_("syntax error, unexpected %s, expecting %s or %s or %s or %s"); -#endif - char *yyfmt; - char const *yyf; - static char const yyunexpected[] = "syntax error, unexpected %s"; - static char const yyexpecting[] = ", expecting %s"; - static char const yyor[] = " or %s"; - char yyformat[sizeof yyunexpected + sizeof yyexpecting - 1 + - ((YYERROR_VERBOSE_ARGS_MAXIMUM - 2) * (sizeof yyor - 1))]; - char const *yyprefix = yyexpecting; - - /* Start YYX at -YYN if negative to avoid negative indexes in - YYCHECK. */ - int yyxbegin = yyn < 0 ? -yyn : 0; - - /* Stay within bounds of both yycheck and yytname. */ - int yychecklim = YYLAST - yyn + 1; - int yyxend = yychecklim < YYNTOKENS ? yychecklim : YYNTOKENS; - int yycount = 1; - - yyarg[0] = yytname[yytype]; - yyfmt = yystpcpy(yyformat, yyunexpected); - - for (yyx = yyxbegin; yyx < yyxend; ++yyx) - if (yycheck[yyx + yyn] == yyx && yyx != YYTERROR) - { - if (yycount == YYERROR_VERBOSE_ARGS_MAXIMUM) - { - yycount = 1; - yysize = yysize0; - yyformat[sizeof yyunexpected - 1] = '\0'; - break; - } - yyarg[yycount++] = yytname[yyx]; - yysize1 = yysize + yytnamerr(0, yytname[yyx]); - yysize_overflow |= (yysize1 < yysize); - yysize = yysize1; - yyfmt = yystpcpy(yyfmt, yyprefix); - yyprefix = yyor; - } - - yyf = YY_(yyformat); - yysize1 = yysize + yystrlen(yyf); - yysize_overflow |= (yysize1 < yysize); - yysize = yysize1; - - if (yysize_overflow) - return YYSIZE_MAXIMUM; - - if (yyresult) - { - /* Avoid sprintf, as that infringes on the user's name space. - Don't have undefined behavior even if the translation - produced a string with the wrong number of "%s"s. */ - char *yyp = yyresult; - int yyi = 0; - while ((*yyp = *yyf) != '\0') - { - if (*yyp == '%' && yyf[1] == 's' && yyi < yycount) - { - yyp += yytnamerr(yyp, yyarg[yyi++]); - yyf += 2; - } - else - { - yyp++; - yyf++; - } - } - } - return yysize; - } -} -#endif /* YYERROR_VERBOSE */ - -/*-----------------------------------------------. -| Release the memory associated to this symbol. | -`-----------------------------------------------*/ - -/*ARGSUSED*/ -#if (defined __STDC__ || defined __C99__FUNC__ || defined __cplusplus || defined _MSC_VER) -static void yydestruct(const char *yymsg, int yytype, YYSTYPE *yyvaluep, YYLTYPE *yylocationp, - std::stack *expr_stack) -#else -static void yydestruct(yymsg, yytype, yyvaluep, yylocationp, expr_stack) const char *yymsg; -int yytype; -YYSTYPE *yyvaluep; -YYLTYPE *yylocationp; -std::stack *expr_stack; -#endif -{ - YYUSE(yyvaluep); - YYUSE(yylocationp); - YYUSE(expr_stack); - - if (!yymsg) - yymsg = "Deleting"; - YY_SYMBOL_PRINT(yymsg, yytype, yyvaluep, yylocationp); - - switch (yytype) - { - - default: - break; - } -} - -/* Prevent warnings from -Wmissing-prototypes. */ - -#ifdef YYPARSE_PARAM -#if defined __STDC__ || defined __cplusplus -int yyparse(void *YYPARSE_PARAM); -#else -int yyparse(); -#endif -#else /* ! YYPARSE_PARAM */ -#if defined __STDC__ || defined __cplusplus -int yyparse(std::stack *expr_stack); -#else -int yyparse(); -#endif -#endif /* ! YYPARSE_PARAM */ - -/* The look-ahead symbol. */ -int yychar; - -/* The semantic value of the look-ahead symbol. */ -YYSTYPE yylval; - -/* Number of syntax errors so far. */ -int yynerrs; -/* Location data for the look-ahead symbol. */ -YYLTYPE yylloc; - -/*----------. -| yyparse. | -`----------*/ - -#ifdef YYPARSE_PARAM -#if (defined __STDC__ || defined __C99__FUNC__ || defined __cplusplus || defined _MSC_VER) -int yyparse(void *YYPARSE_PARAM) -#else -int yyparse(YYPARSE_PARAM) void *YYPARSE_PARAM; -#endif -#else /* ! YYPARSE_PARAM */ -#if (defined __STDC__ || defined __C99__FUNC__ || defined __cplusplus || defined _MSC_VER) -int yyparse(std::stack *expr_stack) -#else -int yyparse(expr_stack) std::stack *expr_stack; -#endif -#endif -{ - - int yystate; - int yyn; - int yyresult; - /* Number of tokens to shift before error messages enabled. */ - int yyerrstatus; - /* Look-ahead token as an internal (translated) token number. */ - int yytoken = 0; -#if YYERROR_VERBOSE - /* Buffer for error messages, and its allocated size. */ - char yymsgbuf[128]; - char *yymsg = yymsgbuf; - YYSIZE_T yymsg_alloc = sizeof yymsgbuf; -#endif - - /* Three stacks and their tools: - `yyss': related to states, - `yyvs': related to semantic values, - `yyls': related to locations. - - Refer to the stacks thru separate pointers, to allow yyoverflow - to reallocate them elsewhere. */ - - /* The state stack. */ - yytype_int16 yyssa[YYINITDEPTH]; - yytype_int16 *yyss = yyssa; - yytype_int16 *yyssp; - - /* The semantic value stack. */ - YYSTYPE yyvsa[YYINITDEPTH]; - YYSTYPE *yyvs = yyvsa; - YYSTYPE *yyvsp; - - /* The location stack. */ - YYLTYPE yylsa[YYINITDEPTH]; - YYLTYPE *yyls = yylsa; - YYLTYPE *yylsp; - /* The locations where the error started and ended. */ - YYLTYPE yyerror_range[2]; - -#define YYPOPSTACK(N) (yyvsp -= (N), yyssp -= (N), yylsp -= (N)) - - YYSIZE_T yystacksize = YYINITDEPTH; - - /* The variables used to return semantic value and location from the - action routines. */ - YYSTYPE yyval; - YYLTYPE yyloc; - - /* The number of symbols on the RHS of the reduced rule. - Keep to zero when no symbol should be popped. */ - int yylen = 0; - - YYDPRINTF((stderr, "Starting parse\n")); - - yystate = 0; - yyerrstatus = 0; - yynerrs = 0; - yychar = YYEMPTY; /* Cause a token to be read. */ - - /* Initialize stack pointers. - Waste one element of value and location stack - so that they stay on the same level as the state stack. - The wasted elements are never initialized. */ - - yyssp = yyss; - yyvsp = yyvs; - yylsp = yyls; -#if defined YYLTYPE_IS_TRIVIAL && YYLTYPE_IS_TRIVIAL - /* Initialize the default location before parsing starts. */ - yylloc.first_line = yylloc.last_line = 1; - yylloc.first_column = yylloc.last_column = 0; -#endif - - goto yysetstate; - - /*------------------------------------------------------------. - | yynewstate -- Push a new state, which is found in yystate. | - `------------------------------------------------------------*/ -yynewstate: - /* In all cases, when you get here, the value and location stacks - have just been pushed. So pushing a state here evens the stacks. */ - yyssp++; - -yysetstate: - *yyssp = yystate; - - if (yyss + yystacksize - 1 <= yyssp) - { - /* Get the current used size of the three stacks, in elements. */ - YYSIZE_T yysize = yyssp - yyss + 1; - -#ifdef yyoverflow - { - /* Give user a chance to reallocate the stack. Use copies of - these so that the &'s don't force the real ones into - memory. */ - YYSTYPE *yyvs1 = yyvs; - yytype_int16 *yyss1 = yyss; - YYLTYPE *yyls1 = yyls; - - /* Each stack pointer address is followed by the size of the - data in use in that stack, in bytes. This used to be a - conditional around just the two extra args, but that might - be undefined if yyoverflow is a macro. */ - yyoverflow(YY_("memory exhausted"), &yyss1, yysize * sizeof(*yyssp), &yyvs1, - yysize * sizeof(*yyvsp), &yyls1, yysize * sizeof(*yylsp), &yystacksize); - yyls = yyls1; - yyss = yyss1; - yyvs = yyvs1; - } -#else /* no yyoverflow */ -#ifndef YYSTACK_RELOCATE - goto yyexhaustedlab; -#else - /* Extend the stack our own way. */ - if (YYMAXDEPTH <= yystacksize) - goto yyexhaustedlab; - yystacksize *= 2; - if (YYMAXDEPTH < yystacksize) - yystacksize = YYMAXDEPTH; - - { - yytype_int16 *yyss1 = yyss; - union yyalloc *yyptr = (union yyalloc *)YYSTACK_ALLOC(YYSTACK_BYTES(yystacksize)); - if (!yyptr) - goto yyexhaustedlab; - YYSTACK_RELOCATE(yyss); - YYSTACK_RELOCATE(yyvs); - YYSTACK_RELOCATE(yyls); -#undef YYSTACK_RELOCATE - if (yyss1 != yyssa) - YYSTACK_FREE(yyss1); - } -#endif -#endif /* no yyoverflow */ - - yyssp = yyss + yysize - 1; - yyvsp = yyvs + yysize - 1; - yylsp = yyls + yysize - 1; - - YYDPRINTF((stderr, "Stack size increased to %lu\n", (unsigned long int)yystacksize)); - - if (yyss + yystacksize - 1 <= yyssp) - YYABORT; - } - - YYDPRINTF((stderr, "Entering state %d\n", yystate)); - - goto yybackup; - -/*-----------. -| yybackup. | -`-----------*/ -yybackup: - - /* Do appropriate processing given the current state. Read a - look-ahead token if we need one and don't already have one. */ - - /* First try to decide what to do without reference to look-ahead token. */ - yyn = yypact[yystate]; - if (yyn == YYPACT_NINF) - goto yydefault; - - /* Not known => get a look-ahead token if don't already have one. */ - - /* YYCHAR is either YYEMPTY or YYEOF or a valid look-ahead symbol. */ - if (yychar == YYEMPTY) - { - YYDPRINTF((stderr, "Reading a token: ")); - yychar = YYLEX; - } - - if (yychar <= YYEOF) - { - yychar = yytoken = YYEOF; - YYDPRINTF((stderr, "Now at end of input.\n")); - } - else - { - yytoken = YYTRANSLATE(yychar); - YY_SYMBOL_PRINT("Next token is", yytoken, &yylval, &yylloc); - } - - /* If the proper action on seeing token YYTOKEN is to reduce or to - detect an error, take that action. */ - yyn += yytoken; - if (yyn < 0 || YYLAST < yyn || yycheck[yyn] != yytoken) - goto yydefault; - yyn = yytable[yyn]; - if (yyn <= 0) - { - if (yyn == 0 || yyn == YYTABLE_NINF) - goto yyerrlab; - yyn = -yyn; - goto yyreduce; - } - - if (yyn == YYFINAL) - YYACCEPT; - - /* Count tokens shifted since error; after three, turn off error - status. */ - if (yyerrstatus) - yyerrstatus--; - - /* Shift the look-ahead token. */ - YY_SYMBOL_PRINT("Shifting", yytoken, &yylval, &yylloc); - - /* Discard the shifted token unless it is eof. */ - if (yychar != YYEOF) - yychar = YYEMPTY; - - yystate = yyn; - *++yyvsp = yylval; - *++yylsp = yylloc; - goto yynewstate; - -/*-----------------------------------------------------------. -| yydefault -- do the default action for the current state. | -`-----------------------------------------------------------*/ -yydefault: - yyn = yydefact[yystate]; - if (yyn == 0) - goto yyerrlab; - goto yyreduce; - -/*-----------------------------. -| yyreduce -- Do a reduction. | -`-----------------------------*/ -yyreduce: - /* yyn is the number of a rule to reduce with. */ - yylen = yyr2[yyn]; - - /* If YYLEN is nonzero, implement the default value of the action: - `$$ = $1'. - - Otherwise, the following line sets YYVAL to garbage. - This behavior is undocumented and Bison - users should not rely upon it. Assigning to YYVAL - unconditionally makes the parser a bit smaller, and it avoids a - GCC warning that YYVAL may be used uninitialized. */ - yyval = yyvsp[1 - yylen]; - - /* Default location. */ - YYLLOC_DEFAULT(yyloc, (yylsp - yylen), yylen); - YY_REDUCE_PRINT(yyn); - switch (yyn) - { - case 2: -#line 51 "parser.y" - { - ; - } - break; - - case 3: -#line 52 "parser.y" - { - ; - } - break; - - case 4: -#line 53 "parser.y" - { - ; - } - break; - - case 5: -#line 54 "parser.y" - { /*yyparse_value = $1->expression;*/ - ; - } - break; - - case 6: -#line 57 "parser.y" - { - adios2::detail::ASTNode::add_lookup_entry((yyvsp[(1) - (2)].sval), (yyvsp[(2) - (2)].sval), - ""); - ; - } - break; - - case 7: -#line 58 "parser.y" - { - adios2::detail::ASTNode::add_lookup_entry((yyvsp[(1) - (3)].sval), (yyvsp[(2) - (3)].sval), - (yyvsp[(3) - (3)].sval)); - ; - } - break; - - case 8: -#line 65 "parser.y" - { - (yyval.ival) = (yyvsp[(1) - (3)].ival) + 1; - ; - } - break; - - case 9: -#line 66 "parser.y" - { - (yyval.ival) = 1; - ; - } - break; - - case 10: -#line 69 "parser.y" - { - (yyval.expr_ptr) = createExpr(expr_stack, "ALIAS", (yyvsp[(1) - (1)].sval), 0, 0); - ; - } - break; - - case 11: -#line 70 "parser.y" - { - createExpr(expr_stack, "ALIAS", (yyvsp[(1) - (2)].sval), 0, 0); - (yyval.expr_ptr) = createExpr(expr_stack, "INDEX", (yyvsp[(2) - (2)].sval), 0, 1); - ; - } - break; - - case 12: -#line 71 "parser.y" - { - (yyval.expr_ptr) = createExpr(expr_stack, "PATH", (yyvsp[(1) - (1)].sval), 0, 0); - ; - } - break; - - case 13: -#line 72 "parser.y" - { - (yyval.expr_ptr) = createExpr(expr_stack, "NUM", "", (yyvsp[(1) - (1)].dval), 0); - ; - } - break; - - case 14: -#line 73 "parser.y" - { - (yyval.expr_ptr) = (yyvsp[(2) - (3)].expr_ptr); - ; - } - break; - - case 15: -#line 74 "parser.y" - { - (yyval.expr_ptr) = createExpr(expr_stack, (yyvsp[(2) - (3)].sval), "", 0, 2); - ; - } - break; - - case 16: -#line 75 "parser.y" - { - (yyval.expr_ptr) = - createExpr(expr_stack, (yyvsp[(1) - (4)].sval), "", 0, (yyvsp[(3) - (4)].ival)); - ; - } - break; - -/* Line 1267 of yacc.c. */ -#line 1480 "parser.cpp" - default: - break; - } - YY_SYMBOL_PRINT("-> $$ =", yyr1[yyn], &yyval, &yyloc); - - YYPOPSTACK(yylen); - yylen = 0; - YY_STACK_PRINT(yyss, yyssp); - - *++yyvsp = yyval; - *++yylsp = yyloc; - - /* Now `shift' the result of the reduction. Determine what state - that goes to, based on the state we popped back to and the rule - number reduced by. */ - - yyn = yyr1[yyn]; - - yystate = yypgoto[yyn - YYNTOKENS] + *yyssp; - if (0 <= yystate && yystate <= YYLAST && yycheck[yystate] == *yyssp) - yystate = yytable[yystate]; - else - yystate = yydefgoto[yyn - YYNTOKENS]; - - goto yynewstate; - -/*------------------------------------. -| yyerrlab -- here on detecting error | -`------------------------------------*/ -yyerrlab: - /* If not already recovering from an error, report this error. */ - if (!yyerrstatus) - { - ++yynerrs; -#if !YYERROR_VERBOSE - yyerror(expr_stack, YY_("syntax error")); -#else - { - YYSIZE_T yysize = yysyntax_error(0, yystate, yychar); - if (yymsg_alloc < yysize && yymsg_alloc < YYSTACK_ALLOC_MAXIMUM) - { - YYSIZE_T yyalloc = 2 * yysize; - if (!(yysize <= yyalloc && yyalloc <= YYSTACK_ALLOC_MAXIMUM)) - yyalloc = YYSTACK_ALLOC_MAXIMUM; - if (yymsg != yymsgbuf) - YYSTACK_FREE(yymsg); - yymsg = (char *)YYSTACK_ALLOC(yyalloc); - if (yymsg) - yymsg_alloc = yyalloc; - else - { - yymsg = yymsgbuf; - yymsg_alloc = sizeof yymsgbuf; - } - } - - if (0 < yysize && yysize <= yymsg_alloc) - { - (void)yysyntax_error(yymsg, yystate, yychar); - yyerror(expr_stack, yymsg); - } - else - { - yyerror(expr_stack, YY_("syntax error")); - if (yysize != 0) - goto yyexhaustedlab; - } - } -#endif - } - - yyerror_range[0] = yylloc; - - if (yyerrstatus == 3) - { - /* If just tried and failed to reuse look-ahead token after an - error, discard it. */ - - if (yychar <= YYEOF) - { - /* Return failure if at end of input. */ - if (yychar == YYEOF) - YYABORT; - } - else - { - yydestruct("Error: discarding", yytoken, &yylval, &yylloc, expr_stack); - yychar = YYEMPTY; - } - } - - /* Else will try to reuse look-ahead token after shifting the error - token. */ - goto yyerrlab1; - -/*---------------------------------------------------. -| yyerrorlab -- error raised explicitly by YYERROR. | -`---------------------------------------------------*/ -yyerrorlab: - - /* Pacify compilers like GCC when the user code never invokes - YYERROR and the label yyerrorlab therefore never appears in user - code. */ - if (/*CONSTCOND*/ 0) - goto yyerrorlab; - - yyerror_range[0] = yylsp[1 - yylen]; - /* Do not reclaim the symbols of the rule which action triggered - this YYERROR. */ - YYPOPSTACK(yylen); - yylen = 0; - YY_STACK_PRINT(yyss, yyssp); - yystate = *yyssp; - goto yyerrlab1; - -/*-------------------------------------------------------------. -| yyerrlab1 -- common code for both syntax error and YYERROR. | -`-------------------------------------------------------------*/ -yyerrlab1: - yyerrstatus = 3; /* Each real token shifted decrements this. */ - - for (;;) - { - yyn = yypact[yystate]; - if (yyn != YYPACT_NINF) - { - yyn += YYTERROR; - if (0 <= yyn && yyn <= YYLAST && yycheck[yyn] == YYTERROR) - { - yyn = yytable[yyn]; - if (0 < yyn) - break; - } - } - - /* Pop the current state because it cannot handle the error token. */ - if (yyssp == yyss) - YYABORT; - - yyerror_range[0] = *yylsp; - yydestruct("Error: popping", yystos[yystate], yyvsp, yylsp, expr_stack); - YYPOPSTACK(1); - yystate = *yyssp; - YY_STACK_PRINT(yyss, yyssp); - } - - if (yyn == YYFINAL) - YYACCEPT; - - *++yyvsp = yylval; - - yyerror_range[1] = yylloc; - /* Using YYLLOC is tempting, but would change the location of - the look-ahead. YYLOC is available though. */ - YYLLOC_DEFAULT(yyloc, (yyerror_range - 1), 2); - *++yylsp = yyloc; - - /* Shift the error token. */ - YY_SYMBOL_PRINT("Shifting", yystos[yyn], yyvsp, yylsp); - - yystate = yyn; - goto yynewstate; - -/*-------------------------------------. -| yyacceptlab -- YYACCEPT comes here. | -`-------------------------------------*/ -yyacceptlab: - yyresult = 0; - goto yyreturn; - -/*-----------------------------------. -| yyabortlab -- YYABORT comes here. | -`-----------------------------------*/ -yyabortlab: - yyresult = 1; - goto yyreturn; - -#ifndef yyoverflow -/*-------------------------------------------------. -| yyexhaustedlab -- memory exhaustion comes here. | -`-------------------------------------------------*/ -yyexhaustedlab: - yyerror(expr_stack, YY_("memory exhausted")); - yyresult = 2; - /* Fall through. */ -#endif - -yyreturn: - if (yychar != YYEOF && yychar != YYEMPTY) - yydestruct("Cleanup: discarding lookahead", yytoken, &yylval, &yylloc, expr_stack); - /* Do not reclaim the symbols of the rule which action triggered - this YYABORT or YYACCEPT. */ - YYPOPSTACK(yylen); - YY_STACK_PRINT(yyss, yyssp); - while (yyssp != yyss) - { - yydestruct("Cleanup: popping", yystos[*yyssp], yyvsp, yylsp, expr_stack); - YYPOPSTACK(1); - } -#ifndef yyoverflow - if (yyss != yyssa) - YYSTACK_FREE(yyss); -#endif -#if YYERROR_VERBOSE - if (yymsg != yymsgbuf) - YYSTACK_FREE(yymsg); -#endif - /* Make sure YYID is used. */ - return YYID(yyresult); -} - -#line 77 "parser.y" - -namespace adios2 -{ -namespace detail -{ - -void *createExpr(std::stack *expr_stack, std::string str_op, const char *name, - double value, size_t numsubexprs) -{ - // std::cout << "Creating ASTNode in function createExpr" << std::endl; - // std::cout << "\tstack size: " << expr_stack->size() << "\n\top: " << str_op << "\n\tname: " - // << name << "\n\tvalue: " << value << "\n\tnumsubexprs: " << numsubexprs << std::endl; - - ExpressionOperator op = get_op(str_op); - - ASTNode *node = new ASTNode(op); - switch (op) - { - case ExpressionOperator::OP_ALIAS: - node = new ASTNode(op, name); - break; - case ExpressionOperator::OP_PATH: - node = new ASTNode(op, name); - break; - case ExpressionOperator::OP_NUM: - node = new ASTNode(op, value); - break; - case ExpressionOperator::OP_INDEX: - // TODO: translate indices - node = new ASTNode(op, name); - break; - default: - node = new ASTNode(op); - }; - node->extend_subexprs(numsubexprs); - for (size_t i = 1; i <= numsubexprs; ++i) - { - ASTNode *subexpr = expr_stack->top(); - node->add_back_subexpr(subexpr, i); - expr_stack->pop(); - } - expr_stack->push(node); - - return &expr_stack->top(); -} - -ASTNode *parse_expression(std::string input) -{ - yy_scan_string(input.c_str()); - std::stack expr_stack; - yyparse(&expr_stack); - - // DEBUGGING - // std::cout << "yyparse complete. Stack size: " << expr_stack.size() << std::endl; - // std::cout << "parser prettyprint:" << std::endl; - // expr_stack.top()->printpretty(""); - return expr_stack.top(); -} - -} -} - -void yyerror(std::stack *expr_stack, const char *msg) -{ - printf("** Line %d: %s\n", yylloc.first_line, msg); -} diff --git a/source/adios2/toolkit/derived/parser/parser.h b/source/adios2/toolkit/derived/parser/parser.h deleted file mode 100644 index 38c3c96d58..0000000000 --- a/source/adios2/toolkit/derived/parser/parser.h +++ /dev/null @@ -1,110 +0,0 @@ -/* A Bison parser, made by GNU Bison 2.3. */ - -/* Skeleton interface for Bison's Yacc-like parsers in C - - Copyright (C) 1984, 1989, 1990, 2000, 2001, 2002, 2003, 2004, 2005, 2006 - Free Software Foundation, Inc. - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2, or (at your option) - any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, - Boston, MA 02110-1301, USA. */ - -/* As a special exception, you may create a larger work that contains - part or all of the Bison parser skeleton and distribute that work - under terms of your choice, so long as that work isn't itself a - parser generator using the skeleton or a modified version thereof - as a parser skeleton. Alternatively, if you modify or redistribute - the parser skeleton itself, you may (at your option) remove this - special exception, which will cause the skeleton and the resulting - Bison output files to be licensed under the GNU General Public - License without this special exception. - - This special exception was added by the Free Software Foundation in - version 2.2 of Bison. */ - -/* Tokens. */ -#ifndef YYTOKENTYPE -#define YYTOKENTYPE -/* Put the tokens into the symbol table, so that GDB and other debuggers - know about them. */ -enum yytokentype -{ - COMMA = 258, - L_PAREN = 259, - R_PAREN = 260, - ENDL = 261, - FUNCTION = 262, - OPERATOR = 263, - INDICES = 264, - NUMBER = 265, - ALIAS = 266, - PATH = 267 -}; -#endif -/* Tokens. */ -#define COMMA 258 -#define L_PAREN 259 -#define R_PAREN 260 -#define ENDL 261 -#define FUNCTION 262 -#define OPERATOR 263 -#define INDICES 264 -#define NUMBER 265 -#define ALIAS 266 -#define PATH 267 - -#if !defined YYSTYPE && !defined YYSTYPE_IS_DECLARED -typedef union YYSTYPE -#line 25 "parser.y" -{ - double dval; - int ival; - char *sval; - void *expr_ptr; -} -/* Line 1529 of yacc.c. */ -#line 80 "parser.h" -YYSTYPE; -#define yystype YYSTYPE /* obsolescent; will be withdrawn */ -#define YYSTYPE_IS_DECLARED 1 -#define YYSTYPE_IS_TRIVIAL 1 -#endif - -extern YYSTYPE yylval; - -#if !defined YYLTYPE && !defined YYLTYPE_IS_DECLARED -typedef struct YYLTYPE -{ - int first_line; - int first_column; - int last_line; - int last_column; -} YYLTYPE; -#define yyltype YYLTYPE /* obsolescent; will be withdrawn */ -#define YYLTYPE_IS_DECLARED 1 -#define YYLTYPE_IS_TRIVIAL 1 -#endif - -extern YYLTYPE yylloc; - -#include "ASTNode.h" -#include - -namespace adios2 -{ -namespace detail -{ -ASTNode *parse_expression(std::string input); -} -} diff --git a/source/adios2/toolkit/derived/parser/parser.output b/source/adios2/toolkit/derived/parser/parser.output deleted file mode 100644 index ae7a9b24a1..0000000000 --- a/source/adios2/toolkit/derived/parser/parser.output +++ /dev/null @@ -1,350 +0,0 @@ -State 5 conflicts: 1 shift/reduce -State 24 conflicts: 1 shift/reduce - - -Grammar - - 0 $accept: input $end - - 1 input: /* empty */ - 2 | ENDL input - 3 | decl input - 4 | exp input - - 5 decl: ALIAS PATH - 6 | ALIAS PATH INDICES - - 7 list: list COMMA exp - 8 | exp - - 9 exp: ALIAS - 10 | ALIAS INDICES - 11 | PATH - 12 | NUMBER - 13 | L_PAREN exp R_PAREN - 14 | exp OPERATOR exp - 15 | FUNCTION L_PAREN list R_PAREN - - -Terminals, with rules where they appear - -$end (0) 0 -error (256) -COMMA (258) 7 -L_PAREN (259) 13 15 -R_PAREN (260) 13 15 -ENDL (261) 2 -FUNCTION (262) 15 -OPERATOR (263) 14 -INDICES (264) 6 10 -NUMBER (265) 12 -ALIAS (266) 5 6 9 10 -PATH (267) 5 6 11 - - -Nonterminals, with rules where they appear - -$accept (13) - on left: 0 -input (14) - on left: 1 2 3 4, on right: 0 2 3 4 -decl (15) - on left: 5 6, on right: 3 -list (16) - on left: 7 8, on right: 7 15 -exp (17) - on left: 9 10 11 12 13 14 15, on right: 4 7 8 13 14 - - -state 0 - - 0 $accept: . input $end - - L_PAREN shift, and go to state 1 - ENDL shift, and go to state 2 - FUNCTION shift, and go to state 3 - NUMBER shift, and go to state 4 - ALIAS shift, and go to state 5 - PATH shift, and go to state 6 - - $default reduce using rule 1 (input) - - input go to state 7 - decl go to state 8 - exp go to state 9 - - -state 1 - - 13 exp: L_PAREN . exp R_PAREN - - L_PAREN shift, and go to state 1 - FUNCTION shift, and go to state 3 - NUMBER shift, and go to state 4 - ALIAS shift, and go to state 10 - PATH shift, and go to state 6 - - exp go to state 11 - - -state 2 - - 2 input: ENDL . input - - L_PAREN shift, and go to state 1 - ENDL shift, and go to state 2 - FUNCTION shift, and go to state 3 - NUMBER shift, and go to state 4 - ALIAS shift, and go to state 5 - PATH shift, and go to state 6 - - $default reduce using rule 1 (input) - - input go to state 12 - decl go to state 8 - exp go to state 9 - - -state 3 - - 15 exp: FUNCTION . L_PAREN list R_PAREN - - L_PAREN shift, and go to state 13 - - -state 4 - - 12 exp: NUMBER . - - $default reduce using rule 12 (exp) - - -state 5 - - 5 decl: ALIAS . PATH - 6 | ALIAS . PATH INDICES - 9 exp: ALIAS . - 10 | ALIAS . INDICES - - INDICES shift, and go to state 14 - PATH shift, and go to state 15 - - PATH [reduce using rule 9 (exp)] - $default reduce using rule 9 (exp) - - -state 6 - - 11 exp: PATH . - - $default reduce using rule 11 (exp) - - -state 7 - - 0 $accept: input . $end - - $end shift, and go to state 16 - - -state 8 - - 3 input: decl . input - - L_PAREN shift, and go to state 1 - ENDL shift, and go to state 2 - FUNCTION shift, and go to state 3 - NUMBER shift, and go to state 4 - ALIAS shift, and go to state 5 - PATH shift, and go to state 6 - - $default reduce using rule 1 (input) - - input go to state 17 - decl go to state 8 - exp go to state 9 - - -state 9 - - 4 input: exp . input - 14 exp: exp . OPERATOR exp - - L_PAREN shift, and go to state 1 - ENDL shift, and go to state 2 - FUNCTION shift, and go to state 3 - OPERATOR shift, and go to state 18 - NUMBER shift, and go to state 4 - ALIAS shift, and go to state 5 - PATH shift, and go to state 6 - - $default reduce using rule 1 (input) - - input go to state 19 - decl go to state 8 - exp go to state 9 - - -state 10 - - 9 exp: ALIAS . - 10 | ALIAS . INDICES - - INDICES shift, and go to state 14 - - $default reduce using rule 9 (exp) - - -state 11 - - 13 exp: L_PAREN exp . R_PAREN - 14 | exp . OPERATOR exp - - R_PAREN shift, and go to state 20 - OPERATOR shift, and go to state 18 - - -state 12 - - 2 input: ENDL input . - - $default reduce using rule 2 (input) - - -state 13 - - 15 exp: FUNCTION L_PAREN . list R_PAREN - - L_PAREN shift, and go to state 1 - FUNCTION shift, and go to state 3 - NUMBER shift, and go to state 4 - ALIAS shift, and go to state 10 - PATH shift, and go to state 6 - - list go to state 21 - exp go to state 22 - - -state 14 - - 10 exp: ALIAS INDICES . - - $default reduce using rule 10 (exp) - - -state 15 - - 5 decl: ALIAS PATH . - 6 | ALIAS PATH . INDICES - - INDICES shift, and go to state 23 - - $default reduce using rule 5 (decl) - - -state 16 - - 0 $accept: input $end . - - $default accept - - -state 17 - - 3 input: decl input . - - $default reduce using rule 3 (input) - - -state 18 - - 14 exp: exp OPERATOR . exp - - L_PAREN shift, and go to state 1 - FUNCTION shift, and go to state 3 - NUMBER shift, and go to state 4 - ALIAS shift, and go to state 10 - PATH shift, and go to state 6 - - exp go to state 24 - - -state 19 - - 4 input: exp input . - - $default reduce using rule 4 (input) - - -state 20 - - 13 exp: L_PAREN exp R_PAREN . - - $default reduce using rule 13 (exp) - - -state 21 - - 7 list: list . COMMA exp - 15 exp: FUNCTION L_PAREN list . R_PAREN - - COMMA shift, and go to state 25 - R_PAREN shift, and go to state 26 - - -state 22 - - 8 list: exp . - 14 exp: exp . OPERATOR exp - - OPERATOR shift, and go to state 18 - - $default reduce using rule 8 (list) - - -state 23 - - 6 decl: ALIAS PATH INDICES . - - $default reduce using rule 6 (decl) - - -state 24 - - 14 exp: exp . OPERATOR exp - 14 | exp OPERATOR exp . - - OPERATOR shift, and go to state 18 - - OPERATOR [reduce using rule 14 (exp)] - $default reduce using rule 14 (exp) - - -state 25 - - 7 list: list COMMA . exp - - L_PAREN shift, and go to state 1 - FUNCTION shift, and go to state 3 - NUMBER shift, and go to state 4 - ALIAS shift, and go to state 10 - PATH shift, and go to state 6 - - exp go to state 27 - - -state 26 - - 15 exp: FUNCTION L_PAREN list R_PAREN . - - $default reduce using rule 15 (exp) - - -state 27 - - 7 list: list COMMA exp . - 14 exp: exp . OPERATOR exp - - OPERATOR shift, and go to state 18 - - $default reduce using rule 7 (list) \ No newline at end of file diff --git a/source/adios2/toolkit/derived/parser/parser.y b/source/adios2/toolkit/derived/parser/parser.y index d9bf172459..a258b34bc1 100644 --- a/source/adios2/toolkit/derived/parser/parser.y +++ b/source/adios2/toolkit/derived/parser/parser.y @@ -1,132 +1,113 @@ -/* calculator. */ -%{ - #include - #include - #include - #include - #include - #include - #include - #include "lexer.h" - #include "ASTNode.h" - - extern int yyparse(std::stack* expr_stack); - - void* createExpr(std::stack*, std::string, const char*, double, size_t); - - static void* yyparse_value; - - void yyerror(std::stack* expr_stack, const char *msg); - -%} - -%parse-param {std::stack* expr_stack} - -%union{ - double dval; - int ival; - char* sval; - void* expr_ptr; +%skeleton "lalr1.cc" +%require "3.8.2" +%header + +%define api.token.raw +%define api.namespace {adios2::detail} +%define api.token.constructor +%define api.value.type variant +%define parse.assert + +%code requires { + #include + #include + #include + namespace adios2 + { + namespace detail + { + class ASTDriver; + } + } } -%error-verbose -%locations +// The parsing context. +%param { ASTDriver& drv } -%start input -%token COMMA L_PAREN R_PAREN ENDL -%token FUNCTION -%token OPERATOR -%token INDICES -%token NUMBER -%token ALIAS PATH -%type NUMBER -%type ALIAS PATH INDICES -%type FUNCTION OPERATOR -%type input exp -%type list - - -%% - -input: {} - | ENDL input {} - | decl input {} - | exp input { /*yyparse_value = $1->expression;*/ } - ; - -decl: ALIAS PATH { ASTNode::add_lookup_entry($1, $2, ""); } - | ALIAS PATH INDICES { ASTNode::add_lookup_entry($1, $2, $3); } - ; - -//index: NUMBER comma index { ASTNode::extend_current_lookup_indices($1); } -// | NUMBER { ASTNode::extend_current_lookup_indices($1); } -// ; - -list: list COMMA exp { $$ = $1 +1; } - | exp { $$ = 1;} - ; - -exp: ALIAS { $$ = createExpr(expr_stack, "ALIAS", $1, 0, 0); } -| ALIAS INDICES { createExpr(expr_stack, "ALIAS", $1, 0, 0); $$ = createExpr(expr_stack, "INDEX", $2, 0, 1); } -| PATH { $$ = createExpr(expr_stack, "PATH", $1, 0, 0); } -| NUMBER { $$ = createExpr(expr_stack, "NUM", "", $1, 0); } -| L_PAREN exp R_PAREN { $$ = $2; } -| exp OPERATOR exp { $$ = createExpr(expr_stack, $2, "", 0, 2); } -| FUNCTION L_PAREN list R_PAREN { $$ = createExpr(expr_stack, $1, "", 0, $3); } - ; -%% +%locations -void* createExpr(std::stack* expr_stack, std::string str_op, const char* name, double value, size_t numsubexprs) { - std::cout << "Creating ASTNode in function createExpr" << std::endl; - std::cout << "\tstack size: " << expr_stack->size() << "\n\top: " << str_op << "\n\tname: " << name << "\n\tvalue: " << value << "\n\tnumsubexprs: " << numsubexprs << std::endl; - - ExprHelper::expr_op op = ExprHelper::get_op(str_op); - - ASTNode *node = new ASTNode(op); - switch(op) { - case ExprHelper::OP_ALIAS: - node = new ASTNode(op, name); - break; - case ExprHelper::OP_PATH: - node = new ASTNode(op, name); - break; - case ExprHelper::OP_NUM: - node = new ASTNode(op, value); - break; - case ExprHelper::OP_INDEX: - // TODO: translate indices - node = new ASTNode(op, name); - break; - default: - node = new ASTNode(op); - }; - node->extend_subexprs(numsubexprs); - for (size_t i = 1; i <= numsubexprs; ++i) - { - ASTNode *subexpr = expr_stack->top(); - node->add_back_subexpr(subexpr,i); - expr_stack->pop(); - } - expr_stack->push(node); +%define parse.trace +%define parse.error detailed +%define parse.lac full - return &expr_stack->top(); +%code { +#include "ASTDriver.h" +#include "ASTNode.h" } -Expression* parse_expression(const char* input) { - yy_scan_string(input); - std::stack expr_stack; - yyparse(&expr_stack); +%define api.token.prefix {TOK_} +%token + ASSIGN "=" + COMMA "," + COLON ":" + L_PAREN "(" + R_PAREN ")" + L_BRACE "[" + R_BRACE "]" +; + +%token OPERATOR +%token IDENTIFIER "identifier" +%token VARNAME +%token INT "number" +%nterm list +%nterm >> indices_list +%nterm > index +%left OPERATOR - // DEBUGGING - std::cout << "yyparse complete. Stack size: " << expr_stack.size() << std::endl; - std::cout << "parser prettyprint:" << std::endl; - expr_stack.top()->printpretty(""); +%% +%start lines; +lines: + assignment lines {} +| exp {} +; + +assignment: + IDENTIFIER ASSIGN VARNAME { drv.add_lookup_entry($1, $3); } +| IDENTIFIER ASSIGN IDENTIFIER { drv.add_lookup_entry($1, $3); } +| IDENTIFIER ASSIGN VARNAME L_BRACE indices_list R_BRACE { drv.add_lookup_entry($1, $3, $5); } +| IDENTIFIER ASSIGN IDENTIFIER L_BRACE indices_list R_BRACE { drv.add_lookup_entry($1, $3, $5); } +; + +exp: + "number" { } +| exp OPERATOR exp { drv.createNode($2, 2); } +| "(" exp ")" { } +| IDENTIFIER "(" list ")" { drv.createNode($1, $3); } +| IDENTIFIER "[" indices_list "]" { drv.createNode($1, $3); } +| IDENTIFIER { drv.createNode($1); } +; + + +indices_list: + %empty { $$ = {}; } +| indices_list COMMA index { $1.push_back($3); $$ = $1; } +| index { $$ = {$1}; } +; + +index: + %empty { $$ = {-1, -1, 1}; } +| INT COLON INT COLON INT { $$ = {$1, $3, $5}; } +| COLON INT COLON INT { $$ = {-1, $2, $4}; } +| INT COLON COLON INT { $$ = {$1, -1, $4}; } +| INT COLON INT COLON { $$ = {$1, $3, 1}; } +| INT COLON INT { $$ = {$1, $3, 1}; } +| COLON COLON INT { $$ = {-1, -1, $3}; } +| COLON INT COLON { $$ = {-1, $2, 1}; } +| COLON INT { $$ = {-1, $2, 1}; } +| INT COLON COLON { $$ = {$1, -1, 1}; } +| INT COLON { $$ = {$1, -1, 1}; } +| INT { $$ = {$1, $1, 1}; } +; + +list: + %empty { $$ = 0; } +| exp COMMA list { $$ = $3 + 1; } +| exp { $$ = 1; } +%% - Expression *dummy_root = new Expression(); - expr_stack.top()->to_expr(dummy_root); - return std::get<0>(dummy_root->sub_exprs[0]); +void +adios2::detail::parser::error (const location_type& l, const std::string& m) +{ + std::cerr << l << ": " << m << '\n'; } - -void yyerror(std::stack* expr_stack, const char *msg) { - printf("** Line %d: %s\n", yylloc.first_line, msg); -} \ No newline at end of file diff --git a/source/adios2/toolkit/derived/parser/pregen-source/lexer.cpp b/source/adios2/toolkit/derived/parser/pregen-source/lexer.cpp new file mode 100644 index 0000000000..11c2e6b66a --- /dev/null +++ b/source/adios2/toolkit/derived/parser/pregen-source/lexer.cpp @@ -0,0 +1,1906 @@ +#line 1 "lexer.cpp" + +#line 3 "lexer.cpp" + +#define YY_INT_ALIGNED short int + +/* A lexical scanner generated by flex */ + +#define FLEX_SCANNER +#define YY_FLEX_MAJOR_VERSION 2 +#define YY_FLEX_MINOR_VERSION 6 +#define YY_FLEX_SUBMINOR_VERSION 4 +#if YY_FLEX_SUBMINOR_VERSION > 0 +#define FLEX_BETA +#endif + +/* First, we deal with platform-specific or compiler-specific issues. */ + +/* begin standard C headers. */ +#include +#include +#include +#include + +/* end standard C headers. */ + +/* flex integer type definitions */ + +#ifndef FLEXINT_H +#define FLEXINT_H + +/* C99 systems have . Non-C99 systems may or may not. */ + +#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L + +/* C99 says to define __STDC_LIMIT_MACROS before including stdint.h, + * if you want the limit (max/min) macros for int types. + */ +#ifndef __STDC_LIMIT_MACROS +#define __STDC_LIMIT_MACROS 1 +#endif + +#include +typedef int8_t flex_int8_t; +typedef uint8_t flex_uint8_t; +typedef int16_t flex_int16_t; +typedef uint16_t flex_uint16_t; +typedef int32_t flex_int32_t; +typedef uint32_t flex_uint32_t; +typedef uint64_t flex_uint64_t; +#else +typedef signed char flex_int8_t; +typedef short int flex_int16_t; +typedef int flex_int32_t; +typedef unsigned char flex_uint8_t; +typedef unsigned short int flex_uint16_t; +typedef unsigned int flex_uint32_t; + +/* Limits of integral types. */ +#ifndef INT8_MIN +#define INT8_MIN (-128) +#endif +#ifndef INT16_MIN +#define INT16_MIN (-32767-1) +#endif +#ifndef INT32_MIN +#define INT32_MIN (-2147483647-1) +#endif +#ifndef INT8_MAX +#define INT8_MAX (127) +#endif +#ifndef INT16_MAX +#define INT16_MAX (32767) +#endif +#ifndef INT32_MAX +#define INT32_MAX (2147483647) +#endif +#ifndef UINT8_MAX +#define UINT8_MAX (255U) +#endif +#ifndef UINT16_MAX +#define UINT16_MAX (65535U) +#endif +#ifndef UINT32_MAX +#define UINT32_MAX (4294967295U) +#endif + +#ifndef SIZE_MAX +#define SIZE_MAX (~(size_t)0) +#endif + +#endif /* ! C99 */ + +#endif /* ! FLEXINT_H */ + +/* begin standard C++ headers. */ + +/* TODO: this is always defined, so inline it */ +#define yyconst const + +#if defined(__GNUC__) && __GNUC__ >= 3 +#define yynoreturn __attribute__((__noreturn__)) +#else +#define yynoreturn +#endif + +/* Returned upon end-of-file. */ +#define YY_NULL 0 + +/* Promotes a possibly negative, possibly signed char to an + * integer in range [0..255] for use as an array index. + */ +#define YY_SC_TO_UI(c) ((YY_CHAR) (c)) + +/* Enter a start condition. This macro really ought to take a parameter, + * but we do it the disgusting crufty way forced on us by the ()-less + * definition of BEGIN. + */ +#define BEGIN (yy_start) = 1 + 2 * +/* Translate the current start state into a value that can be later handed + * to BEGIN to return to the state. The YYSTATE alias is for lex + * compatibility. + */ +#define YY_START (((yy_start) - 1) / 2) +#define YYSTATE YY_START +/* Action number for EOF rule of a given start state. */ +#define YY_STATE_EOF(state) (YY_END_OF_BUFFER + state + 1) +/* Special action meaning "start processing a new file". */ +#define YY_NEW_FILE yyrestart( yyin ) +#define YY_END_OF_BUFFER_CHAR 0 + +/* Size of default input buffer. */ +#ifndef YY_BUF_SIZE +#ifdef __ia64__ +/* On IA-64, the buffer size is 16k, not 8k. + * Moreover, YY_BUF_SIZE is 2*YY_READ_BUF_SIZE in the general case. + * Ditto for the __ia64__ case accordingly. + */ +#define YY_BUF_SIZE 32768 +#else +#define YY_BUF_SIZE 16384 +#endif /* __ia64__ */ +#endif + +/* The state buf must be large enough to hold one state per character in the main buffer. + */ +#define YY_STATE_BUF_SIZE ((YY_BUF_SIZE + 2) * sizeof(yy_state_type)) + +#ifndef YY_TYPEDEF_YY_BUFFER_STATE +#define YY_TYPEDEF_YY_BUFFER_STATE +typedef struct yy_buffer_state *YY_BUFFER_STATE; +#endif + +#ifndef YY_TYPEDEF_YY_SIZE_T +#define YY_TYPEDEF_YY_SIZE_T +typedef size_t yy_size_t; +#endif + +extern yy_size_t yyleng; + +extern FILE *yyin, *yyout; + +#define EOB_ACT_CONTINUE_SCAN 0 +#define EOB_ACT_END_OF_FILE 1 +#define EOB_ACT_LAST_MATCH 2 + + #define YY_LESS_LINENO(n) + #define YY_LINENO_REWIND_TO(ptr) + +/* Return all but the first "n" matched characters back to the input stream. */ +#define yyless(n) \ + do \ + { \ + /* Undo effects of setting up yytext. */ \ + int yyless_macro_arg = (n); \ + YY_LESS_LINENO(yyless_macro_arg);\ + *yy_cp = (yy_hold_char); \ + YY_RESTORE_YY_MORE_OFFSET \ + (yy_c_buf_p) = yy_cp = yy_bp + yyless_macro_arg - YY_MORE_ADJ; \ + YY_DO_BEFORE_ACTION; /* set up yytext again */ \ + } \ + while ( 0 ) +#define unput(c) yyunput( c, (yytext_ptr) ) + +#ifndef YY_STRUCT_YY_BUFFER_STATE +#define YY_STRUCT_YY_BUFFER_STATE +struct yy_buffer_state + { + FILE *yy_input_file; + + char *yy_ch_buf; /* input buffer */ + char *yy_buf_pos; /* current position in input buffer */ + + /* Size of input buffer in bytes, not including room for EOB + * characters. + */ + int yy_buf_size; + + /* Number of characters read into yy_ch_buf, not including EOB + * characters. + */ + yy_size_t yy_n_chars; + + /* Whether we "own" the buffer - i.e., we know we created it, + * and can realloc() it to grow it, and should free() it to + * delete it. + */ + int yy_is_our_buffer; + + /* Whether this is an "interactive" input source; if so, and + * if we're using stdio for input, then we want to use getc() + * instead of fread(), to make sure we stop fetching input after + * each newline. + */ + int yy_is_interactive; + + /* Whether we're considered to be at the beginning of a line. + * If so, '^' rules will be active on the next match, otherwise + * not. + */ + int yy_at_bol; + + int yy_bs_lineno; /**< The line count. */ + int yy_bs_column; /**< The column count. */ + + /* Whether to try to fill the input buffer when we reach the + * end of it. + */ + int yy_fill_buffer; + + int yy_buffer_status; + +#define YY_BUFFER_NEW 0 +#define YY_BUFFER_NORMAL 1 + /* When an EOF's been seen but there's still some text to process + * then we mark the buffer as YY_EOF_PENDING, to indicate that we + * shouldn't try reading from the input source any more. We might + * still have a bunch of tokens to match, though, because of + * possible backing-up. + * + * When we actually see the EOF, we change the status to "new" + * (via yyrestart()), so that the user can continue scanning by + * just pointing yyin at a new input file. + */ +#define YY_BUFFER_EOF_PENDING 2 + + }; +#endif /* !YY_STRUCT_YY_BUFFER_STATE */ + +/* Stack of input buffers. */ +static size_t yy_buffer_stack_top = 0; /**< index of top of stack. */ +static size_t yy_buffer_stack_max = 0; /**< capacity of stack. */ +static YY_BUFFER_STATE * yy_buffer_stack = NULL; /**< Stack as an array. */ + +/* We provide macros for accessing buffer states in case in the + * future we want to put the buffer states in a more general + * "scanner state". + * + * Returns the top of the stack, or NULL. + */ +#define YY_CURRENT_BUFFER ( (yy_buffer_stack) \ + ? (yy_buffer_stack)[(yy_buffer_stack_top)] \ + : NULL) +/* Same as previous macro, but useful when we know that the buffer stack is not + * NULL or when we need an lvalue. For internal use only. + */ +#define YY_CURRENT_BUFFER_LVALUE (yy_buffer_stack)[(yy_buffer_stack_top)] + +/* yy_hold_char holds the character lost when yytext is formed. */ +static char yy_hold_char; +static yy_size_t yy_n_chars; /* number of characters read into yy_ch_buf */ +yy_size_t yyleng; + +/* Points to current character in buffer. */ +static char *yy_c_buf_p = NULL; +static int yy_init = 0; /* whether we need to initialize */ +static int yy_start = 0; /* start state number */ + +/* Flag which is used to allow yywrap()'s to do buffer switches + * instead of setting up a fresh yyin. A bit of a hack ... + */ +static int yy_did_buffer_switch_on_eof; + +void yyrestart ( FILE *input_file ); +void yy_switch_to_buffer ( YY_BUFFER_STATE new_buffer ); +YY_BUFFER_STATE yy_create_buffer ( FILE *file, int size ); +void yy_delete_buffer ( YY_BUFFER_STATE b ); +void yy_flush_buffer ( YY_BUFFER_STATE b ); +void yypush_buffer_state ( YY_BUFFER_STATE new_buffer ); +void yypop_buffer_state ( void ); + +static void yyensure_buffer_stack ( void ); +static void yy_load_buffer_state ( void ); +static void yy_init_buffer ( YY_BUFFER_STATE b, FILE *file ); +#define YY_FLUSH_BUFFER yy_flush_buffer( YY_CURRENT_BUFFER ) + +YY_BUFFER_STATE yy_scan_buffer ( char *base, yy_size_t size ); +YY_BUFFER_STATE yy_scan_string ( const char *yy_str ); +YY_BUFFER_STATE yy_scan_bytes ( const char *bytes, yy_size_t len ); + +void *yyalloc ( yy_size_t ); +void *yyrealloc ( void *, yy_size_t ); +void yyfree ( void * ); + +#define yy_new_buffer yy_create_buffer +#define yy_set_interactive(is_interactive) \ + { \ + if ( ! YY_CURRENT_BUFFER ){ \ + yyensure_buffer_stack (); \ + YY_CURRENT_BUFFER_LVALUE = \ + yy_create_buffer( yyin, YY_BUF_SIZE ); \ + } \ + YY_CURRENT_BUFFER_LVALUE->yy_is_interactive = is_interactive; \ + } +#define yy_set_bol(at_bol) \ + { \ + if ( ! YY_CURRENT_BUFFER ){\ + yyensure_buffer_stack (); \ + YY_CURRENT_BUFFER_LVALUE = \ + yy_create_buffer( yyin, YY_BUF_SIZE ); \ + } \ + YY_CURRENT_BUFFER_LVALUE->yy_at_bol = at_bol; \ + } +#define YY_AT_BOL() (YY_CURRENT_BUFFER_LVALUE->yy_at_bol) + +/* Begin user sect3 */ + +#define yywrap() (/*CONSTCOND*/1) +#define YY_SKIP_YYWRAP +typedef flex_uint8_t YY_CHAR; + +FILE *yyin = NULL, *yyout = NULL; + +typedef int yy_state_type; + +extern int yylineno; +int yylineno = 1; + +extern char *yytext; +#ifdef yytext_ptr +#undef yytext_ptr +#endif +#define yytext_ptr yytext + +static yy_state_type yy_get_previous_state ( void ); +static yy_state_type yy_try_NUL_trans ( yy_state_type current_state ); +static int yy_get_next_buffer ( void ); +static void yynoreturn yy_fatal_error ( const char* msg ); + +/* Done after the current pattern has been matched and before the + * corresponding action - sets up yytext. + */ +#define YY_DO_BEFORE_ACTION \ + (yytext_ptr) = yy_bp; \ + yyleng = (yy_size_t) (yy_cp - yy_bp); \ + (yy_hold_char) = *yy_cp; \ + *yy_cp = '\0'; \ + (yy_c_buf_p) = yy_cp; +#define YY_NUM_RULES 15 +#define YY_END_OF_BUFFER 16 +/* This struct is not used in this scanner, + but its presence is necessary. */ +struct yy_trans_info + { + flex_int32_t yy_verify; + flex_int32_t yy_nxt; + }; +static const flex_int16_t yy_accept[23] = + { 0, + 0, 0, 16, 14, 1, 2, 6, 7, 11, 4, + 10, 5, 3, 12, 8, 9, 1, 2, 10, 13, + 12, 0 + } ; + +static const YY_CHAR yy_ec[256] = + { 0, + 1, 1, 1, 1, 1, 1, 1, 1, 2, 3, + 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 2, 1, 1, 1, 1, 1, 1, 1, 4, + 5, 6, 6, 7, 8, 9, 8, 10, 10, 10, + 10, 10, 10, 10, 10, 10, 10, 11, 1, 1, + 12, 1, 1, 1, 13, 13, 13, 13, 13, 13, + 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, + 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, + 14, 9, 15, 6, 16, 1, 13, 13, 13, 13, + + 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, + 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, + 13, 13, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1 + } ; + +static const YY_CHAR yy_meta[17] = + { 0, + 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, + 1, 1, 2, 1, 1, 2 + } ; + +static const flex_int16_t yy_base[25] = + { 0, + 0, 0, 29, 30, 26, 24, 30, 30, 30, 30, + 16, 30, 30, 9, 30, 30, 23, 21, 13, 0, + 11, 30, 20, 19 + } ; + +static const flex_int16_t yy_def[25] = + { 0, + 22, 1, 22, 22, 22, 22, 22, 22, 22, 22, + 22, 22, 22, 23, 22, 22, 22, 22, 22, 24, + 23, 0, 22, 22 + } ; + +static const flex_int16_t yy_nxt[47] = + { 0, + 4, 5, 6, 7, 8, 9, 10, 9, 4, 11, + 12, 13, 14, 15, 16, 4, 20, 20, 20, 20, + 20, 21, 19, 18, 17, 19, 18, 17, 22, 3, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 22 + } ; + +static const flex_int16_t yy_chk[47] = + { 0, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 14, 14, 21, 21, + 24, 23, 19, 18, 17, 11, 6, 5, 3, 22, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 22 + } ; + +static yy_state_type yy_last_accepting_state; +static char *yy_last_accepting_cpos; + +extern int yy_flex_debug; +int yy_flex_debug = 0; + +/* The intent behind this definition is that it'll catch + * any uses of REJECT which flex missed. + */ +#define REJECT reject_used_but_not_detected +#define yymore() yymore_used_but_not_detected +#define YY_MORE_ADJ 0 +#define YY_RESTORE_YY_MORE_OFFSET +char *yytext; +#line 1 "../lexer.l" +#line 2 "../lexer.l" +#include +#include +#include +#include // strerror +#include +#include "ASTDriver.h" +#include "parser.h" +#if defined(_MSC_VER) +#include +#define YY_NO_UNISTD_H +#define strdup _strdup +#define isatty _isatty +#define fileno _fileno +#include +#include +#endif +#line 477 "lexer.cpp" +#line 21 "../lexer.l" +#if defined __clang__ +# define CLANG_VERSION (__clang_major__ * 100 + __clang_minor__) +#endif + +// Clang and ICC like to pretend they are GCC. +#if defined __GNUC__ && !defined __clang__ && !defined __ICC +# define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) +#endif + +// Pacify warnings in yy_init_buffer (observed with Flex 2.6.4) +// and GCC 6.4.0, 7.3.0 with -O3. +#if defined GCC_VERSION && 600 <= GCC_VERSION +# pragma GCC diagnostic ignored "-Wnull-dereference" +#endif + +// This example uses Flex's C back end, yet compiles it as C++. +// So expect warnings about C style casts and NULL. +#if defined CLANG_VERSION && 500 <= CLANG_VERSION +# pragma clang diagnostic ignored "-Wold-style-cast" +# pragma clang diagnostic ignored "-Wzero-as-null-pointer-constant" +#elif defined GCC_VERSION && 407 <= GCC_VERSION +# pragma GCC diagnostic ignored "-Wold-style-cast" +# pragma GCC diagnostic ignored "-Wzero-as-null-pointer-constant" +#endif + +#define FLEX_VERSION (YY_FLEX_MAJOR_VERSION * 100 + YY_FLEX_MINOR_VERSION) + +// Old versions of Flex (2.5.35) generate an incomplete documentation comment. +// +// In file included from src/scan-code-c.c:3: +// src/scan-code.c:2198:21: error: empty paragraph passed to '@param' command +// [-Werror,-Wdocumentation] +// * @param line_number +// ~~~~~~~~~~~~~~~~~^ +// 1 error generated. +#if FLEX_VERSION < 206 && defined CLANG_VERSION +# pragma clang diagnostic ignored "-Wdocumentation" +#endif + +// Old versions of Flex (2.5.35) use 'register'. Warnings introduced in +// GCC 7 and Clang 6. +#if FLEX_VERSION < 206 +# if defined CLANG_VERSION && 600 <= CLANG_VERSION +# pragma clang diagnostic ignored "-Wdeprecated-register" +# elif defined GCC_VERSION && 700 <= GCC_VERSION +# pragma GCC diagnostic ignored "-Wregister" +# endif +#endif + +#if FLEX_VERSION < 206 +# if defined CLANG_VERSION +# pragma clang diagnostic ignored "-Wconversion" +# pragma clang diagnostic ignored "-Wdocumentation" +# pragma clang diagnostic ignored "-Wshorten-64-to-32" +# pragma clang diagnostic ignored "-Wsign-conversion" +# elif defined GCC_VERSION +# pragma GCC diagnostic ignored "-Wconversion" +# pragma GCC diagnostic ignored "-Wsign-conversion" +# endif +#endif + +// Flex 2.6.4, GCC 9 +// warning: useless cast to type 'int' [-Wuseless-cast] +// 1361 | YY_CURRENT_BUFFER_LVALUE->yy_buf_size = (int) (new_size - 2); +// | ^ +#if defined GCC_VERSION && 900 <= GCC_VERSION +# pragma GCC diagnostic ignored "-Wuseless-cast" +#endif +#line 547 "lexer.cpp" +#define YY_NO_INPUT 1 +#line 94 "../lexer.l" + // A number symbol corresponding to the value in S. + adios2::detail::parser::symbol_type + make_INT (const std::string &s, const adios2::detail::parser::location_type& loc); +#line 553 "lexer.cpp" +#line 106 "../lexer.l" + // Code run each time a pattern is matched. + # define YY_USER_ACTION loc.columns (yyleng); +#line 557 "lexer.cpp" +#line 558 "lexer.cpp" + +#define INITIAL 0 + +#ifndef YY_NO_UNISTD_H +/* Special case for "unistd.h", since it is non-ANSI. We include it way + * down here because we want the user's section 1 to have been scanned first. + * The user has a chance to override it with an option. + */ +#include +#endif + +#ifndef YY_EXTRA_TYPE +#define YY_EXTRA_TYPE void * +#endif + +static int yy_init_globals ( void ); + +/* Accessor methods to globals. + These are made visible to non-reentrant scanners for convenience. */ + +int yylex_destroy ( void ); + +int yyget_debug ( void ); + +void yyset_debug ( int debug_flag ); + +YY_EXTRA_TYPE yyget_extra ( void ); + +void yyset_extra ( YY_EXTRA_TYPE user_defined ); + +FILE *yyget_in ( void ); + +void yyset_in ( FILE * _in_str ); + +FILE *yyget_out ( void ); + +void yyset_out ( FILE * _out_str ); + + yy_size_t yyget_leng ( void ); + +char *yyget_text ( void ); + +int yyget_lineno ( void ); + +void yyset_lineno ( int _line_number ); + +/* Macros after this point can all be overridden by user definitions in + * section 1. + */ + +#ifndef YY_SKIP_YYWRAP +#ifdef __cplusplus +extern "C" int yywrap ( void ); +#else +extern int yywrap ( void ); +#endif +#endif + +#ifndef YY_NO_UNPUT + +#endif + +#ifndef yytext_ptr +static void yy_flex_strncpy ( char *, const char *, int ); +#endif + +#ifdef YY_NEED_STRLEN +static int yy_flex_strlen ( const char * ); +#endif + +#ifndef YY_NO_INPUT +#ifdef __cplusplus +static int yyinput ( void ); +#else +static int input ( void ); +#endif + +#endif + +/* Amount of stuff to slurp up with each read. */ +#ifndef YY_READ_BUF_SIZE +#ifdef __ia64__ +/* On IA-64, the buffer size is 16k, not 8k */ +#define YY_READ_BUF_SIZE 16384 +#else +#define YY_READ_BUF_SIZE 8192 +#endif /* __ia64__ */ +#endif + +/* Copy whatever the last rule matched to the standard output. */ +#ifndef ECHO +/* This used to be an fputs(), but since the string might contain NUL's, + * we now use fwrite(). + */ +#define ECHO do { if (fwrite( yytext, (size_t) yyleng, 1, yyout )) {} } while (0) +#endif + +/* Gets input and stuffs it into "buf". number of characters read, or YY_NULL, + * is returned in "result". + */ +#ifndef YY_INPUT +#define YY_INPUT(buf,result,max_size) \ + if ( YY_CURRENT_BUFFER_LVALUE->yy_is_interactive ) \ + { \ + int c = '*'; \ + yy_size_t n; \ + for ( n = 0; n < max_size && \ + (c = getc( yyin )) != EOF && c != '\n'; ++n ) \ + buf[n] = (char) c; \ + if ( c == '\n' ) \ + buf[n++] = (char) c; \ + if ( c == EOF && ferror( yyin ) ) \ + YY_FATAL_ERROR( "input in flex scanner failed" ); \ + result = n; \ + } \ + else \ + { \ + errno=0; \ + while ( (result = (int) fread(buf, 1, (yy_size_t) max_size, yyin)) == 0 && ferror(yyin)) \ + { \ + if( errno != EINTR) \ + { \ + YY_FATAL_ERROR( "input in flex scanner failed" ); \ + break; \ + } \ + errno=0; \ + clearerr(yyin); \ + } \ + }\ +\ + +#endif + +/* No semi-colon after return; correct usage is to write "yyterminate();" - + * we don't want an extra ';' after the "return" because that will cause + * some compilers to complain about unreachable statements. + */ +#ifndef yyterminate +#define yyterminate() return YY_NULL +#endif + +/* Number of entries by which start-condition stack grows. */ +#ifndef YY_START_STACK_INCR +#define YY_START_STACK_INCR 25 +#endif + +/* Report a fatal error. */ +#ifndef YY_FATAL_ERROR +#define YY_FATAL_ERROR(msg) yy_fatal_error( msg ) +#endif + +/* end tables serialization structures and prototypes */ + +/* Default declaration of generated scanner - a define so the user can + * easily add parameters. + */ +#ifndef YY_DECL +#define YY_DECL_IS_OURS 1 + +extern int yylex (void); + +#define YY_DECL int yylex (void) +#endif /* !YY_DECL */ + +/* Code executed at the beginning of each rule, after yytext and yyleng + * have been set up. + */ +#ifndef YY_USER_ACTION +#define YY_USER_ACTION +#endif + +/* Code executed at the end of each rule. */ +#ifndef YY_BREAK +#define YY_BREAK /*LINTED*/break; +#endif + +#define YY_RULE_SETUP \ + YY_USER_ACTION + +/** The main scanner function which does all the work. + */ +YY_DECL +{ + yy_state_type yy_current_state; + char *yy_cp, *yy_bp; + int yy_act; + + if ( !(yy_init) ) + { + (yy_init) = 1; + +#ifdef YY_USER_INIT + YY_USER_INIT; +#endif + + if ( ! (yy_start) ) + (yy_start) = 1; /* first start state */ + + if ( ! yyin ) + yyin = stdin; + + if ( ! yyout ) + yyout = stdout; + + if ( ! YY_CURRENT_BUFFER ) { + yyensure_buffer_stack (); + YY_CURRENT_BUFFER_LVALUE = + yy_create_buffer( yyin, YY_BUF_SIZE ); + } + + yy_load_buffer_state( ); + } + + { +#line 109 "../lexer.l" + + +#line 112 "../lexer.l" + // A handy shortcut to the location held by the adios2::detail::ASTDriver. + adios2::detail::location& loc = drv.location; + // Code run each time yylex is called. + loc.step (); + +#line 782 "lexer.cpp" + + while ( /*CONSTCOND*/1 ) /* loops until end-of-file is reached */ + { + yy_cp = (yy_c_buf_p); + + /* Support of yytext. */ + *yy_cp = (yy_hold_char); + + /* yy_bp points to the position in yy_ch_buf of the start of + * the current run. + */ + yy_bp = yy_cp; + + yy_current_state = (yy_start); +yy_match: + do + { + YY_CHAR yy_c = yy_ec[YY_SC_TO_UI(*yy_cp)] ; + if ( yy_accept[yy_current_state] ) + { + (yy_last_accepting_state) = yy_current_state; + (yy_last_accepting_cpos) = yy_cp; + } + while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) + { + yy_current_state = (int) yy_def[yy_current_state]; + if ( yy_current_state >= 23 ) + yy_c = yy_meta[yy_c]; + } + yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c]; + ++yy_cp; + } + while ( yy_current_state != 22 ); + yy_cp = (yy_last_accepting_cpos); + yy_current_state = (yy_last_accepting_state); + +yy_find_action: + yy_act = yy_accept[yy_current_state]; + + YY_DO_BEFORE_ACTION; + +do_action: /* This label is used only to access EOF actions. */ + + switch ( yy_act ) + { /* beginning of action switch */ + case 0: /* must back up */ + /* undo the effects of YY_DO_BEFORE_ACTION */ + *yy_cp = (yy_hold_char); + yy_cp = (yy_last_accepting_cpos); + yy_current_state = (yy_last_accepting_state); + goto yy_find_action; + +case 1: +YY_RULE_SETUP +#line 117 "../lexer.l" +loc.step (); + YY_BREAK +case 2: +/* rule 2 can match eol */ +YY_RULE_SETUP +#line 118 "../lexer.l" +loc.lines (yyleng); loc.step (); + YY_BREAK +case 3: +YY_RULE_SETUP +#line 120 "../lexer.l" +return adios2::detail::parser::make_ASSIGN (loc); + YY_BREAK +case 4: +YY_RULE_SETUP +#line 121 "../lexer.l" +return adios2::detail::parser::make_COMMA (loc); + YY_BREAK +case 5: +YY_RULE_SETUP +#line 122 "../lexer.l" +return adios2::detail::parser::make_COLON (loc); + YY_BREAK +case 6: +YY_RULE_SETUP +#line 123 "../lexer.l" +return adios2::detail::parser::make_L_PAREN (loc); + YY_BREAK +case 7: +YY_RULE_SETUP +#line 124 "../lexer.l" +return adios2::detail::parser::make_R_PAREN (loc); + YY_BREAK +case 8: +YY_RULE_SETUP +#line 125 "../lexer.l" +return adios2::detail::parser::make_L_BRACE (loc); + YY_BREAK +case 9: +YY_RULE_SETUP +#line 126 "../lexer.l" +return adios2::detail::parser::make_R_BRACE (loc); + YY_BREAK +case 10: +YY_RULE_SETUP +#line 128 "../lexer.l" +return make_INT (yytext, loc); + YY_BREAK +case 11: +YY_RULE_SETUP +#line 129 "../lexer.l" +return adios2::detail::parser::make_OPERATOR (yytext, loc); + YY_BREAK +case 12: +YY_RULE_SETUP +#line 130 "../lexer.l" +return adios2::detail::parser::make_IDENTIFIER (yytext, loc); + YY_BREAK +case 13: +YY_RULE_SETUP +#line 131 "../lexer.l" +return adios2::detail::parser::make_VARNAME (yytext, loc); + YY_BREAK +case 14: +YY_RULE_SETUP +#line 132 "../lexer.l" +{ + throw adios2::detail::parser::syntax_error + (loc, "invalid character: " + std::string(yytext)); +} + YY_BREAK +case YY_STATE_EOF(INITIAL): +#line 136 "../lexer.l" +return adios2::detail::parser::make_YYEOF (loc); + YY_BREAK +case 15: +YY_RULE_SETUP +#line 137 "../lexer.l" +ECHO; + YY_BREAK +#line 918 "lexer.cpp" + + case YY_END_OF_BUFFER: + { + /* Amount of text matched not including the EOB char. */ + int yy_amount_of_matched_text = (int) (yy_cp - (yytext_ptr)) - 1; + + /* Undo the effects of YY_DO_BEFORE_ACTION. */ + *yy_cp = (yy_hold_char); + YY_RESTORE_YY_MORE_OFFSET + + if ( YY_CURRENT_BUFFER_LVALUE->yy_buffer_status == YY_BUFFER_NEW ) + { + /* We're scanning a new file or input source. It's + * possible that this happened because the user + * just pointed yyin at a new source and called + * yylex(). If so, then we have to assure + * consistency between YY_CURRENT_BUFFER and our + * globals. Here is the right place to do so, because + * this is the first action (other than possibly a + * back-up) that will match for the new input source. + */ + (yy_n_chars) = YY_CURRENT_BUFFER_LVALUE->yy_n_chars; + YY_CURRENT_BUFFER_LVALUE->yy_input_file = yyin; + YY_CURRENT_BUFFER_LVALUE->yy_buffer_status = YY_BUFFER_NORMAL; + } + + /* Note that here we test for yy_c_buf_p "<=" to the position + * of the first EOB in the buffer, since yy_c_buf_p will + * already have been incremented past the NUL character + * (since all states make transitions on EOB to the + * end-of-buffer state). Contrast this with the test + * in input(). + */ + if ( (yy_c_buf_p) <= &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[(yy_n_chars)] ) + { /* This was really a NUL. */ + yy_state_type yy_next_state; + + (yy_c_buf_p) = (yytext_ptr) + yy_amount_of_matched_text; + + yy_current_state = yy_get_previous_state( ); + + /* Okay, we're now positioned to make the NUL + * transition. We couldn't have + * yy_get_previous_state() go ahead and do it + * for us because it doesn't know how to deal + * with the possibility of jamming (and we don't + * want to build jamming into it because then it + * will run more slowly). + */ + + yy_next_state = yy_try_NUL_trans( yy_current_state ); + + yy_bp = (yytext_ptr) + YY_MORE_ADJ; + + if ( yy_next_state ) + { + /* Consume the NUL. */ + yy_cp = ++(yy_c_buf_p); + yy_current_state = yy_next_state; + goto yy_match; + } + + else + { + yy_cp = (yy_last_accepting_cpos); + yy_current_state = (yy_last_accepting_state); + goto yy_find_action; + } + } + + else switch ( yy_get_next_buffer( ) ) + { + case EOB_ACT_END_OF_FILE: + { + (yy_did_buffer_switch_on_eof) = 0; + + if ( yywrap( ) ) + { + /* Note: because we've taken care in + * yy_get_next_buffer() to have set up + * yytext, we can now set up + * yy_c_buf_p so that if some total + * hoser (like flex itself) wants to + * call the scanner after we return the + * YY_NULL, it'll still work - another + * YY_NULL will get returned. + */ + (yy_c_buf_p) = (yytext_ptr) + YY_MORE_ADJ; + + yy_act = YY_STATE_EOF(YY_START); + goto do_action; + } + + else + { + if ( ! (yy_did_buffer_switch_on_eof) ) + YY_NEW_FILE; + } + break; + } + + case EOB_ACT_CONTINUE_SCAN: + (yy_c_buf_p) = + (yytext_ptr) + yy_amount_of_matched_text; + + yy_current_state = yy_get_previous_state( ); + + yy_cp = (yy_c_buf_p); + yy_bp = (yytext_ptr) + YY_MORE_ADJ; + goto yy_match; + + case EOB_ACT_LAST_MATCH: + (yy_c_buf_p) = + &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[(yy_n_chars)]; + + yy_current_state = yy_get_previous_state( ); + + yy_cp = (yy_c_buf_p); + yy_bp = (yytext_ptr) + YY_MORE_ADJ; + goto yy_find_action; + } + break; + } + + default: + YY_FATAL_ERROR( + "fatal flex scanner internal error--no action found" ); + } /* end of action switch */ + } /* end of scanning one token */ + } /* end of user's declarations */ +} /* end of yylex */ + +/* yy_get_next_buffer - try to read in a new buffer + * + * Returns a code representing an action: + * EOB_ACT_LAST_MATCH - + * EOB_ACT_CONTINUE_SCAN - continue scanning from current position + * EOB_ACT_END_OF_FILE - end of file + */ +static int yy_get_next_buffer (void) +{ + char *dest = YY_CURRENT_BUFFER_LVALUE->yy_ch_buf; + char *source = (yytext_ptr); + int number_to_move, i; + int ret_val; + + if ( (yy_c_buf_p) > &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[(yy_n_chars) + 1] ) + YY_FATAL_ERROR( + "fatal flex scanner internal error--end of buffer missed" ); + + if ( YY_CURRENT_BUFFER_LVALUE->yy_fill_buffer == 0 ) + { /* Don't try to fill the buffer, so this is an EOF. */ + if ( (yy_c_buf_p) - (yytext_ptr) - YY_MORE_ADJ == 1 ) + { + /* We matched a single character, the EOB, so + * treat this as a final EOF. + */ + return EOB_ACT_END_OF_FILE; + } + + else + { + /* We matched some text prior to the EOB, first + * process it. + */ + return EOB_ACT_LAST_MATCH; + } + } + + /* Try to read more data. */ + + /* First move last chars to start of buffer. */ + number_to_move = (int) ((yy_c_buf_p) - (yytext_ptr) - 1); + + for ( i = 0; i < number_to_move; ++i ) + *(dest++) = *(source++); + + if ( YY_CURRENT_BUFFER_LVALUE->yy_buffer_status == YY_BUFFER_EOF_PENDING ) + /* don't do the read, it's not guaranteed to return an EOF, + * just force an EOF + */ + YY_CURRENT_BUFFER_LVALUE->yy_n_chars = (yy_n_chars) = 0; + + else + { + yy_size_t num_to_read = + YY_CURRENT_BUFFER_LVALUE->yy_buf_size - number_to_move - 1; + + while ( num_to_read <= 0 ) + { /* Not enough room in the buffer - grow it. */ + + /* just a shorter name for the current buffer */ + YY_BUFFER_STATE b = YY_CURRENT_BUFFER_LVALUE; + + int yy_c_buf_p_offset = + (int) ((yy_c_buf_p) - b->yy_ch_buf); + + if ( b->yy_is_our_buffer ) + { + yy_size_t new_size = b->yy_buf_size * 2; + + if ( new_size <= 0 ) + b->yy_buf_size += b->yy_buf_size / 8; + else + b->yy_buf_size *= 2; + + b->yy_ch_buf = (char *) + /* Include room in for 2 EOB chars. */ + yyrealloc( (void *) b->yy_ch_buf, + (yy_size_t) (b->yy_buf_size + 2) ); + } + else + /* Can't grow it, we don't own it. */ + b->yy_ch_buf = NULL; + + if ( ! b->yy_ch_buf ) + YY_FATAL_ERROR( + "fatal error - scanner input buffer overflow" ); + + (yy_c_buf_p) = &b->yy_ch_buf[yy_c_buf_p_offset]; + + num_to_read = YY_CURRENT_BUFFER_LVALUE->yy_buf_size - + number_to_move - 1; + + } + + if ( num_to_read > YY_READ_BUF_SIZE ) + num_to_read = YY_READ_BUF_SIZE; + + /* Read in more data. */ + YY_INPUT( (&YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[number_to_move]), + (yy_n_chars), num_to_read ); + + YY_CURRENT_BUFFER_LVALUE->yy_n_chars = (yy_n_chars); + } + + if ( (yy_n_chars) == 0 ) + { + if ( number_to_move == YY_MORE_ADJ ) + { + ret_val = EOB_ACT_END_OF_FILE; + yyrestart( yyin ); + } + + else + { + ret_val = EOB_ACT_LAST_MATCH; + YY_CURRENT_BUFFER_LVALUE->yy_buffer_status = + YY_BUFFER_EOF_PENDING; + } + } + + else + ret_val = EOB_ACT_CONTINUE_SCAN; + + if (((yy_n_chars) + number_to_move) > YY_CURRENT_BUFFER_LVALUE->yy_buf_size) { + /* Extend the array by 50%, plus the number we really need. */ + yy_size_t new_size = (yy_n_chars) + number_to_move + ((yy_n_chars) >> 1); + YY_CURRENT_BUFFER_LVALUE->yy_ch_buf = (char *) yyrealloc( + (void *) YY_CURRENT_BUFFER_LVALUE->yy_ch_buf, (yy_size_t) new_size ); + if ( ! YY_CURRENT_BUFFER_LVALUE->yy_ch_buf ) + YY_FATAL_ERROR( "out of dynamic memory in yy_get_next_buffer()" ); + /* "- 2" to take care of EOB's */ + YY_CURRENT_BUFFER_LVALUE->yy_buf_size = (int) (new_size - 2); + } + + (yy_n_chars) += number_to_move; + YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[(yy_n_chars)] = YY_END_OF_BUFFER_CHAR; + YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[(yy_n_chars) + 1] = YY_END_OF_BUFFER_CHAR; + + (yytext_ptr) = &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[0]; + + return ret_val; +} + +/* yy_get_previous_state - get the state just before the EOB char was reached */ + + static yy_state_type yy_get_previous_state (void) +{ + yy_state_type yy_current_state; + char *yy_cp; + + yy_current_state = (yy_start); + + for ( yy_cp = (yytext_ptr) + YY_MORE_ADJ; yy_cp < (yy_c_buf_p); ++yy_cp ) + { + YY_CHAR yy_c = (*yy_cp ? yy_ec[YY_SC_TO_UI(*yy_cp)] : 1); + if ( yy_accept[yy_current_state] ) + { + (yy_last_accepting_state) = yy_current_state; + (yy_last_accepting_cpos) = yy_cp; + } + while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) + { + yy_current_state = (int) yy_def[yy_current_state]; + if ( yy_current_state >= 23 ) + yy_c = yy_meta[yy_c]; + } + yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c]; + } + + return yy_current_state; +} + +/* yy_try_NUL_trans - try to make a transition on the NUL character + * + * synopsis + * next_state = yy_try_NUL_trans( current_state ); + */ + static yy_state_type yy_try_NUL_trans (yy_state_type yy_current_state ) +{ + int yy_is_jam; + char *yy_cp = (yy_c_buf_p); + + YY_CHAR yy_c = 1; + if ( yy_accept[yy_current_state] ) + { + (yy_last_accepting_state) = yy_current_state; + (yy_last_accepting_cpos) = yy_cp; + } + while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) + { + yy_current_state = (int) yy_def[yy_current_state]; + if ( yy_current_state >= 23 ) + yy_c = yy_meta[yy_c]; + } + yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c]; + yy_is_jam = (yy_current_state == 22); + + return yy_is_jam ? 0 : yy_current_state; +} + +#ifndef YY_NO_UNPUT + +#endif + +#ifndef YY_NO_INPUT +#ifdef __cplusplus + static int yyinput (void) +#else + static int input (void) +#endif + +{ + int c; + + *(yy_c_buf_p) = (yy_hold_char); + + if ( *(yy_c_buf_p) == YY_END_OF_BUFFER_CHAR ) + { + /* yy_c_buf_p now points to the character we want to return. + * If this occurs *before* the EOB characters, then it's a + * valid NUL; if not, then we've hit the end of the buffer. + */ + if ( (yy_c_buf_p) < &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[(yy_n_chars)] ) + /* This was really a NUL. */ + *(yy_c_buf_p) = '\0'; + + else + { /* need more input */ + yy_size_t offset = (yy_c_buf_p) - (yytext_ptr); + ++(yy_c_buf_p); + + switch ( yy_get_next_buffer( ) ) + { + case EOB_ACT_LAST_MATCH: + /* This happens because yy_g_n_b() + * sees that we've accumulated a + * token and flags that we need to + * try matching the token before + * proceeding. But for input(), + * there's no matching to consider. + * So convert the EOB_ACT_LAST_MATCH + * to EOB_ACT_END_OF_FILE. + */ + + /* Reset buffer status. */ + yyrestart( yyin ); + + /*FALLTHROUGH*/ + + case EOB_ACT_END_OF_FILE: + { + if ( yywrap( ) ) + return 0; + + if ( ! (yy_did_buffer_switch_on_eof) ) + YY_NEW_FILE; +#ifdef __cplusplus + return yyinput(); +#else + return input(); +#endif + } + + case EOB_ACT_CONTINUE_SCAN: + (yy_c_buf_p) = (yytext_ptr) + offset; + break; + } + } + } + + c = *(unsigned char *) (yy_c_buf_p); /* cast for 8-bit char's */ + *(yy_c_buf_p) = '\0'; /* preserve yytext */ + (yy_hold_char) = *++(yy_c_buf_p); + + return c; +} +#endif /* ifndef YY_NO_INPUT */ + +/** Immediately switch to a different input stream. + * @param input_file A readable stream. + * + * @note This function does not reset the start condition to @c INITIAL . + */ + void yyrestart (FILE * input_file ) +{ + + if ( ! YY_CURRENT_BUFFER ){ + yyensure_buffer_stack (); + YY_CURRENT_BUFFER_LVALUE = + yy_create_buffer( yyin, YY_BUF_SIZE ); + } + + yy_init_buffer( YY_CURRENT_BUFFER, input_file ); + yy_load_buffer_state( ); +} + +/** Switch to a different input buffer. + * @param new_buffer The new input buffer. + * + */ + void yy_switch_to_buffer (YY_BUFFER_STATE new_buffer ) +{ + + /* TODO. We should be able to replace this entire function body + * with + * yypop_buffer_state(); + * yypush_buffer_state(new_buffer); + */ + yyensure_buffer_stack (); + if ( YY_CURRENT_BUFFER == new_buffer ) + return; + + if ( YY_CURRENT_BUFFER ) + { + /* Flush out information for old buffer. */ + *(yy_c_buf_p) = (yy_hold_char); + YY_CURRENT_BUFFER_LVALUE->yy_buf_pos = (yy_c_buf_p); + YY_CURRENT_BUFFER_LVALUE->yy_n_chars = (yy_n_chars); + } + + YY_CURRENT_BUFFER_LVALUE = new_buffer; + yy_load_buffer_state( ); + + /* We don't actually know whether we did this switch during + * EOF (yywrap()) processing, but the only time this flag + * is looked at is after yywrap() is called, so it's safe + * to go ahead and always set it. + */ + (yy_did_buffer_switch_on_eof) = 1; +} + +static void yy_load_buffer_state (void) +{ + (yy_n_chars) = YY_CURRENT_BUFFER_LVALUE->yy_n_chars; + (yytext_ptr) = (yy_c_buf_p) = YY_CURRENT_BUFFER_LVALUE->yy_buf_pos; + yyin = YY_CURRENT_BUFFER_LVALUE->yy_input_file; + (yy_hold_char) = *(yy_c_buf_p); +} + +/** Allocate and initialize an input buffer state. + * @param file A readable stream. + * @param size The character buffer size in bytes. When in doubt, use @c YY_BUF_SIZE. + * + * @return the allocated buffer state. + */ + YY_BUFFER_STATE yy_create_buffer (FILE * file, int size ) +{ + YY_BUFFER_STATE b; + + b = (YY_BUFFER_STATE) yyalloc( sizeof( struct yy_buffer_state ) ); + if ( ! b ) + YY_FATAL_ERROR( "out of dynamic memory in yy_create_buffer()" ); + + b->yy_buf_size = size; + + /* yy_ch_buf has to be 2 characters longer than the size given because + * we need to put in 2 end-of-buffer characters. + */ + b->yy_ch_buf = (char *) yyalloc( (yy_size_t) (b->yy_buf_size + 2) ); + if ( ! b->yy_ch_buf ) + YY_FATAL_ERROR( "out of dynamic memory in yy_create_buffer()" ); + + b->yy_is_our_buffer = 1; + + yy_init_buffer( b, file ); + + return b; +} + +/** Destroy the buffer. + * @param b a buffer created with yy_create_buffer() + * + */ + void yy_delete_buffer (YY_BUFFER_STATE b ) +{ + + if ( ! b ) + return; + + if ( b == YY_CURRENT_BUFFER ) /* Not sure if we should pop here. */ + YY_CURRENT_BUFFER_LVALUE = (YY_BUFFER_STATE) 0; + + if ( b->yy_is_our_buffer ) + yyfree( (void *) b->yy_ch_buf ); + + yyfree( (void *) b ); +} + +/* Initializes or reinitializes a buffer. + * This function is sometimes called more than once on the same buffer, + * such as during a yyrestart() or at EOF. + */ + static void yy_init_buffer (YY_BUFFER_STATE b, FILE * file ) + +{ + int oerrno = errno; + + yy_flush_buffer( b ); + + b->yy_input_file = file; + b->yy_fill_buffer = 1; + + /* If b is the current buffer, then yy_init_buffer was _probably_ + * called from yyrestart() or through yy_get_next_buffer. + * In that case, we don't want to reset the lineno or column. + */ + if (b != YY_CURRENT_BUFFER){ + b->yy_bs_lineno = 1; + b->yy_bs_column = 0; + } + + b->yy_is_interactive = file ? (isatty( fileno(file) ) > 0) : 0; + + errno = oerrno; +} + +/** Discard all buffered characters. On the next scan, YY_INPUT will be called. + * @param b the buffer state to be flushed, usually @c YY_CURRENT_BUFFER. + * + */ + void yy_flush_buffer (YY_BUFFER_STATE b ) +{ + if ( ! b ) + return; + + b->yy_n_chars = 0; + + /* We always need two end-of-buffer characters. The first causes + * a transition to the end-of-buffer state. The second causes + * a jam in that state. + */ + b->yy_ch_buf[0] = YY_END_OF_BUFFER_CHAR; + b->yy_ch_buf[1] = YY_END_OF_BUFFER_CHAR; + + b->yy_buf_pos = &b->yy_ch_buf[0]; + + b->yy_at_bol = 1; + b->yy_buffer_status = YY_BUFFER_NEW; + + if ( b == YY_CURRENT_BUFFER ) + yy_load_buffer_state( ); +} + +/** Pushes the new state onto the stack. The new state becomes + * the current state. This function will allocate the stack + * if necessary. + * @param new_buffer The new state. + * + */ +void yypush_buffer_state (YY_BUFFER_STATE new_buffer ) +{ + if (new_buffer == NULL) + return; + + yyensure_buffer_stack(); + + /* This block is copied from yy_switch_to_buffer. */ + if ( YY_CURRENT_BUFFER ) + { + /* Flush out information for old buffer. */ + *(yy_c_buf_p) = (yy_hold_char); + YY_CURRENT_BUFFER_LVALUE->yy_buf_pos = (yy_c_buf_p); + YY_CURRENT_BUFFER_LVALUE->yy_n_chars = (yy_n_chars); + } + + /* Only push if top exists. Otherwise, replace top. */ + if (YY_CURRENT_BUFFER) + (yy_buffer_stack_top)++; + YY_CURRENT_BUFFER_LVALUE = new_buffer; + + /* copied from yy_switch_to_buffer. */ + yy_load_buffer_state( ); + (yy_did_buffer_switch_on_eof) = 1; +} + +/** Removes and deletes the top of the stack, if present. + * The next element becomes the new top. + * + */ +void yypop_buffer_state (void) +{ + if (!YY_CURRENT_BUFFER) + return; + + yy_delete_buffer(YY_CURRENT_BUFFER ); + YY_CURRENT_BUFFER_LVALUE = NULL; + if ((yy_buffer_stack_top) > 0) + --(yy_buffer_stack_top); + + if (YY_CURRENT_BUFFER) { + yy_load_buffer_state( ); + (yy_did_buffer_switch_on_eof) = 1; + } +} + +/* Allocates the stack if it does not exist. + * Guarantees space for at least one push. + */ +static void yyensure_buffer_stack (void) +{ + yy_size_t num_to_alloc; + + if (!(yy_buffer_stack)) { + + /* First allocation is just for 2 elements, since we don't know if this + * scanner will even need a stack. We use 2 instead of 1 to avoid an + * immediate realloc on the next call. + */ + num_to_alloc = 1; /* After all that talk, this was set to 1 anyways... */ + (yy_buffer_stack) = (struct yy_buffer_state**)yyalloc + (num_to_alloc * sizeof(struct yy_buffer_state*) + ); + if ( ! (yy_buffer_stack) ) + YY_FATAL_ERROR( "out of dynamic memory in yyensure_buffer_stack()" ); + + memset((yy_buffer_stack), 0, num_to_alloc * sizeof(struct yy_buffer_state*)); + + (yy_buffer_stack_max) = num_to_alloc; + (yy_buffer_stack_top) = 0; + return; + } + + if ((yy_buffer_stack_top) >= ((yy_buffer_stack_max)) - 1){ + + /* Increase the buffer to prepare for a possible push. */ + yy_size_t grow_size = 8 /* arbitrary grow size */; + + num_to_alloc = (yy_buffer_stack_max) + grow_size; + (yy_buffer_stack) = (struct yy_buffer_state**)yyrealloc + ((yy_buffer_stack), + num_to_alloc * sizeof(struct yy_buffer_state*) + ); + if ( ! (yy_buffer_stack) ) + YY_FATAL_ERROR( "out of dynamic memory in yyensure_buffer_stack()" ); + + /* zero only the new slots.*/ + memset((yy_buffer_stack) + (yy_buffer_stack_max), 0, grow_size * sizeof(struct yy_buffer_state*)); + (yy_buffer_stack_max) = num_to_alloc; + } +} + +/** Setup the input buffer state to scan directly from a user-specified character buffer. + * @param base the character buffer + * @param size the size in bytes of the character buffer + * + * @return the newly allocated buffer state object. + */ +YY_BUFFER_STATE yy_scan_buffer (char * base, yy_size_t size ) +{ + YY_BUFFER_STATE b; + + if ( size < 2 || + base[size-2] != YY_END_OF_BUFFER_CHAR || + base[size-1] != YY_END_OF_BUFFER_CHAR ) + /* They forgot to leave room for the EOB's. */ + return NULL; + + b = (YY_BUFFER_STATE) yyalloc( sizeof( struct yy_buffer_state ) ); + if ( ! b ) + YY_FATAL_ERROR( "out of dynamic memory in yy_scan_buffer()" ); + + b->yy_buf_size = (int) (size - 2); /* "- 2" to take care of EOB's */ + b->yy_buf_pos = b->yy_ch_buf = base; + b->yy_is_our_buffer = 0; + b->yy_input_file = NULL; + b->yy_n_chars = b->yy_buf_size; + b->yy_is_interactive = 0; + b->yy_at_bol = 1; + b->yy_fill_buffer = 0; + b->yy_buffer_status = YY_BUFFER_NEW; + + yy_switch_to_buffer( b ); + + return b; +} + +/** Setup the input buffer state to scan a string. The next call to yylex() will + * scan from a @e copy of @a str. + * @param yystr a NUL-terminated string to scan + * + * @return the newly allocated buffer state object. + * @note If you want to scan bytes that may contain NUL values, then use + * yy_scan_bytes() instead. + */ +YY_BUFFER_STATE yy_scan_string (const char * yystr ) +{ + + return yy_scan_bytes( yystr, (int) strlen(yystr) ); +} + +/** Setup the input buffer state to scan the given bytes. The next call to yylex() will + * scan from a @e copy of @a bytes. + * @param yybytes the byte buffer to scan + * @param _yybytes_len the number of bytes in the buffer pointed to by @a bytes. + * + * @return the newly allocated buffer state object. + */ +YY_BUFFER_STATE yy_scan_bytes (const char * yybytes, yy_size_t _yybytes_len ) +{ + YY_BUFFER_STATE b; + char *buf; + yy_size_t n; + yy_size_t i; + + /* Get memory for full buffer, including space for trailing EOB's. */ + n = (yy_size_t) (_yybytes_len + 2); + buf = (char *) yyalloc( n ); + if ( ! buf ) + YY_FATAL_ERROR( "out of dynamic memory in yy_scan_bytes()" ); + + for ( i = 0; i < _yybytes_len; ++i ) + buf[i] = yybytes[i]; + + buf[_yybytes_len] = buf[_yybytes_len+1] = YY_END_OF_BUFFER_CHAR; + + b = yy_scan_buffer( buf, n ); + if ( ! b ) + YY_FATAL_ERROR( "bad buffer in yy_scan_bytes()" ); + + /* It's okay to grow etc. this buffer, and we should throw it + * away when we're done. + */ + b->yy_is_our_buffer = 1; + + return b; +} + +#ifndef YY_EXIT_FAILURE +#define YY_EXIT_FAILURE 2 +#endif + +static void yynoreturn yy_fatal_error (const char* msg ) +{ + fprintf( stderr, "%s\n", msg ); + exit( YY_EXIT_FAILURE ); +} + +/* Redefine yyless() so it works in section 3 code. */ + +#undef yyless +#define yyless(n) \ + do \ + { \ + /* Undo effects of setting up yytext. */ \ + yy_size_t yyless_macro_arg = (n); \ + YY_LESS_LINENO(yyless_macro_arg);\ + yytext[yyleng] = (yy_hold_char); \ + (yy_c_buf_p) = yytext + yyless_macro_arg; \ + (yy_hold_char) = *(yy_c_buf_p); \ + *(yy_c_buf_p) = '\0'; \ + yyleng = yyless_macro_arg; \ + } \ + while ( 0 ) + +/* Accessor methods (get/set functions) to struct members. */ + +/** Get the current line number. + * + */ +int yyget_lineno (void) +{ + + return yylineno; +} + +/** Get the input stream. + * + */ +FILE *yyget_in (void) +{ + return yyin; +} + +/** Get the output stream. + * + */ +FILE *yyget_out (void) +{ + return yyout; +} + +/** Get the length of the current token. + * + */ +yy_size_t yyget_leng (void) +{ + return yyleng; +} + +/** Get the current token. + * + */ + +char *yyget_text (void) +{ + return yytext; +} + +/** Set the current line number. + * @param _line_number line number + * + */ +void yyset_lineno (int _line_number ) +{ + + yylineno = _line_number; +} + +/** Set the input stream. This does not discard the current + * input buffer. + * @param _in_str A readable stream. + * + * @see yy_switch_to_buffer + */ +void yyset_in (FILE * _in_str ) +{ + yyin = _in_str ; +} + +void yyset_out (FILE * _out_str ) +{ + yyout = _out_str ; +} + +int yyget_debug (void) +{ + return yy_flex_debug; +} + +void yyset_debug (int _bdebug ) +{ + yy_flex_debug = _bdebug ; +} + +static int yy_init_globals (void) +{ + /* Initialization is the same as for the non-reentrant scanner. + * This function is called from yylex_destroy(), so don't allocate here. + */ + + (yy_buffer_stack) = NULL; + (yy_buffer_stack_top) = 0; + (yy_buffer_stack_max) = 0; + (yy_c_buf_p) = NULL; + (yy_init) = 0; + (yy_start) = 0; + +/* Defined in main.c */ +#ifdef YY_STDINIT + yyin = stdin; + yyout = stdout; +#else + yyin = NULL; + yyout = NULL; +#endif + + /* For future reference: Set errno on error, since we are called by + * yylex_init() + */ + return 0; +} + +/* yylex_destroy is for both reentrant and non-reentrant scanners. */ +int yylex_destroy (void) +{ + + /* Pop the buffer stack, destroying each element. */ + while(YY_CURRENT_BUFFER){ + yy_delete_buffer( YY_CURRENT_BUFFER ); + YY_CURRENT_BUFFER_LVALUE = NULL; + yypop_buffer_state(); + } + + /* Destroy the stack itself. */ + yyfree((yy_buffer_stack) ); + (yy_buffer_stack) = NULL; + + /* Reset the globals. This is important in a non-reentrant scanner so the next time + * yylex() is called, initialization will occur. */ + yy_init_globals( ); + + return 0; +} + +/* + * Internal utility routines. + */ + +#ifndef yytext_ptr +static void yy_flex_strncpy (char* s1, const char * s2, int n ) +{ + + int i; + for ( i = 0; i < n; ++i ) + s1[i] = s2[i]; +} +#endif + +#ifdef YY_NEED_STRLEN +static int yy_flex_strlen (const char * s ) +{ + int n; + for ( n = 0; s[n]; ++n ) + ; + + return n; +} +#endif + +void *yyalloc (yy_size_t size ) +{ + return malloc(size); +} + +void *yyrealloc (void * ptr, yy_size_t size ) +{ + + /* The cast to (char *) in the following accommodates both + * implementations that use char* generic pointers, and those + * that use void* generic pointers. It works with the latter + * because both ANSI C and C++ allow castless assignment from + * any pointer type to void*, and deal with argument conversions + * as though doing an assignment. + */ + return realloc(ptr, size); +} + +void yyfree (void * ptr ) +{ + free( (char *) ptr ); /* see yyrealloc() for (char *) cast */ +} + +#define YYTABLES_NAME "yytables" + +#line 137 "../lexer.l" + + +adios2::detail::parser::symbol_type +make_INT (const std::string &s, const adios2::detail::parser::location_type& loc) +{ + errno = 0; + long n = strtol (s.c_str(), NULL, 10); + if (! (INT_MIN <= n && n <= INT_MAX && errno != ERANGE)) + throw adios2::detail::parser::syntax_error (loc, "integer is out of range: " + s); + return adios2::detail::parser::make_INT ((int) n, loc); +} + +void +adios2::detail::ASTDriver::parse (const std::string input) +{ + adios2::detail::parser parse (*this); + yy_scan_string(input.c_str()); + parse.set_debug_level (trace_parsing); + parse (); +} + diff --git a/source/adios2/toolkit/derived/parser/lexer.h b/source/adios2/toolkit/derived/parser/pregen-source/lexer.h similarity index 71% rename from source/adios2/toolkit/derived/parser/lexer.h rename to source/adios2/toolkit/derived/parser/pregen-source/lexer.h index 75c3a14657..c83d2213a8 100644 --- a/source/adios2/toolkit/derived/parser/lexer.h +++ b/source/adios2/toolkit/derived/parser/pregen-source/lexer.h @@ -6,7 +6,7 @@ #line 7 "lexer.h" -#define YY_INT_ALIGNED short int +#define YY_INT_ALIGNED short int /* A lexical scanner generated by flex */ @@ -21,10 +21,10 @@ /* First, we deal with platform-specific or compiler-specific issues. */ /* begin standard C headers. */ -#include #include -#include #include +#include +#include /* end standard C headers. */ @@ -35,10 +35,10 @@ /* C99 systems have . Non-C99 systems may or may not. */ -#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L +#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 says to define __STDC_LIMIT_MACROS before including stdint.h, - * if you want the limit (max/min) macros for int types. + * if you want the limit (max/min) macros for int types. */ #ifndef __STDC_LIMIT_MACROS #define __STDC_LIMIT_MACROS 1 @@ -56,41 +56,41 @@ typedef uint64_t flex_uint64_t; typedef signed char flex_int8_t; typedef short int flex_int16_t; typedef int flex_int32_t; -typedef unsigned char flex_uint8_t; +typedef unsigned char flex_uint8_t; typedef unsigned short int flex_uint16_t; typedef unsigned int flex_uint32_t; /* Limits of integral types. */ #ifndef INT8_MIN -#define INT8_MIN (-128) +#define INT8_MIN (-128) #endif #ifndef INT16_MIN -#define INT16_MIN (-32767 - 1) +#define INT16_MIN (-32767-1) #endif #ifndef INT32_MIN -#define INT32_MIN (-2147483647 - 1) +#define INT32_MIN (-2147483647-1) #endif #ifndef INT8_MAX -#define INT8_MAX (127) +#define INT8_MAX (127) #endif #ifndef INT16_MAX -#define INT16_MAX (32767) +#define INT16_MAX (32767) #endif #ifndef INT32_MAX -#define INT32_MAX (2147483647) +#define INT32_MAX (2147483647) #endif #ifndef UINT8_MAX -#define UINT8_MAX (255U) +#define UINT8_MAX (255U) #endif #ifndef UINT16_MAX -#define UINT16_MAX (65535U) +#define UINT16_MAX (65535U) #endif #ifndef UINT32_MAX -#define UINT32_MAX (4294967295U) +#define UINT32_MAX (4294967295U) #endif #ifndef SIZE_MAX -#define SIZE_MAX (~(size_t)0) +#define SIZE_MAX (~(size_t)0) #endif #endif /* ! C99 */ @@ -138,72 +138,73 @@ extern FILE *yyin, *yyout; #ifndef YY_STRUCT_YY_BUFFER_STATE #define YY_STRUCT_YY_BUFFER_STATE struct yy_buffer_state -{ - FILE *yy_input_file; - - char *yy_ch_buf; /* input buffer */ - char *yy_buf_pos; /* current position in input buffer */ - - /* Size of input buffer in bytes, not including room for EOB - * characters. - */ - int yy_buf_size; - - /* Number of characters read into yy_ch_buf, not including EOB - * characters. - */ - yy_size_t yy_n_chars; - - /* Whether we "own" the buffer - i.e., we know we created it, - * and can realloc() it to grow it, and should free() it to - * delete it. - */ - int yy_is_our_buffer; - - /* Whether this is an "interactive" input source; if so, and - * if we're using stdio for input, then we want to use getc() - * instead of fread(), to make sure we stop fetching input after - * each newline. - */ - int yy_is_interactive; - - /* Whether we're considered to be at the beginning of a line. - * If so, '^' rules will be active on the next match, otherwise - * not. - */ - int yy_at_bol; + { + FILE *yy_input_file; + + char *yy_ch_buf; /* input buffer */ + char *yy_buf_pos; /* current position in input buffer */ + + /* Size of input buffer in bytes, not including room for EOB + * characters. + */ + int yy_buf_size; + + /* Number of characters read into yy_ch_buf, not including EOB + * characters. + */ + yy_size_t yy_n_chars; + + /* Whether we "own" the buffer - i.e., we know we created it, + * and can realloc() it to grow it, and should free() it to + * delete it. + */ + int yy_is_our_buffer; + + /* Whether this is an "interactive" input source; if so, and + * if we're using stdio for input, then we want to use getc() + * instead of fread(), to make sure we stop fetching input after + * each newline. + */ + int yy_is_interactive; + + /* Whether we're considered to be at the beginning of a line. + * If so, '^' rules will be active on the next match, otherwise + * not. + */ + int yy_at_bol; int yy_bs_lineno; /**< The line count. */ int yy_bs_column; /**< The column count. */ - /* Whether to try to fill the input buffer when we reach the - * end of it. - */ - int yy_fill_buffer; + /* Whether to try to fill the input buffer when we reach the + * end of it. + */ + int yy_fill_buffer; + + int yy_buffer_status; - int yy_buffer_status; -}; + }; #endif /* !YY_STRUCT_YY_BUFFER_STATE */ -void yyrestart(FILE *input_file); -void yy_switch_to_buffer(YY_BUFFER_STATE new_buffer); -YY_BUFFER_STATE yy_create_buffer(FILE *file, int size); -void yy_delete_buffer(YY_BUFFER_STATE b); -void yy_flush_buffer(YY_BUFFER_STATE b); -void yypush_buffer_state(YY_BUFFER_STATE new_buffer); -void yypop_buffer_state(void); +void yyrestart ( FILE *input_file ); +void yy_switch_to_buffer ( YY_BUFFER_STATE new_buffer ); +YY_BUFFER_STATE yy_create_buffer ( FILE *file, int size ); +void yy_delete_buffer ( YY_BUFFER_STATE b ); +void yy_flush_buffer ( YY_BUFFER_STATE b ); +void yypush_buffer_state ( YY_BUFFER_STATE new_buffer ); +void yypop_buffer_state ( void ); -YY_BUFFER_STATE yy_scan_buffer(char *base, yy_size_t size); -YY_BUFFER_STATE yy_scan_string(const char *yy_str); -YY_BUFFER_STATE yy_scan_bytes(const char *bytes, yy_size_t len); +YY_BUFFER_STATE yy_scan_buffer ( char *base, yy_size_t size ); +YY_BUFFER_STATE yy_scan_string ( const char *yy_str ); +YY_BUFFER_STATE yy_scan_bytes ( const char *bytes, yy_size_t len ); -void *yyalloc(yy_size_t); -void *yyrealloc(void *, yy_size_t); -void yyfree(void *); +void *yyalloc ( yy_size_t ); +void *yyrealloc ( void *, yy_size_t ); +void yyfree ( void * ); /* Begin user sect3 */ -#define yywrap() (/*CONSTCOND*/ 1) +#define yywrap() (/*CONSTCOND*/1) #define YY_SKIP_YYWRAP extern int yylineno; @@ -234,31 +235,31 @@ extern char *yytext; /* Accessor methods to globals. These are made visible to non-reentrant scanners for convenience. */ -int yylex_destroy(void); +int yylex_destroy ( void ); -int yyget_debug(void); +int yyget_debug ( void ); -void yyset_debug(int debug_flag); +void yyset_debug ( int debug_flag ); -YY_EXTRA_TYPE yyget_extra(void); +YY_EXTRA_TYPE yyget_extra ( void ); -void yyset_extra(YY_EXTRA_TYPE user_defined); +void yyset_extra ( YY_EXTRA_TYPE user_defined ); -FILE *yyget_in(void); +FILE *yyget_in ( void ); -void yyset_in(FILE *_in_str); +void yyset_in ( FILE * _in_str ); -FILE *yyget_out(void); +FILE *yyget_out ( void ); -void yyset_out(FILE *_out_str); +void yyset_out ( FILE * _out_str ); -yy_size_t yyget_leng(void); + yy_size_t yyget_leng ( void ); -char *yyget_text(void); +char *yyget_text ( void ); -int yyget_lineno(void); +int yyget_lineno ( void ); -void yyset_lineno(int _line_number); +void yyset_lineno ( int _line_number ); /* Macros after this point can all be overridden by user definitions in * section 1. @@ -266,18 +267,18 @@ void yyset_lineno(int _line_number); #ifndef YY_SKIP_YYWRAP #ifdef __cplusplus -extern "C" int yywrap(void); +extern "C" int yywrap ( void ); #else -extern int yywrap(void); +extern int yywrap ( void ); #endif #endif #ifndef yytext_ptr -static void yy_flex_strncpy(char *, const char *, int); +static void yy_flex_strncpy ( char *, const char *, int ); #endif #ifdef YY_NEED_STRLEN -static int yy_flex_strlen(const char *); +static int yy_flex_strlen ( const char * ); #endif #ifndef YY_NO_INPUT @@ -305,9 +306,9 @@ static int yy_flex_strlen(const char *); #ifndef YY_DECL #define YY_DECL_IS_OURS 1 -extern int yylex(void); +extern int yylex (void); -#define YY_DECL int yylex(void) +#define YY_DECL int yylex (void) #endif /* !YY_DECL */ /* yy_get_previous_state - get the state just before the EOB char was reached */ @@ -469,7 +470,8 @@ extern int yylex(void); #undef yyTABLES_NAME #endif -#line 47 "lexer.l" +#line 137 "../lexer.l" + #line 476 "lexer.h" #undef yyIN_HEADER diff --git a/source/adios2/toolkit/derived/parser/pregen-source/location.hh b/source/adios2/toolkit/derived/parser/pregen-source/location.hh new file mode 100644 index 0000000000..923c49c812 --- /dev/null +++ b/source/adios2/toolkit/derived/parser/pregen-source/location.hh @@ -0,0 +1,306 @@ +// A Bison parser, made by GNU Bison 3.8.2. + +// Locations for Bison parsers in C++ + +// Copyright (C) 2002-2015, 2018-2021 Free Software Foundation, Inc. + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +// As a special exception, you may create a larger work that contains +// part or all of the Bison parser skeleton and distribute that work +// under terms of your choice, so long as that work isn't itself a +// parser generator using the skeleton or a modified version thereof +// as a parser skeleton. Alternatively, if you modify or redistribute +// the parser skeleton itself, you may (at your option) remove this +// special exception, which will cause the skeleton and the resulting +// Bison output files to be licensed under the GNU General Public +// License without this special exception. + +// This special exception was added by the Free Software Foundation in +// version 2.2 of Bison. + +/** + ** \file location.hh + ** Define the adios2::detail::location class. + */ + +#ifndef YY_YY_LOCATION_HH_INCLUDED +# define YY_YY_LOCATION_HH_INCLUDED + +# include +# include + +# ifndef YY_NULLPTR +# if defined __cplusplus +# if 201103L <= __cplusplus +# define YY_NULLPTR nullptr +# else +# define YY_NULLPTR 0 +# endif +# else +# define YY_NULLPTR ((void*)0) +# endif +# endif + +#line 6 "../parser.y" +namespace adios2 { namespace detail { +#line 59 "location.hh" + + /// A point in a source file. + class position + { + public: + /// Type for file name. + typedef const std::string filename_type; + /// Type for line and column numbers. + typedef int counter_type; + + /// Construct a position. + explicit position (filename_type* f = YY_NULLPTR, + counter_type l = 1, + counter_type c = 1) + : filename (f) + , line (l) + , column (c) + {} + + + /// Initialization. + void initialize (filename_type* fn = YY_NULLPTR, + counter_type l = 1, + counter_type c = 1) + { + filename = fn; + line = l; + column = c; + } + + /** \name Line and Column related manipulators + ** \{ */ + /// (line related) Advance to the COUNT next lines. + void lines (counter_type count = 1) + { + if (count) + { + column = 1; + line = add_ (line, count, 1); + } + } + + /// (column related) Advance to the COUNT next columns. + void columns (counter_type count = 1) + { + column = add_ (column, count, 1); + } + /** \} */ + + /// File name to which this position refers. + filename_type* filename; + /// Current line number. + counter_type line; + /// Current column number. + counter_type column; + + private: + /// Compute max (min, lhs+rhs). + static counter_type add_ (counter_type lhs, counter_type rhs, counter_type min) + { + return lhs + rhs < min ? min : lhs + rhs; + } + }; + + /// Add \a width columns, in place. + inline position& + operator+= (position& res, position::counter_type width) + { + res.columns (width); + return res; + } + + /// Add \a width columns. + inline position + operator+ (position res, position::counter_type width) + { + return res += width; + } + + /// Subtract \a width columns, in place. + inline position& + operator-= (position& res, position::counter_type width) + { + return res += -width; + } + + /// Subtract \a width columns. + inline position + operator- (position res, position::counter_type width) + { + return res -= width; + } + + /** \brief Intercept output stream redirection. + ** \param ostr the destination output stream + ** \param pos a reference to the position to redirect + */ + template + std::basic_ostream& + operator<< (std::basic_ostream& ostr, const position& pos) + { + if (pos.filename) + ostr << *pos.filename << ':'; + return ostr << pos.line << '.' << pos.column; + } + + /// Two points in a source file. + class location + { + public: + /// Type for file name. + typedef position::filename_type filename_type; + /// Type for line and column numbers. + typedef position::counter_type counter_type; + + /// Construct a location from \a b to \a e. + location (const position& b, const position& e) + : begin (b) + , end (e) + {} + + /// Construct a 0-width location in \a p. + explicit location (const position& p = position ()) + : begin (p) + , end (p) + {} + + /// Construct a 0-width location in \a f, \a l, \a c. + explicit location (filename_type* f, + counter_type l = 1, + counter_type c = 1) + : begin (f, l, c) + , end (f, l, c) + {} + + + /// Initialization. + void initialize (filename_type* f = YY_NULLPTR, + counter_type l = 1, + counter_type c = 1) + { + begin.initialize (f, l, c); + end = begin; + } + + /** \name Line and Column related manipulators + ** \{ */ + public: + /// Reset initial location to final location. + void step () + { + begin = end; + } + + /// Extend the current location to the COUNT next columns. + void columns (counter_type count = 1) + { + end += count; + } + + /// Extend the current location to the COUNT next lines. + void lines (counter_type count = 1) + { + end.lines (count); + } + /** \} */ + + + public: + /// Beginning of the located region. + position begin; + /// End of the located region. + position end; + }; + + /// Join two locations, in place. + inline location& + operator+= (location& res, const location& end) + { + res.end = end.end; + return res; + } + + /// Join two locations. + inline location + operator+ (location res, const location& end) + { + return res += end; + } + + /// Add \a width columns to the end position, in place. + inline location& + operator+= (location& res, location::counter_type width) + { + res.columns (width); + return res; + } + + /// Add \a width columns to the end position. + inline location + operator+ (location res, location::counter_type width) + { + return res += width; + } + + /// Subtract \a width columns to the end position, in place. + inline location& + operator-= (location& res, location::counter_type width) + { + return res += -width; + } + + /// Subtract \a width columns to the end position. + inline location + operator- (location res, location::counter_type width) + { + return res -= width; + } + + /** \brief Intercept output stream redirection. + ** \param ostr the destination output stream + ** \param loc a reference to the location to redirect + ** + ** Avoid duplicate information. + */ + template + std::basic_ostream& + operator<< (std::basic_ostream& ostr, const location& loc) + { + location::counter_type end_col + = 0 < loc.end.column ? loc.end.column - 1 : 0; + ostr << loc.begin; + if (loc.end.filename + && (!loc.begin.filename + || *loc.begin.filename != *loc.end.filename)) + ostr << '-' << loc.end.filename << ':' << loc.end.line << '.' << end_col; + else if (loc.begin.line < loc.end.line) + ostr << '-' << loc.end.line << '.' << end_col; + else if (loc.begin.column < end_col) + ostr << '-' << end_col; + return ostr; + } + +#line 6 "../parser.y" +} } // adios2::detail +#line 305 "location.hh" + +#endif // !YY_YY_LOCATION_HH_INCLUDED diff --git a/source/adios2/toolkit/derived/parser/pregen-source/parser.cpp b/source/adios2/toolkit/derived/parser/pregen-source/parser.cpp new file mode 100644 index 0000000000..e11401c54d --- /dev/null +++ b/source/adios2/toolkit/derived/parser/pregen-source/parser.cpp @@ -0,0 +1,1414 @@ +// A Bison parser, made by GNU Bison 3.8.2. + +// Skeleton implementation for Bison LALR(1) parsers in C++ + +// Copyright (C) 2002-2015, 2018-2021 Free Software Foundation, Inc. + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +// As a special exception, you may create a larger work that contains +// part or all of the Bison parser skeleton and distribute that work +// under terms of your choice, so long as that work isn't itself a +// parser generator using the skeleton or a modified version thereof +// as a parser skeleton. Alternatively, if you modify or redistribute +// the parser skeleton itself, you may (at your option) remove this +// special exception, which will cause the skeleton and the resulting +// Bison output files to be licensed under the GNU General Public +// License without this special exception. + +// This special exception was added by the Free Software Foundation in +// version 2.2 of Bison. + +// DO NOT RELY ON FEATURES THAT ARE NOT DOCUMENTED in the manual, +// especially those whose name start with YY_ or yy_. They are +// private implementation details that can be changed or removed. + + + + + +#include "parser.h" + + +// Unqualified %code blocks. +#line 33 "../parser.y" + +#include "ASTDriver.h" +#include "ASTNode.h" + +#line 51 "parser.cpp" + + +#ifndef YY_ +# if defined YYENABLE_NLS && YYENABLE_NLS +# if ENABLE_NLS +# include // FIXME: INFRINGES ON USER NAME SPACE. +# define YY_(msgid) dgettext ("bison-runtime", msgid) +# endif +# endif +# ifndef YY_ +# define YY_(msgid) msgid +# endif +#endif + + +// Whether we are compiled with exception support. +#ifndef YY_EXCEPTIONS +# if defined __GNUC__ && !defined __EXCEPTIONS +# define YY_EXCEPTIONS 0 +# else +# define YY_EXCEPTIONS 1 +# endif +#endif + +#define YYRHSLOC(Rhs, K) ((Rhs)[K].location) +/* YYLLOC_DEFAULT -- Set CURRENT to span from RHS[1] to RHS[N]. + If N is 0, then set CURRENT to the empty location which ends + the previous symbol: RHS[0] (always defined). */ + +# ifndef YYLLOC_DEFAULT +# define YYLLOC_DEFAULT(Current, Rhs, N) \ + do \ + if (N) \ + { \ + (Current).begin = YYRHSLOC (Rhs, 1).begin; \ + (Current).end = YYRHSLOC (Rhs, N).end; \ + } \ + else \ + { \ + (Current).begin = (Current).end = YYRHSLOC (Rhs, 0).end; \ + } \ + while (false) +# endif + + +// Enable debugging if requested. +#if YYDEBUG + +// A pseudo ostream that takes yydebug_ into account. +# define YYCDEBUG if (yydebug_) (*yycdebug_) + +# define YY_SYMBOL_PRINT(Title, Symbol) \ + do { \ + if (yydebug_) \ + { \ + *yycdebug_ << Title << ' '; \ + yy_print_ (*yycdebug_, Symbol); \ + *yycdebug_ << '\n'; \ + } \ + } while (false) + +# define YY_REDUCE_PRINT(Rule) \ + do { \ + if (yydebug_) \ + yy_reduce_print_ (Rule); \ + } while (false) + +# define YY_STACK_PRINT() \ + do { \ + if (yydebug_) \ + yy_stack_print_ (); \ + } while (false) + +#else // !YYDEBUG + +# define YYCDEBUG if (false) std::cerr +# define YY_SYMBOL_PRINT(Title, Symbol) YY_USE (Symbol) +# define YY_REDUCE_PRINT(Rule) static_cast (0) +# define YY_STACK_PRINT() static_cast (0) + +#endif // !YYDEBUG + +#define yyerrok (yyerrstatus_ = 0) +#define yyclearin (yyla.clear ()) + +#define YYACCEPT goto yyacceptlab +#define YYABORT goto yyabortlab +#define YYERROR goto yyerrorlab +#define YYRECOVERING() (!!yyerrstatus_) + +#line 6 "../parser.y" +namespace adios2 { namespace detail { +#line 144 "parser.cpp" + + /// Build a parser object. + parser::parser (ASTDriver& drv_yyarg) +#if YYDEBUG + : yydebug_ (false), + yycdebug_ (&std::cerr), +#else + : +#endif + yy_lac_established_ (false), + drv (drv_yyarg) + {} + + parser::~parser () + {} + + parser::syntax_error::~syntax_error () YY_NOEXCEPT YY_NOTHROW + {} + + /*---------. + | symbol. | + `---------*/ + + + + // by_state. + parser::by_state::by_state () YY_NOEXCEPT + : state (empty_state) + {} + + parser::by_state::by_state (const by_state& that) YY_NOEXCEPT + : state (that.state) + {} + + void + parser::by_state::clear () YY_NOEXCEPT + { + state = empty_state; + } + + void + parser::by_state::move (by_state& that) + { + state = that.state; + that.clear (); + } + + parser::by_state::by_state (state_type s) YY_NOEXCEPT + : state (s) + {} + + parser::symbol_kind_type + parser::by_state::kind () const YY_NOEXCEPT + { + if (state == empty_state) + return symbol_kind::S_YYEMPTY; + else + return YY_CAST (symbol_kind_type, yystos_[+state]); + } + + parser::stack_symbol_type::stack_symbol_type () + {} + + parser::stack_symbol_type::stack_symbol_type (YY_RVREF (stack_symbol_type) that) + : super_type (YY_MOVE (that.state), YY_MOVE (that.location)) + { + switch (that.kind ()) + { + case symbol_kind::S_INT: // "number" + case symbol_kind::S_list: // list + value.YY_MOVE_OR_COPY< int > (YY_MOVE (that.value)); + break; + + case symbol_kind::S_OPERATOR: // OPERATOR + case symbol_kind::S_IDENTIFIER: // "identifier" + case symbol_kind::S_VARNAME: // VARNAME + value.YY_MOVE_OR_COPY< std::string > (YY_MOVE (that.value)); + break; + + case symbol_kind::S_index: // index + value.YY_MOVE_OR_COPY< std::tuple > (YY_MOVE (that.value)); + break; + + case symbol_kind::S_indices_list: // indices_list + value.YY_MOVE_OR_COPY< std::vector> > (YY_MOVE (that.value)); + break; + + default: + break; + } + +#if 201103L <= YY_CPLUSPLUS + // that is emptied. + that.state = empty_state; +#endif + } + + parser::stack_symbol_type::stack_symbol_type (state_type s, YY_MOVE_REF (symbol_type) that) + : super_type (s, YY_MOVE (that.location)) + { + switch (that.kind ()) + { + case symbol_kind::S_INT: // "number" + case symbol_kind::S_list: // list + value.move< int > (YY_MOVE (that.value)); + break; + + case symbol_kind::S_OPERATOR: // OPERATOR + case symbol_kind::S_IDENTIFIER: // "identifier" + case symbol_kind::S_VARNAME: // VARNAME + value.move< std::string > (YY_MOVE (that.value)); + break; + + case symbol_kind::S_index: // index + value.move< std::tuple > (YY_MOVE (that.value)); + break; + + case symbol_kind::S_indices_list: // indices_list + value.move< std::vector> > (YY_MOVE (that.value)); + break; + + default: + break; + } + + // that is emptied. + that.kind_ = symbol_kind::S_YYEMPTY; + } + +#if YY_CPLUSPLUS < 201103L + parser::stack_symbol_type& + parser::stack_symbol_type::operator= (const stack_symbol_type& that) + { + state = that.state; + switch (that.kind ()) + { + case symbol_kind::S_INT: // "number" + case symbol_kind::S_list: // list + value.copy< int > (that.value); + break; + + case symbol_kind::S_OPERATOR: // OPERATOR + case symbol_kind::S_IDENTIFIER: // "identifier" + case symbol_kind::S_VARNAME: // VARNAME + value.copy< std::string > (that.value); + break; + + case symbol_kind::S_index: // index + value.copy< std::tuple > (that.value); + break; + + case symbol_kind::S_indices_list: // indices_list + value.copy< std::vector> > (that.value); + break; + + default: + break; + } + + location = that.location; + return *this; + } + + parser::stack_symbol_type& + parser::stack_symbol_type::operator= (stack_symbol_type& that) + { + state = that.state; + switch (that.kind ()) + { + case symbol_kind::S_INT: // "number" + case symbol_kind::S_list: // list + value.move< int > (that.value); + break; + + case symbol_kind::S_OPERATOR: // OPERATOR + case symbol_kind::S_IDENTIFIER: // "identifier" + case symbol_kind::S_VARNAME: // VARNAME + value.move< std::string > (that.value); + break; + + case symbol_kind::S_index: // index + value.move< std::tuple > (that.value); + break; + + case symbol_kind::S_indices_list: // indices_list + value.move< std::vector> > (that.value); + break; + + default: + break; + } + + location = that.location; + // that is emptied. + that.state = empty_state; + return *this; + } +#endif + + template + void + parser::yy_destroy_ (const char* yymsg, basic_symbol& yysym) const + { + if (yymsg) + YY_SYMBOL_PRINT (yymsg, yysym); + } + +#if YYDEBUG + template + void + parser::yy_print_ (std::ostream& yyo, const basic_symbol& yysym) const + { + std::ostream& yyoutput = yyo; + YY_USE (yyoutput); + if (yysym.empty ()) + yyo << "empty symbol"; + else + { + symbol_kind_type yykind = yysym.kind (); + yyo << (yykind < YYNTOKENS ? "token" : "nterm") + << ' ' << yysym.name () << " (" + << yysym.location << ": "; + YY_USE (yykind); + yyo << ')'; + } + } +#endif + + void + parser::yypush_ (const char* m, YY_MOVE_REF (stack_symbol_type) sym) + { + if (m) + YY_SYMBOL_PRINT (m, sym); + yystack_.push (YY_MOVE (sym)); + } + + void + parser::yypush_ (const char* m, state_type s, YY_MOVE_REF (symbol_type) sym) + { +#if 201103L <= YY_CPLUSPLUS + yypush_ (m, stack_symbol_type (s, std::move (sym))); +#else + stack_symbol_type ss (s, sym); + yypush_ (m, ss); +#endif + } + + void + parser::yypop_ (int n) YY_NOEXCEPT + { + yystack_.pop (n); + } + +#if YYDEBUG + std::ostream& + parser::debug_stream () const + { + return *yycdebug_; + } + + void + parser::set_debug_stream (std::ostream& o) + { + yycdebug_ = &o; + } + + + parser::debug_level_type + parser::debug_level () const + { + return yydebug_; + } + + void + parser::set_debug_level (debug_level_type l) + { + yydebug_ = l; + } +#endif // YYDEBUG + + parser::state_type + parser::yy_lr_goto_state_ (state_type yystate, int yysym) + { + int yyr = yypgoto_[yysym - YYNTOKENS] + yystate; + if (0 <= yyr && yyr <= yylast_ && yycheck_[yyr] == yystate) + return yytable_[yyr]; + else + return yydefgoto_[yysym - YYNTOKENS]; + } + + bool + parser::yy_pact_value_is_default_ (int yyvalue) YY_NOEXCEPT + { + return yyvalue == yypact_ninf_; + } + + bool + parser::yy_table_value_is_error_ (int yyvalue) YY_NOEXCEPT + { + return yyvalue == yytable_ninf_; + } + + int + parser::operator() () + { + return parse (); + } + + int + parser::parse () + { + int yyn; + /// Length of the RHS of the rule being reduced. + int yylen = 0; + + // Error handling. + int yynerrs_ = 0; + int yyerrstatus_ = 0; + + /// The lookahead symbol. + symbol_type yyla; + + /// The locations where the error started and ended. + stack_symbol_type yyerror_range[3]; + + /// The return value of parse (). + int yyresult; + + // Discard the LAC context in case there still is one left from a + // previous invocation. + yy_lac_discard_ ("init"); + +#if YY_EXCEPTIONS + try +#endif // YY_EXCEPTIONS + { + YYCDEBUG << "Starting parse\n"; + + + /* Initialize the stack. The initial state will be set in + yynewstate, since the latter expects the semantical and the + location values to have been already stored, initialize these + stacks with a primary value. */ + yystack_.clear (); + yypush_ (YY_NULLPTR, 0, YY_MOVE (yyla)); + + /*-----------------------------------------------. + | yynewstate -- push a new symbol on the stack. | + `-----------------------------------------------*/ + yynewstate: + YYCDEBUG << "Entering state " << int (yystack_[0].state) << '\n'; + YY_STACK_PRINT (); + + // Accept? + if (yystack_[0].state == yyfinal_) + YYACCEPT; + + goto yybackup; + + + /*-----------. + | yybackup. | + `-----------*/ + yybackup: + // Try to take a decision without lookahead. + yyn = yypact_[+yystack_[0].state]; + if (yy_pact_value_is_default_ (yyn)) + goto yydefault; + + // Read a lookahead token. + if (yyla.empty ()) + { + YYCDEBUG << "Reading a token\n"; +#if YY_EXCEPTIONS + try +#endif // YY_EXCEPTIONS + { + symbol_type yylookahead (yylex (drv)); + yyla.move (yylookahead); + } +#if YY_EXCEPTIONS + catch (const syntax_error& yyexc) + { + YYCDEBUG << "Caught exception: " << yyexc.what() << '\n'; + error (yyexc); + goto yyerrlab1; + } +#endif // YY_EXCEPTIONS + } + YY_SYMBOL_PRINT ("Next token is", yyla); + + if (yyla.kind () == symbol_kind::S_YYerror) + { + // The scanner already issued an error message, process directly + // to error recovery. But do not keep the error token as + // lookahead, it is too special and may lead us to an endless + // loop in error recovery. */ + yyla.kind_ = symbol_kind::S_YYUNDEF; + goto yyerrlab1; + } + + /* If the proper action on seeing token YYLA.TYPE is to reduce or + to detect an error, take that action. */ + yyn += yyla.kind (); + if (yyn < 0 || yylast_ < yyn || yycheck_[yyn] != yyla.kind ()) + { + if (!yy_lac_establish_ (yyla.kind ())) + goto yyerrlab; + goto yydefault; + } + + // Reduce or error. + yyn = yytable_[yyn]; + if (yyn <= 0) + { + if (yy_table_value_is_error_ (yyn)) + goto yyerrlab; + if (!yy_lac_establish_ (yyla.kind ())) + goto yyerrlab; + + yyn = -yyn; + goto yyreduce; + } + + // Count tokens shifted since error; after three, turn off error status. + if (yyerrstatus_) + --yyerrstatus_; + + // Shift the lookahead token. + yypush_ ("Shifting", state_type (yyn), YY_MOVE (yyla)); + yy_lac_discard_ ("shift"); + goto yynewstate; + + + /*-----------------------------------------------------------. + | yydefault -- do the default action for the current state. | + `-----------------------------------------------------------*/ + yydefault: + yyn = yydefact_[+yystack_[0].state]; + if (yyn == 0) + goto yyerrlab; + goto yyreduce; + + + /*-----------------------------. + | yyreduce -- do a reduction. | + `-----------------------------*/ + yyreduce: + yylen = yyr2_[yyn]; + { + stack_symbol_type yylhs; + yylhs.state = yy_lr_goto_state_ (yystack_[yylen].state, yyr1_[yyn]); + /* Variants are always initialized to an empty instance of the + correct type. The default '$$ = $1' action is NOT applied + when using variants. */ + switch (yyr1_[yyn]) + { + case symbol_kind::S_INT: // "number" + case symbol_kind::S_list: // list + yylhs.value.emplace< int > (); + break; + + case symbol_kind::S_OPERATOR: // OPERATOR + case symbol_kind::S_IDENTIFIER: // "identifier" + case symbol_kind::S_VARNAME: // VARNAME + yylhs.value.emplace< std::string > (); + break; + + case symbol_kind::S_index: // index + yylhs.value.emplace< std::tuple > (); + break; + + case symbol_kind::S_indices_list: // indices_list + yylhs.value.emplace< std::vector> > (); + break; + + default: + break; + } + + + // Default location. + { + stack_type::slice range (yystack_, yylen); + YYLLOC_DEFAULT (yylhs.location, range, yylen); + yyerror_range[1].location = yylhs.location; + } + + // Perform the reduction. + YY_REDUCE_PRINT (yyn); +#if YY_EXCEPTIONS + try +#endif // YY_EXCEPTIONS + { + switch (yyn) + { + case 2: // lines: assignment lines +#line 61 "../parser.y" + {} +#line 644 "parser.cpp" + break; + + case 3: // lines: exp +#line 62 "../parser.y" + {} +#line 650 "parser.cpp" + break; + + case 4: // assignment: "identifier" "=" VARNAME +#line 66 "../parser.y" + { drv.add_lookup_entry(yystack_[2].value.as < std::string > (), yystack_[0].value.as < std::string > ()); } +#line 656 "parser.cpp" + break; + + case 5: // assignment: "identifier" "=" "identifier" +#line 67 "../parser.y" + { drv.add_lookup_entry(yystack_[2].value.as < std::string > (), yystack_[0].value.as < std::string > ()); } +#line 662 "parser.cpp" + break; + + case 6: // assignment: "identifier" "=" VARNAME "[" indices_list "]" +#line 68 "../parser.y" + { drv.add_lookup_entry(yystack_[5].value.as < std::string > (), yystack_[3].value.as < std::string > (), yystack_[1].value.as < std::vector> > ()); } +#line 668 "parser.cpp" + break; + + case 7: // assignment: "identifier" "=" "identifier" "[" indices_list "]" +#line 69 "../parser.y" + { drv.add_lookup_entry(yystack_[5].value.as < std::string > (), yystack_[3].value.as < std::string > (), yystack_[1].value.as < std::vector> > ()); } +#line 674 "parser.cpp" + break; + + case 8: // exp: "number" +#line 73 "../parser.y" + { } +#line 680 "parser.cpp" + break; + + case 9: // exp: exp OPERATOR exp +#line 74 "../parser.y" + { drv.createNode(yystack_[1].value.as < std::string > (), 2); } +#line 686 "parser.cpp" + break; + + case 10: // exp: "(" exp ")" +#line 75 "../parser.y" + { } +#line 692 "parser.cpp" + break; + + case 11: // exp: "identifier" "(" list ")" +#line 76 "../parser.y" + { drv.createNode(yystack_[3].value.as < std::string > (), yystack_[1].value.as < int > ()); } +#line 698 "parser.cpp" + break; + + case 12: // exp: "identifier" "[" indices_list "]" +#line 77 "../parser.y" + { drv.createNode(yystack_[3].value.as < std::string > (), yystack_[1].value.as < std::vector> > ()); } +#line 704 "parser.cpp" + break; + + case 13: // exp: "identifier" +#line 78 "../parser.y" + { drv.createNode(yystack_[0].value.as < std::string > ()); } +#line 710 "parser.cpp" + break; + + case 14: // indices_list: %empty +#line 83 "../parser.y" + { yylhs.value.as < std::vector> > () = {}; } +#line 716 "parser.cpp" + break; + + case 15: // indices_list: indices_list "," index +#line 84 "../parser.y" + { yystack_[2].value.as < std::vector> > ().push_back(yystack_[0].value.as < std::tuple > ()); yylhs.value.as < std::vector> > () = yystack_[2].value.as < std::vector> > (); } +#line 722 "parser.cpp" + break; + + case 16: // indices_list: index +#line 85 "../parser.y" + { yylhs.value.as < std::vector> > () = {yystack_[0].value.as < std::tuple > ()}; } +#line 728 "parser.cpp" + break; + + case 17: // index: %empty +#line 89 "../parser.y" + { yylhs.value.as < std::tuple > () = {-1, -1, 1}; } +#line 734 "parser.cpp" + break; + + case 18: // index: "number" ":" "number" ":" "number" +#line 90 "../parser.y" + { yylhs.value.as < std::tuple > () = {yystack_[4].value.as < int > (), yystack_[2].value.as < int > (), yystack_[0].value.as < int > ()}; } +#line 740 "parser.cpp" + break; + + case 19: // index: ":" "number" ":" "number" +#line 91 "../parser.y" + { yylhs.value.as < std::tuple > () = {-1, yystack_[2].value.as < int > (), yystack_[0].value.as < int > ()}; } +#line 746 "parser.cpp" + break; + + case 20: // index: "number" ":" ":" "number" +#line 92 "../parser.y" + { yylhs.value.as < std::tuple > () = {yystack_[3].value.as < int > (), -1, yystack_[0].value.as < int > ()}; } +#line 752 "parser.cpp" + break; + + case 21: // index: "number" ":" "number" ":" +#line 93 "../parser.y" + { yylhs.value.as < std::tuple > () = {yystack_[3].value.as < int > (), yystack_[1].value.as < int > (), 1}; } +#line 758 "parser.cpp" + break; + + case 22: // index: "number" ":" "number" +#line 94 "../parser.y" + { yylhs.value.as < std::tuple > () = {yystack_[2].value.as < int > (), yystack_[0].value.as < int > (), 1}; } +#line 764 "parser.cpp" + break; + + case 23: // index: ":" ":" "number" +#line 95 "../parser.y" + { yylhs.value.as < std::tuple > () = {-1, -1, yystack_[0].value.as < int > ()}; } +#line 770 "parser.cpp" + break; + + case 24: // index: ":" "number" ":" +#line 96 "../parser.y" + { yylhs.value.as < std::tuple > () = {-1, yystack_[1].value.as < int > (), 1}; } +#line 776 "parser.cpp" + break; + + case 25: // index: ":" "number" +#line 97 "../parser.y" + { yylhs.value.as < std::tuple > () = {-1, yystack_[0].value.as < int > (), 1}; } +#line 782 "parser.cpp" + break; + + case 26: // index: "number" ":" ":" +#line 98 "../parser.y" + { yylhs.value.as < std::tuple > () = {yystack_[2].value.as < int > (), -1, 1}; } +#line 788 "parser.cpp" + break; + + case 27: // index: "number" ":" +#line 99 "../parser.y" + { yylhs.value.as < std::tuple > () = {yystack_[1].value.as < int > (), -1, 1}; } +#line 794 "parser.cpp" + break; + + case 28: // index: "number" +#line 100 "../parser.y" + { yylhs.value.as < std::tuple > () = {yystack_[0].value.as < int > (), yystack_[0].value.as < int > (), 1}; } +#line 800 "parser.cpp" + break; + + case 29: // list: %empty +#line 104 "../parser.y" + { yylhs.value.as < int > () = 0; } +#line 806 "parser.cpp" + break; + + case 30: // list: exp "," list +#line 105 "../parser.y" + { yylhs.value.as < int > () = yystack_[0].value.as < int > () + 1; } +#line 812 "parser.cpp" + break; + + case 31: // list: exp +#line 106 "../parser.y" + { yylhs.value.as < int > () = 1; } +#line 818 "parser.cpp" + break; + + +#line 822 "parser.cpp" + + default: + break; + } + } +#if YY_EXCEPTIONS + catch (const syntax_error& yyexc) + { + YYCDEBUG << "Caught exception: " << yyexc.what() << '\n'; + error (yyexc); + YYERROR; + } +#endif // YY_EXCEPTIONS + YY_SYMBOL_PRINT ("-> $$ =", yylhs); + yypop_ (yylen); + yylen = 0; + + // Shift the result of the reduction. + yypush_ (YY_NULLPTR, YY_MOVE (yylhs)); + } + goto yynewstate; + + + /*--------------------------------------. + | yyerrlab -- here on detecting error. | + `--------------------------------------*/ + yyerrlab: + // If not already recovering from an error, report this error. + if (!yyerrstatus_) + { + ++yynerrs_; + context yyctx (*this, yyla); + std::string msg = yysyntax_error_ (yyctx); + error (yyla.location, YY_MOVE (msg)); + } + + + yyerror_range[1].location = yyla.location; + if (yyerrstatus_ == 3) + { + /* If just tried and failed to reuse lookahead token after an + error, discard it. */ + + // Return failure if at end of input. + if (yyla.kind () == symbol_kind::S_YYEOF) + YYABORT; + else if (!yyla.empty ()) + { + yy_destroy_ ("Error: discarding", yyla); + yyla.clear (); + } + } + + // Else will try to reuse lookahead token after shifting the error token. + goto yyerrlab1; + + + /*---------------------------------------------------. + | yyerrorlab -- error raised explicitly by YYERROR. | + `---------------------------------------------------*/ + yyerrorlab: + /* Pacify compilers when the user code never invokes YYERROR and + the label yyerrorlab therefore never appears in user code. */ + if (false) + YYERROR; + + /* Do not reclaim the symbols of the rule whose action triggered + this YYERROR. */ + yypop_ (yylen); + yylen = 0; + YY_STACK_PRINT (); + goto yyerrlab1; + + + /*-------------------------------------------------------------. + | yyerrlab1 -- common code for both syntax error and YYERROR. | + `-------------------------------------------------------------*/ + yyerrlab1: + yyerrstatus_ = 3; // Each real token shifted decrements this. + // Pop stack until we find a state that shifts the error token. + for (;;) + { + yyn = yypact_[+yystack_[0].state]; + if (!yy_pact_value_is_default_ (yyn)) + { + yyn += symbol_kind::S_YYerror; + if (0 <= yyn && yyn <= yylast_ + && yycheck_[yyn] == symbol_kind::S_YYerror) + { + yyn = yytable_[yyn]; + if (0 < yyn) + break; + } + } + + // Pop the current state because it cannot handle the error token. + if (yystack_.size () == 1) + YYABORT; + + yyerror_range[1].location = yystack_[0].location; + yy_destroy_ ("Error: popping", yystack_[0]); + yypop_ (); + YY_STACK_PRINT (); + } + { + stack_symbol_type error_token; + + yyerror_range[2].location = yyla.location; + YYLLOC_DEFAULT (error_token.location, yyerror_range, 2); + + // Shift the error token. + yy_lac_discard_ ("error recovery"); + error_token.state = state_type (yyn); + yypush_ ("Shifting", YY_MOVE (error_token)); + } + goto yynewstate; + + + /*-------------------------------------. + | yyacceptlab -- YYACCEPT comes here. | + `-------------------------------------*/ + yyacceptlab: + yyresult = 0; + goto yyreturn; + + + /*-----------------------------------. + | yyabortlab -- YYABORT comes here. | + `-----------------------------------*/ + yyabortlab: + yyresult = 1; + goto yyreturn; + + + /*-----------------------------------------------------. + | yyreturn -- parsing is finished, return the result. | + `-----------------------------------------------------*/ + yyreturn: + if (!yyla.empty ()) + yy_destroy_ ("Cleanup: discarding lookahead", yyla); + + /* Do not reclaim the symbols of the rule whose action triggered + this YYABORT or YYACCEPT. */ + yypop_ (yylen); + YY_STACK_PRINT (); + while (1 < yystack_.size ()) + { + yy_destroy_ ("Cleanup: popping", yystack_[0]); + yypop_ (); + } + + return yyresult; + } +#if YY_EXCEPTIONS + catch (...) + { + YYCDEBUG << "Exception caught: cleaning lookahead and stack\n"; + // Do not try to display the values of the reclaimed symbols, + // as their printers might throw an exception. + if (!yyla.empty ()) + yy_destroy_ (YY_NULLPTR, yyla); + + while (1 < yystack_.size ()) + { + yy_destroy_ (YY_NULLPTR, yystack_[0]); + yypop_ (); + } + throw; + } +#endif // YY_EXCEPTIONS + } + + void + parser::error (const syntax_error& yyexc) + { + error (yyexc.location, yyexc.what ()); + } + + const char * + parser::symbol_name (symbol_kind_type yysymbol) + { + static const char *const yy_sname[] = + { + "end of file", "error", "invalid token", "=", ",", ":", "(", ")", "[", + "]", "OPERATOR", "identifier", "VARNAME", "number", "$accept", "lines", + "assignment", "exp", "indices_list", "index", "list", YY_NULLPTR + }; + return yy_sname[yysymbol]; + } + + + + // parser::context. + parser::context::context (const parser& yyparser, const symbol_type& yyla) + : yyparser_ (yyparser) + , yyla_ (yyla) + {} + + int + parser::context::expected_tokens (symbol_kind_type yyarg[], int yyargn) const + { + // Actual number of expected tokens + int yycount = 0; + +#if YYDEBUG + // Execute LAC once. We don't care if it is successful, we + // only do it for the sake of debugging output. + if (!yyparser_.yy_lac_established_) + yyparser_.yy_lac_check_ (yyla_.kind ()); +#endif + + for (int yyx = 0; yyx < YYNTOKENS; ++yyx) + { + symbol_kind_type yysym = YY_CAST (symbol_kind_type, yyx); + if (yysym != symbol_kind::S_YYerror + && yysym != symbol_kind::S_YYUNDEF + && yyparser_.yy_lac_check_ (yysym)) + { + if (!yyarg) + ++yycount; + else if (yycount == yyargn) + return 0; + else + yyarg[yycount++] = yysym; + } + } + if (yyarg && yycount == 0 && 0 < yyargn) + yyarg[0] = symbol_kind::S_YYEMPTY; + return yycount; + } + + + + + bool + parser::yy_lac_check_ (symbol_kind_type yytoken) const + { + // Logically, the yylac_stack's lifetime is confined to this function. + // Clear it, to get rid of potential left-overs from previous call. + yylac_stack_.clear (); + // Reduce until we encounter a shift and thereby accept the token. +#if YYDEBUG + YYCDEBUG << "LAC: checking lookahead " << symbol_name (yytoken) << ':'; +#endif + std::ptrdiff_t lac_top = 0; + while (true) + { + state_type top_state = (yylac_stack_.empty () + ? yystack_[lac_top].state + : yylac_stack_.back ()); + int yyrule = yypact_[+top_state]; + if (yy_pact_value_is_default_ (yyrule) + || (yyrule += yytoken) < 0 || yylast_ < yyrule + || yycheck_[yyrule] != yytoken) + { + // Use the default action. + yyrule = yydefact_[+top_state]; + if (yyrule == 0) + { + YYCDEBUG << " Err\n"; + return false; + } + } + else + { + // Use the action from yytable. + yyrule = yytable_[yyrule]; + if (yy_table_value_is_error_ (yyrule)) + { + YYCDEBUG << " Err\n"; + return false; + } + if (0 < yyrule) + { + YYCDEBUG << " S" << yyrule << '\n'; + return true; + } + yyrule = -yyrule; + } + // By now we know we have to simulate a reduce. + YYCDEBUG << " R" << yyrule - 1; + // Pop the corresponding number of values from the stack. + { + std::ptrdiff_t yylen = yyr2_[yyrule]; + // First pop from the LAC stack as many tokens as possible. + std::ptrdiff_t lac_size = std::ptrdiff_t (yylac_stack_.size ()); + if (yylen < lac_size) + { + yylac_stack_.resize (std::size_t (lac_size - yylen)); + yylen = 0; + } + else if (lac_size) + { + yylac_stack_.clear (); + yylen -= lac_size; + } + // Only afterwards look at the main stack. + // We simulate popping elements by incrementing lac_top. + lac_top += yylen; + } + // Keep top_state in sync with the updated stack. + top_state = (yylac_stack_.empty () + ? yystack_[lac_top].state + : yylac_stack_.back ()); + // Push the resulting state of the reduction. + state_type state = yy_lr_goto_state_ (top_state, yyr1_[yyrule]); + YYCDEBUG << " G" << int (state); + yylac_stack_.push_back (state); + } + } + + // Establish the initial context if no initial context currently exists. + bool + parser::yy_lac_establish_ (symbol_kind_type yytoken) + { + /* Establish the initial context for the current lookahead if no initial + context is currently established. + + We define a context as a snapshot of the parser stacks. We define + the initial context for a lookahead as the context in which the + parser initially examines that lookahead in order to select a + syntactic action. Thus, if the lookahead eventually proves + syntactically unacceptable (possibly in a later context reached via a + series of reductions), the initial context can be used to determine + the exact set of tokens that would be syntactically acceptable in the + lookahead's place. Moreover, it is the context after which any + further semantic actions would be erroneous because they would be + determined by a syntactically unacceptable token. + + yy_lac_establish_ should be invoked when a reduction is about to be + performed in an inconsistent state (which, for the purposes of LAC, + includes consistent states that don't know they're consistent because + their default reductions have been disabled). + + For parse.lac=full, the implementation of yy_lac_establish_ is as + follows. If no initial context is currently established for the + current lookahead, then check if that lookahead can eventually be + shifted if syntactic actions continue from the current context. */ + if (yy_lac_established_) + return true; + else + { +#if YYDEBUG + YYCDEBUG << "LAC: initial context established for " + << symbol_name (yytoken) << '\n'; +#endif + yy_lac_established_ = true; + return yy_lac_check_ (yytoken); + } + } + + // Discard any previous initial lookahead context. + void + parser::yy_lac_discard_ (const char* event) + { + /* Discard any previous initial lookahead context because of Event, + which may be a lookahead change or an invalidation of the currently + established initial context for the current lookahead. + + The most common example of a lookahead change is a shift. An example + of both cases is syntax error recovery. That is, a syntax error + occurs when the lookahead is syntactically erroneous for the + currently established initial context, so error recovery manipulates + the parser stacks to try to find a new initial context in which the + current lookahead is syntactically acceptable. If it fails to find + such a context, it discards the lookahead. */ + if (yy_lac_established_) + { + YYCDEBUG << "LAC: initial context discarded due to " + << event << '\n'; + yy_lac_established_ = false; + } + } + + + int + parser::yy_syntax_error_arguments_ (const context& yyctx, + symbol_kind_type yyarg[], int yyargn) const + { + /* There are many possibilities here to consider: + - If this state is a consistent state with a default action, then + the only way this function was invoked is if the default action + is an error action. In that case, don't check for expected + tokens because there are none. + - The only way there can be no lookahead present (in yyla) is + if this state is a consistent state with a default action. + Thus, detecting the absence of a lookahead is sufficient to + determine that there is no unexpected or expected token to + report. In that case, just report a simple "syntax error". + - Don't assume there isn't a lookahead just because this state is + a consistent state with a default action. There might have + been a previous inconsistent state, consistent state with a + non-default action, or user semantic action that manipulated + yyla. (However, yyla is currently not documented for users.) + In the first two cases, it might appear that the current syntax + error should have been detected in the previous state when + yy_lac_check was invoked. However, at that time, there might + have been a different syntax error that discarded a different + initial context during error recovery, leaving behind the + current lookahead. + */ + + if (!yyctx.lookahead ().empty ()) + { + if (yyarg) + yyarg[0] = yyctx.token (); + int yyn = yyctx.expected_tokens (yyarg ? yyarg + 1 : yyarg, yyargn - 1); + return yyn + 1; + } + return 0; + } + + // Generate an error message. + std::string + parser::yysyntax_error_ (const context& yyctx) const + { + // Its maximum. + enum { YYARGS_MAX = 5 }; + // Arguments of yyformat. + symbol_kind_type yyarg[YYARGS_MAX]; + int yycount = yy_syntax_error_arguments_ (yyctx, yyarg, YYARGS_MAX); + + char const* yyformat = YY_NULLPTR; + switch (yycount) + { +#define YYCASE_(N, S) \ + case N: \ + yyformat = S; \ + break + default: // Avoid compiler warnings. + YYCASE_ (0, YY_("syntax error")); + YYCASE_ (1, YY_("syntax error, unexpected %s")); + YYCASE_ (2, YY_("syntax error, unexpected %s, expecting %s")); + YYCASE_ (3, YY_("syntax error, unexpected %s, expecting %s or %s")); + YYCASE_ (4, YY_("syntax error, unexpected %s, expecting %s or %s or %s")); + YYCASE_ (5, YY_("syntax error, unexpected %s, expecting %s or %s or %s or %s")); +#undef YYCASE_ + } + + std::string yyres; + // Argument number. + std::ptrdiff_t yyi = 0; + for (char const* yyp = yyformat; *yyp; ++yyp) + if (yyp[0] == '%' && yyp[1] == 's' && yyi < yycount) + { + yyres += symbol_name (yyarg[yyi++]); + ++yyp; + } + else + yyres += *yyp; + return yyres; + } + + + const signed char parser::yypact_ninf_ = -7; + + const signed char parser::yytable_ninf_ = -1; + + const signed char + parser::yypact_[] = + { + 6, 9, 22, -7, 5, 6, 13, 27, -6, -4, + 9, -3, -7, -7, 9, -7, 30, 31, 14, 24, + -2, 35, 12, -7, -7, -3, -3, 9, -7, 28, + 37, 1, -3, -7, 23, 25, -7, -7, 32, 33, + 38, -7, -7, -7, -7, -7, 34, -7 + }; + + const signed char + parser::yydefact_[] = + { + 0, 0, 13, 8, 0, 0, 3, 13, 0, 0, + 29, 14, 1, 2, 0, 10, 5, 4, 31, 0, + 0, 28, 0, 16, 9, 14, 14, 29, 11, 0, + 25, 27, 17, 12, 0, 0, 30, 23, 24, 26, + 22, 15, 7, 6, 19, 20, 21, 18 + }; + + const signed char + parser::yypgoto_[] = + { + -7, 39, -7, -1, 11, 16, 26 + }; + + const signed char + parser::yydefgoto_[] = + { + 0, 4, 5, 6, 22, 23, 19 + }; + + const signed char + parser::yytable_[] = + { + 8, 15, 20, 29, 14, 12, 39, 16, 17, 18, + 21, 30, 1, 24, 40, 1, 32, 2, 27, 3, + 7, 33, 3, 14, 14, 9, 18, 32, 10, 32, + 11, 28, 42, 10, 43, 11, 34, 35, 25, 26, + 31, 37, 38, 46, 13, 44, 45, 47, 41, 0, + 0, 0, 0, 36 + }; + + const signed char + parser::yycheck_[] = + { + 1, 7, 5, 5, 10, 0, 5, 11, 12, 10, + 13, 13, 6, 14, 13, 6, 4, 11, 4, 13, + 11, 9, 13, 10, 10, 3, 27, 4, 6, 4, + 8, 7, 9, 6, 9, 8, 25, 26, 8, 8, + 5, 13, 5, 5, 5, 13, 13, 13, 32, -1, + -1, -1, -1, 27 + }; + + const signed char + parser::yystos_[] = + { + 0, 6, 11, 13, 15, 16, 17, 11, 17, 3, + 6, 8, 0, 15, 10, 7, 11, 12, 17, 20, + 5, 13, 18, 19, 17, 8, 8, 4, 7, 5, + 13, 5, 4, 9, 18, 18, 20, 13, 5, 5, + 13, 19, 9, 9, 13, 13, 5, 13 + }; + + const signed char + parser::yyr1_[] = + { + 0, 14, 15, 15, 16, 16, 16, 16, 17, 17, + 17, 17, 17, 17, 18, 18, 18, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 20, + 20, 20 + }; + + const signed char + parser::yyr2_[] = + { + 0, 2, 2, 1, 3, 3, 6, 6, 1, 3, + 3, 4, 4, 1, 0, 3, 1, 0, 5, 4, + 4, 4, 3, 3, 3, 2, 3, 2, 1, 0, + 3, 1 + }; + + + + +#if YYDEBUG + const signed char + parser::yyrline_[] = + { + 0, 61, 61, 62, 66, 67, 68, 69, 73, 74, + 75, 76, 77, 78, 83, 84, 85, 89, 90, 91, + 92, 93, 94, 95, 96, 97, 98, 99, 100, 104, + 105, 106 + }; + + void + parser::yy_stack_print_ () const + { + *yycdebug_ << "Stack now"; + for (stack_type::const_iterator + i = yystack_.begin (), + i_end = yystack_.end (); + i != i_end; ++i) + *yycdebug_ << ' ' << int (i->state); + *yycdebug_ << '\n'; + } + + void + parser::yy_reduce_print_ (int yyrule) const + { + int yylno = yyrline_[yyrule]; + int yynrhs = yyr2_[yyrule]; + // Print the symbols being reduced, and their result. + *yycdebug_ << "Reducing stack by rule " << yyrule - 1 + << " (line " << yylno << "):\n"; + // The symbols being reduced. + for (int yyi = 0; yyi < yynrhs; yyi++) + YY_SYMBOL_PRINT (" $" << yyi + 1 << " =", + yystack_[(yynrhs) - (yyi + 1)]); + } +#endif // YYDEBUG + + +#line 6 "../parser.y" +} } // adios2::detail +#line 1406 "parser.cpp" + +#line 107 "../parser.y" + + +void +adios2::detail::parser::error (const location_type& l, const std::string& m) +{ + std::cerr << l << ": " << m << '\n'; +} diff --git a/source/adios2/toolkit/derived/parser/pregen-source/parser.h b/source/adios2/toolkit/derived/parser/pregen-source/parser.h new file mode 100644 index 0000000000..819381e379 --- /dev/null +++ b/source/adios2/toolkit/derived/parser/pregen-source/parser.h @@ -0,0 +1,1609 @@ +// A Bison parser, made by GNU Bison 3.8.2. + +// Skeleton interface for Bison LALR(1) parsers in C++ + +// Copyright (C) 2002-2015, 2018-2021 Free Software Foundation, Inc. + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +// As a special exception, you may create a larger work that contains +// part or all of the Bison parser skeleton and distribute that work +// under terms of your choice, so long as that work isn't itself a +// parser generator using the skeleton or a modified version thereof +// as a parser skeleton. Alternatively, if you modify or redistribute +// the parser skeleton itself, you may (at your option) remove this +// special exception, which will cause the skeleton and the resulting +// Bison output files to be licensed under the GNU General Public +// License without this special exception. + +// This special exception was added by the Free Software Foundation in +// version 2.2 of Bison. + + +/** + ** \file parser.h + ** Define the adios2::detail::parser class. + */ + +// C++ LALR(1) parser skeleton written by Akim Demaille. + +// DO NOT RELY ON FEATURES THAT ARE NOT DOCUMENTED in the manual, +// especially those whose name start with YY_ or yy_. They are +// private implementation details that can be changed or removed. + +#ifndef YY_YY_PARSER_H_INCLUDED +# define YY_YY_PARSER_H_INCLUDED +// "%code requires" blocks. +#line 11 "../parser.y" + + #include + #include + #include + namespace adios2 + { + namespace detail + { + class ASTDriver; + } + } + +#line 62 "parser.h" + +# include +# include // std::abort +# include +# include +# include +# include + +#if defined __cplusplus +# define YY_CPLUSPLUS __cplusplus +#else +# define YY_CPLUSPLUS 199711L +#endif + +// Support move semantics when possible. +#if 201103L <= YY_CPLUSPLUS +# define YY_MOVE std::move +# define YY_MOVE_OR_COPY move +# define YY_MOVE_REF(Type) Type&& +# define YY_RVREF(Type) Type&& +# define YY_COPY(Type) Type +#else +# define YY_MOVE +# define YY_MOVE_OR_COPY copy +# define YY_MOVE_REF(Type) Type& +# define YY_RVREF(Type) const Type& +# define YY_COPY(Type) const Type& +#endif + +// Support noexcept when possible. +#if 201103L <= YY_CPLUSPLUS +# define YY_NOEXCEPT noexcept +# define YY_NOTHROW +#else +# define YY_NOEXCEPT +# define YY_NOTHROW throw () +#endif + +// Support constexpr when possible. +#if 201703 <= YY_CPLUSPLUS +# define YY_CONSTEXPR constexpr +#else +# define YY_CONSTEXPR +#endif +# include "location.hh" +#include +#ifndef YY_ASSERT +# include +# define YY_ASSERT assert +#endif + + +#ifndef YY_ATTRIBUTE_PURE +# if defined __GNUC__ && 2 < __GNUC__ + (96 <= __GNUC_MINOR__) +# define YY_ATTRIBUTE_PURE __attribute__ ((__pure__)) +# else +# define YY_ATTRIBUTE_PURE +# endif +#endif + +#ifndef YY_ATTRIBUTE_UNUSED +# if defined __GNUC__ && 2 < __GNUC__ + (7 <= __GNUC_MINOR__) +# define YY_ATTRIBUTE_UNUSED __attribute__ ((__unused__)) +# else +# define YY_ATTRIBUTE_UNUSED +# endif +#endif + +/* Suppress unused-variable warnings by "using" E. */ +#if ! defined lint || defined __GNUC__ +# define YY_USE(E) ((void) (E)) +#else +# define YY_USE(E) /* empty */ +#endif + +/* Suppress an incorrect diagnostic about yylval being uninitialized. */ +#if defined __GNUC__ && ! defined __ICC && 406 <= __GNUC__ * 100 + __GNUC_MINOR__ +# if __GNUC__ * 100 + __GNUC_MINOR__ < 407 +# define YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN \ + _Pragma ("GCC diagnostic push") \ + _Pragma ("GCC diagnostic ignored \"-Wuninitialized\"") +# else +# define YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN \ + _Pragma ("GCC diagnostic push") \ + _Pragma ("GCC diagnostic ignored \"-Wuninitialized\"") \ + _Pragma ("GCC diagnostic ignored \"-Wmaybe-uninitialized\"") +# endif +# define YY_IGNORE_MAYBE_UNINITIALIZED_END \ + _Pragma ("GCC diagnostic pop") +#else +# define YY_INITIAL_VALUE(Value) Value +#endif +#ifndef YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN +# define YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN +# define YY_IGNORE_MAYBE_UNINITIALIZED_END +#endif +#ifndef YY_INITIAL_VALUE +# define YY_INITIAL_VALUE(Value) /* Nothing. */ +#endif + +#if defined __cplusplus && defined __GNUC__ && ! defined __ICC && 6 <= __GNUC__ +# define YY_IGNORE_USELESS_CAST_BEGIN \ + _Pragma ("GCC diagnostic push") \ + _Pragma ("GCC diagnostic ignored \"-Wuseless-cast\"") +# define YY_IGNORE_USELESS_CAST_END \ + _Pragma ("GCC diagnostic pop") +#endif +#ifndef YY_IGNORE_USELESS_CAST_BEGIN +# define YY_IGNORE_USELESS_CAST_BEGIN +# define YY_IGNORE_USELESS_CAST_END +#endif + +# ifndef YY_CAST +# ifdef __cplusplus +# define YY_CAST(Type, Val) static_cast (Val) +# define YY_REINTERPRET_CAST(Type, Val) reinterpret_cast (Val) +# else +# define YY_CAST(Type, Val) ((Type) (Val)) +# define YY_REINTERPRET_CAST(Type, Val) ((Type) (Val)) +# endif +# endif +# ifndef YY_NULLPTR +# if defined __cplusplus +# if 201103L <= __cplusplus +# define YY_NULLPTR nullptr +# else +# define YY_NULLPTR 0 +# endif +# else +# define YY_NULLPTR ((void*)0) +# endif +# endif + +/* Debug traces. */ +#ifndef YYDEBUG +# define YYDEBUG 1 +#endif + +#line 6 "../parser.y" +namespace adios2 { namespace detail { +#line 203 "parser.h" + + + + + /// A Bison parser. + class parser + { + public: +#ifdef YYSTYPE +# ifdef __GNUC__ +# pragma GCC message "bison: do not #define YYSTYPE in C++, use %define api.value.type" +# endif + typedef YYSTYPE value_type; +#else + /// A buffer to store and retrieve objects. + /// + /// Sort of a variant, but does not keep track of the nature + /// of the stored data, since that knowledge is available + /// via the current parser state. + class value_type + { + public: + /// Type of *this. + typedef value_type self_type; + + /// Empty construction. + value_type () YY_NOEXCEPT + : yyraw_ () + , yytypeid_ (YY_NULLPTR) + {} + + /// Construct and fill. + template + value_type (YY_RVREF (T) t) + : yytypeid_ (&typeid (T)) + { + YY_ASSERT (sizeof (T) <= size); + new (yyas_ ()) T (YY_MOVE (t)); + } + +#if 201103L <= YY_CPLUSPLUS + /// Non copyable. + value_type (const self_type&) = delete; + /// Non copyable. + self_type& operator= (const self_type&) = delete; +#endif + + /// Destruction, allowed only if empty. + ~value_type () YY_NOEXCEPT + { + YY_ASSERT (!yytypeid_); + } + +# if 201103L <= YY_CPLUSPLUS + /// Instantiate a \a T in here from \a t. + template + T& + emplace (U&&... u) + { + YY_ASSERT (!yytypeid_); + YY_ASSERT (sizeof (T) <= size); + yytypeid_ = & typeid (T); + return *new (yyas_ ()) T (std::forward (u)...); + } +# else + /// Instantiate an empty \a T in here. + template + T& + emplace () + { + YY_ASSERT (!yytypeid_); + YY_ASSERT (sizeof (T) <= size); + yytypeid_ = & typeid (T); + return *new (yyas_ ()) T (); + } + + /// Instantiate a \a T in here from \a t. + template + T& + emplace (const T& t) + { + YY_ASSERT (!yytypeid_); + YY_ASSERT (sizeof (T) <= size); + yytypeid_ = & typeid (T); + return *new (yyas_ ()) T (t); + } +# endif + + /// Instantiate an empty \a T in here. + /// Obsolete, use emplace. + template + T& + build () + { + return emplace (); + } + + /// Instantiate a \a T in here from \a t. + /// Obsolete, use emplace. + template + T& + build (const T& t) + { + return emplace (t); + } + + /// Accessor to a built \a T. + template + T& + as () YY_NOEXCEPT + { + YY_ASSERT (yytypeid_); + YY_ASSERT (*yytypeid_ == typeid (T)); + YY_ASSERT (sizeof (T) <= size); + return *yyas_ (); + } + + /// Const accessor to a built \a T (for %printer). + template + const T& + as () const YY_NOEXCEPT + { + YY_ASSERT (yytypeid_); + YY_ASSERT (*yytypeid_ == typeid (T)); + YY_ASSERT (sizeof (T) <= size); + return *yyas_ (); + } + + /// Swap the content with \a that, of same type. + /// + /// Both variants must be built beforehand, because swapping the actual + /// data requires reading it (with as()), and this is not possible on + /// unconstructed variants: it would require some dynamic testing, which + /// should not be the variant's responsibility. + /// Swapping between built and (possibly) non-built is done with + /// self_type::move (). + template + void + swap (self_type& that) YY_NOEXCEPT + { + YY_ASSERT (yytypeid_); + YY_ASSERT (*yytypeid_ == *that.yytypeid_); + std::swap (as (), that.as ()); + } + + /// Move the content of \a that to this. + /// + /// Destroys \a that. + template + void + move (self_type& that) + { +# if 201103L <= YY_CPLUSPLUS + emplace (std::move (that.as ())); +# else + emplace (); + swap (that); +# endif + that.destroy (); + } + +# if 201103L <= YY_CPLUSPLUS + /// Move the content of \a that to this. + template + void + move (self_type&& that) + { + emplace (std::move (that.as ())); + that.destroy (); + } +#endif + + /// Copy the content of \a that to this. + template + void + copy (const self_type& that) + { + emplace (that.as ()); + } + + /// Destroy the stored \a T. + template + void + destroy () + { + as ().~T (); + yytypeid_ = YY_NULLPTR; + } + + private: +#if YY_CPLUSPLUS < 201103L + /// Non copyable. + value_type (const self_type&); + /// Non copyable. + self_type& operator= (const self_type&); +#endif + + /// Accessor to raw memory as \a T. + template + T* + yyas_ () YY_NOEXCEPT + { + void *yyp = yyraw_; + return static_cast (yyp); + } + + /// Const accessor to raw memory as \a T. + template + const T* + yyas_ () const YY_NOEXCEPT + { + const void *yyp = yyraw_; + return static_cast (yyp); + } + + /// An auxiliary type to compute the largest semantic type. + union union_type + { + // "number" + // list + char dummy1[sizeof (int)]; + + // OPERATOR + // "identifier" + // VARNAME + char dummy2[sizeof (std::string)]; + + // index + char dummy3[sizeof (std::tuple)]; + + // indices_list + char dummy4[sizeof (std::vector>)]; + }; + + /// The size of the largest semantic type. + enum { size = sizeof (union_type) }; + + /// A buffer to store semantic values. + union + { + /// Strongest alignment constraints. + long double yyalign_me_; + /// A buffer large enough to store any of the semantic values. + char yyraw_[size]; + }; + + /// Whether the content is built: if defined, the name of the stored type. + const std::type_info *yytypeid_; + }; + +#endif + /// Backward compatibility (Bison 3.8). + typedef value_type semantic_type; + + /// Symbol locations. + typedef location location_type; + + /// Syntax errors thrown from user actions. + struct syntax_error : std::runtime_error + { + syntax_error (const location_type& l, const std::string& m) + : std::runtime_error (m) + , location (l) + {} + + syntax_error (const syntax_error& s) + : std::runtime_error (s.what ()) + , location (s.location) + {} + + ~syntax_error () YY_NOEXCEPT YY_NOTHROW; + + location_type location; + }; + + /// Token kinds. + struct token + { + enum token_kind_type + { + TOK_YYEMPTY = -2, + TOK_YYEOF = 0, // "end of file" + TOK_YYerror = 1, // error + TOK_YYUNDEF = 2, // "invalid token" + TOK_ASSIGN = 3, // "=" + TOK_COMMA = 4, // "," + TOK_COLON = 5, // ":" + TOK_L_PAREN = 6, // "(" + TOK_R_PAREN = 7, // ")" + TOK_L_BRACE = 8, // "[" + TOK_R_BRACE = 9, // "]" + TOK_OPERATOR = 10, // OPERATOR + TOK_IDENTIFIER = 11, // "identifier" + TOK_VARNAME = 12, // VARNAME + TOK_INT = 13 // "number" + }; + /// Backward compatibility alias (Bison 3.6). + typedef token_kind_type yytokentype; + }; + + /// Token kind, as returned by yylex. + typedef token::token_kind_type token_kind_type; + + /// Backward compatibility alias (Bison 3.6). + typedef token_kind_type token_type; + + /// Symbol kinds. + struct symbol_kind + { + enum symbol_kind_type + { + YYNTOKENS = 14, ///< Number of tokens. + S_YYEMPTY = -2, + S_YYEOF = 0, // "end of file" + S_YYerror = 1, // error + S_YYUNDEF = 2, // "invalid token" + S_ASSIGN = 3, // "=" + S_COMMA = 4, // "," + S_COLON = 5, // ":" + S_L_PAREN = 6, // "(" + S_R_PAREN = 7, // ")" + S_L_BRACE = 8, // "[" + S_R_BRACE = 9, // "]" + S_OPERATOR = 10, // OPERATOR + S_IDENTIFIER = 11, // "identifier" + S_VARNAME = 12, // VARNAME + S_INT = 13, // "number" + S_YYACCEPT = 14, // $accept + S_lines = 15, // lines + S_assignment = 16, // assignment + S_exp = 17, // exp + S_indices_list = 18, // indices_list + S_index = 19, // index + S_list = 20 // list + }; + }; + + /// (Internal) symbol kind. + typedef symbol_kind::symbol_kind_type symbol_kind_type; + + /// The number of tokens. + static const symbol_kind_type YYNTOKENS = symbol_kind::YYNTOKENS; + + /// A complete symbol. + /// + /// Expects its Base type to provide access to the symbol kind + /// via kind (). + /// + /// Provide access to semantic value and location. + template + struct basic_symbol : Base + { + /// Alias to Base. + typedef Base super_type; + + /// Default constructor. + basic_symbol () YY_NOEXCEPT + : value () + , location () + {} + +#if 201103L <= YY_CPLUSPLUS + /// Move constructor. + basic_symbol (basic_symbol&& that) + : Base (std::move (that)) + , value () + , location (std::move (that.location)) + { + switch (this->kind ()) + { + case symbol_kind::S_INT: // "number" + case symbol_kind::S_list: // list + value.move< int > (std::move (that.value)); + break; + + case symbol_kind::S_OPERATOR: // OPERATOR + case symbol_kind::S_IDENTIFIER: // "identifier" + case symbol_kind::S_VARNAME: // VARNAME + value.move< std::string > (std::move (that.value)); + break; + + case symbol_kind::S_index: // index + value.move< std::tuple > (std::move (that.value)); + break; + + case symbol_kind::S_indices_list: // indices_list + value.move< std::vector> > (std::move (that.value)); + break; + + default: + break; + } + + } +#endif + + /// Copy constructor. + basic_symbol (const basic_symbol& that); + + /// Constructors for typed symbols. +#if 201103L <= YY_CPLUSPLUS + basic_symbol (typename Base::kind_type t, location_type&& l) + : Base (t) + , location (std::move (l)) + {} +#else + basic_symbol (typename Base::kind_type t, const location_type& l) + : Base (t) + , location (l) + {} +#endif + +#if 201103L <= YY_CPLUSPLUS + basic_symbol (typename Base::kind_type t, int&& v, location_type&& l) + : Base (t) + , value (std::move (v)) + , location (std::move (l)) + {} +#else + basic_symbol (typename Base::kind_type t, const int& v, const location_type& l) + : Base (t) + , value (v) + , location (l) + {} +#endif + +#if 201103L <= YY_CPLUSPLUS + basic_symbol (typename Base::kind_type t, std::string&& v, location_type&& l) + : Base (t) + , value (std::move (v)) + , location (std::move (l)) + {} +#else + basic_symbol (typename Base::kind_type t, const std::string& v, const location_type& l) + : Base (t) + , value (v) + , location (l) + {} +#endif + +#if 201103L <= YY_CPLUSPLUS + basic_symbol (typename Base::kind_type t, std::tuple&& v, location_type&& l) + : Base (t) + , value (std::move (v)) + , location (std::move (l)) + {} +#else + basic_symbol (typename Base::kind_type t, const std::tuple& v, const location_type& l) + : Base (t) + , value (v) + , location (l) + {} +#endif + +#if 201103L <= YY_CPLUSPLUS + basic_symbol (typename Base::kind_type t, std::vector>&& v, location_type&& l) + : Base (t) + , value (std::move (v)) + , location (std::move (l)) + {} +#else + basic_symbol (typename Base::kind_type t, const std::vector>& v, const location_type& l) + : Base (t) + , value (v) + , location (l) + {} +#endif + + /// Destroy the symbol. + ~basic_symbol () + { + clear (); + } + + + + /// Destroy contents, and record that is empty. + void clear () YY_NOEXCEPT + { + // User destructor. + symbol_kind_type yykind = this->kind (); + basic_symbol& yysym = *this; + (void) yysym; + switch (yykind) + { + default: + break; + } + + // Value type destructor. +switch (yykind) + { + case symbol_kind::S_INT: // "number" + case symbol_kind::S_list: // list + value.template destroy< int > (); + break; + + case symbol_kind::S_OPERATOR: // OPERATOR + case symbol_kind::S_IDENTIFIER: // "identifier" + case symbol_kind::S_VARNAME: // VARNAME + value.template destroy< std::string > (); + break; + + case symbol_kind::S_index: // index + value.template destroy< std::tuple > (); + break; + + case symbol_kind::S_indices_list: // indices_list + value.template destroy< std::vector> > (); + break; + + default: + break; + } + + Base::clear (); + } + + /// The user-facing name of this symbol. + const char *name () const YY_NOEXCEPT + { + return parser::symbol_name (this->kind ()); + } + + /// Backward compatibility (Bison 3.6). + symbol_kind_type type_get () const YY_NOEXCEPT; + + /// Whether empty. + bool empty () const YY_NOEXCEPT; + + /// Destructive move, \a s is emptied into this. + void move (basic_symbol& s); + + /// The semantic value. + value_type value; + + /// The location. + location_type location; + + private: +#if YY_CPLUSPLUS < 201103L + /// Assignment operator. + basic_symbol& operator= (const basic_symbol& that); +#endif + }; + + /// Type access provider for token (enum) based symbols. + struct by_kind + { + /// The symbol kind as needed by the constructor. + typedef token_kind_type kind_type; + + /// Default constructor. + by_kind () YY_NOEXCEPT; + +#if 201103L <= YY_CPLUSPLUS + /// Move constructor. + by_kind (by_kind&& that) YY_NOEXCEPT; +#endif + + /// Copy constructor. + by_kind (const by_kind& that) YY_NOEXCEPT; + + /// Constructor from (external) token numbers. + by_kind (kind_type t) YY_NOEXCEPT; + + + + /// Record that this symbol is empty. + void clear () YY_NOEXCEPT; + + /// Steal the symbol kind from \a that. + void move (by_kind& that); + + /// The (internal) type number (corresponding to \a type). + /// \a empty when empty. + symbol_kind_type kind () const YY_NOEXCEPT; + + /// Backward compatibility (Bison 3.6). + symbol_kind_type type_get () const YY_NOEXCEPT; + + /// The symbol kind. + /// \a S_YYEMPTY when empty. + symbol_kind_type kind_; + }; + + /// Backward compatibility for a private implementation detail (Bison 3.6). + typedef by_kind by_type; + + /// "External" symbols: returned by the scanner. + struct symbol_type : basic_symbol + { + /// Superclass. + typedef basic_symbol super_type; + + /// Empty symbol. + symbol_type () YY_NOEXCEPT {} + + /// Constructor for valueless symbols, and symbols from each type. +#if 201103L <= YY_CPLUSPLUS + symbol_type (int tok, location_type l) + : super_type (token_kind_type (tok), std::move (l)) +#else + symbol_type (int tok, const location_type& l) + : super_type (token_kind_type (tok), l) +#endif + { +#if !defined _MSC_VER || defined __clang__ + YY_ASSERT (tok == token::TOK_YYEOF + || (token::TOK_YYerror <= tok && tok <= token::TOK_R_BRACE)); +#endif + } +#if 201103L <= YY_CPLUSPLUS + symbol_type (int tok, int v, location_type l) + : super_type (token_kind_type (tok), std::move (v), std::move (l)) +#else + symbol_type (int tok, const int& v, const location_type& l) + : super_type (token_kind_type (tok), v, l) +#endif + { +#if !defined _MSC_VER || defined __clang__ + YY_ASSERT (tok == token::TOK_INT); +#endif + } +#if 201103L <= YY_CPLUSPLUS + symbol_type (int tok, std::string v, location_type l) + : super_type (token_kind_type (tok), std::move (v), std::move (l)) +#else + symbol_type (int tok, const std::string& v, const location_type& l) + : super_type (token_kind_type (tok), v, l) +#endif + { +#if !defined _MSC_VER || defined __clang__ + YY_ASSERT ((token::TOK_OPERATOR <= tok && tok <= token::TOK_VARNAME)); +#endif + } + }; + + /// Build a parser object. + parser (ASTDriver& drv_yyarg); + virtual ~parser (); + +#if 201103L <= YY_CPLUSPLUS + /// Non copyable. + parser (const parser&) = delete; + /// Non copyable. + parser& operator= (const parser&) = delete; +#endif + + /// Parse. An alias for parse (). + /// \returns 0 iff parsing succeeded. + int operator() (); + + /// Parse. + /// \returns 0 iff parsing succeeded. + virtual int parse (); + +#if YYDEBUG + /// The current debugging stream. + std::ostream& debug_stream () const YY_ATTRIBUTE_PURE; + /// Set the current debugging stream. + void set_debug_stream (std::ostream &); + + /// Type for debugging levels. + typedef int debug_level_type; + /// The current debugging level. + debug_level_type debug_level () const YY_ATTRIBUTE_PURE; + /// Set the current debugging level. + void set_debug_level (debug_level_type l); +#endif + + /// Report a syntax error. + /// \param loc where the syntax error is found. + /// \param msg a description of the syntax error. + virtual void error (const location_type& loc, const std::string& msg); + + /// Report a syntax error. + void error (const syntax_error& err); + + /// The user-facing name of the symbol whose (internal) number is + /// YYSYMBOL. No bounds checking. + static const char *symbol_name (symbol_kind_type yysymbol); + + // Implementation of make_symbol for each token kind. +#if 201103L <= YY_CPLUSPLUS + static + symbol_type + make_YYEOF (location_type l) + { + return symbol_type (token::TOK_YYEOF, std::move (l)); + } +#else + static + symbol_type + make_YYEOF (const location_type& l) + { + return symbol_type (token::TOK_YYEOF, l); + } +#endif +#if 201103L <= YY_CPLUSPLUS + static + symbol_type + make_YYerror (location_type l) + { + return symbol_type (token::TOK_YYerror, std::move (l)); + } +#else + static + symbol_type + make_YYerror (const location_type& l) + { + return symbol_type (token::TOK_YYerror, l); + } +#endif +#if 201103L <= YY_CPLUSPLUS + static + symbol_type + make_YYUNDEF (location_type l) + { + return symbol_type (token::TOK_YYUNDEF, std::move (l)); + } +#else + static + symbol_type + make_YYUNDEF (const location_type& l) + { + return symbol_type (token::TOK_YYUNDEF, l); + } +#endif +#if 201103L <= YY_CPLUSPLUS + static + symbol_type + make_ASSIGN (location_type l) + { + return symbol_type (token::TOK_ASSIGN, std::move (l)); + } +#else + static + symbol_type + make_ASSIGN (const location_type& l) + { + return symbol_type (token::TOK_ASSIGN, l); + } +#endif +#if 201103L <= YY_CPLUSPLUS + static + symbol_type + make_COMMA (location_type l) + { + return symbol_type (token::TOK_COMMA, std::move (l)); + } +#else + static + symbol_type + make_COMMA (const location_type& l) + { + return symbol_type (token::TOK_COMMA, l); + } +#endif +#if 201103L <= YY_CPLUSPLUS + static + symbol_type + make_COLON (location_type l) + { + return symbol_type (token::TOK_COLON, std::move (l)); + } +#else + static + symbol_type + make_COLON (const location_type& l) + { + return symbol_type (token::TOK_COLON, l); + } +#endif +#if 201103L <= YY_CPLUSPLUS + static + symbol_type + make_L_PAREN (location_type l) + { + return symbol_type (token::TOK_L_PAREN, std::move (l)); + } +#else + static + symbol_type + make_L_PAREN (const location_type& l) + { + return symbol_type (token::TOK_L_PAREN, l); + } +#endif +#if 201103L <= YY_CPLUSPLUS + static + symbol_type + make_R_PAREN (location_type l) + { + return symbol_type (token::TOK_R_PAREN, std::move (l)); + } +#else + static + symbol_type + make_R_PAREN (const location_type& l) + { + return symbol_type (token::TOK_R_PAREN, l); + } +#endif +#if 201103L <= YY_CPLUSPLUS + static + symbol_type + make_L_BRACE (location_type l) + { + return symbol_type (token::TOK_L_BRACE, std::move (l)); + } +#else + static + symbol_type + make_L_BRACE (const location_type& l) + { + return symbol_type (token::TOK_L_BRACE, l); + } +#endif +#if 201103L <= YY_CPLUSPLUS + static + symbol_type + make_R_BRACE (location_type l) + { + return symbol_type (token::TOK_R_BRACE, std::move (l)); + } +#else + static + symbol_type + make_R_BRACE (const location_type& l) + { + return symbol_type (token::TOK_R_BRACE, l); + } +#endif +#if 201103L <= YY_CPLUSPLUS + static + symbol_type + make_OPERATOR (std::string v, location_type l) + { + return symbol_type (token::TOK_OPERATOR, std::move (v), std::move (l)); + } +#else + static + symbol_type + make_OPERATOR (const std::string& v, const location_type& l) + { + return symbol_type (token::TOK_OPERATOR, v, l); + } +#endif +#if 201103L <= YY_CPLUSPLUS + static + symbol_type + make_IDENTIFIER (std::string v, location_type l) + { + return symbol_type (token::TOK_IDENTIFIER, std::move (v), std::move (l)); + } +#else + static + symbol_type + make_IDENTIFIER (const std::string& v, const location_type& l) + { + return symbol_type (token::TOK_IDENTIFIER, v, l); + } +#endif +#if 201103L <= YY_CPLUSPLUS + static + symbol_type + make_VARNAME (std::string v, location_type l) + { + return symbol_type (token::TOK_VARNAME, std::move (v), std::move (l)); + } +#else + static + symbol_type + make_VARNAME (const std::string& v, const location_type& l) + { + return symbol_type (token::TOK_VARNAME, v, l); + } +#endif +#if 201103L <= YY_CPLUSPLUS + static + symbol_type + make_INT (int v, location_type l) + { + return symbol_type (token::TOK_INT, std::move (v), std::move (l)); + } +#else + static + symbol_type + make_INT (const int& v, const location_type& l) + { + return symbol_type (token::TOK_INT, v, l); + } +#endif + + + class context + { + public: + context (const parser& yyparser, const symbol_type& yyla); + const symbol_type& lookahead () const YY_NOEXCEPT { return yyla_; } + symbol_kind_type token () const YY_NOEXCEPT { return yyla_.kind (); } + const location_type& location () const YY_NOEXCEPT { return yyla_.location; } + + /// Put in YYARG at most YYARGN of the expected tokens, and return the + /// number of tokens stored in YYARG. If YYARG is null, return the + /// number of expected tokens (guaranteed to be less than YYNTOKENS). + int expected_tokens (symbol_kind_type yyarg[], int yyargn) const; + + private: + const parser& yyparser_; + const symbol_type& yyla_; + }; + + private: +#if YY_CPLUSPLUS < 201103L + /// Non copyable. + parser (const parser&); + /// Non copyable. + parser& operator= (const parser&); +#endif + + /// Check the lookahead yytoken. + /// \returns true iff the token will be eventually shifted. + bool yy_lac_check_ (symbol_kind_type yytoken) const; + /// Establish the initial context if no initial context currently exists. + /// \returns true iff the token will be eventually shifted. + bool yy_lac_establish_ (symbol_kind_type yytoken); + /// Discard any previous initial lookahead context because of event. + /// \param event the event which caused the lookahead to be discarded. + /// Only used for debbuging output. + void yy_lac_discard_ (const char* event); + + /// Stored state numbers (used for stacks). + typedef signed char state_type; + + /// The arguments of the error message. + int yy_syntax_error_arguments_ (const context& yyctx, + symbol_kind_type yyarg[], int yyargn) const; + + /// Generate an error message. + /// \param yyctx the context in which the error occurred. + virtual std::string yysyntax_error_ (const context& yyctx) const; + /// Compute post-reduction state. + /// \param yystate the current state + /// \param yysym the nonterminal to push on the stack + static state_type yy_lr_goto_state_ (state_type yystate, int yysym); + + /// Whether the given \c yypact_ value indicates a defaulted state. + /// \param yyvalue the value to check + static bool yy_pact_value_is_default_ (int yyvalue) YY_NOEXCEPT; + + /// Whether the given \c yytable_ value indicates a syntax error. + /// \param yyvalue the value to check + static bool yy_table_value_is_error_ (int yyvalue) YY_NOEXCEPT; + + static const signed char yypact_ninf_; + static const signed char yytable_ninf_; + + /// Convert a scanner token kind \a t to a symbol kind. + /// In theory \a t should be a token_kind_type, but character literals + /// are valid, yet not members of the token_kind_type enum. + static symbol_kind_type yytranslate_ (int t) YY_NOEXCEPT; + + + + // Tables. + // YYPACT[STATE-NUM] -- Index in YYTABLE of the portion describing + // STATE-NUM. + static const signed char yypact_[]; + + // YYDEFACT[STATE-NUM] -- Default reduction number in state STATE-NUM. + // Performed when YYTABLE does not specify something else to do. Zero + // means the default is an error. + static const signed char yydefact_[]; + + // YYPGOTO[NTERM-NUM]. + static const signed char yypgoto_[]; + + // YYDEFGOTO[NTERM-NUM]. + static const signed char yydefgoto_[]; + + // YYTABLE[YYPACT[STATE-NUM]] -- What to do in state STATE-NUM. If + // positive, shift that token. If negative, reduce the rule whose + // number is the opposite. If YYTABLE_NINF, syntax error. + static const signed char yytable_[]; + + static const signed char yycheck_[]; + + // YYSTOS[STATE-NUM] -- The symbol kind of the accessing symbol of + // state STATE-NUM. + static const signed char yystos_[]; + + // YYR1[RULE-NUM] -- Symbol kind of the left-hand side of rule RULE-NUM. + static const signed char yyr1_[]; + + // YYR2[RULE-NUM] -- Number of symbols on the right-hand side of rule RULE-NUM. + static const signed char yyr2_[]; + + +#if YYDEBUG + // YYRLINE[YYN] -- Source line where rule number YYN was defined. + static const signed char yyrline_[]; + /// Report on the debug stream that the rule \a r is going to be reduced. + virtual void yy_reduce_print_ (int r) const; + /// Print the state stack on the debug stream. + virtual void yy_stack_print_ () const; + + /// Debugging level. + int yydebug_; + /// Debug stream. + std::ostream* yycdebug_; + + /// \brief Display a symbol kind, value and location. + /// \param yyo The output stream. + /// \param yysym The symbol. + template + void yy_print_ (std::ostream& yyo, const basic_symbol& yysym) const; +#endif + + /// \brief Reclaim the memory associated to a symbol. + /// \param yymsg Why this token is reclaimed. + /// If null, print nothing. + /// \param yysym The symbol. + template + void yy_destroy_ (const char* yymsg, basic_symbol& yysym) const; + + private: + /// Type access provider for state based symbols. + struct by_state + { + /// Default constructor. + by_state () YY_NOEXCEPT; + + /// The symbol kind as needed by the constructor. + typedef state_type kind_type; + + /// Constructor. + by_state (kind_type s) YY_NOEXCEPT; + + /// Copy constructor. + by_state (const by_state& that) YY_NOEXCEPT; + + /// Record that this symbol is empty. + void clear () YY_NOEXCEPT; + + /// Steal the symbol kind from \a that. + void move (by_state& that); + + /// The symbol kind (corresponding to \a state). + /// \a symbol_kind::S_YYEMPTY when empty. + symbol_kind_type kind () const YY_NOEXCEPT; + + /// The state number used to denote an empty symbol. + /// We use the initial state, as it does not have a value. + enum { empty_state = 0 }; + + /// The state. + /// \a empty when empty. + state_type state; + }; + + /// "Internal" symbol: element of the stack. + struct stack_symbol_type : basic_symbol + { + /// Superclass. + typedef basic_symbol super_type; + /// Construct an empty symbol. + stack_symbol_type (); + /// Move or copy construction. + stack_symbol_type (YY_RVREF (stack_symbol_type) that); + /// Steal the contents from \a sym to build this. + stack_symbol_type (state_type s, YY_MOVE_REF (symbol_type) sym); +#if YY_CPLUSPLUS < 201103L + /// Assignment, needed by push_back by some old implementations. + /// Moves the contents of that. + stack_symbol_type& operator= (stack_symbol_type& that); + + /// Assignment, needed by push_back by other implementations. + /// Needed by some other old implementations. + stack_symbol_type& operator= (const stack_symbol_type& that); +#endif + }; + + /// A stack with random access from its top. + template > + class stack + { + public: + // Hide our reversed order. + typedef typename S::iterator iterator; + typedef typename S::const_iterator const_iterator; + typedef typename S::size_type size_type; + typedef typename std::ptrdiff_t index_type; + + stack (size_type n = 200) YY_NOEXCEPT + : seq_ (n) + {} + +#if 201103L <= YY_CPLUSPLUS + /// Non copyable. + stack (const stack&) = delete; + /// Non copyable. + stack& operator= (const stack&) = delete; +#endif + + /// Random access. + /// + /// Index 0 returns the topmost element. + const T& + operator[] (index_type i) const + { + return seq_[size_type (size () - 1 - i)]; + } + + /// Random access. + /// + /// Index 0 returns the topmost element. + T& + operator[] (index_type i) + { + return seq_[size_type (size () - 1 - i)]; + } + + /// Steal the contents of \a t. + /// + /// Close to move-semantics. + void + push (YY_MOVE_REF (T) t) + { + seq_.push_back (T ()); + operator[] (0).move (t); + } + + /// Pop elements from the stack. + void + pop (std::ptrdiff_t n = 1) YY_NOEXCEPT + { + for (; 0 < n; --n) + seq_.pop_back (); + } + + /// Pop all elements from the stack. + void + clear () YY_NOEXCEPT + { + seq_.clear (); + } + + /// Number of elements on the stack. + index_type + size () const YY_NOEXCEPT + { + return index_type (seq_.size ()); + } + + /// Iterator on top of the stack (going downwards). + const_iterator + begin () const YY_NOEXCEPT + { + return seq_.begin (); + } + + /// Bottom of the stack. + const_iterator + end () const YY_NOEXCEPT + { + return seq_.end (); + } + + /// Present a slice of the top of a stack. + class slice + { + public: + slice (const stack& stack, index_type range) YY_NOEXCEPT + : stack_ (stack) + , range_ (range) + {} + + const T& + operator[] (index_type i) const + { + return stack_[range_ - i]; + } + + private: + const stack& stack_; + index_type range_; + }; + + private: +#if YY_CPLUSPLUS < 201103L + /// Non copyable. + stack (const stack&); + /// Non copyable. + stack& operator= (const stack&); +#endif + /// The wrapped container. + S seq_; + }; + + + /// Stack type. + typedef stack stack_type; + + /// The stack. + stack_type yystack_; + /// The stack for LAC. + /// Logically, the yy_lac_stack's lifetime is confined to the function + /// yy_lac_check_. We just store it as a member of this class to hold + /// on to the memory and to avoid frequent reallocations. + /// Since yy_lac_check_ is const, this member must be mutable. + mutable std::vector yylac_stack_; + /// Whether an initial LAC context was established. + bool yy_lac_established_; + + + /// Push a new state on the stack. + /// \param m a debug message to display + /// if null, no trace is output. + /// \param sym the symbol + /// \warning the contents of \a s.value is stolen. + void yypush_ (const char* m, YY_MOVE_REF (stack_symbol_type) sym); + + /// Push a new look ahead token on the state on the stack. + /// \param m a debug message to display + /// if null, no trace is output. + /// \param s the state + /// \param sym the symbol (for its value and location). + /// \warning the contents of \a sym.value is stolen. + void yypush_ (const char* m, state_type s, YY_MOVE_REF (symbol_type) sym); + + /// Pop \a n symbols from the stack. + void yypop_ (int n = 1) YY_NOEXCEPT; + + /// Constants. + enum + { + yylast_ = 53, ///< Last index in yytable_. + yynnts_ = 7, ///< Number of nonterminal symbols. + yyfinal_ = 12 ///< Termination state number. + }; + + + // User arguments. + ASTDriver& drv; + + }; + + inline + parser::symbol_kind_type + parser::yytranslate_ (int t) YY_NOEXCEPT + { + return static_cast (t); + } + + // basic_symbol. + template + parser::basic_symbol::basic_symbol (const basic_symbol& that) + : Base (that) + , value () + , location (that.location) + { + switch (this->kind ()) + { + case symbol_kind::S_INT: // "number" + case symbol_kind::S_list: // list + value.copy< int > (YY_MOVE (that.value)); + break; + + case symbol_kind::S_OPERATOR: // OPERATOR + case symbol_kind::S_IDENTIFIER: // "identifier" + case symbol_kind::S_VARNAME: // VARNAME + value.copy< std::string > (YY_MOVE (that.value)); + break; + + case symbol_kind::S_index: // index + value.copy< std::tuple > (YY_MOVE (that.value)); + break; + + case symbol_kind::S_indices_list: // indices_list + value.copy< std::vector> > (YY_MOVE (that.value)); + break; + + default: + break; + } + + } + + + + + template + parser::symbol_kind_type + parser::basic_symbol::type_get () const YY_NOEXCEPT + { + return this->kind (); + } + + + template + bool + parser::basic_symbol::empty () const YY_NOEXCEPT + { + return this->kind () == symbol_kind::S_YYEMPTY; + } + + template + void + parser::basic_symbol::move (basic_symbol& s) + { + super_type::move (s); + switch (this->kind ()) + { + case symbol_kind::S_INT: // "number" + case symbol_kind::S_list: // list + value.move< int > (YY_MOVE (s.value)); + break; + + case symbol_kind::S_OPERATOR: // OPERATOR + case symbol_kind::S_IDENTIFIER: // "identifier" + case symbol_kind::S_VARNAME: // VARNAME + value.move< std::string > (YY_MOVE (s.value)); + break; + + case symbol_kind::S_index: // index + value.move< std::tuple > (YY_MOVE (s.value)); + break; + + case symbol_kind::S_indices_list: // indices_list + value.move< std::vector> > (YY_MOVE (s.value)); + break; + + default: + break; + } + + location = YY_MOVE (s.location); + } + + // by_kind. + inline + parser::by_kind::by_kind () YY_NOEXCEPT + : kind_ (symbol_kind::S_YYEMPTY) + {} + +#if 201103L <= YY_CPLUSPLUS + inline + parser::by_kind::by_kind (by_kind&& that) YY_NOEXCEPT + : kind_ (that.kind_) + { + that.clear (); + } +#endif + + inline + parser::by_kind::by_kind (const by_kind& that) YY_NOEXCEPT + : kind_ (that.kind_) + {} + + inline + parser::by_kind::by_kind (token_kind_type t) YY_NOEXCEPT + : kind_ (yytranslate_ (t)) + {} + + + + inline + void + parser::by_kind::clear () YY_NOEXCEPT + { + kind_ = symbol_kind::S_YYEMPTY; + } + + inline + void + parser::by_kind::move (by_kind& that) + { + kind_ = that.kind_; + that.clear (); + } + + inline + parser::symbol_kind_type + parser::by_kind::kind () const YY_NOEXCEPT + { + return kind_; + } + + + inline + parser::symbol_kind_type + parser::by_kind::type_get () const YY_NOEXCEPT + { + return this->kind (); + } + + +#line 6 "../parser.y" +} } // adios2::detail +#line 1605 "parser.h" + + + + +#endif // !YY_YY_PARSER_H_INCLUDED diff --git a/source/adios2/toolkit/format/bp5/BP5Deserializer.cpp b/source/adios2/toolkit/format/bp5/BP5Deserializer.cpp index 0e7574a17c..9c1a127408 100644 --- a/source/adios2/toolkit/format/bp5/BP5Deserializer.cpp +++ b/source/adios2/toolkit/format/bp5/BP5Deserializer.cpp @@ -1994,6 +1994,8 @@ BP5Deserializer::~BP5Deserializer() m_Engine->m_IO.RemoveVariable(VarRec.second->VarName); free(VarRec.second->VarName); + if (VarRec.second->ExprStr) + free(VarRec.second->ExprStr); if (VarRec.second->Operator) free(VarRec.second->Operator); if (VarRec.second->Def) diff --git a/testing/adios2/bindings/fortran/CMakeLists.txt b/testing/adios2/bindings/fortran/CMakeLists.txt index c026628343..f1664f3d1f 100644 --- a/testing/adios2/bindings/fortran/CMakeLists.txt +++ b/testing/adios2/bindings/fortran/CMakeLists.txt @@ -15,6 +15,9 @@ macro(fortran_add_test_helper testname mpi) add_executable(${tgt} Test${testname}.F90) set_target_properties(${tgt} PROPERTIES LINKER_LANGUAGE Fortran) target_link_libraries(${tgt} adios2::fortran) + if (ADIOS2_HAVE_Derived_Variable) + target_compile_definitions(${tgt} PRIVATE -DADIOS2_HAVE_Derived_Variable=1) + endif() add_test( NAME ${pfx}${testname}.Serial COMMAND ${tgt} @@ -26,6 +29,9 @@ macro(fortran_add_test_helper testname mpi) add_executable(${tgt} Test${testname}.F90) set_target_properties(${tgt} PROPERTIES LINKER_LANGUAGE Fortran) target_link_libraries(${tgt} adios2::fortran_mpi MPI::MPI_Fortran) + if (ADIOS2_HAVE_Derived_Variable) + target_compile_definitions(${tgt} PRIVATE -DADIOS2_HAVE_Derived_Variable=1) + endif() add_test( NAME ${pfx}${testname}.MPI COMMAND ${MPIEXEC_COMMAND} $ diff --git a/testing/adios2/bindings/fortran/TestBPWriteTypes.F90 b/testing/adios2/bindings/fortran/TestBPWriteTypes.F90 index 7cb02991e0..2145baedf8 100644 --- a/testing/adios2/bindings/fortran/TestBPWriteTypes.F90 +++ b/testing/adios2/bindings/fortran/TestBPWriteTypes.F90 @@ -51,14 +51,26 @@ program TestBPWriteTypes start_dims(1) = irank*inx count_dims(1) = inx - if( adios%valid .eqv. .true. ) stop 'Invalid adios default' - if( ioWrite%valid .eqv. .true. ) stop 'Invalid io default' + if( adios%valid .eqv. .true. ) then + write(*,*) 'Invalid adios default' + stop 1 + end if + if( ioWrite%valid .eqv. .true. ) then + write(*,*) 'Invalid io default' + stop 1 + end if do i=1,12 - if( variables(i)%valid .eqv. .true. ) stop 'Invalid variables default' + if( variables(i)%valid .eqv. .true. ) then + write(*,*) 'Invalid variables default' + stop 1 + end if end do - if( bpWriter%valid .eqv. .true. ) stop 'Invalid engine default' + if( bpWriter%valid .eqv. .true. ) then + write(*,*) 'Invalid engine default' + stop 1 + end if ! Create adios handler passing the communicator and error flag @@ -67,38 +79,62 @@ program TestBPWriteTypes #else call adios2_init(adios, ierr) #endif - if( adios%valid .eqv. .false. ) stop 'Invalid adios2_init' + if( adios%valid .eqv. .false. ) then + write(*,*) 'Invalid adios2_init' + stop 1 + end if !!!!!!!!!!!!!!!!!!!!!!!! WRITER !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ! Declare an IO process configuration inside adios call adios2_declare_io(ioWrite, adios, "ioWrite", ierr) - if( ioWrite%valid .eqv. .false. ) stop 'Invalid adios2_declare_io' + if( ioWrite%valid .eqv. .false. ) then + write(*,*) 'Invalid adios2_declare_io' + stop 1 + end if call adios2_at_io(ioWrite, adios, "ioWrite", ierr) - if( ioWrite%valid .eqv. .false. ) stop 'Invalid adios2_at_io' + if( ioWrite%valid .eqv. .false. ) then + write(*,*) 'Invalid adios2_at_io' + stop 1 + end if call adios2_in_config_file(result, ioWrite, ierr) - if( result .eqv. .true. ) stop 'Invalid ioWrite adios2_in_config_file' + if( result .eqv. .true. ) then + write(*,*) 'Invalid ioWrite adios2_in_config_file' + stop 1 + end if call adios2_set_engine(ioWrite, 'File', ierr) call adios2_set_parameter(ioWrite, 'ProfileUnits', 'Microseconds', ierr) call adios2_get_parameter(param_value, ioWrite, 'ProfileUnits', ierr) - if( param_value /= "Microseconds") stop 'Failed adios2_get_parameter ProfileUnits' + if( param_value /= "Microseconds") then + write(*,*) 'Failed adios2_get_parameter ProfileUnits' + stop 1 + end if call adios2_set_parameters(ioWrite, 'Threads=2, CollectiveMetadata = OFF', ierr) call adios2_get_parameter(param_value, ioWrite, 'Threads', ierr) - if( param_value /= "2") stop 'Failed adios2_get_parameter Threads' + if( param_value /= "2") then + write(*,*) 'Failed adios2_get_parameter Threads' + stop 1 + end if call adios2_get_parameter(param_value, ioWrite, 'CollectiveMetadata', ierr) - if( param_value /= "OFF") stop 'Failed adios2_get_parameter CollectiveMetadata' + if( param_value /= "OFF") then + write(*,*) 'Failed adios2_get_parameter CollectiveMetadata' + stop 1 + end if ! set back the default to make sure writing/reading test works call adios2_clear_parameters(ioWrite, ierr) call adios2_get_parameter(param_value, ioWrite, 'CollectiveMetadata', ierr) - if( param_value /= "") stop 'Still Could retrieve parameter CollectiveMetadata after clearing all parameters' + if( param_value /= "") then + write(*,*) 'Still Could retrieve parameter CollectiveMetadata after clearing all parameters' + stop 1 + end if deallocate(param_value) @@ -166,41 +202,71 @@ program TestBPWriteTypes ! derived variable call adios2_define_derived_variable(derived_variable, ioWrite, "derived/magnitude_of_var_R64", & - "x:var_R64 y:var_R64 z:var_R64 magnitude(x,y,z)", adios2_derived_var_type_metadata_only, ierr) - + "x=var_R64 y=var_R64 z=var_R64 magnitude(x,y,z)", adios2_derived_var_type_metadata_only, ierr) +#if ADIOS2_HAVE_Derived_Variable +#define TOTAL_VAR_COUNT 15 +#else +#define TOTAL_VAR_COUNT 14 +#endif do i=1,13 - if( variables(i)%valid .eqv. .false. ) stop 'Invalid adios2_define_variable' + if( variables(i)%valid .eqv. .false. ) then + write(*,*) 'Invalid adios2_define_variable' + stop 1 + end if end do ! Testing adios2_variable_name for just two cases call adios2_variable_name(varName, variables(1), ierr) - if (varName /= 'var_I8') stop 'Invalid adios2_variable_name' + if (varName /= 'var_I8') then + write(*,*) 'Invalid adios2_variable_name' + stop 1 + end if call adios2_variable_name(varName, variables(2), ierr) - if (varName /= 'var_I16') stop 'Invalid adios2_variable_name' + if (varName /= 'var_I16') then + write(*,*) 'Invalid adios2_variable_name' + stop 1 + end if deallocate(varName) ! Open myVector_f.bp in write mode, this launches an engine - if( ioWrite%valid .eqv. .false. ) stop 'Invalid adios2_io' - if( bpWriter%valid .eqv. .true. ) stop 'Invalid adios2_engine pre-open' + if( ioWrite%valid .eqv. .false. ) then + write(*,*) 'Invalid adios2_io' + stop 1 + end if + if( bpWriter%valid .eqv. .true. ) then + write(*,*) 'Invalid adios2_engine pre-open' + stop 1 + end if call adios2_open(bpWriter, ioWrite, "ftypes.bp", adios2_mode_write, ierr) - if( bpWriter%valid .eqv. .false. ) stop 'Invalid adios2_engine post-open' - if( TRIM(bpWriter%name) /= "ftypes.bp") stop 'Invalid adios2_engine name' + if( bpWriter%valid .eqv. .false. ) then + write(*,*) 'Invalid adios2_engine post-open' + stop 1 + end if + if( TRIM(bpWriter%name) /= "ftypes.bp") then + write(*,*) 'Invalid adios2_engine name' + stop 1 + end if if( TRIM(bpWriter%type) /= 'BP5Writer') then write(*,*) 'Engine Type ', TRIM(bpWriter%type) - stop 'Invalid adios2_engine type' + write(*,*) 'Invalid adios2_engine type' + stop 1 end if call adios2_io_engine_type(engineType, ioWrite, ierr) if( engineType /= 'File') then ! FIXME, different from the above! write(*,*) 'Engine Type ', engineType - stop 'Invalid type from adios2_engine_type' + write(*,*) 'Invalid type from adios2_engine_type' + stop 1 end if - if( bpWriter%mode /= adios2_mode_write) stop 'Invalid adios2_engine mode' + if( bpWriter%mode /= adios2_mode_write) then + write(*,*) 'Invalid adios2_engine mode' + stop 1 + end if ! Put array contents to bp buffer, based on var1 metadata do i = 1, 3 @@ -232,7 +298,10 @@ program TestBPWriteTypes ! Closes engine1 and deallocates it, becomes unreachable call adios2_close(bpWriter, ierr) - if( bpWriter%valid .eqv. .true. ) stop 'Invalid adios2_close' + if( bpWriter%valid .eqv. .true. ) then + write(*,*) 'Invalid adios2_close' + stop 1 + end if !!!!!!!!!!!!!!!!!!!!!!!! READER !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ! Declare io reader @@ -241,80 +310,179 @@ program TestBPWriteTypes call adios2_open(bpReader, ioRead, "ftypes.bp", adios2_mode_readRandomAccess, ierr) call adios2_steps(nsteps, bpReader, ierr) - if(nsteps /= 3) stop 'ftypes.bp must have 3 steps' + if(nsteps /= 3) then + write(*,*) 'ftypes.bp must have 3 steps' + stop 1 + end if call adios2_available_variables(ioRead, namestruct, ierr) - if (ierr /= 0) stop 'adios2_available_variables returned with error' - if (.not.namestruct%valid) stop 'adios2_available_variables returned invalid struct' + if (ierr /= 0) then + write(*,*) 'adios2_available_variables returned with error' + stop 1 + end if + if (.not.namestruct%valid) then + write(*,*) 'adios2_available_variables returned invalid struct' + stop 1 + end if write(*,*) 'Number of variables = ', namestruct%count write(*,*) 'Max name length = ', namestruct%max_name_len - if (namestruct%count /= 14) stop 'adios2_available_variables returned not the expected 14' + if (namestruct%count /= TOTAL_VAR_COUNT) then + write(*,*) 'adios2_available_variables returned not the expected 14' + stop 1 + end if allocate(varnamelist(namestruct%count)) call adios2_retrieve_names(namestruct, varnamelist, ierr) - if (ierr /= 0) stop 'adios2_retrieve_names returned with error' + if (ierr /= 0) then + write(*,*) 'adios2_retrieve_names returned with error' + stop 1 + end if do i=1,namestruct%count write(*,'("Var[",i2,"] = ",a12)') i, varnamelist(i) end do deallocate(varnamelist) - if (namestruct%f2c /= 0_8) stop 'namestruct f2c pointer is not null after adios2_retrieve_names()' - if (namestruct%valid) stop 'namestruct is not invalidated after adios2_retrieve_names()' + if (namestruct%f2c /= 0_8) then + write(*,*) 'namestruct f2c pointer is not null after adios2_retrieve_names()' + stop 1 + end if + if (namestruct%valid) then + write(*,*) 'namestruct is not invalidated after adios2_retrieve_names()' + stop 1 + end if call adios2_inquire_variable(variables(1), ioRead, "var_I8", ierr) - if (variables(1)%name /= 'var_I8') stop 'var_I8 not recognized' - if (variables(1)%type /= adios2_type_integer1) stop 'var_I8 type not recognized' + if (variables(1)%name /= 'var_I8') then + write(*,*) 'var_I8 not recognized' + stop 1 + end if + if (variables(1)%type /= adios2_type_integer1) then + write(*,*) 'var_I8 type not recognized' + stop 1 + end if call adios2_variable_shape(shape_in, ndims, variables(1), ierr) - if (ndims /= 1) stop 'var_I8 ndims is not 1' - if (shape_in(1) /= isize*inx) stop 'var_I8 shape_in read failed' + if (ndims /= 1) then + write(*,*) 'var_I8 ndims is not 1' + stop 1 + end if + if (shape_in(1) /= isize*inx) then + write(*,*) 'var_I8 shape_in read failed' + stop 1 + end if call adios2_inquire_variable(variables(2), ioRead, "var_I16", ierr) - if (variables(2)%name /= 'var_I16') stop 'var_I16 not recognized' - if (variables(2)%type /= adios2_type_integer2) stop 'var_I16 type not recognized' + if (variables(2)%name /= 'var_I16') then + write(*,*) 'var_I16 not recognized' + stop 1 + end if + if (variables(2)%type /= adios2_type_integer2) then + write(*,*) 'var_I16 type not recognized' + stop 1 + end if call adios2_variable_shape(shape_in, ndims, variables(2), ierr) - if (ndims /= 1) stop 'var_I16 ndims is not 1' - if (shape_in(1) /= isize*inx) stop 'var_I16 shape_in read failed' + if (ndims /= 1) then + write(*,*) 'var_I16 ndims is not 1' + stop 1 + end if + if (shape_in(1) /= isize*inx) then + write(*,*) 'var_I16 shape_in read failed' + stop 1 + end if call adios2_inquire_variable(variables(3), ioRead, "var_I32", ierr) - if (variables(3)%name /= 'var_I32') stop 'var_I32 not recognized' - if (variables(3)%type /= adios2_type_integer4) stop 'var_I32 type not recognized' + if (variables(3)%name /= 'var_I32') then + write(*,*) 'var_I32 not recognized' + stop 1 + end if + if (variables(3)%type /= adios2_type_integer4) then + write(*,*) 'var_I32 type not recognized' + stop 1 + end if call adios2_variable_shape(shape_in, ndims, variables(3), ierr) - if (ndims /= 1) stop 'var_I32 ndims is not 1' - if (shape_in(1) /= isize*inx) stop 'var_I32 shape_in read failed' + if (ndims /= 1) then + write(*,*) 'var_I32 ndims is not 1' + stop 1 + end if + if (shape_in(1) /= isize*inx) then + write(*,*) 'var_I32 shape_in read failed' + stop 1 + end if call adios2_inquire_variable(variables(4), ioRead, "var_I64", ierr) - if (variables(4)%name /= 'var_I64') stop 'var_I64 not recognized' - if (variables(4)%type /= adios2_type_integer8) stop 'var_I64 type not recognized' + if (variables(4)%name /= 'var_I64') then + write(*,*) 'var_I64 not recognized' + stop 1 + end if + if (variables(4)%type /= adios2_type_integer8) then + write(*,*) 'var_I64 type not recognized' + stop 1 + end if call adios2_variable_shape(shape_in, ndims, variables(4), ierr) - if (ndims /= 1) stop 'var_I64 ndims is not 1' - if (shape_in(1) /= isize*inx) stop 'var_I64 shape_in read failed' + if (ndims /= 1) then + write(*,*) 'var_I64 ndims is not 1' + stop 1 + end if + if (shape_in(1) /= isize*inx) then + write(*,*) 'var_I64 shape_in read failed' + stop 1 + end if call adios2_inquire_variable(variables(5), ioRead, "var_R32", ierr) - if (variables(5)%name /= 'var_R32') stop 'var_R32 not recognized' - if (variables(5)%type /= adios2_type_real) stop 'var_R32 type not recognized' + if (variables(5)%name /= 'var_R32') then + write(*,*) 'var_R32 not recognized' + stop 1 + end if + if (variables(5)%type /= adios2_type_real) then + write(*,*) 'var_R32 type not recognized' + stop 1 + end if call adios2_variable_shape(shape_in, ndims, variables(5), ierr) - if (ndims /= 1) stop 'var_R32 ndims is not 1' - if (shape_in(1) /= isize*inx) stop 'var_R32 shape_in read failed' + if (ndims /= 1) then + write(*,*) 'var_R32 ndims is not 1' + stop 1 + end if + if (shape_in(1) /= isize*inx) then + write(*,*) 'var_R32 shape_in read failed' + stop 1 + end if call adios2_inquire_variable(variables(6), ioRead, "var_R64", ierr) - if (variables(6)%name /= 'var_R64') stop 'var_R64 not recognized' - if (variables(6)%type /= adios2_type_dp) stop 'var_R64 type not recognized' + if (variables(6)%name /= 'var_R64') then + write(*,*) 'var_R64 not recognized' + stop 1 + end if + if (variables(6)%type /= adios2_type_dp) then + write(*,*) 'var_R64 type not recognized' + stop 1 + end if call adios2_variable_shape(shape_in, ndims, variables(6), ierr) - if (ndims /= 1) stop 'var_R64 ndims is not 1' - if (shape_in(1) /= isize*inx) stop 'var_R64 shape_in read failed' + if (ndims /= 1) then + write(*,*) 'var_R64 ndims is not 1' + stop 1 + end if + if (shape_in(1) /= isize*inx) then + write(*,*) 'var_R64 shape_in read failed' + stop 1 + end if call adios2_inquire_variable(variables(13), ioRead, "gvar_Str", ierr) call adios2_get(bpReader, variables(13), inString, ierr) call adios2_perform_gets(bpReader, ierr) - if( inString /= data_Strings(1) ) stop 'gvar_Str read failed' + if( inString /= data_Strings(1) ) then + write(*,*) 'gvar_Str read failed' + stop 1 + end if call adios2_inquire_variable(variables(14), ioRead, "lvar_i32", ierr) allocate(inRanks(isize)) call adios2_get(bpReader, variables(14), inRanks, ierr) call adios2_perform_gets(bpReader, ierr) - if( inRanks(irank+1) /= irank ) stop 'lvar_i32 read failed' + if( inRanks(irank+1) /= irank ) then + write(*,*) 'lvar_i32 read failed' + stop 1 + end if deallocate(inRanks) call adios2_close(bpReader, ierr) diff --git a/testing/adios2/derived/TestBPDerivedCorrectness.cpp b/testing/adios2/derived/TestBPDerivedCorrectness.cpp index 002015b843..886b942db3 100644 --- a/testing/adios2/derived/TestBPDerivedCorrectness.cpp +++ b/testing/adios2/derived/TestBPDerivedCorrectness.cpp @@ -3,6 +3,7 @@ #include #include +#include #include #include #include @@ -36,25 +37,21 @@ TEST(DerivedCorrectness, AddCorrectnessTest) std::vector varname = {"sim1/Ux", "sim1/Uy", "sim1/Uz"}; std::string derivedname = "derived/addU"; - std::cout << "Define Variable " << varname[0] << std::endl; auto Ux = bpOut.DefineVariable(varname[0], {Nx, Ny, Nz}, {0, 0, 0}, {Nx, Ny, Nz}); - std::cout << "Define Variable " << varname[1] << std::endl; auto Uy = bpOut.DefineVariable(varname[1], {Nx, Ny, Nz}, {0, 0, 0}, {Nx, Ny, Nz}); - std::cout << "Define Variable " << varname[2] << std::endl; auto Uz = bpOut.DefineVariable(varname[2], {Nx, Ny, Nz}, {0, 0, 0}, {Nx, Ny, Nz}); - std::cout << "Define Derived Variable " << derivedname << std::endl; // clang-format off - auto addU = bpOut.DefineDerivedVariable(derivedname, - "x:" + varname[0] + " \n" - "y:" + varname[1] + " \n" - "z:" + varname[2] + " \n" - "x+y+z", - adios2::DerivedVarType::StoreData); + bpOut.DefineDerivedVariable(derivedname, + "x =" + varname[0] + " \n" + "y =" + varname[1] + " \n" + "z =" + varname[2] + " \n" + "x+y+z", + adios2::DerivedVarType::StoreData); // clang-format on std::string filename = "expAdd.bp"; adios2::Engine bpFileWriter = bpOut.Open(filename, adios2::Mode::Write); - for (int i = 0; i < steps; i++) + for (size_t i = 0; i < steps; i++) { bpFileWriter.BeginStep(); bpFileWriter.Put(Ux, simArray1.data()); @@ -73,8 +70,8 @@ TEST(DerivedCorrectness, AddCorrectnessTest) std::vector readAdd; float calcA; - float epsilon = 0.01; - for (int i = 0; i < steps; i++) + float epsilon = (float)0.01; + for (size_t i = 0; i < steps; i++) { bpFileReader.BeginStep(); bpFileReader.Get(varname[0], readUx); @@ -119,17 +116,17 @@ TEST(DerivedCorrectness, MagCorrectnessTest) auto Uy = bpOut.DefineVariable(varname[1], {Nx, Ny, Nz}, {0, 0, 0}, {Nx, Ny, Nz}); auto Uz = bpOut.DefineVariable(varname[2], {Nx, Ny, Nz}, {0, 0, 0}, {Nx, Ny, Nz}); // clang-format off - auto magU = bpOut.DefineDerivedVariable(derivedname, - "x:" + varname[0] + " \n" - "y:" + varname[1] + " \n" - "z:" + varname[2] + " \n" - "magnitude(x,y,z)", - adios2::DerivedVarType::StoreData); + bpOut.DefineDerivedVariable(derivedname, + "x =" + varname[0] + " \n" + "y =" + varname[1] + " \n" + "z =" + varname[2] + " \n" + "magnitude(x,y,z)", + adios2::DerivedVarType::StoreData); // clang-format on std::string filename = "expMagnitude.bp"; adios2::Engine bpFileWriter = bpOut.Open(filename, adios2::Mode::Write); - for (int i = 0; i < steps; i++) + for (size_t i = 0; i < steps; i++) { bpFileWriter.BeginStep(); bpFileWriter.Put(Ux, simArray1.data()); @@ -148,8 +145,8 @@ TEST(DerivedCorrectness, MagCorrectnessTest) std::vector readMag; float calcM; - float epsilon = 0.01; - for (int i = 0; i < steps; i++) + float epsilon = (float)0.01; + for (size_t i = 0; i < steps; i++) { bpFileReader.BeginStep(); auto varx = bpIn.InquireVariable(varname[0]); @@ -165,7 +162,7 @@ TEST(DerivedCorrectness, MagCorrectnessTest) for (size_t ind = 0; ind < Nx * Ny * Nz; ++ind) { - calcM = sqrt(pow(readUx[ind], 2) + pow(readUy[ind], 2) + pow(readUz[ind], 2)); + calcM = (float)sqrt(pow(readUx[ind], 2) + pow(readUy[ind], 2) + pow(readUz[ind], 2)); EXPECT_TRUE(fabs(calcM - readMag[ind]) < epsilon); } } From 5c6fac3282f4781a5f366fa37890371656397175 Mon Sep 17 00:00:00 2001 From: Greg Eisenhauer Date: Wed, 20 Mar 2024 10:27:42 -0400 Subject: [PATCH 090/124] WIP: Make Fortran tests fail with a non-zero exit code (#4097) * Make Fortran tests fail with a non-zero exit code Co-authored-by: anagainaru --- bindings/C/adios2/c/adios2_c_variable.cpp | 8 - source/adios2/common/ADIOSTypes.cpp | 17 + source/adios2/common/ADIOSTypes.h | 1 + source/adios2/core/VariableBase.h | 4 - .../adios2/engine/dataman/DataManReader.tcc | 5 +- .../adios2/engine/dataman/DataManWriter.tcc | 5 +- .../format/dataman/DataManSerializer.h | 4 +- .../format/dataman/DataManSerializer.tcc | 9 +- .../fortran/TestAdios2BindingsFortranIO.F90 | 35 +- .../bindings/fortran/TestBPMemorySpace.F90 | 30 +- .../bindings/fortran/TestBPMemorySpaceGPU.F90 | 30 +- .../fortran/TestBPReadGlobalsByName.F90 | 11 +- .../TestBPWriteMemorySelectionRead2D.F90 | 210 +++++++-- .../TestBPWriteMemorySelectionRead3D.F90 | 210 +++++++-- .../fortran/TestBPWriteReadAttributes.F90 | 407 ++++++++++++++---- .../fortran/TestBPWriteReadHeatMap2D.F90 | 30 +- .../fortran/TestBPWriteReadHeatMap3D.F90 | 30 +- .../fortran/TestBPWriteReadHeatMap4D.F90 | 30 +- .../fortran/TestBPWriteReadHeatMap5D.F90 | 30 +- .../fortran/TestBPWriteReadHeatMap6D.F90 | 30 +- .../fortran/TestBPWriteTypesByName.F90 | 120 ++++-- .../fortran/TestBPWriteTypesLocal.F90 | 185 ++++++-- .../fortran/TestBPWriteVariableAttributes.F90 | 12 +- .../bindings/fortran/TestNullEngine.F90 | 13 +- .../adios2/bindings/fortran/TestRemove.F90 | 135 ++++-- .../fortran/operation/TestBPWriteReadSZ2D.F90 | 40 +- .../fortran/operation/TestBPWriteReadSZ3D.F90 | 40 +- .../operation/TestBPWriteReadZfp2D.F90 | 40 +- .../operation/TestBPWriteReadZfp2DRemove.F90 | 20 +- .../engine/bp/TestBPFortranToCppWriter.F90 | 5 +- .../engine/staging-common/TestCommonReadF.F90 | 225 ++++++++-- 31 files changed, 1560 insertions(+), 411 deletions(-) diff --git a/bindings/C/adios2/c/adios2_c_variable.cpp b/bindings/C/adios2/c/adios2_c_variable.cpp index f17b8c44ae..a8d02bf5ad 100644 --- a/bindings/C/adios2/c/adios2_c_variable.cpp +++ b/bindings/C/adios2/c/adios2_c_variable.cpp @@ -79,11 +79,7 @@ adios2_error adios2_set_shape(adios2_variable *variable, const size_t ndims, con adios2::MemorySpace adios2_ToMemorySpace(const adios2_memory_space Cmem) { -#ifdef ADIOS2_HAVE_GPU_SUPPORT adios2::MemorySpace mem = adios2::MemorySpace::Detect; -#else - adios2::MemorySpace mem = adios2::MemorySpace::Host; -#endif switch (Cmem) { @@ -104,11 +100,7 @@ adios2::MemorySpace adios2_ToMemorySpace(const adios2_memory_space Cmem) adios2_memory_space adios2_FromMemorySpace(const adios2::MemorySpace mem) { -#ifdef ADIOS2_HAVE_GPU_SUPPORT adios2_memory_space Cmem = adios2_memory_space_detect; -#else - adios2_memory_space Cmem = adios2_memory_space_host; -#endif switch (mem) { diff --git a/source/adios2/common/ADIOSTypes.cpp b/source/adios2/common/ADIOSTypes.cpp index 5fbdb69216..51dc8b2ca4 100644 --- a/source/adios2/common/ADIOSTypes.cpp +++ b/source/adios2/common/ADIOSTypes.cpp @@ -18,6 +18,23 @@ namespace adios2 { +std::string ToString(MemorySpace value) +{ + switch (value) + { + case MemorySpace::Detect: + return "MemorySpace::Detect"; + case MemorySpace::Host: + return "MemorySpace::Host"; +#ifdef ADIOS2_HAVE_GPU_SUPPORT + case MemorySpace::GPU: + return "MemorySpace::GPU"; +#endif + default: + return "ToString: Unknown MemorySpace"; + } +} + std::string ToString(ShapeID value) { switch (value) diff --git a/source/adios2/common/ADIOSTypes.h b/source/adios2/common/ADIOSTypes.h index 817e4da2b1..5d9eeb2624 100644 --- a/source/adios2/common/ADIOSTypes.h +++ b/source/adios2/common/ADIOSTypes.h @@ -355,6 +355,7 @@ std::string ToString(SelectionType value); std::string ToString(DataType type); std::string ToString(const Dims &dims); std::string ToString(const Box &box); +std::string ToString(const MemorySpace value); /** * os << [adios2_type] enables output of adios2 enums/classes directly diff --git a/source/adios2/core/VariableBase.h b/source/adios2/core/VariableBase.h index dc410ec78d..5b585d46a2 100644 --- a/source/adios2/core/VariableBase.h +++ b/source/adios2/core/VariableBase.h @@ -52,11 +52,7 @@ class VariableBase const size_t m_ElementSize; /* User requested memory space */ -#ifdef ADIOS2_HAVE_GPU_SUPPORT MemorySpace m_MemSpace = MemorySpace::Detect; -#else - MemorySpace m_MemSpace = MemorySpace::Host; -#endif #if defined(ADIOS2_HAVE_KOKKOS) || defined(ADIOS2_HAVE_GPU_SUPPORT) ArrayOrdering m_BaseLayout; ArrayOrdering m_ArrayLayout = ArrayOrdering::Auto; diff --git a/source/adios2/engine/dataman/DataManReader.tcc b/source/adios2/engine/dataman/DataManReader.tcc index 9edd962b96..df1af3ac67 100644 --- a/source/adios2/engine/dataman/DataManReader.tcc +++ b/source/adios2/engine/dataman/DataManReader.tcc @@ -31,12 +31,13 @@ void DataManReader::GetSyncCommon(Variable &variable, T *data) template void DataManReader::GetDeferredCommon(Variable &variable, T *data) { + auto varMemSpace = variable.GetMemorySpace(data); if (m_IO.m_ArrayOrder == ArrayOrdering::RowMajor) { while (true) { int ret = m_Serializer.GetData(data, variable.m_Name, variable.m_Start, - variable.m_Count, m_CurrentStep, variable.m_MemSpace, + variable.m_Count, m_CurrentStep, varMemSpace, variable.m_MemoryStart, variable.m_MemoryCount); if (ret == 0) { @@ -57,7 +58,7 @@ void DataManReader::GetDeferredCommon(Variable &variable, T *data) while (true) { int ret = m_Serializer.GetData(data, variable.m_Name, start, count, m_CurrentStep, - variable.m_MemSpace, memstart, memcount); + varMemSpace, memstart, memcount); if (ret == 0) { break; diff --git a/source/adios2/engine/dataman/DataManWriter.tcc b/source/adios2/engine/dataman/DataManWriter.tcc index f9996e4678..ba5f1a6974 100644 --- a/source/adios2/engine/dataman/DataManWriter.tcc +++ b/source/adios2/engine/dataman/DataManWriter.tcc @@ -31,10 +31,11 @@ void DataManWriter::PutSyncCommon(Variable &variable, const T *values) template void DataManWriter::PutDeferredCommon(Variable &variable, const T *values) { + auto varMemSpace = variable.GetMemorySpace(values); variable.SetData(values); if (m_IO.m_ArrayOrder == ArrayOrdering::RowMajor) { - m_Serializer.PutData(variable, m_Name, CurrentStep(), m_MpiRank, ""); + m_Serializer.PutData(variable, m_Name, CurrentStep(), m_MpiRank, varMemSpace, ""); } else { @@ -49,7 +50,7 @@ void DataManWriter::PutDeferredCommon(Variable &variable, const T *values) std::reverse(memstart.begin(), memstart.end()); std::reverse(memcount.begin(), memcount.end()); m_Serializer.PutData(variable.m_Data, variable.m_Name, shape, start, count, memstart, - memcount, variable.m_MemSpace, m_Name, CurrentStep(), m_MpiRank, "", + memcount, varMemSpace, m_Name, CurrentStep(), m_MpiRank, "", variable.m_Operations); } diff --git a/source/adios2/toolkit/format/dataman/DataManSerializer.h b/source/adios2/toolkit/format/dataman/DataManSerializer.h index 4e2c2ada6a..8e3247667f 100644 --- a/source/adios2/toolkit/format/dataman/DataManSerializer.h +++ b/source/adios2/toolkit/format/dataman/DataManSerializer.h @@ -110,8 +110,8 @@ class DataManSerializer // another wrapper for PutData which accepts adios2::core::Variable template void PutData(const core::Variable &variable, const std::string &doid, const size_t step, - const int rank, const std::string &address, VecPtr localBuffer = nullptr, - JsonPtr metadataJson = nullptr); + const int rank, const MemorySpace varMemSpace, const std::string &address, + VecPtr localBuffer = nullptr, JsonPtr metadataJson = nullptr); // attach attributes to local pack void AttachAttributesToLocalPack(); diff --git a/source/adios2/toolkit/format/dataman/DataManSerializer.tcc b/source/adios2/toolkit/format/dataman/DataManSerializer.tcc index e3fd9b29d3..4060b93681 100644 --- a/source/adios2/toolkit/format/dataman/DataManSerializer.tcc +++ b/source/adios2/toolkit/format/dataman/DataManSerializer.tcc @@ -76,13 +76,14 @@ void DataManSerializer::CalculateMinMax(const T *data, const Dims &count, template void DataManSerializer::PutData(const core::Variable &variable, const std::string &doid, - const size_t step, const int rank, const std::string &address, - VecPtr localBuffer, JsonPtr metadataJson) + const size_t step, const int rank, const MemorySpace memSpace, + const std::string &address, VecPtr localBuffer, + JsonPtr metadataJson) { PERFSTUBS_SCOPED_TIMER_FUNC(); PutData(variable.GetData(), variable.m_Name, variable.m_Shape, variable.m_Start, - variable.m_Count, variable.m_MemoryStart, variable.m_MemoryCount, variable.m_MemSpace, - doid, step, rank, address, variable.m_Operations, localBuffer, metadataJson); + variable.m_Count, variable.m_MemoryStart, variable.m_MemoryCount, memSpace, doid, step, + rank, address, variable.m_Operations, localBuffer, metadataJson); } template diff --git a/testing/adios2/bindings/fortran/TestAdios2BindingsFortranIO.F90 b/testing/adios2/bindings/fortran/TestAdios2BindingsFortranIO.F90 index 24c648d10f..7bb5ffb3a6 100644 --- a/testing/adios2/bindings/fortran/TestAdios2BindingsFortranIO.F90 +++ b/testing/adios2/bindings/fortran/TestAdios2BindingsFortranIO.F90 @@ -60,7 +60,10 @@ subroutine testing_adios_io_finalize() ! FIXME, shouldn't we be able to do this by handle? call adios2_remove_io(result, adios, "TestIo", ierr) - if ((ierr /= 0) .or. (result .neqv. .true.)) stop "FAIL: adios2_remove_io" + if ((ierr /= 0) .or. (result .neqv. .true.)) then + write(*,*) "FAIL: adios2_remove_io" + stop 1 + end if call testing_adios_finalize() end subroutine testing_adios_io_finalize @@ -88,18 +91,27 @@ subroutine testing_adios_io_engine() call adios2_set_engine(io, "file", ierr) call adios2_io_engine_type(engine_type, io, ierr) - if (engine_type /= "file") stop "FAIL adios2_io_engine_type" + if (engine_type /= "file") then + write(*,*) "FAIL adios2_io_engine_type" + stop 1 + end if deallocate(engine_type) call adios2_open(engine, io, "ftypes.bp", adios2_mode_write, ierr) - if (engine%type /= "BP5Writer") stop "FAIL engine%type" + if (engine%type /= "BP5Writer") then + write(*,*) "FAIL engine%type" + stop 1 + end if ! // FIXME, I'd like to check that the engine type itself is correct, but ! // there's no (function-style) API to get it ! // FIXME, I'd like to check the engine's name, but there's no API to get it call adios2_io_engine_type(engine_type, io, ierr) - if (engine_type /= "file") stop "FAIL adios2_io_engine_type" + if (engine_type /= "file") then + write(*,*) "FAIL adios2_io_engine_type" + stop 1 + end if deallocate(engine_type) call testing_adios_io_finalize() @@ -123,18 +135,27 @@ subroutine testing_adios_io_engine_default() call adios2_set_engine(io, "", ierr) call adios2_io_engine_type(engine_type, io, ierr) - if (engine_type /= "") stop "FAIL adios2_io_engine_type" + if (engine_type /= "") then + write(*,*) "FAIL adios2_io_engine_type" + stop 1 + end if deallocate(engine_type) call adios2_open(engine, io, "ftypes.bp", adios2_mode_write, ierr) - if (engine%type /= "BP5Writer") stop "FAIL engine%type" + if (engine%type /= "BP5Writer") then + write(*,*) "FAIL engine%type" + stop 1 + end if ! // FIXME, I'd like to check that the engine type itself is correct, but ! // there's no (function-style) API to get it ! // FIXME, I'd like to check the engine's name, but there's no API to get it call adios2_io_engine_type(engine_type, io, ierr) - if (engine_type /= "") stop "FAIL adios2_io_engine_type" + if (engine_type /= "") then + write(*,*) "FAIL adios2_io_engine_type" + stop 1 + end if deallocate(engine_type) call testing_adios_io_finalize diff --git a/testing/adios2/bindings/fortran/TestBPMemorySpace.F90 b/testing/adios2/bindings/fortran/TestBPMemorySpace.F90 index e22cb67438..fbec8267c4 100644 --- a/testing/adios2/bindings/fortran/TestBPMemorySpace.F90 +++ b/testing/adios2/bindings/fortran/TestBPMemorySpace.F90 @@ -13,14 +13,26 @@ program TestBPMemorySpace start_dims(1) = 0 count_dims(1) = 10 - if( adios%valid .eqv. .true. ) stop 'Invalid adios default' - if( variable%valid .eqv. .true. ) stop 'Invalid variables default' + if( adios%valid .eqv. .true. ) then + write(*,*) 'Invalid adios default' + stop 1 + end if + if( variable%valid .eqv. .true. ) then + write(*,*) 'Invalid variables default' + stop 1 + end if call adios2_init(adios, ierr) - if( adios%valid .eqv. .false. ) stop 'Invalid adios2_init' + if( adios%valid .eqv. .false. ) then + write(*,*) 'Invalid adios2_init' + stop 1 + end if call adios2_declare_io(ioWrite, adios, "ioWrite", ierr) - if( ioWrite%valid .eqv. .false. ) stop 'Invalid adios2_declare_io' + if( ioWrite%valid .eqv. .false. ) then + write(*,*) 'Invalid adios2_declare_io' + stop 1 + end if call adios2_set_engine(ioWrite, 'File', ierr) @@ -31,11 +43,17 @@ program TestBPMemorySpace ! check that the default execution space is Detect call adios2_get_memory_space(mem, variable, ierr) - if (mem /= adios2_memory_space_detect) stop 'Invalid adios2_memory_space' + if (mem /= adios2_memory_space_detect) then + write(*,*) 'Invalid default adios2_memory_space' + stop 1 + end if ! check that the execution space is updated to Host call adios2_set_memory_space(variable, adios2_memory_space_host, ierr) call adios2_get_memory_space(mem, variable, ierr) - if (mem /= adios2_memory_space_host) stop 'Invalid adios2_memory_space' + if (mem /= adios2_memory_space_host) then + write(*,*) 'Invalid set adios2_memory_space' + stop 1 + end if end program TestBPMemorySpace diff --git a/testing/adios2/bindings/fortran/TestBPMemorySpaceGPU.F90 b/testing/adios2/bindings/fortran/TestBPMemorySpaceGPU.F90 index 64fb26755f..bffc935753 100644 --- a/testing/adios2/bindings/fortran/TestBPMemorySpaceGPU.F90 +++ b/testing/adios2/bindings/fortran/TestBPMemorySpaceGPU.F90 @@ -13,14 +13,26 @@ program TestBPMemorySpace start_dims(1) = 0 count_dims(1) = 10 - if( adios%valid .eqv. .true. ) stop 'Invalid adios default' - if( variable%valid .eqv. .true. ) stop 'Invalid variables default' + if( adios%valid .eqv. .true. ) then + write(*,*) 'Invalid adios default' + stop 1 + end if + if( variable%valid .eqv. .true. ) then + write(*,*) 'Invalid variables default' + stop 1 + end if call adios2_init(adios, ierr) - if( adios%valid .eqv. .false. ) stop 'Invalid adios2_init' + if( adios%valid .eqv. .false. ) then + write(*,*) 'Invalid adios2_init' + stop 1 + end if call adios2_declare_io(ioWrite, adios, "ioWrite", ierr) - if( ioWrite%valid .eqv. .false. ) stop 'Invalid adios2_declare_io' + if( ioWrite%valid .eqv. .false. ) then + write(*,*) 'Invalid adios2_declare_io' + stop 1 + end if call adios2_set_engine(ioWrite, 'File', ierr) @@ -31,11 +43,17 @@ program TestBPMemorySpace ! check that the default execution space is Detect call adios2_get_memory_space(mem, variable, ierr) - if (mem /= adios2_memory_space_detect) stop 'Invalid adios2_memory_space' + if (mem /= adios2_memory_space_detect) then + write(*,*) 'Invalid default adios2_memory_space' + stop 1 + end if ! check that the execution space is updated to GPU call adios2_set_memory_space(variable, adios2_memory_space_gpu, ierr) call adios2_get_memory_space(mem, variable, ierr) - if (mem /= adios2_memory_space_gpu) stop 'Invalid adios2_memory_space' + if (mem /= adios2_memory_space_gpu) then + write(*,*) 'Invalid set adios2_memory_space' + stop 1 + end if end program TestBPMemorySpace diff --git a/testing/adios2/bindings/fortran/TestBPReadGlobalsByName.F90 b/testing/adios2/bindings/fortran/TestBPReadGlobalsByName.F90 index 6c71157c77..d2fba7b634 100644 --- a/testing/adios2/bindings/fortran/TestBPReadGlobalsByName.F90 +++ b/testing/adios2/bindings/fortran/TestBPReadGlobalsByName.F90 @@ -38,7 +38,10 @@ program TestBPReadGlobalsByName call adios2_put(writer, "sml_outpsi", 0.295477_8, ierr) call adios2_close(writer, ierr) - if(ierr /= 0) stop 'Problems writing' + if(ierr /= 0) then + write(*,*) 'Problems writing' + stop 1 + end if call MPI_Barrier(MPI_COMM_WORLD, ierr) ! reader @@ -60,11 +63,13 @@ program TestBPReadGlobalsByName print *, irank, 'sml_outpsi', sml_outpsi if( diag_1d_nsp /= 1 ) then - stop 'diag_1d_nsp is not 1' + write(*,*) 'diag_1d_nsp is not 1' + stop 1 end if if( sml_outpsi /= 0.295477_8 ) then - stop 'sml_outpsi is not 0.295477' + write(*,*) 'sml_outpsi is not 0.295477' + stop 1 end if call MPI_Finalize(ierr) diff --git a/testing/adios2/bindings/fortran/TestBPWriteMemorySelectionRead2D.F90 b/testing/adios2/bindings/fortran/TestBPWriteMemorySelectionRead2D.F90 index fe56398890..6e37bc44b7 100644 --- a/testing/adios2/bindings/fortran/TestBPWriteMemorySelectionRead2D.F90 +++ b/testing/adios2/bindings/fortran/TestBPWriteMemorySelectionRead2D.F90 @@ -156,64 +156,172 @@ program TestBPWriteMemorySelectionRead2D expected_max = current_step call adios2_inquire_variable(vars_in(1), ioGet, 'var_i1', ierr) - if( vars_in(1)%valid .eqv. .false. ) stop 'i1 invalid error' - if( vars_in(1)%name /= 'var_i1' ) stop 'i1 name error' - if( vars_in(1)%type /= adios2_type_integer1 ) stop 'i1 type error' - if( vars_in(1)%ndims /= 2 ) stop 'i1 dims error' + if( vars_in(1)%valid .eqv. .false. ) then + write(*,*) 'i1 invalid error' + stop 1 + end if + if( vars_in(1)%name /= 'var_i1' ) then + write(*,*) 'i1 name error' + stop 1 + end if + if( vars_in(1)%type /= adios2_type_integer1 ) then + write(*,*) 'i1 type error' + stop 1 + end if + if( vars_in(1)%ndims /= 2 ) then + write(*,*) 'i1 dims error' + stop 1 + end if call adios2_variable_min(min_i1, vars_in(1), ierr) - if(min_i1 /= expected_min ) stop 'i1 min error' + if(min_i1 /= expected_min ) then + write(*,*) 'i1 min error' + stop 1 + end if call adios2_variable_max(max_i1, vars_in(1), ierr) - if(max_i1 /= expected_max ) stop 'i1 max error' + if(max_i1 /= expected_max ) then + write(*,*) 'i1 max error' + stop 1 + end if call adios2_inquire_variable(vars_in(2), ioGet, 'var_i2', ierr) - if( vars_in(2)%valid .eqv. .false. ) stop 'i2 invalid error' - if( vars_in(2)%name /= 'var_i2' ) stop 'i2 name error' - if( vars_in(2)%type /= adios2_type_integer2 ) stop 'i2 type error' - if( vars_in(2)%ndims /= 2 ) stop 'i2 dims error' + if( vars_in(2)%valid .eqv. .false. ) then + write(*,*) 'i2 invalid error' + stop 1 + end if + if( vars_in(2)%name /= 'var_i2' ) then + write(*,*) 'i2 name error' + stop 1 + end if + if( vars_in(2)%type /= adios2_type_integer2 ) then + write(*,*) 'i2 type error' + stop 1 + end if + if( vars_in(2)%ndims /= 2 ) then + write(*,*) 'i2 dims error' + stop 1 + end if call adios2_variable_min(min_i2, vars_in(2), ierr) - if(min_i2 /= expected_min ) stop 'i2 min error' + if(min_i2 /= expected_min ) then + write(*,*) 'i2 min error' + stop 1 + end if call adios2_variable_max(max_i2, vars_in(2), ierr) - if(max_i2 /= expected_max ) stop 'i2 max error' + if(max_i2 /= expected_max ) then + write(*,*) 'i2 max error' + stop 1 + end if call adios2_inquire_variable(vars_in(3), ioGet, 'var_i4', ierr) - if( vars_in(3)%valid .eqv. .false. ) stop 'i4 invalid error' - if( vars_in(3)%name /= 'var_i4' ) stop 'i4 name error' - if( vars_in(3)%type /= adios2_type_integer4 ) stop 'i4 type error' - if( vars_in(3)%ndims /= 2 ) stop 'i4 dims error' + if( vars_in(3)%valid .eqv. .false. ) then + write(*,*) 'i4 invalid error' + stop 1 + end if + if( vars_in(3)%name /= 'var_i4' ) then + write(*,*) 'i4 name error' + stop 1 + end if + if( vars_in(3)%type /= adios2_type_integer4 ) then + write(*,*) 'i4 type error' + stop 1 + end if + if( vars_in(3)%ndims /= 2 ) then + write(*,*) 'i4 dims error' + stop 1 + end if call adios2_variable_min(min_i4, vars_in(3), ierr) - if(min_i4 /= expected_min ) stop 'i4 min error' + if(min_i4 /= expected_min ) then + write(*,*) 'i4 min error' + stop 1 + end if call adios2_variable_max(max_i4, vars_in(3), ierr) - if(max_i4 /= expected_max ) stop 'i4 max error' + if(max_i4 /= expected_max ) then + write(*,*) 'i4 max error' + stop 1 + end if call adios2_inquire_variable(vars_in(4), ioGet, 'var_i8', ierr) - if( vars_in(4)%valid .eqv. .false. ) stop 'i8 invalid error' - if( vars_in(4)%name /= 'var_i8' ) stop 'i8 name error' - if( vars_in(4)%type /= adios2_type_integer8 ) stop 'i8 type error' - if( vars_in(4)%ndims /= 2 ) stop 'i8 dims error' + if( vars_in(4)%valid .eqv. .false. ) then + write(*,*) 'i8 invalid error' + stop 1 + end if + if( vars_in(4)%name /= 'var_i8' ) then + write(*,*) 'i8 name error' + stop 1 + end if + if( vars_in(4)%type /= adios2_type_integer8 ) then + write(*,*) 'i8 type error' + stop 1 + end if + if( vars_in(4)%ndims /= 2 ) then + write(*,*) 'i8 dims error' + stop 1 + end if call adios2_variable_min(min_i8, vars_in(4), ierr) - if(min_i8 /= expected_min ) stop 'i8 min error' + if(min_i8 /= expected_min ) then + write(*,*) 'i8 min error' + stop 1 + end if call adios2_variable_max(max_i8, vars_in(4), ierr) - if(max_i8 /= expected_max ) stop 'i8 max error' + if(max_i8 /= expected_max ) then + write(*,*) 'i8 max error' + stop 1 + end if call adios2_inquire_variable(vars_in(5), ioGet, 'var_r4', ierr) - if( vars_in(5)%valid .eqv. .false. ) stop 'r4 invalid error' - if( vars_in(5)%name /= 'var_r4' ) stop 'r4 name error' - if( vars_in(5)%type /= adios2_type_real ) stop 'r4 type error' - if( vars_in(5)%ndims /= 2 ) stop 'r4 dims error' + if( vars_in(5)%valid .eqv. .false. ) then + write(*,*) 'r4 invalid error' + stop 1 + end if + if( vars_in(5)%name /= 'var_r4' ) then + write(*,*) 'r4 name error' + stop 1 + end if + if( vars_in(5)%type /= adios2_type_real ) then + write(*,*) 'r4 type error' + stop 1 + end if + if( vars_in(5)%ndims /= 2 ) then + write(*,*) 'r4 dims error' + stop 1 + end if call adios2_variable_min(min_r4, vars_in(5), ierr) - if(min_r4 /= float(expected_min) ) stop 'r4 min error' + if(min_r4 /= float(expected_min) ) then + write(*,*) 'r4 min error' + stop 1 + end if call adios2_variable_max(max_r4, vars_in(5), ierr) - if(max_r4 /= float(expected_max) ) stop 'r4 max error' + if(max_r4 /= float(expected_max) ) then + write(*,*) 'r4 max error' + stop 1 + end if call adios2_inquire_variable(vars_in(6), ioGet, 'var_r8', ierr) - if( vars_in(6)%valid .eqv. .false. ) stop 'r8 invalid error' - if( vars_in(6)%name /= 'var_r8' ) stop 'r8 name error' - if( vars_in(6)%type /= adios2_type_dp ) stop 'r8 type error' - if( vars_in(6)%ndims /= 2 ) stop 'r8 dims error' + if( vars_in(6)%valid .eqv. .false. ) then + write(*,*) 'r8 invalid error' + stop 1 + end if + if( vars_in(6)%name /= 'var_r8' ) then + write(*,*) 'r8 name error' + stop 1 + end if + if( vars_in(6)%type /= adios2_type_dp ) then + write(*,*) 'r8 type error' + stop 1 + end if + if( vars_in(6)%ndims /= 2 ) then + write(*,*) 'r8 dims error' + stop 1 + end if call adios2_variable_min(min_r8, vars_in(6), ierr) - if(min_r8 /= float(expected_min) ) stop 'r8 min error' + if(min_r8 /= float(expected_min) ) then + write(*,*) 'r8 min error' + stop 1 + end if call adios2_variable_max(max_r8, vars_in(6), ierr) - if(max_r8 /= float(expected_max) ) stop 'r8 max error' + if(max_r8 /= float(expected_max) ) then + write(*,*) 'r8 max error' + stop 1 + end if call adios2_get(bpReader, 'var_i1', in_data_i1, ierr) call adios2_get(bpReader, 'var_i2', in_data_i2, ierr) @@ -225,12 +333,30 @@ program TestBPWriteMemorySelectionRead2D do j=1,isize*ny do i=1,nx - if(in_data_i1(i,j) /= current_step) stop 'i1 read error' - if(in_data_i2(i,j) /= current_step) stop 'i2 read error' - if(in_data_i4(i,j) /= current_step) stop 'i4 read error' - if(in_data_i8(i,j) /= current_step) stop 'i8 read rerror' - if(in_data_r4(i,j) /= REAL(current_step, 4)) stop 'r4 read error' - if(in_data_r8(i,j) /= current_step) stop 'r8 read rerror' + if(in_data_i1(i,j) /= current_step) then + write(*,*) 'i1 read error' + stop 1 + end if + if(in_data_i2(i,j) /= current_step) then + write(*,*) 'i2 read error' + stop 1 + end if + if(in_data_i4(i,j) /= current_step) then + write(*,*) 'i4 read error' + stop 1 + end if + if(in_data_i8(i,j) /= current_step) then + write(*,*) 'i8 read rerror' + stop 1 + end if + if(in_data_r4(i,j) /= REAL(current_step, 4)) then + write(*,*) 'r4 read error' + stop 1 + end if + if(in_data_r8(i,j) /= current_step) then + write(*,*) 'r8 read rerror' + stop 1 + end if end do end do diff --git a/testing/adios2/bindings/fortran/TestBPWriteMemorySelectionRead3D.F90 b/testing/adios2/bindings/fortran/TestBPWriteMemorySelectionRead3D.F90 index 498c815f0e..a4721bf7dc 100644 --- a/testing/adios2/bindings/fortran/TestBPWriteMemorySelectionRead3D.F90 +++ b/testing/adios2/bindings/fortran/TestBPWriteMemorySelectionRead3D.F90 @@ -156,64 +156,172 @@ program TestBPWriteMemorySelectionRead3D expected_max = current_step call adios2_inquire_variable(vars_in(1), ioGet, 'var_i1', ierr) - if( vars_in(1)%valid .eqv. .false. ) stop 'i1 invalid error' - if( vars_in(1)%name /= 'var_i1' ) stop 'i1 name error' - if( vars_in(1)%type /= adios2_type_integer1 ) stop 'i1 type error' - if( vars_in(1)%ndims /= 3 ) stop 'i1 dims error' + if( vars_in(1)%valid .eqv. .false. ) then + write(*,*) 'i1 invalid error' + stop 1 + end if + if( vars_in(1)%name /= 'var_i1' ) then + write(*,*) 'i1 name error' + stop 1 + end if + if( vars_in(1)%type /= adios2_type_integer1 ) then + write(*,*) 'i1 type error' + stop 1 + end if + if( vars_in(1)%ndims /= 3 ) then + write(*,*) 'i1 dims error' + stop 1 + end if call adios2_variable_min(min_i1, vars_in(1), ierr) - if(min_i1 /= expected_min ) stop 'i1 min error' + if(min_i1 /= expected_min ) then + write(*,*) 'i1 min error' + stop 1 + end if call adios2_variable_max(max_i1, vars_in(1), ierr) - if(max_i1 /= expected_max ) stop 'i1 max error' + if(max_i1 /= expected_max ) then + write(*,*) 'i1 max error' + stop 1 + end if call adios2_inquire_variable(vars_in(2), ioGet, 'var_i2', ierr) - if( vars_in(2)%valid .eqv. .false. ) stop 'i2 invalid error' - if( vars_in(2)%name /= 'var_i2' ) stop 'i2 name error' - if( vars_in(2)%type /= adios2_type_integer2 ) stop 'i2 type error' - if( vars_in(2)%ndims /= 3 ) stop 'i2 dims error' + if( vars_in(2)%valid .eqv. .false. ) then + write(*,*) 'i2 invalid error' + stop 1 + end if + if( vars_in(2)%name /= 'var_i2' ) then + write(*,*) 'i2 name error' + stop 1 + end if + if( vars_in(2)%type /= adios2_type_integer2 ) then + write(*,*) 'i2 type error' + stop 1 + end if + if( vars_in(2)%ndims /= 3 ) then + write(*,*) 'i2 dims error' + stop 1 + end if call adios2_variable_min(min_i2, vars_in(2), ierr) - if(min_i2 /= expected_min ) stop 'i2 min error' + if(min_i2 /= expected_min ) then + write(*,*) 'i2 min error' + stop 1 + end if call adios2_variable_max(max_i2, vars_in(2), ierr) - if(max_i2 /= expected_max ) stop 'i2 max error' + if(max_i2 /= expected_max ) then + write(*,*) 'i2 max error' + stop 1 + end if call adios2_inquire_variable(vars_in(3), ioGet, 'var_i4', ierr) - if( vars_in(3)%valid .eqv. .false. ) stop 'i4 invalid error' - if( vars_in(3)%name /= 'var_i4' ) stop 'i4 name error' - if( vars_in(3)%type /= adios2_type_integer4 ) stop 'i4 type error' - if( vars_in(3)%ndims /= 3 ) stop 'i4 dims error' + if( vars_in(3)%valid .eqv. .false. ) then + write(*,*) 'i4 invalid error' + stop 1 + end if + if( vars_in(3)%name /= 'var_i4' ) then + write(*,*) 'i4 name error' + stop 1 + end if + if( vars_in(3)%type /= adios2_type_integer4 ) then + write(*,*) 'i4 type error' + stop 1 + end if + if( vars_in(3)%ndims /= 3 ) then + write(*,*) 'i4 dims error' + stop 1 + end if call adios2_variable_min(min_i4, vars_in(3), ierr) - if(min_i4 /= expected_min ) stop 'i4 min error' + if(min_i4 /= expected_min ) then + write(*,*) 'i4 min error' + stop 1 + end if call adios2_variable_max(max_i4, vars_in(3), ierr) - if(max_i4 /= expected_max ) stop 'i4 max error' + if(max_i4 /= expected_max ) then + write(*,*) 'i4 max error' + stop 1 + end if call adios2_inquire_variable(vars_in(4), ioGet, 'var_i8', ierr) - if( vars_in(4)%valid .eqv. .false. ) stop 'i8 invalid error' - if( vars_in(4)%name /= 'var_i8' ) stop 'i8 name error' - if( vars_in(4)%type /= adios2_type_integer8 ) stop 'i8 type error' - if( vars_in(4)%ndims /= 3 ) stop 'i8 dims error' + if( vars_in(4)%valid .eqv. .false. ) then + write(*,*) 'i8 invalid error' + stop 1 + end if + if( vars_in(4)%name /= 'var_i8' ) then + write(*,*) 'i8 name error' + stop 1 + end if + if( vars_in(4)%type /= adios2_type_integer8 ) then + write(*,*) 'i8 type error' + stop 1 + end if + if( vars_in(4)%ndims /= 3 ) then + write(*,*) 'i8 dims error' + stop 1 + end if call adios2_variable_min(min_i8, vars_in(4), ierr) - if(min_i8 /= expected_min ) stop 'i8 min error' + if(min_i8 /= expected_min ) then + write(*,*) 'i8 min error' + stop 1 + end if call adios2_variable_max(max_i8, vars_in(4), ierr) - if(max_i8 /= expected_max ) stop 'i8 max error' + if(max_i8 /= expected_max ) then + write(*,*) 'i8 max error' + stop 1 + end if call adios2_inquire_variable(vars_in(5), ioGet, 'var_r4', ierr) - if( vars_in(5)%valid .eqv. .false. ) stop 'r4 invalid error' - if( vars_in(5)%name /= 'var_r4' ) stop 'r4 name error' - if( vars_in(5)%type /= adios2_type_real ) stop 'r4 type error' - if( vars_in(5)%ndims /= 3 ) stop 'r4 dims error' + if( vars_in(5)%valid .eqv. .false. ) then + write(*,*) 'r4 invalid error' + stop 1 + end if + if( vars_in(5)%name /= 'var_r4' ) then + write(*,*) 'r4 name error' + stop 1 + end if + if( vars_in(5)%type /= adios2_type_real ) then + write(*,*) 'r4 type error' + stop 1 + end if + if( vars_in(5)%ndims /= 3 ) then + write(*,*) 'r4 dims error' + stop 1 + end if call adios2_variable_min(min_r4, vars_in(5), ierr) - if(min_r4 /= float(expected_min) ) stop 'r4 min error' + if(min_r4 /= float(expected_min) ) then + write(*,*) 'r4 min error' + stop 1 + end if call adios2_variable_max(max_r4, vars_in(5), ierr) - if(max_r4 /= float(expected_max) ) stop 'r4 max error' + if(max_r4 /= float(expected_max) ) then + write(*,*) 'r4 max error' + stop 1 + end if call adios2_inquire_variable(vars_in(6), ioGet, 'var_r8', ierr) - if( vars_in(6)%valid .eqv. .false. ) stop 'r8 invalid error' - if( vars_in(6)%name /= 'var_r8' ) stop 'r8 name error' - if( vars_in(6)%type /= adios2_type_dp ) stop 'r8 type error' - if( vars_in(6)%ndims /= 3 ) stop 'r8 dims error' + if( vars_in(6)%valid .eqv. .false. ) then + write(*,*) 'r8 invalid error' + stop 1 + end if + if( vars_in(6)%name /= 'var_r8' ) then + write(*,*) 'r8 name error' + stop 1 + end if + if( vars_in(6)%type /= adios2_type_dp ) then + write(*,*) 'r8 type error' + stop 1 + end if + if( vars_in(6)%ndims /= 3 ) then + write(*,*) 'r8 dims error' + stop 1 + end if call adios2_variable_min(min_r8, vars_in(6), ierr) - if(min_r8 /= float(expected_min) ) stop 'r8 min error' + if(min_r8 /= float(expected_min) ) then + write(*,*) 'r8 min error' + stop 1 + end if call adios2_variable_max(max_r8, vars_in(6), ierr) - if(max_r8 /= float(expected_max) ) stop 'r8 max error' + if(max_r8 /= float(expected_max) ) then + write(*,*) 'r8 max error' + stop 1 + end if call adios2_get(bpReader, 'var_i1', in_data_i1, ierr) call adios2_get(bpReader, 'var_i2', in_data_i2, ierr) @@ -226,12 +334,30 @@ program TestBPWriteMemorySelectionRead3D do k=1,isize*nz do j=1,ny do i=1,nx - if(in_data_i1(i,j,k) /= current_step) stop 'i1 read error' - if(in_data_i2(i,j,k) /= current_step) stop 'i2 read error' - if(in_data_i4(i,j,k) /= current_step) stop 'i4 read error' - if(in_data_i8(i,j,k) /= current_step) stop 'i8 read error' - if(in_data_r4(i,j,k) /= REAL(current_step, 4)) stop 'r4 read error' - if(in_data_r8(i,j,k) /= current_step) stop 'r8 read error' + if(in_data_i1(i,j,k) /= current_step) then + write(*,*) 'i1 read error' + stop 1 + end if + if(in_data_i2(i,j,k) /= current_step) then + write(*,*) 'i2 read error' + stop 1 + end if + if(in_data_i4(i,j,k) /= current_step) then + write(*,*) 'i4 read error' + stop 1 + end if + if(in_data_i8(i,j,k) /= current_step) then + write(*,*) 'i8 read error' + stop 1 + end if + if(in_data_r4(i,j,k) /= REAL(current_step, 4)) then + write(*,*) 'r4 read error' + stop 1 + end if + if(in_data_r8(i,j,k) /= current_step) then + write(*,*) 'r8 read error' + stop 1 + end if end do end do end do diff --git a/testing/adios2/bindings/fortran/TestBPWriteReadAttributes.F90 b/testing/adios2/bindings/fortran/TestBPWriteReadAttributes.F90 index d08ac47a5a..583aa83b77 100644 --- a/testing/adios2/bindings/fortran/TestBPWriteReadAttributes.F90 +++ b/testing/adios2/bindings/fortran/TestBPWriteReadAttributes.F90 @@ -45,7 +45,10 @@ program TestBPWriteAttributes call adios2_declare_io(ioWrite, adios, "ioWrite", ierr) do i=1,14 - if( attributes(i)%valid .eqv. .true. ) stop 'Invalid attribute default' + if( attributes(i)%valid .eqv. .true. ) then + write(*,*) 'Invalid attribute default' + stop 1 + end if end do ! single value @@ -93,12 +96,18 @@ program TestBPWriteAttributes data_R64, 3, ierr) do i=1,14 - if( attributes(i)%valid .eqv. .false. ) stop 'Invalid adios2_define_attribute' + if( attributes(i)%valid .eqv. .false. ) then + write(*,*) 'Invalid adios2_define_attribute' + stop 1 + end if end do ! Testing adios2_attribute_name for just one case call adios2_attribute_name(attrName, attributes(1), ierr) - if (attrName /= 'att_String') stop 'Invalid adios2_attribute_name' + if (attrName /= 'att_String') then + write(*,*) 'Invalid adios2_attribute_name' + stop 1 + end if deallocate(attrName) call adios2_open(bpWriter, ioWrite, "fattr_types.bp", adios2_mode_write, & @@ -116,23 +125,41 @@ program TestBPWriteAttributes ! Test getting list of attribute names call adios2_available_attributes(ioRead, namestruct, ierr) - if (ierr /= 0) stop 'adios2_available_attributes returned with error' - if (.not.namestruct%valid) stop 'adios2_available_attributes returned invalid struct' + if (ierr /= 0) then + write(*,*) 'adios2_available_attributes returned with error' + stop 1 + end if + if (.not.namestruct%valid) then + write(*,*) 'adios2_available_attributes returned invalid struct' + stop 1 + end if write(*,*) 'Number of attributes = ', namestruct%count write(*,*) 'Max name length = ', namestruct%max_name_len - if (namestruct%count /= 14) stop 'adios2_available_attributes returned not the expected 14' + if (namestruct%count /= 14) then + write(*,*) 'adios2_available_attributes returned not the expected 14' + stop 1 + end if allocate(attrnamelist(namestruct%count)) call adios2_retrieve_names(namestruct, attrnamelist, ierr) - if (ierr /= 0) stop 'adios2_retrieve_names returned with error' + if (ierr /= 0) then + write(*,*) 'adios2_retrieve_names returned with error' + stop 1 + end if do i=1,namestruct%count write(*,'("Attr[",i2,"] = ",a20)') i, attrnamelist(i) end do deallocate(attrnamelist) - if (namestruct%f2c /= 0_8) stop 'namestruct f2c pointer is not null after adios2_retrieve_names()' - if (namestruct%valid) stop 'namestruct is not invalidated after adios2_retrieve_names()' + if (namestruct%f2c /= 0_8) then + write(*,*) 'namestruct f2c pointer is not null after adios2_retrieve_names()' + stop 1 + end if + if (namestruct%valid) then + write(*,*) 'namestruct is not invalidated after adios2_retrieve_names()' + stop 1 + end if call adios2_inquire_attribute(attributes_in(1), ioRead, 'att_String', ierr) @@ -143,54 +170,159 @@ program TestBPWriteAttributes call adios2_inquire_attribute(attributes_in(6), ioRead, 'att_r32', ierr) call adios2_inquire_attribute(attributes_in(7), ioRead, 'att_r64', ierr) - if(attributes_in(1)%valid .eqv. .false.) stop 'attribute iString not found' - if(attributes_in(1)%type /= adios2_type_string) stop 'attribute iString wrong type' - if(attributes_in(1)%length /= 1) stop 'attribute iString length is not 1' - if(attributes_in(1)%is_value .eqv. .false.) stop 'attribute iString must be value' + if(attributes_in(1)%valid .eqv. .false.) then + write(*,*) 'attribute iString not found' + stop 1 + end if + if(attributes_in(1)%type /= adios2_type_string) then + write(*,*) 'attribute iString wrong type' + stop 1 + end if + if(attributes_in(1)%length /= 1) then + write(*,*) 'attribute iString length is not 1' + stop 1 + end if + if(attributes_in(1)%is_value .eqv. .false.) then + write(*,*) 'attribute iString must be value' + stop 1 + end if call adios2_attribute_data( iString_value, attributes_in(1), ierr) - if( iString_value /= 'ADIOS2 String attribute' ) stop 'attribute iString data error' - - if(attributes_in(2)%valid .eqv. .false.) stop 'attribute i8 not found' - if(attributes_in(2)%type /= adios2_type_integer1) stop 'attribute i8 wrong type' - if(attributes_in(2)%length /= 1) stop 'attribute i8 length is not 1' - if(attributes_in(2)%is_value .eqv. .false.) stop 'attribute i8 must be value' + if( iString_value /= 'ADIOS2 String attribute' ) then + write(*,*) 'attribute iString data error' + stop 1 + end if + + if(attributes_in(2)%valid .eqv. .false.) then + write(*,*) 'attribute i8 not found' + stop 1 + end if + if(attributes_in(2)%type /= adios2_type_integer1) then + write(*,*) 'attribute i8 wrong type' + stop 1 + end if + if(attributes_in(2)%length /= 1) then + write(*,*) 'attribute i8 length is not 1' + stop 1 + end if + if(attributes_in(2)%is_value .eqv. .false.) then + write(*,*) 'attribute i8 must be value' + stop 1 + end if call adios2_attribute_data( i8_value, attributes_in(2), ierr) - if( i8_value /= data_I8(1) ) stop 'attribute i8 data error' - - if(attributes_in(3)%valid .eqv. .false.) stop 'attribute i16 not found' - if(attributes_in(3)%type /= adios2_type_integer2) stop 'attribute i16 wrong type' - if(attributes_in(3)%length /= 1) stop 'attribute i16 length is not 1' - if(attributes_in(3)%is_value .eqv. .false.) stop 'attribute i16 must be value' + if( i8_value /= data_I8(1) ) then + write(*,*) 'attribute i8 data error' + stop 1 + end if + + if(attributes_in(3)%valid .eqv. .false.) then + write(*,*) 'attribute i16 not found' + stop 1 + end if + if(attributes_in(3)%type /= adios2_type_integer2) then + write(*,*) 'attribute i16 wrong type' + stop 1 + end if + if(attributes_in(3)%length /= 1) then + write(*,*) 'attribute i16 length is not 1' + stop 1 + end if + if(attributes_in(3)%is_value .eqv. .false.) then + write(*,*) 'attribute i16 must be value' + stop 1 + end if call adios2_attribute_data( i16_value, attributes_in(3), ierr) - if( i16_value /= data_I16(1) ) stop 'attribute i16 data error' - - if(attributes_in(4)%valid .eqv. .false.) stop 'attribute i32 not found' - if(attributes_in(4)%type /= adios2_type_integer4) stop 'attribute i32 wrong type' - if(attributes_in(4)%length /= 1) stop 'attribute i32 length is not 1' - if(attributes_in(4)%is_value .eqv. .false.) stop 'attribute i32 must be value' + if( i16_value /= data_I16(1) ) then + write(*,*) 'attribute i16 data error' + stop 1 + end if + + if(attributes_in(4)%valid .eqv. .false.) then + write(*,*) 'attribute i32 not found' + stop 1 + end if + if(attributes_in(4)%type /= adios2_type_integer4) then + write(*,*) 'attribute i32 wrong type' + stop 1 + end if + if(attributes_in(4)%length /= 1) then + write(*,*) 'attribute i32 length is not 1' + stop 1 + end if + if(attributes_in(4)%is_value .eqv. .false.) then + write(*,*) 'attribute i32 must be value' + stop 1 + end if call adios2_attribute_data( i32_value, attributes_in(4), ierr) - if( i32_value /= data_I32(1) ) stop 'attribute i32 data error' - - if(attributes_in(5)%valid .eqv. .false.) stop 'attribute i64 not found' - if(attributes_in(5)%type /= adios2_type_integer8) stop 'attribute i64 wrong type' - if(attributes_in(5)%length /= 1) stop 'attribute i64 length is not 1' - if(attributes_in(5)%is_value .eqv. .false.) stop 'attribute i64 must be value' + if( i32_value /= data_I32(1) ) then + write(*,*) 'attribute i32 data error' + stop 1 + end if + + if(attributes_in(5)%valid .eqv. .false.) then + write(*,*) 'attribute i64 not found' + stop 1 + end if + if(attributes_in(5)%type /= adios2_type_integer8) then + write(*,*) 'attribute i64 wrong type' + stop 1 + end if + if(attributes_in(5)%length /= 1) then + write(*,*) 'attribute i64 length is not 1' + stop 1 + end if + if(attributes_in(5)%is_value .eqv. .false.) then + write(*,*) 'attribute i64 must be value' + stop 1 + end if call adios2_attribute_data( i64_value, attributes_in(5), ierr) - if( i64_value /= data_I64(1) ) stop 'attribute i64 data error' - - if(attributes_in(6)%valid .eqv. .false.) stop 'attribute r32 not found' - if(attributes_in(6)%type /= adios2_type_real) stop 'attribute r32 wrong type' - if(attributes_in(6)%length /= 1) stop 'attribute r32 length is not 1' - if(attributes_in(6)%is_value .eqv. .false.) stop 'attribute r32 must be value' + if( i64_value /= data_I64(1) ) then + write(*,*) 'attribute i64 data error' + stop 1 + end if + + if(attributes_in(6)%valid .eqv. .false.) then + write(*,*) 'attribute r32 not found' + stop 1 + end if + if(attributes_in(6)%type /= adios2_type_real) then + write(*,*) 'attribute r32 wrong type' + stop 1 + end if + if(attributes_in(6)%length /= 1) then + write(*,*) 'attribute r32 length is not 1' + stop 1 + end if + if(attributes_in(6)%is_value .eqv. .false.) then + write(*,*) 'attribute r32 must be value' + stop 1 + end if call adios2_attribute_data( r32_value, attributes_in(6), ierr) - if( r32_value /= data_R32(1) ) stop 'attribute r32 data error' - - if(attributes_in(7)%valid .eqv. .false.) stop 'attribute r64 not found' - if(attributes_in(7)%type /= adios2_type_dp) stop 'attribute r64 wrong type' - if(attributes_in(7)%length /= 1) stop 'attribute r64 length is not 1' - if(attributes_in(7)%is_value .eqv. .false.) stop 'attribute r64 must be value' + if( r32_value /= data_R32(1) ) then + write(*,*) 'attribute r32 data error' + stop 1 + end if + + if(attributes_in(7)%valid .eqv. .false.) then + write(*,*) 'attribute r64 not found' + stop 1 + end if + if(attributes_in(7)%type /= adios2_type_dp) then + write(*,*) 'attribute r64 wrong type' + stop 1 + end if + if(attributes_in(7)%length /= 1) then + write(*,*) 'attribute r64 length is not 1' + stop 1 + end if + if(attributes_in(7)%is_value .eqv. .false.) then + write(*,*) 'attribute r64 must be value' + stop 1 + end if call adios2_attribute_data( r64_value, attributes_in(7), ierr) - if( r64_value /= data_R64(1) ) stop 'attribute r64 data error' + if( r64_value /= data_R64(1) ) then + write(*,*) 'attribute r64 data error' + stop 1 + end if ! Array call adios2_inquire_attribute(attributes_in(8), ioRead, 'att_Strings_array', ierr) @@ -201,67 +333,172 @@ program TestBPWriteAttributes call adios2_inquire_attribute(attributes_in(13), ioRead, 'att_r32_array', ierr) call adios2_inquire_attribute(attributes_in(14), ioRead, 'att_r64_array', ierr) - if(attributes_in(8)%valid .eqv. .false.) stop 'attribute string array not found' - if(attributes_in(8)%type /= adios2_type_string) stop 'attribute string array wrong type' - if(attributes_in(8)%length /= 3) stop 'attribute string array length is not 3' - if(attributes_in(8)%is_value .eqv. .true.) stop 'attribute string array must be array' + if(attributes_in(8)%valid .eqv. .false.) then + write(*,*) 'attribute string array not found' + stop 1 + end if + if(attributes_in(8)%type /= adios2_type_string) then + write(*,*) 'attribute string array wrong type' + stop 1 + end if + if(attributes_in(8)%length /= 3) then + write(*,*) 'attribute string array length is not 3' + stop 1 + end if + if(attributes_in(8)%is_value .eqv. .true.) then + write(*,*) 'attribute string array must be array' + stop 1 + end if call adios2_attribute_data( iString_array, attributes_in(8), ierr) do i=1,3 - if( iString_array(i) /= data_Strings(i) ) stop 'attribute string array data error' + if( iString_array(i) /= data_Strings(i) ) then + write(*,*) 'attribute string array data error' + stop 1 + end if end do - if(attributes_in(9)%valid .eqv. .false.) stop 'attribute i8 array not found' - if(attributes_in(9)%type /= adios2_type_integer1) stop 'attribute i8 array wrong type' - if(attributes_in(9)%length /= 3) stop 'attribute i8 array length is not 3' - if(attributes_in(9)%is_value .eqv. .true.) stop 'attribute i8 array must be array' + if(attributes_in(9)%valid .eqv. .false.) then + write(*,*) 'attribute i8 array not found' + stop 1 + end if + if(attributes_in(9)%type /= adios2_type_integer1) then + write(*,*) 'attribute i8 array wrong type' + stop 1 + end if + if(attributes_in(9)%length /= 3) then + write(*,*) 'attribute i8 array length is not 3' + stop 1 + end if + if(attributes_in(9)%is_value .eqv. .true.) then + write(*,*) 'attribute i8 array must be array' + stop 1 + end if call adios2_attribute_data( i8_array, attributes_in(9), ierr) do i=1,3 - if( i8_array(i) /= data_I8(i) ) stop 'attribute i8 array data error' + if( i8_array(i) /= data_I8(i) ) then + write(*,*) 'attribute i8 array data error' + stop 1 + end if end do - if(attributes_in(10)%valid .eqv. .false.) stop 'attribute i16 array not found' - if(attributes_in(10)%type /= adios2_type_integer2) stop 'attribute i16 array wrong type' - if(attributes_in(10)%length /= 3) stop 'attribute i16 array length is not 3' - if(attributes_in(10)%is_value .eqv. .true.) stop 'attribute i16 array must be array' + if(attributes_in(10)%valid .eqv. .false.) then + write(*,*) 'attribute i16 array not found' + stop 1 + end if + if(attributes_in(10)%type /= adios2_type_integer2) then + write(*,*) 'attribute i16 array wrong type' + stop 1 + end if + if(attributes_in(10)%length /= 3) then + write(*,*) 'attribute i16 array length is not 3' + stop 1 + end if + if(attributes_in(10)%is_value .eqv. .true.) then + write(*,*) 'attribute i16 array must be array' + stop 1 + end if call adios2_attribute_data( i16_array, attributes_in(10), ierr) do i=1,3 - if( i16_array(i) /= data_I16(i) ) stop 'attribute i16 array data error' + if( i16_array(i) /= data_I16(i) ) then + write(*,*) 'attribute i16 array data error' + stop 1 + end if end do - if(attributes_in(11)%valid .eqv. .false.) stop 'attribute i32 array not found' - if(attributes_in(11)%type /= adios2_type_integer4) stop 'attribute i32 array wrong type' - if(attributes_in(11)%length /= 3) stop 'attribute i32 array length is not 3' - if(attributes_in(11)%is_value .eqv. .true.) stop 'attribute i32 array must be array' + if(attributes_in(11)%valid .eqv. .false.) then + write(*,*) 'attribute i32 array not found' + stop 1 + end if + if(attributes_in(11)%type /= adios2_type_integer4) then + write(*,*) 'attribute i32 array wrong type' + stop 1 + end if + if(attributes_in(11)%length /= 3) then + write(*,*) 'attribute i32 array length is not 3' + stop 1 + end if + if(attributes_in(11)%is_value .eqv. .true.) then + write(*,*) 'attribute i32 array must be array' + stop 1 + end if call adios2_attribute_data( i32_array, attributes_in(11), ierr) do i=1,3 - if( i32_array(i) /= data_I32(i) ) stop 'attribute i32 array data error' + if( i32_array(i) /= data_I32(i) ) then + write(*,*) 'attribute i32 array data error' + stop 1 + end if end do - if(attributes_in(12)%valid .eqv. .false.) stop 'attribute i64 array not found' - if(attributes_in(12)%type /= adios2_type_integer8) stop 'attribute i64 array wrong type' - if(attributes_in(12)%length /= 3) stop 'attribute i64 array length is not 3' - if(attributes_in(12)%is_value .eqv. .true.) stop 'attribute i64 array must be array' + if(attributes_in(12)%valid .eqv. .false.) then + write(*,*) 'attribute i64 array not found' + stop 1 + end if + if(attributes_in(12)%type /= adios2_type_integer8) then + write(*,*) 'attribute i64 array wrong type' + stop 1 + end if + if(attributes_in(12)%length /= 3) then + write(*,*) 'attribute i64 array length is not 3' + stop 1 + end if + if(attributes_in(12)%is_value .eqv. .true.) then + write(*,*) 'attribute i64 array must be array' + stop 1 + end if call adios2_attribute_data( i64_array, attributes_in(12), ierr) do i=1,3 - if( i64_array(i) /= data_I64(i) ) stop 'attribute i64 array data error' + if( i64_array(i) /= data_I64(i) ) then + write(*,*) 'attribute i64 array data error' + stop 1 + end if end do - if(attributes_in(13)%valid .eqv. .false.) stop 'attribute r32 array not found' - if(attributes_in(13)%type /= adios2_type_real) stop 'attribute r32 array wrong type' - if(attributes_in(13)%length /= 3) stop 'attribute r32 array length is not 3' - if(attributes_in(13)%is_value .eqv. .true.) stop 'attribute r32 array must be array' + if(attributes_in(13)%valid .eqv. .false.) then + write(*,*) 'attribute r32 array not found' + stop 1 + end if + if(attributes_in(13)%type /= adios2_type_real) then + write(*,*) 'attribute r32 array wrong type' + stop 1 + end if + if(attributes_in(13)%length /= 3) then + write(*,*) 'attribute r32 array length is not 3' + stop 1 + end if + if(attributes_in(13)%is_value .eqv. .true.) then + write(*,*) 'attribute r32 array must be array' + stop 1 + end if call adios2_attribute_data( r32_array, attributes_in(13), ierr) do i=1,3 - if( r32_array(i) /= data_R32(i) ) stop 'attribute r32 array data error' + if( r32_array(i) /= data_R32(i) ) then + write(*,*) 'attribute r32 array data error' + stop 1 + end if end do - if(attributes_in(14)%valid .eqv. .false.) stop 'attribute r64 array not found' - if(attributes_in(14)%type /= adios2_type_dp) stop 'attribute r64 array wrong type' - if(attributes_in(14)%length /= 3) stop 'attribute r64 array length is not 3' - if(attributes_in(14)%is_value .eqv. .true.) stop 'attribute r64 array must be array' + if(attributes_in(14)%valid .eqv. .false.) then + write(*,*) 'attribute r64 array not found' + stop 1 + end if + if(attributes_in(14)%type /= adios2_type_dp) then + write(*,*) 'attribute r64 array wrong type' + stop 1 + end if + if(attributes_in(14)%length /= 3) then + write(*,*) 'attribute r64 array length is not 3' + stop 1 + end if + if(attributes_in(14)%is_value .eqv. .true.) then + write(*,*) 'attribute r64 array must be array' + stop 1 + end if call adios2_attribute_data( r64_array, attributes_in(14), ierr) do i=1,3 - if( r64_array(i) /= data_R64(i) ) stop 'attribute r64 array data error' + if( r64_array(i) /= data_R64(i) ) then + write(*,*) 'attribute r64 array data error' + stop 1 + end if end do call adios2_close(bpReader, ierr) diff --git a/testing/adios2/bindings/fortran/TestBPWriteReadHeatMap2D.F90 b/testing/adios2/bindings/fortran/TestBPWriteReadHeatMap2D.F90 index 510a7d07e2..c18fe044bf 100644 --- a/testing/adios2/bindings/fortran/TestBPWriteReadHeatMap2D.F90 +++ b/testing/adios2/bindings/fortran/TestBPWriteReadHeatMap2D.F90 @@ -187,12 +187,30 @@ program TestBPWriteReadHeatMap2D end do end do - if (sum_i1 /= 50*isize) stop 'Test failed integer*1' - if (sum_i2 /= 50*isize) stop 'Test failed integer*2' - if (sum(sel_temperatures_i4) /= 50*isize) stop 'Test failed integer*4' - if (sum(sel_temperatures_i8) /= 50*isize) stop 'Test failed integer*8' - if (sum(sel_temperatures_r4) /= 50*isize) stop 'Test failed real*4' - if (sum(sel_temperatures_r8) /= 50*isize) stop 'Test failed real*8' + if (sum_i1 /= 50*isize) then + write(*,*) 'Test failed integer*1' + stop 1 + end if + if (sum_i2 /= 50*isize) then + write(*,*) 'Test failed integer*2' + stop 1 + end if + if (sum(sel_temperatures_i4) /= 50*isize) then + write(*,*) 'Test failed integer*4' + stop 1 + end if + if (sum(sel_temperatures_i8) /= 50*isize) then + write(*,*) 'Test failed integer*8' + stop 1 + end if + if (sum(sel_temperatures_r4) /= 50*isize) then + write(*,*) 'Test failed real*4' + stop 1 + end if + if (sum(sel_temperatures_r8) /= 50*isize) then + write(*,*) 'Test failed real*8' + stop 1 + end if if (allocated(sel_temperatures_i1)) deallocate (sel_temperatures_i1) if (allocated(sel_temperatures_i2)) deallocate (sel_temperatures_i2) diff --git a/testing/adios2/bindings/fortran/TestBPWriteReadHeatMap3D.F90 b/testing/adios2/bindings/fortran/TestBPWriteReadHeatMap3D.F90 index 06b7954b0c..55df80789f 100644 --- a/testing/adios2/bindings/fortran/TestBPWriteReadHeatMap3D.F90 +++ b/testing/adios2/bindings/fortran/TestBPWriteReadHeatMap3D.F90 @@ -190,12 +190,30 @@ program TestBPWriteReadHeatMap3D end do end do - if (sum_i1 /= 500*isize) stop 'Test failed integer*1' - if (sum_i2 /= 500*isize) stop 'Test failed integer*2' - if (sum(sel_temperatures_i4) /= 500*isize) stop 'Test failed integer*4' - if (sum(sel_temperatures_i8) /= 500*isize) stop 'Test failed integer*8' - if (sum(sel_temperatures_r4) /= 500*isize) stop 'Test failed real*4' - if (sum(sel_temperatures_r8) /= 500*isize) stop 'Test failed real*8' + if (sum_i1 /= 500*isize) then + write(*,*) 'Test failed integer*1' + stop 1 + end if + if (sum_i2 /= 500*isize) then + write(*,*) 'Test failed integer*2' + stop 1 + end if + if (sum(sel_temperatures_i4) /= 500*isize) then + write(*,*) 'Test failed integer*4' + stop 1 + end if + if (sum(sel_temperatures_i8) /= 500*isize) then + write(*,*) 'Test failed integer*8' + stop 1 + end if + if (sum(sel_temperatures_r4) /= 500*isize) then + write(*,*) 'Test failed real*4' + stop 1 + end if + if (sum(sel_temperatures_r8) /= 500*isize) then + write(*,*) 'Test failed real*8' + stop 1 + end if if (allocated(sel_temperatures_i1)) deallocate (sel_temperatures_i1) if (allocated(sel_temperatures_i2)) deallocate (sel_temperatures_i2) diff --git a/testing/adios2/bindings/fortran/TestBPWriteReadHeatMap4D.F90 b/testing/adios2/bindings/fortran/TestBPWriteReadHeatMap4D.F90 index 3111567d62..ce2711ad7c 100644 --- a/testing/adios2/bindings/fortran/TestBPWriteReadHeatMap4D.F90 +++ b/testing/adios2/bindings/fortran/TestBPWriteReadHeatMap4D.F90 @@ -194,12 +194,30 @@ program TestBPWriteReadHeatMap4D end do end do - if (sum_i1 /= 10000*isize) stop 'Test failed integer*1' - if (sum_i2 /= 10000*isize) stop 'Test failed integer*2' - if (sum(sel_temperatures_i4) /= 10000*isize) stop 'Test failed integer*4' - if (sum(sel_temperatures_i8) /= 10000*isize) stop 'Test failed integer*8' - if (sum(sel_temperatures_r4) /= 10000*isize) stop 'Test failed real*4' - if (sum(sel_temperatures_r8) /= 10000*isize) stop 'Test failed real*8' + if (sum_i1 /= 10000*isize) then + write(*,*) 'Test failed integer*1' + stop 1 + end if + if (sum_i2 /= 10000*isize) then + write(*,*) 'Test failed integer*2' + stop 1 + end if + if (sum(sel_temperatures_i4) /= 10000*isize) then + write(*,*) 'Test failed integer*4' + stop 1 + end if + if (sum(sel_temperatures_i8) /= 10000*isize) then + write(*,*) 'Test failed integer*8' + stop 1 + end if + if (sum(sel_temperatures_r4) /= 10000*isize) then + write(*,*) 'Test failed real*4' + stop 1 + end if + if (sum(sel_temperatures_r8) /= 10000*isize) then + write(*,*) 'Test failed real*8' + stop 1 + end if if (allocated(sel_temperatures_i1)) deallocate (sel_temperatures_i1) if (allocated(sel_temperatures_i2)) deallocate (sel_temperatures_i2) diff --git a/testing/adios2/bindings/fortran/TestBPWriteReadHeatMap5D.F90 b/testing/adios2/bindings/fortran/TestBPWriteReadHeatMap5D.F90 index 20eed9cee1..bc6b970217 100644 --- a/testing/adios2/bindings/fortran/TestBPWriteReadHeatMap5D.F90 +++ b/testing/adios2/bindings/fortran/TestBPWriteReadHeatMap5D.F90 @@ -203,12 +203,30 @@ program TestBPWriteReadHeatMap5D end do end do - if (sum_i1 /= 100000*isize) stop 'Test failed integer*1' - if (sum_i2 /= 100000*isize) stop 'Test failed integer*2' - if (sum(sel_temperatures_i4) /= 100000*isize) stop 'Test failed integer*4' - if (sum(sel_temperatures_i8) /= 100000*isize) stop 'Test failed integer*8' - if (sum(sel_temperatures_r4) /= 100000*isize) stop 'Test failed real*4' - if (sum(sel_temperatures_r8) /= 100000*isize) stop 'Test failed real*8' + if (sum_i1 /= 100000*isize) then + write(*,*) 'Test failed integer*1' + stop 1 + end if + if (sum_i2 /= 100000*isize) then + write(*,*) 'Test failed integer*2' + stop 1 + end if + if (sum(sel_temperatures_i4) /= 100000*isize) then + write(*,*) 'Test failed integer*4' + stop 1 + end if + if (sum(sel_temperatures_i8) /= 100000*isize) then + write(*,*) 'Test failed integer*8' + stop 1 + end if + if (sum(sel_temperatures_r4) /= 100000*isize) then + write(*,*) 'Test failed real*4' + stop 1 + end if + if (sum(sel_temperatures_r8) /= 100000*isize) then + write(*,*) 'Test failed real*8' + stop 1 + end if if (allocated(sel_temperatures_i1)) deallocate (sel_temperatures_i1) if (allocated(sel_temperatures_i2)) deallocate (sel_temperatures_i2) diff --git a/testing/adios2/bindings/fortran/TestBPWriteReadHeatMap6D.F90 b/testing/adios2/bindings/fortran/TestBPWriteReadHeatMap6D.F90 index f749102237..ac9f035156 100644 --- a/testing/adios2/bindings/fortran/TestBPWriteReadHeatMap6D.F90 +++ b/testing/adios2/bindings/fortran/TestBPWriteReadHeatMap6D.F90 @@ -207,12 +207,30 @@ program TestBPWriteReadHeatMap6D end do end do - if (sum_i1 /= 1000000*isize) stop 'Test failed integer*1' - if (sum_i2 /= 1000000*isize) stop 'Test failed integer*2' - if (sum(sel_temperatures_i4) /= 1000000*isize) stop 'Test failed integer*4' - if (sum(sel_temperatures_i8) /= 1000000*isize) stop 'Test failed integer*8' - if (sum(sel_temperatures_r4) /= 1000000*isize) stop 'Test failed real*4' - if (sum(sel_temperatures_r8) /= 1000000*isize) stop 'Test failed real*8' + if (sum_i1 /= 1000000*isize) then + write(*,*) 'Test failed integer*1' + stop 1 + end if + if (sum_i2 /= 1000000*isize) then + write(*,*) 'Test failed integer*2' + stop 1 + end if + if (sum(sel_temperatures_i4) /= 1000000*isize) then + write(*,*) 'Test failed integer*4' + stop 1 + end if + if (sum(sel_temperatures_i8) /= 1000000*isize) then + write(*,*) 'Test failed integer*8' + stop 1 + end if + if (sum(sel_temperatures_r4) /= 1000000*isize) then + write(*,*) 'Test failed real*4' + stop 1 + end if + if (sum(sel_temperatures_r8) /= 1000000*isize) then + write(*,*) 'Test failed real*8' + stop 1 + end if if (allocated(sel_temperatures_i1)) deallocate (sel_temperatures_i1) if (allocated(sel_temperatures_i2)) deallocate (sel_temperatures_i2) diff --git a/testing/adios2/bindings/fortran/TestBPWriteTypesByName.F90 b/testing/adios2/bindings/fortran/TestBPWriteTypesByName.F90 index 13d1bf047d..0c7c74afdc 100644 --- a/testing/adios2/bindings/fortran/TestBPWriteTypesByName.F90 +++ b/testing/adios2/bindings/fortran/TestBPWriteTypesByName.F90 @@ -127,46 +127,118 @@ program TestBPWriteTypes step_status, ierr) call adios2_inquire_variable(variables(1), ioRead, "var_I8", ierr) - if (variables(1)%name /= 'var_I8') stop 'var_I8 not recognized' - if (variables(1)%type /= adios2_type_integer1) stop 'var_I8 type not recognized' + if (variables(1)%name /= 'var_I8') then + write(*,*) 'var_I8 not recognized' + stop 1 + end if + if (variables(1)%type /= adios2_type_integer1) then + write(*,*) 'var_I8 type not recognized' + stop 1 + end if call adios2_variable_shape(shape_in, ndims, variables(1), ierr) - if (ndims /= 1) stop 'var_I8 ndims is not 1' - if (shape_in(1) /= isize*inx) stop 'var_I8 shape_in read failed' + if (ndims /= 1) then + write(*,*) 'var_I8 ndims is not 1' + stop 1 + end if + if (shape_in(1) /= isize*inx) then + write(*,*) 'var_I8 shape_in read failed' + stop 1 + end if call adios2_inquire_variable(variables(2), ioRead, "var_I16", ierr) - if (variables(2)%name /= 'var_I16') stop 'var_I16 not recognized' - if (variables(2)%type /= adios2_type_integer2) stop 'var_I16 type not recognized' + if (variables(2)%name /= 'var_I16') then + write(*,*) 'var_I16 not recognized' + stop 1 + end if + if (variables(2)%type /= adios2_type_integer2) then + write(*,*) 'var_I16 type not recognized' + stop 1 + end if call adios2_variable_shape( shape_in, ndims,variables(2),ierr) - if (ndims /= 1) stop 'var_I16 ndims is not 1' - if (shape_in(1) /= isize*inx) stop 'var_I16 shape_in read failed' + if (ndims /= 1) then + write(*,*) 'var_I16 ndims is not 1' + stop 1 + end if + if (shape_in(1) /= isize*inx) then + write(*,*) 'var_I16 shape_in read failed' + stop 1 + end if call adios2_inquire_variable(variables(3), ioRead, "var_I32", ierr) - if (variables(3)%name /= 'var_I32') stop 'var_I32 not recognized' - if (variables(3)%type /= adios2_type_integer4) stop 'var_I32 type not recognized' + if (variables(3)%name /= 'var_I32') then + write(*,*) 'var_I32 not recognized' + stop 1 + end if + if (variables(3)%type /= adios2_type_integer4) then + write(*,*) 'var_I32 type not recognized' + stop 1 + end if call adios2_variable_shape( shape_in, ndims, variables(3),ierr) - if (ndims /= 1) stop 'var_I32 ndims is not 1' - if (shape_in(1) /= isize*inx) stop 'var_I32 shape_in read failed' + if (ndims /= 1) then + write(*,*) 'var_I32 ndims is not 1' + stop 1 + end if + if (shape_in(1) /= isize*inx) then + write(*,*) 'var_I32 shape_in read failed' + stop 1 + end if call adios2_inquire_variable(variables(4), ioRead, "var_I64", ierr) - if (variables(4)%name /= 'var_I64') stop 'var_I64 not recognized' - if (variables(4)%type /= adios2_type_integer8) stop 'var_I64 type not recognized' + if (variables(4)%name /= 'var_I64') then + write(*,*) 'var_I64 not recognized' + stop 1 + end if + if (variables(4)%type /= adios2_type_integer8) then + write(*,*) 'var_I64 type not recognized' + stop 1 + end if call adios2_variable_shape(shape_in, ndims, variables(4),ierr) - if (ndims /= 1) stop 'var_I64 ndims is not 1' - if (shape_in(1) /= isize*inx) stop 'var_I64 shape_in read failed' + if (ndims /= 1) then + write(*,*) 'var_I64 ndims is not 1' + stop 1 + end if + if (shape_in(1) /= isize*inx) then + write(*,*) 'var_I64 shape_in read failed' + stop 1 + end if call adios2_inquire_variable(variables(5), ioRead, "var_R32", ierr) - if (variables(5)%name /= 'var_R32') stop 'var_R32 not recognized' - if (variables(5)%type /= adios2_type_real) stop 'var_R32 type not recognized' + if (variables(5)%name /= 'var_R32') then + write(*,*) 'var_R32 not recognized' + stop 1 + end if + if (variables(5)%type /= adios2_type_real) then + write(*,*) 'var_R32 type not recognized' + stop 1 + end if call adios2_variable_shape( shape_in, ndims, variables(5), ierr) - if (ndims /= 1) stop 'var_R32 ndims is not 1' - if (shape_in(1) /= isize*inx) stop 'var_R32 shape_in read failed' + if (ndims /= 1) then + write(*,*) 'var_R32 ndims is not 1' + stop 1 + end if + if (shape_in(1) /= isize*inx) then + write(*,*) 'var_R32 shape_in read failed' + stop 1 + end if call adios2_inquire_variable(variables(6), ioRead, "var_R64", ierr) - if (variables(6)%name /= 'var_R64') stop 'var_R64 not recognized' - if (variables(6)%type /= adios2_type_dp) stop 'var_R64 type not recognized' + if (variables(6)%name /= 'var_R64') then + write(*,*) 'var_R64 not recognized' + stop 1 + end if + if (variables(6)%type /= adios2_type_dp) then + write(*,*) 'var_R64 type not recognized' + stop 1 + end if call adios2_variable_shape( shape_in, ndims, variables(6), ierr) - if (ndims /= 1) stop 'var_R64 ndims is not 1' - if (shape_in(1) /= isize*inx) stop 'var_R64 shape_in read failed' + if (ndims /= 1) then + write(*,*) 'var_R64 ndims is not 1' + stop 1 + end if + if (shape_in(1) /= isize*inx) then + write(*,*) 'var_R64 shape_in read failed' + stop 1 + end if call adios2_end_step(bpReader, ierr) diff --git a/testing/adios2/bindings/fortran/TestBPWriteTypesLocal.F90 b/testing/adios2/bindings/fortran/TestBPWriteTypesLocal.F90 index 41081593e9..6c8ef24210 100644 --- a/testing/adios2/bindings/fortran/TestBPWriteTypesLocal.F90 +++ b/testing/adios2/bindings/fortran/TestBPWriteTypesLocal.F90 @@ -104,15 +104,24 @@ program TestBPWriteTypes adios2_variable_dims, ierr) write (*, *) "Engine type: ", ioWrite%engine_type - if (TRIM(ioWrite%engine_type) /= 'File') stop 'Wrong engine_type' + if (TRIM(ioWrite%engine_type) /= 'File') then + write(*,*) 'Wrong engine_type' + stop 1 + end if call adios2_set_engine(ioWrite, "SST", ierr) write (*, *) "Engine type: ", ioWrite%engine_type - if (TRIM(ioWrite%engine_type) /= 'SST') stop 'Wrong engine_type' + if (TRIM(ioWrite%engine_type) /= 'SST') then + write(*,*) 'Wrong engine_type' + stop 1 + end if call adios2_at_io(ioDummy, adios, "ioWrite", ierr) write (*, *) "Engine type: ", ioDummy%engine_type - if (TRIM(ioDummy%engine_type) /= 'SST') stop 'Wrong engine_type' + if (TRIM(ioDummy%engine_type) /= 'SST') then + write(*,*) 'Wrong engine_type' + stop 1 + end if call adios2_set_engine(ioWrite, "BPFile", ierr) @@ -124,7 +133,10 @@ program TestBPWriteTypes call adios2_begin_step(bpWriter, ierr) call adios2_current_step(current_step, bpWriter, ierr) - if (current_step /= s - 1) stop 'wrong current step' + if (current_step /= s - 1) then + write(*,*) 'wrong current step' + stop 1 + end if if (irank == 0 .and. s == 1) then call adios2_put(bpWriter, variables(7), data_I8(1), ierr) @@ -181,57 +193,135 @@ program TestBPWriteTypes if (current_step == 0) then call adios2_inquire_variable(variables(7), ioRead, "gvar_I8", ierr) - if (variables(7)%name /= 'gvar_I8') stop 'gvar_I8 name not recognized' - if (variables(7)%type /= adios2_type_integer1) stop 'gvar_I8 type not recognized' + if (variables(7)%name /= 'gvar_I8') then + write(*,*) 'gvar_I8 name not recognized' + stop 1 + end if + if (variables(7)%type /= adios2_type_integer1) then + write(*,*) 'gvar_I8 type not recognized' + stop 1 + end if call adios2_inquire_variable(variables(8), ioRead, "gvar_I16", ierr) - if (variables(8)%name /= 'gvar_I16') stop 'gvar_I16 name not recognized' - if (variables(8)%type /= adios2_type_integer2) stop 'gvar_I16 type not recognized' + if (variables(8)%name /= 'gvar_I16') then + write(*,*) 'gvar_I16 name not recognized' + stop 1 + end if + if (variables(8)%type /= adios2_type_integer2) then + write(*,*) 'gvar_I16 type not recognized' + stop 1 + end if call adios2_inquire_variable(variables(9), ioRead, "gvar_I32", ierr) - if (variables(9)%name /= 'gvar_I32') stop 'gvar_I32 name not recognized' - if (variables(9)%type /= adios2_type_integer4) stop 'gvar_I32 type not recognized' + if (variables(9)%name /= 'gvar_I32') then + write(*,*) 'gvar_I32 name not recognized' + stop 1 + end if + if (variables(9)%type /= adios2_type_integer4) then + write(*,*) 'gvar_I32 type not recognized' + stop 1 + end if call adios2_inquire_variable(variables(10), ioRead, "gvar_I64", ierr) - if (variables(10)%name /= 'gvar_I64') stop 'gvar_I64 name not recognized' - if (variables(10)%type /= adios2_type_integer8) stop 'gvar_I64 type not recognized' + if (variables(10)%name /= 'gvar_I64') then + write(*,*) 'gvar_I64 name not recognized' + stop 1 + end if + if (variables(10)%type /= adios2_type_integer8) then + write(*,*) 'gvar_I64 type not recognized' + stop 1 + end if call adios2_inquire_variable(variables(11), ioRead, "gvar_R32", ierr) - if (variables(11)%name /= 'gvar_R32') stop 'gvar_R32 name not recognized' - if (variables(11)%type /= adios2_type_real) stop 'gvar_I64 type not recognized' + if (variables(11)%name /= 'gvar_R32') then + write(*,*) 'gvar_R32 name not recognized' + stop 1 + end if + if (variables(11)%type /= adios2_type_real) then + write(*,*) 'gvar_I64 type not recognized' + stop 1 + end if call adios2_inquire_variable(variables(12), ioRead, "gvar_R64", ierr) - if (variables(12)%name /= 'gvar_R64') stop 'gvar_R64 name not recognized' - if (variables(12)%type /= adios2_type_dp) stop 'gvar_I64 type not recognized' + if (variables(12)%name /= 'gvar_R64') then + write(*,*) 'gvar_R64 name not recognized' + stop 1 + end if + if (variables(12)%type /= adios2_type_dp) then + write(*,*) 'gvar_I64 type not recognized' + stop 1 + end if end if call adios2_inquire_variable(variables(1), ioRead, "var_I8", ierr) - if (variables(1)%name /= 'var_I8') stop 'var_I8 not recognized' - if (variables(1)%type /= adios2_type_integer1) stop 'var_I8 type not recognized' + if (variables(1)%name /= 'var_I8') then + write(*,*) 'var_I8 not recognized' + stop 1 + end if + if (variables(1)%type /= adios2_type_integer1) then + write(*,*) 'var_I8 type not recognized' + stop 1 + end if call adios2_inquire_variable(variables(2), ioRead, "var_I16", ierr) - if (variables(2)%name /= 'var_I16') stop 'var_I16 not recognized' - if (variables(2)%type /= adios2_type_integer2) stop 'var_I16 type not recognized' + if (variables(2)%name /= 'var_I16') then + write(*,*) 'var_I16 not recognized' + stop 1 + end if + if (variables(2)%type /= adios2_type_integer2) then + write(*,*) 'var_I16 type not recognized' + stop 1 + end if call adios2_inquire_variable(variables(3), ioRead, "var_I32", ierr) - if (variables(3)%name /= 'var_I32') stop 'var_I32 not recognized' - if (variables(3)%type /= adios2_type_integer4) stop 'var_I32 type not recognized' + if (variables(3)%name /= 'var_I32') then + write(*,*) 'var_I32 not recognized' + stop 1 + end if + if (variables(3)%type /= adios2_type_integer4) then + write(*,*) 'var_I32 type not recognized' + stop 1 + end if call adios2_inquire_variable(variables(4), ioRead, "var_I64", ierr) - if (variables(4)%name /= 'var_I64') stop 'var_I64 not recognized' - if (variables(4)%type /= adios2_type_integer8) stop 'var_I64 type not recognized' + if (variables(4)%name /= 'var_I64') then + write(*,*) 'var_I64 not recognized' + stop 1 + end if + if (variables(4)%type /= adios2_type_integer8) then + write(*,*) 'var_I64 type not recognized' + stop 1 + end if call adios2_inquire_variable(variables(5), ioRead, "var_R32", ierr) - if (variables(5)%name /= 'var_R32') stop 'var_R32 not recognized' - if (variables(5)%type /= adios2_type_real) stop 'var_R32 type not recognized' + if (variables(5)%name /= 'var_R32') then + write(*,*) 'var_R32 not recognized' + stop 1 + end if + if (variables(5)%type /= adios2_type_real) then + write(*,*) 'var_R32 type not recognized' + stop 1 + end if call adios2_inquire_variable(variables(6), ioRead, "var_R64", ierr) - if (variables(6)%name /= 'var_R64') stop 'var_R64 not recognized' - if (variables(6)%type /= adios2_type_dp) stop 'var_R64 type not recognized' + if (variables(6)%name /= 'var_R64') then + write(*,*) 'var_R64 not recognized' + stop 1 + end if + if (variables(6)%type /= adios2_type_dp) then + write(*,*) 'var_R64 type not recognized' + stop 1 + end if call adios2_inquire_variable(variables(13), ioRead, "var_changingR64", ierr) - if (variables(13)%name /= 'var_changingR64') stop 'var_changingR64 not recognized' - if (variables(13)%type /= adios2_type_dp) stop 'var_R64 type not recognized' + if (variables(13)%name /= 'var_changingR64') then + write(*,*) 'var_changingR64 not recognized' + stop 1 + end if + if (variables(13)%type /= adios2_type_dp) then + write(*,*) 'var_R64 type not recognized' + stop 1 + end if do block_id = 0, isize - 1 @@ -262,15 +352,36 @@ program TestBPWriteTypes end do do i = 1, inx - if (I8(i) /= inI8(i)) stop 'Error reading var_I8' - if (I16(i) /= inI16(i)) stop 'Error reading var_I16' - if (I32(i) /= inI32(i)) stop 'Error reading var_I32' - if (I64(i) /= inI64(i)) stop 'Error reading var_I64' - if (R32(i) /= inR32(i)) stop 'Error reading var_R32' - if (R64(i) /= inR64(i)) stop 'Error reading var_R64' + if (I8(i) /= inI8(i)) then + write(*,*) 'Error reading var_I8' + stop 1 + end if + if (I16(i) /= inI16(i)) then + write(*,*) 'Error reading var_I16' + stop 1 + end if + if (I32(i) /= inI32(i)) then + write(*,*) 'Error reading var_I32' + stop 1 + end if + if (I64(i) /= inI64(i)) then + write(*,*) 'Error reading var_I64' + stop 1 + end if + if (R32(i) /= inR32(i)) then + write(*,*) 'Error reading var_R32' + stop 1 + end if + if (R64(i) /= inR64(i)) then + write(*,*) 'Error reading var_R64' + stop 1 + end if if( i < current_step) then - if (R64(i) /= inchangingR64(i)) stop 'Error reading var_changingR64' + if (R64(i) /= inchangingR64(i)) then + write(*,*) 'Error reading var_changingR64' + stop 1 + end if end if end do diff --git a/testing/adios2/bindings/fortran/TestBPWriteVariableAttributes.F90 b/testing/adios2/bindings/fortran/TestBPWriteVariableAttributes.F90 index 4c750e19bb..ae95a9aa96 100644 --- a/testing/adios2/bindings/fortran/TestBPWriteVariableAttributes.F90 +++ b/testing/adios2/bindings/fortran/TestBPWriteVariableAttributes.F90 @@ -34,13 +34,16 @@ program TestBPWriteVariableAttributes ! Failed with ci/circleci: suse-pgi-openmpi (exceptions trigger abort) ! call adios2_define_attribute(failed_att, ioWrite, 'att_String', & ! 'ADIOS2 String attribute', 'myVar2', '/', ierr) -! if(ierr == 0) stop 'myVar2 does not exist, should not create attribute att_String' +! if(ierr == 0) then 'myVar2 does not exist, should not create attribute att_String' ! if(failed_att%valid .eqv. .true.) then ! stop 'failed attribute must not exist ' ! end if do i=1,14 - if( attributes(i)%valid .eqv. .true. ) stop 'Invalid attribute default' + if( attributes(i)%valid .eqv. .true. ) then + write(*,*) 'Invalid attribute default' + stop 1 + end if end do ! single value @@ -89,7 +92,10 @@ program TestBPWriteVariableAttributes data_R64, 3, var%name, ierr) do i=1,14 - if( attributes(i)%valid .eqv. .false. ) stop 'Invalid adios2_define_attribute' + if( attributes(i)%valid .eqv. .false. ) then + write(*,*) 'Invalid adios2_define_attribute' + stop 1 + end if end do call adios2_open(bpWriter, ioWrite, "fvarattr_types.bp", adios2_mode_write, & diff --git a/testing/adios2/bindings/fortran/TestNullEngine.F90 b/testing/adios2/bindings/fortran/TestNullEngine.F90 index 668430ef34..f6e1a9f76b 100644 --- a/testing/adios2/bindings/fortran/TestNullEngine.F90 +++ b/testing/adios2/bindings/fortran/TestNullEngine.F90 @@ -39,7 +39,10 @@ program TestNullEngine ! Declare an IO process configuration inside adios call adios2_declare_io(ioWrite, adios, "nullWriter", ierr) call adios2_set_engine(ioWrite, "NULL", ierr) - if (TRIM(ioWrite%engine_type) /= "NULL") stop 'Wrong io engine_type' + if (TRIM(ioWrite%engine_type) /= "NULL") then + write(*,*) 'Wrong io engine_type' + stop 1 + end if ! Defines a variable to be written in bp format call adios2_define_variable(var, ioWrite, "var_R64", & @@ -70,11 +73,15 @@ program TestNullEngine call adios2_begin_step(nullReader, adios2_step_mode_read, -1.0, & step_status, ierr) if (step_status /= adios2_step_status_end_of_stream) then - stop 'null engine status failed' + write(*,*) 'null engine status failed' + stop 1 end if call adios2_inquire_variable(varIn, ioRead, "var_R64", ierr) - if (varIn%valid .eqv. .true.) stop 'var_R64 inquire error' + if (varIn%valid .eqv. .true.) then + write(*,*) 'var_R64 inquire error' + stop 1 + end if call adios2_get(nullReader, varIn, inR64, ierr) call adios2_perform_gets(nullReader, ierr) diff --git a/testing/adios2/bindings/fortran/TestRemove.F90 b/testing/adios2/bindings/fortran/TestRemove.F90 index adce243c74..ff909e7e15 100644 --- a/testing/adios2/bindings/fortran/TestRemove.F90 +++ b/testing/adios2/bindings/fortran/TestRemove.F90 @@ -97,67 +97,148 @@ program TestRemove call adios2_define_variable(variables(12), ioWrite, "gvar_R64", & adios2_type_dp, ierr) - if (variables(1)%valid .eqv. .false. ) stop 'var_I8 not defined' - if (variables(2)%valid .eqv. .false. ) stop 'var_I16 not defined' - if (variables(3)%valid .eqv. .false. ) stop 'var_I32 not defined' - if (variables(4)%valid .eqv. .false. ) stop 'var_I64 not defined' - if (variables(5)%valid .eqv. .false. ) stop 'var_R32 not defined' - if (variables(6)%valid .eqv. .false. ) stop 'var_R64 not defined' - if (variables(7)%valid .eqv. .false. ) stop 'gvar_I8 not defined' - if (variables(8)%valid .eqv. .false. ) stop 'gvar_I16 not defined' - if (variables(9)%valid .eqv. .false. ) stop 'gvar_I32 not defined' - if (variables(10)%valid .eqv. .false. ) stop 'gvar_I64 not defined' - if (variables(11)%valid .eqv. .false. ) stop 'gvar_R32 not defined' - if (variables(12)%valid .eqv. .false. ) stop 'gvar_IR64 not defined' + if (variables(1)%valid .eqv. .false. ) then + write(*,*) 'var_I8 not defined' + stop 1 + end if + if (variables(2)%valid .eqv. .false. ) then + write(*,*) 'var_I16 not defined' + stop 1 + end if + if (variables(3)%valid .eqv. .false. ) then + write(*,*) 'var_I32 not defined' + stop 1 + end if + if (variables(4)%valid .eqv. .false. ) then + write(*,*) 'var_I64 not defined' + stop 1 + end if + if (variables(5)%valid .eqv. .false. ) then + write(*,*) 'var_R32 not defined' + stop 1 + end if + if (variables(6)%valid .eqv. .false. ) then + write(*,*) 'var_R64 not defined' + stop 1 + end if + if (variables(7)%valid .eqv. .false. ) then + write(*,*) 'gvar_I8 not defined' + stop 1 + end if + if (variables(8)%valid .eqv. .false. ) then + write(*,*) 'gvar_I16 not defined' + stop 1 + end if + if (variables(9)%valid .eqv. .false. ) then + write(*,*) 'gvar_I32 not defined' + stop 1 + end if + if (variables(10)%valid .eqv. .false. ) then + write(*,*) 'gvar_I64 not defined' + stop 1 + end if + if (variables(11)%valid .eqv. .false. ) then + write(*,*) 'gvar_R32 not defined' + stop 1 + end if + if (variables(12)%valid .eqv. .false. ) then + write(*,*) 'gvar_IR64 not defined' + stop 1 + end if ! remove piece call adios2_remove_variable(res, ioWrite, "gvar_R64", ierr) - if( res .eqv. .false. ) stop 'adios2_remove_variable failed' + if( res .eqv. .false. ) then + write(*,*) 'adios2_remove_variable failed' + stop 1 + end if call adios2_inquire_variable(variables(12), ioWrite, "gvar_R64", ierr) - if (variables(12)%valid .eqv. .true. ) stop 'gvar_R64 found with inquire, not removed' + if (variables(12)%valid .eqv. .true. ) then + write(*,*) 'gvar_R64 found with inquire, not removed' + stop 1 + end if ! remove all call adios2_remove_all_variables(ioWrite, ierr) call adios2_inquire_variable(variables(1), ioWrite, "var_I8", ierr) - if (variables(1)%valid .eqv. .true. ) stop 'var_I8 found' + if (variables(1)%valid .eqv. .true. ) then + write(*,*) 'var_I8 found' + stop 1 + end if call adios2_inquire_variable(variables(2), ioWrite, "var_I16", ierr) - if (variables(2)%valid .eqv. .true.) stop 'var_I16 found' + if (variables(2)%valid .eqv. .true.) then + write(*,*) 'var_I16 found' + stop 1 + end if call adios2_inquire_variable(variables(3), ioWrite, "var_I32", ierr) - if (variables(3)%valid .eqv. .true.) stop 'var_I32 found' + if (variables(3)%valid .eqv. .true.) then + write(*,*) 'var_I32 found' + stop 1 + end if call adios2_inquire_variable(variables(4), ioWrite, "var_I64", ierr) - if (variables(4)%valid .eqv. .true.) stop 'var_I64 found' + if (variables(4)%valid .eqv. .true.) then + write(*,*) 'var_I64 found' + stop 1 + end if call adios2_inquire_variable(variables(5), ioWrite, "var_R32", ierr) - if (variables(5)%valid .eqv. .true.) stop 'var_R32 found' + if (variables(5)%valid .eqv. .true.) then + write(*,*) 'var_R32 found' + stop 1 + end if call adios2_inquire_variable(variables(6), ioWrite, "var_R64", ierr) - if (variables(6)%valid .eqv. .true.) stop 'var_R64 found' + if (variables(6)%valid .eqv. .true.) then + write(*,*) 'var_R64 found' + stop 1 + end if call adios2_inquire_variable(variables(7), ioWrite, "gvar_I8", ierr) - if (variables(7)%valid .eqv. .true.) stop 'gvar_I8 found' + if (variables(7)%valid .eqv. .true.) then + write(*,*) 'gvar_I8 found' + stop 1 + end if call adios2_inquire_variable(variables(8), ioWrite, "gvar_I16", ierr) - if (variables(8)%valid .eqv. .true.) stop 'gvar_I16 found' + if (variables(8)%valid .eqv. .true.) then + write(*,*) 'gvar_I16 found' + stop 1 + end if call adios2_inquire_variable(variables(9), ioWrite, "gvar_I32", ierr) - if (variables(9)%valid .eqv. .true.) stop 'gvar_I32 found' + if (variables(9)%valid .eqv. .true.) then + write(*,*) 'gvar_I32 found' + stop 1 + end if call adios2_inquire_variable(variables(10), ioWrite, "gvar_I64", ierr) - if (variables(10)%valid .eqv. .true.) stop 'gvar_I64 found' + if (variables(10)%valid .eqv. .true.) then + write(*,*) 'gvar_I64 found' + stop 1 + end if call adios2_inquire_variable(variables(11), ioWrite, "gvar_R32", ierr) - if (variables(11)%valid .eqv. .true.) stop 'gvar_R32 found' + if (variables(11)%valid .eqv. .true.) then + write(*,*) 'gvar_R32 found' + stop 1 + end if call adios2_remove_io(res, adios, 'ioWrite', ierr) - if( res .neqv. .true. ) stop 'could not remove ioWrite' + if( res .neqv. .true. ) then + write(*,*) 'could not remove ioWrite' + stop 1 + end if call adios2_at_io(ioWrite, adios, 'ioWrite', ierr) - if( ioWrite%valid .eqv. .true. ) stop 'did not remove ioWrite correctly' + if( ioWrite%valid .eqv. .true. ) then + write(*,*) 'did not remove ioWrite correctly' + stop 1 + end if call adios2_finalize(adios, ierr) diff --git a/testing/adios2/bindings/fortran/operation/TestBPWriteReadSZ2D.F90 b/testing/adios2/bindings/fortran/operation/TestBPWriteReadSZ2D.F90 index 685b6ba2c3..ccaf30a890 100644 --- a/testing/adios2/bindings/fortran/operation/TestBPWriteReadSZ2D.F90 +++ b/testing/adios2/bindings/fortran/operation/TestBPWriteReadSZ2D.F90 @@ -96,7 +96,10 @@ program TestBPWriteReadHeatMapSZ2D call adios2_add_operation(operation_id, var_temperatures(5), & sz_operator, 'accuracy', '0.01', ierr) - if( operation_id /= 0 ) stop 'operation_id not added for real type' + if( operation_id /= 0 ) then + write(*,*) 'operation_id not added for real type' + stop 1 + end if call adios2_define_variable(var_temperatures(6), ioPut, & @@ -106,7 +109,10 @@ program TestBPWriteReadHeatMapSZ2D call adios2_add_operation(operation_id, var_temperatures(6), & sz_operator, '', '', ierr) - if( operation_id /= 0 ) stop 'operation_id not added for dp type' + if( operation_id /= 0 ) then + write(*,*) 'operation_id not added for dp type' + stop 1 + end if call adios2_set_operation_parameter( var_temperatures(6), operation_id, & 'accuracy', '0.01', ierr) @@ -205,12 +211,30 @@ program TestBPWriteReadHeatMapSZ2D end do end do - if (sum_i1 /= 100*isize) stop 'Test failed integer*1' - if (sum_i2 /= 100*isize) stop 'Test failed integer*2' - if (sum(sel_temperatures_i4) /= 100*isize) stop 'Test failed integer*4' - if (sum(sel_temperatures_i8) /= 100*isize) stop 'Test failed integer*8' - if (sum(sel_temperatures_r4) /= 100*isize) stop 'Test failed real*4' - if (sum(sel_temperatures_r8) /= 100*isize) stop 'Test failed real*8' + if (sum_i1 /= 100*isize) then + write(*,*) 'Test failed integer*1' + stop 1 + end if + if (sum_i2 /= 100*isize) then + write(*,*) 'Test failed integer*2' + stop 1 + end if + if (sum(sel_temperatures_i4) /= 100*isize) then + write(*,*) 'Test failed integer*4' + stop 1 + end if + if (sum(sel_temperatures_i8) /= 100*isize) then + write(*,*) 'Test failed integer*8' + stop 1 + end if + if (sum(sel_temperatures_r4) /= 100*isize) then + write(*,*) 'Test failed real*4' + stop 1 + end if + if (sum(sel_temperatures_r8) /= 100*isize) then + write(*,*) 'Test failed real*8' + stop 1 + end if if (allocated(sel_temperatures_i1)) deallocate (sel_temperatures_i1) if (allocated(sel_temperatures_i2)) deallocate (sel_temperatures_i2) diff --git a/testing/adios2/bindings/fortran/operation/TestBPWriteReadSZ3D.F90 b/testing/adios2/bindings/fortran/operation/TestBPWriteReadSZ3D.F90 index 9838122aa7..f7ad41ac96 100644 --- a/testing/adios2/bindings/fortran/operation/TestBPWriteReadSZ3D.F90 +++ b/testing/adios2/bindings/fortran/operation/TestBPWriteReadSZ3D.F90 @@ -96,7 +96,10 @@ program TestBPWriteReadHeatMapSZ3D call adios2_add_operation(operation_id, var_temperatures(5), & sz_operator, 'accuracy', '0.01', ierr) - if( operation_id /= 0 ) stop 'operation_id not added for real type' + if( operation_id /= 0 ) then + write(*,*) 'operation_id not added for real type' + stop 1 + end if call adios2_define_variable(var_temperatures(6), ioPut, & @@ -107,7 +110,10 @@ program TestBPWriteReadHeatMapSZ3D call adios2_add_operation(operation_id, var_temperatures(6), & sz_operator, 'accuracy', '0.01', ierr) - if( operation_id /= 0 ) stop 'operation_id not added for dp type' + if( operation_id /= 0 ) then + write(*,*) 'operation_id not added for dp type' + stop 1 + end if call adios2_open(bpWriter, ioPut, 'HeatMapSZ3D_f.bp', adios2_mode_write, & ierr) @@ -205,12 +211,30 @@ program TestBPWriteReadHeatMapSZ3D end do end do - if (sum_i1 /= 1000*isize) stop 'Test failed integer*1' - if (sum_i2 /= 1000*isize) stop 'Test failed integer*2' - if (sum(sel_temperatures_i4) /= 1000*isize) stop 'Test failed integer*4' - if (sum(sel_temperatures_i8) /= 1000*isize) stop 'Test failed integer*8' - if (sum(sel_temperatures_r4) /= 1000*isize) stop 'Test failed real*4' - if (sum(sel_temperatures_r8) /= 1000*isize) stop 'Test failed real*8' + if (sum_i1 /= 1000*isize) then + write(*,*) 'Test failed integer*1' + stop 1 + end if + if (sum_i2 /= 1000*isize) then + write(*,*) 'Test failed integer*2' + stop 1 + end if + if (sum(sel_temperatures_i4) /= 1000*isize) then + write(*,*) 'Test failed integer*4' + stop 1 + end if + if (sum(sel_temperatures_i8) /= 1000*isize) then + write(*,*) 'Test failed integer*8' + stop 1 + end if + if (sum(sel_temperatures_r4) /= 1000*isize) then + write(*,*) 'Test failed real*4' + stop 1 + end if + if (sum(sel_temperatures_r8) /= 1000*isize) then + write(*,*) 'Test failed real*8' + stop 1 + end if if (allocated(sel_temperatures_i1)) deallocate (sel_temperatures_i1) if (allocated(sel_temperatures_i2)) deallocate (sel_temperatures_i2) diff --git a/testing/adios2/bindings/fortran/operation/TestBPWriteReadZfp2D.F90 b/testing/adios2/bindings/fortran/operation/TestBPWriteReadZfp2D.F90 index d564e1e307..38801f439d 100644 --- a/testing/adios2/bindings/fortran/operation/TestBPWriteReadZfp2D.F90 +++ b/testing/adios2/bindings/fortran/operation/TestBPWriteReadZfp2D.F90 @@ -96,7 +96,10 @@ program TestBPWriteReadHeatMapZfp2D call adios2_add_operation(operation_id, var_temperatures(5), & zfp_operator, 'rate', '8', ierr) - if( operation_id /= 0 ) stop 'operation_id not added for real type' + if( operation_id /= 0 ) then + write(*,*) 'operation_id not added for real type' + stop 1 + end if call adios2_define_variable(var_temperatures(6), ioPut, & @@ -107,7 +110,10 @@ program TestBPWriteReadHeatMapZfp2D call adios2_add_operation(operation_id, var_temperatures(6), & zfp_operator, 'rate', '8', ierr) - if( operation_id /= 0 ) stop 'operation_id not added for dp type' + if( operation_id /= 0 ) then + write(*,*) 'operation_id not added for dp type' + stop 1 + end if call adios2_open(bpWriter, ioPut, 'HeatMapZfp2D_f.bp', adios2_mode_write, & @@ -204,12 +210,30 @@ program TestBPWriteReadHeatMapZfp2D end do end do - if (sum_i1 /= 100*isize) stop 'Test failed integer*1' - if (sum_i2 /= 100*isize) stop 'Test failed integer*2' - if (sum(sel_temperatures_i4) /= 100*isize) stop 'Test failed integer*4' - if (sum(sel_temperatures_i8) /= 100*isize) stop 'Test failed integer*8' - if (sum(sel_temperatures_r4) /= 100*isize) stop 'Test failed real*4' - if (sum(sel_temperatures_r8) /= 100*isize) stop 'Test failed real*8' + if (sum_i1 /= 100*isize) then + write(*,*) 'Test failed integer*1' + stop 1 + end if + if (sum_i2 /= 100*isize) then + write(*,*) 'Test failed integer*2' + stop 1 + end if + if (sum(sel_temperatures_i4) /= 100*isize) then + write(*,*) 'Test failed integer*4' + stop 1 + end if + if (sum(sel_temperatures_i8) /= 100*isize) then + write(*,*) 'Test failed integer*8' + stop 1 + end if + if (sum(sel_temperatures_r4) /= 100*isize) then + write(*,*) 'Test failed real*4' + stop 1 + end if + if (sum(sel_temperatures_r8) /= 100*isize) then + write(*,*) 'Test failed real*8' + stop 1 + end if if (allocated(sel_temperatures_i1)) deallocate (sel_temperatures_i1) if (allocated(sel_temperatures_i2)) deallocate (sel_temperatures_i2) diff --git a/testing/adios2/bindings/fortran/operation/TestBPWriteReadZfp2DRemove.F90 b/testing/adios2/bindings/fortran/operation/TestBPWriteReadZfp2DRemove.F90 index 70d962dde3..9d624c986c 100644 --- a/testing/adios2/bindings/fortran/operation/TestBPWriteReadZfp2DRemove.F90 +++ b/testing/adios2/bindings/fortran/operation/TestBPWriteReadZfp2DRemove.F90 @@ -67,11 +67,17 @@ program TestBPWriteReadHeatMapZfp2DRemove if( mod(i,2) == 0 ) then call adios2_add_operation(operation_id, var_temperatures(1), & zfp_operator, 'rate', '8', ierr) - if( operation_id /= 0 ) stop 'operation_id not added for real type' + if( operation_id /= 0 ) then + write(*,*) 'operation_id not added for real type' + stop 1 + end if call adios2_add_operation(operation_id, var_temperatures(2), & zfp_operator, 'rate', '8', ierr) - if( operation_id /= 0 ) stop 'operation_id not added for dp type' + if( operation_id /= 0 ) then + write(*,*) 'operation_id not added for dp type' + stop 1 + end if else call adios2_remove_operations(var_temperatures(1), ierr) @@ -125,8 +131,14 @@ program TestBPWriteReadHeatMapZfp2DRemove call adios2_get(bpReader, var_temperaturesIn(2), sel_temperatures_r8, ierr) call adios2_end_step(bpReader, ierr) - if (sum(sel_temperatures_r4) /= 100*isize) stop 'Test failed real*4' - if (sum(sel_temperatures_r8) /= 100*isize) stop 'Test failed real*8' + if (sum(sel_temperatures_r4) /= 100*isize) then + write(*,*) 'Test failed real*4' + stop 1 + end if + if (sum(sel_temperatures_r8) /= 100*isize) then + write(*,*) 'Test failed real*8' + stop 1 + end if end do diff --git a/testing/adios2/engine/bp/TestBPFortranToCppWriter.F90 b/testing/adios2/engine/bp/TestBPFortranToCppWriter.F90 index 68debc2b3b..2fc0a46e9c 100644 --- a/testing/adios2/engine/bp/TestBPFortranToCppWriter.F90 +++ b/testing/adios2/engine/bp/TestBPFortranToCppWriter.F90 @@ -83,7 +83,10 @@ end function iargc call adios2_begin_step(bpWriter, ierr) call adios2_current_step(current_step, bpWriter, ierr) - if (current_step /= s - 1) stop 'wrong current step' + if (current_step /= s - 1) then + write(*,*) 'wrong current step' + stop 1 + end if if (irank == 0 .and. s == 1) then call adios2_put(bpWriter, vGlobalValue, inx, ierr) diff --git a/testing/adios2/engine/staging-common/TestCommonReadF.F90 b/testing/adios2/engine/staging-common/TestCommonReadF.F90 index 2e30795a44..7aeeaad16e 100644 --- a/testing/adios2/engine/staging-common/TestCommonReadF.F90 +++ b/testing/adios2/engine/staging-common/TestCommonReadF.F90 @@ -113,91 +113,223 @@ program TestSstRead call adios2_inquire_variable(variables(1), ioRead, "i8", ierr) - if (variables(1)%name /= 'i8') stop 'i8 not recognized' - if (variables(1)%type /= adios2_type_integer1) stop 'i8 type not recognized' + if (variables(1)%name /= 'i8') then + write(*,*) 'i8 not recognized' + stop 1 + end if + if (variables(1)%type /= adios2_type_integer1) then + write(*,*) 'i8 type not recognized' + stop 1 + end if call adios2_variable_shape(shape_in, ndims, variables(1), ierr) - if (ndims /= 1) stop 'i8 ndims is not 1' - if (modulo(shape_in(1), int(nx, 8)) /= 0) stop 'i8 shape_in read failed' + if (ndims /= 1) then + write(*,*) 'i8 ndims is not 1' + stop 1 + end if + if (modulo(shape_in(1), int(nx, 8)) /= 0) then + write(*,*) 'i8 shape_in read failed' + stop 1 + end if writerSize = INT(shape_in(1)) / nx deallocate(shape_in) call adios2_inquire_variable(variables(2), ioRead, "i16", ierr) - if (variables(2)%name /= 'i16') stop 'i16 not recognized' - if (variables(2)%type /= adios2_type_integer2) stop 'i16 type not recognized' + if (variables(2)%name /= 'i16') then + write(*,*) 'i16 not recognized' + stop 1 + end if + if (variables(2)%type /= adios2_type_integer2) then + write(*,*) 'i16 type not recognized' + stop 1 + end if call adios2_variable_shape( shape_in, ndims, variables(2), ierr) - if (ndims /= 1) stop 'i16 ndims is not 1' - if (shape_in(1) /= nx*writerSize) stop 'i16 shape_in read failed' + if (ndims /= 1) then + write(*,*) 'i16 ndims is not 1' + stop 1 + end if + if (shape_in(1) /= nx*writerSize) then + write(*,*) 'i16 shape_in read failed' + stop 1 + end if deallocate(shape_in) call adios2_inquire_variable(variables(3), ioRead, "i32", ierr) - if (variables(3)%name /= 'i32') stop 'i32 not recognized' - if (variables(3)%type /= adios2_type_integer4) stop 'i32 type not recognized' + if (variables(3)%name /= 'i32') then + write(*,*) 'i32 not recognized' + stop 1 + end if + if (variables(3)%type /= adios2_type_integer4) then + write(*,*) 'i32 type not recognized' + stop 1 + end if call adios2_variable_shape(shape_in, ndims, variables(3), ierr) - if (ndims /= 1) stop 'i32 ndims is not 1' - if (shape_in(1) /= nx*writerSize) stop 'i32 shape_in read failed' + if (ndims /= 1) then + write(*,*) 'i32 ndims is not 1' + stop 1 + end if + if (shape_in(1) /= nx*writerSize) then + write(*,*) 'i32 shape_in read failed' + stop 1 + end if deallocate(shape_in) call adios2_inquire_variable(variables(4), ioRead, "i64", ierr) - if (variables(4)%name /= 'i64') stop 'i64 not recognized' - if (variables(4)%type /= adios2_type_integer8) stop 'i64 type not recognized' + if (variables(4)%name /= 'i64') then + write(*,*) 'i64 not recognized' + stop 1 + end if + if (variables(4)%type /= adios2_type_integer8) then + write(*,*) 'i64 type not recognized' + stop 1 + end if call adios2_variable_shape(shape_in, ndims, variables(4), ierr) - if (ndims /= 1) stop 'i64 ndims is not 1' - if (shape_in(1) /= nx*writerSize) stop 'i64 shape_in read failed' + if (ndims /= 1) then + write(*,*) 'i64 ndims is not 1' + stop 1 + end if + if (shape_in(1) /= nx*writerSize) then + write(*,*) 'i64 shape_in read failed' + stop 1 + end if deallocate(shape_in) call adios2_inquire_variable(variables(5), ioRead, "r32", ierr) - if (variables(5)%name /= 'r32') stop 'r32 not recognized' - if (variables(5)%type /= adios2_type_real) stop 'r32 type not recognized' + if (variables(5)%name /= 'r32') then + write(*,*) 'r32 not recognized' + stop 1 + end if + if (variables(5)%type /= adios2_type_real) then + write(*,*) 'r32 type not recognized' + stop 1 + end if call adios2_variable_shape(shape_in, ndims, variables(5), ierr) - if (ndims /= 1) stop 'r32 ndims is not 1' - if (shape_in(1) /= nx*writerSize) stop 'r32 shape_in read failed' + if (ndims /= 1) then + write(*,*) 'r32 ndims is not 1' + stop 1 + end if + if (shape_in(1) /= nx*writerSize) then + write(*,*) 'r32 shape_in read failed' + stop 1 + end if deallocate(shape_in) call adios2_inquire_variable(variables(6), ioRead, "r64", ierr) - if (variables(6)%name /= 'r64') stop 'r64 not recognized' - if (variables(6)%type /= adios2_type_dp) stop 'r64 type not recognized' + if (variables(6)%name /= 'r64') then + write(*,*) 'r64 not recognized' + stop 1 + end if + if (variables(6)%type /= adios2_type_dp) then + write(*,*) 'r64 type not recognized' + stop 1 + end if call adios2_variable_shape(shape_in, ndims, variables(6), ierr) - if (ndims /= 1) stop 'r64 ndims is not 1' - if (shape_in(1) /= nx*writerSize) stop 'r64 shape_in read failed' + if (ndims /= 1) then + write(*,*) 'r64 ndims is not 1' + stop 1 + end if + if (shape_in(1) /= nx*writerSize) then + write(*,*) 'r64 shape_in read failed' + stop 1 + end if deallocate(shape_in) call adios2_inquire_variable(variables(7), ioRead, "r64_2d", ierr) - if (variables(7)%name /= 'r64_2d') stop 'r64_2d not recognized' - if (variables(7)%type /= adios2_type_dp) stop 'r64_2d type not recognized' + if (variables(7)%name /= 'r64_2d') then + write(*,*) 'r64_2d not recognized' + stop 1 + end if + if (variables(7)%type /= adios2_type_dp) then + write(*,*) 'r64_2d type not recognized' + stop 1 + end if call adios2_variable_shape( shape_in, ndims, variables(7), ierr) - if (ndims /= 2) stop 'r64_2d ndims is not 2' - if (shape_in(1) /= 2) stop 'r64_2d shape_in(1) read failed' - if (shape_in(2) /= nx*writerSize) stop 'r64_2d shape_in(2) read failed' + if (ndims /= 2) then + write(*,*) 'r64_2d ndims is not 2' + stop 1 + end if + if (shape_in(1) /= 2) then + write(*,*) 'r64_2d shape_in(1) read failed' + stop 1 + end if + if (shape_in(2) /= nx*writerSize) then + write(*,*) 'r64_2d shape_in(2) read failed' + stop 1 + end if deallocate(shape_in) call adios2_inquire_variable(variables(8), ioRead, "r64_2d_rev", ierr) - if (variables(8)%name /= 'r64_2d_rev') stop 'r64_2d_rev not recognized' - if (variables(8)%type /= adios2_type_dp) stop 'r64_2d_rev type not recognized' + if (variables(8)%name /= 'r64_2d_rev') then + write(*,*) 'r64_2d_rev not recognized' + stop 1 + end if + if (variables(8)%type /= adios2_type_dp) then + write(*,*) 'r64_2d_rev type not recognized' + stop 1 + end if call adios2_variable_shape(shape_in, ndims, variables(8), ierr) - if (ndims /= 2) stop 'r64_2d_rev ndims is not 2' - if (shape_in(1) /= nx*writerSize) stop 'r64_2d_rev shape_in(2) read failed' - if (shape_in(2) /= 2) stop 'r64_2d_rev shape_in(1) read failed' + if (ndims /= 2) then + write(*,*) 'r64_2d_rev ndims is not 2' + stop 1 + end if + if (shape_in(1) /= nx*writerSize) then + write(*,*) 'r64_2d_rev shape_in(2) read failed' + stop 1 + end if + if (shape_in(2) /= 2) then + write(*,*) 'r64_2d_rev shape_in(1) read failed' + stop 1 + end if deallocate(shape_in) call adios2_inquire_variable(variables(10), ioRead, "c32", ierr) - if (variables(10)%name /= 'c32') stop 'c32 not recognized' - if (variables(10)%type /= adios2_type_complex) stop 'c32 type not recognized' + if (variables(10)%name /= 'c32') then + write(*,*) 'c32 not recognized' + stop 1 + end if + if (variables(10)%type /= adios2_type_complex) then + write(*,*) 'c32 type not recognized' + stop 1 + end if call adios2_variable_shape(shape_in, ndims, variables(10), ierr) - if (ndims /= 1) stop 'c32 ndims is not 1' - if (shape_in(1) /= nx*writerSize) stop 'c32 shape_in read failed' + if (ndims /= 1) then + write(*,*) 'c32 ndims is not 1' + stop 1 + end if + if (shape_in(1) /= nx*writerSize) then + write(*,*) 'c32 shape_in read failed' + stop 1 + end if deallocate(shape_in) call adios2_inquire_variable(variables(11), ioRead, "c64", ierr) - if (variables(11)%name /= 'c64') stop 'c64 not recognized' - if (variables(11)%type /= adios2_type_complex_dp) stop 'c64 type not recognized' + if (variables(11)%name /= 'c64') then + write(*,*) 'c64 not recognized' + stop 1 + end if + if (variables(11)%type /= adios2_type_complex_dp) then + write(*,*) 'c64 type not recognized' + stop 1 + end if call adios2_variable_shape(shape_in, ndims, variables(11), ierr) - if (ndims /= 1) stop 'c64 ndims is not 1' - if (shape_in(1) /= nx*writerSize) stop 'c64 shape_in read failed' + if (ndims /= 1) then + write(*,*) 'c64 ndims is not 1' + stop 1 + end if + if (shape_in(1) /= nx*writerSize) then + write(*,*) 'c64 shape_in read failed' + stop 1 + end if deallocate(shape_in) call adios2_inquire_variable(variables(12), ioRead, "scalar_r64", ierr) - if (variables(12)%name /= 'scalar_r64') stop 'scalar_r64 not recognized' - if (variables(12)%type /= adios2_type_dp) stop 'scalar_r64 type not recognized' + if (variables(12)%name /= 'scalar_r64') then + write(*,*) 'scalar_r64 not recognized' + stop 1 + end if + if (variables(12)%type /= adios2_type_dp) then + write(*,*) 'scalar_r64 type not recognized' + stop 1 + end if myStart = (writerSize * Nx / isize) * irank myLength = ((writerSize * Nx + isize - 1) / isize) @@ -274,7 +406,10 @@ program TestSstRead ! Deallocates adios and calls its destructor call adios2_finalize(adios, ierr) - if( adios%valid .eqv. .true. ) stop 'Invalid adios2_finalize' + if( adios%valid .eqv. .true. ) then + write(*,*) 'Invalid adios2_finalize' + stop 1 + end if #if ADIOS2_USE_MPI From ed94245fef83c00d4ea78ed70c98dc2a778f72c6 Mon Sep 17 00:00:00 2001 From: Jamie J Quinn Date: Thu, 21 Mar 2024 16:36:47 +0000 Subject: [PATCH 091/124] Fix typo in fortran.rst (#4102) --- docs/user_guide/source/api_full/fortran.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/user_guide/source/api_full/fortran.rst b/docs/user_guide/source/api_full/fortran.rst index f45648bf98..026a55d4d2 100644 --- a/docs/user_guide/source/api_full/fortran.rst +++ b/docs/user_guide/source/api_full/fortran.rst @@ -746,11 +746,11 @@ ADIOS2 Fortran bindings handlers are mapped 1-to-1 to the ADIOS2 components desc integer, intent(out) :: ierr -* :f90:`subroutine adios2_set_steps_selection` set a list of steps by specifying the starting step and the step count +* :f90:`subroutine adios2_set_step_selection` set a list of steps by specifying the starting step and the step count .. code-block:: fortran - subroutine adios2_set_selection(variable, step_start, step_count, ierr) + subroutine adios2_set_step_selection(variable, step_start, step_count, ierr) ! WHERE From af56f95df5445c968607b0f108857d8b56a98d37 Mon Sep 17 00:00:00 2001 From: Vicente Bolea Date: Sun, 24 Mar 2024 14:31:56 -0400 Subject: [PATCH 092/124] ci: add ccache job summary (#4101) --- .github/workflows/everything.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/everything.yml b/.github/workflows/everything.yml index 4bd22408f5..af2f464df2 100644 --- a/.github/workflows/everything.yml +++ b/.github/workflows/everything.yml @@ -188,7 +188,7 @@ jobs: - name: Build run: gha/scripts/ci/gh-actions/run.sh build - name: Print ccache statistics - run: ccache -s + run: ccache -s | tee $GITHUB_STEP_SUMMARY - name: Save cache uses: actions/cache/save@v3 if: ${{ github.ref_name == 'master' && steps.restore-cache.outputs.cache-hit != 'true' }} From b585f9691e71f9e8b61fdedd3127fbd06a7e2fca Mon Sep 17 00:00:00 2001 From: Vicente Bolea Date: Mon, 25 Mar 2024 08:46:30 -0400 Subject: [PATCH 093/124] Fix static blosc2 build (#4093) * blosc2: require >=2.10.1 This is needed to simplify our dependency to Blosc2, supporting prior to 2.10.1 requires us to support to types of blosc2 cmake dependencies (CONFIG and MODULE) and code this per each version. * compress: Address blosc2 compress warnings in Windows * ci: use blosc2 in builds --- cmake/DetectOptions.cmake | 31 +++---- cmake/FindBlosc2.cmake | 91 ------------------- cmake/adios2-config-common.cmake.in | 5 + scripts/ci/cmake/ci-ascent-xl.cmake | 2 +- scripts/ci/cmake/ci-crusher-cray.cmake | 2 +- scripts/ci/cmake/ci-el7-spack.cmake | 4 +- scripts/ci/cmake/ci-el8-icc-mpich.cmake | 2 +- scripts/ci/cmake/ci-el8-icc-ompi.cmake | 2 +- scripts/ci/cmake/ci-el8-icc-serial.cmake | 2 +- scripts/ci/cmake/ci-el8-oneapi-mpich.cmake | 2 +- scripts/ci/cmake/ci-el8-oneapi-ompi.cmake | 2 +- scripts/ci/cmake/ci-el8-oneapi-serial.cmake | 2 +- ...i-macos-12-xcode13_4_1-static-serial.cmake | 2 +- .../cmake/ci-ubuntu20.04-clang10-mpich.cmake | 2 +- .../cmake/ci-ubuntu20.04-clang10-ompi.cmake | 2 +- .../cmake/ci-ubuntu20.04-clang10-serial.cmake | 2 +- .../cmake/ci-ubuntu20.04-clang6-mpich.cmake | 2 +- .../ci/cmake/ci-ubuntu20.04-clang6-ompi.cmake | 2 +- .../cmake/ci-ubuntu20.04-clang6-serial.cmake | 2 +- .../ci-ubuntu20.04-clang6-static-ompi.cmake | 2 +- .../ci/cmake/ci-ubuntu20.04-gcc10-mpich.cmake | 2 +- .../ci/cmake/ci-ubuntu20.04-gcc10-ompi.cmake | 2 +- .../cmake/ci-ubuntu20.04-gcc10-serial.cmake | 2 +- .../ci/cmake/ci-ubuntu20.04-gcc11-mpich.cmake | 2 +- .../ci/cmake/ci-ubuntu20.04-gcc11-ompi.cmake | 2 +- .../cmake/ci-ubuntu20.04-gcc11-serial.cmake | 2 +- .../ci/cmake/ci-ubuntu20.04-gcc8-mpich.cmake | 2 +- .../ci/cmake/ci-ubuntu20.04-gcc8-ompi.cmake | 2 +- .../ci-ubuntu20.04-gcc8-serial-codeql.cmake | 2 +- .../ci/cmake/ci-ubuntu20.04-gcc8-serial.cmake | 2 +- .../ci-ubuntu20.04-gcc8-static-ompi.cmake | 2 +- .../ci-ubuntu20.04-gcc8-static-serial.cmake | 2 +- .../ci/cmake/ci-ubuntu20.04-gcc9-mpich.cmake | 2 +- .../ci/cmake/ci-ubuntu20.04-gcc9-ompi.cmake | 2 +- .../ci/cmake/ci-ubuntu20.04-gcc9-serial.cmake | 2 +- .../ci/cmake/ci-win2019-vs2019-ninja.cmake | 1 + scripts/ci/cmake/ci-win2019-vs2019-ompi.cmake | 1 + .../ci/cmake/ci-win2019-vs2019-serial.cmake | 1 + scripts/ci/cmake/ci-win2022-vs2022-ompi.cmake | 1 + .../ci/cmake/ci-win2022-vs2022-serial.cmake | 1 + .../ci-win2022-vs2022-static-serial.cmake | 2 + scripts/ci/gh-actions/windows-setup.ps1 | 10 ++ scripts/ci/setup-run/ci-Windows.sh | 5 + .../operator/compress/CompressBlosc.cpp | 11 ++- 44 files changed, 79 insertions(+), 147 deletions(-) delete mode 100644 cmake/FindBlosc2.cmake create mode 100755 scripts/ci/setup-run/ci-Windows.sh diff --git a/cmake/DetectOptions.cmake b/cmake/DetectOptions.cmake index 2b2bd3b285..0dedcb61ca 100644 --- a/cmake/DetectOptions.cmake +++ b/cmake/DetectOptions.cmake @@ -72,31 +72,26 @@ find_package(Threads REQUIRED) # Blosc2 if(ADIOS2_USE_Blosc2 STREQUAL AUTO) - # Prefect CONFIG mode - find_package(Blosc2 2.4 CONFIG QUIET) - if(NOT Blosc2_FOUND) - find_package(Blosc2 2.4 MODULE QUIET) - endif() + find_package(Blosc2 2.10.1 QUIET) elseif(ADIOS2_USE_Blosc2) - # Prefect CONFIG mode - find_package(Blosc2 2.4 CONFIG) - if(NOT Blosc2_FOUND) - find_package(Blosc2 2.4 MODULE REQUIRED) - endif() + find_package(Blosc2 2.10.1) endif() if(Blosc2_FOUND) set(ADIOS2_HAVE_Blosc2 TRUE) if(TARGET Blosc2::blosc2_shared) - set(Blosc2_shlib_available ON) + set(blosc2_shlib_available ON) endif() - set(adios2_blosc2_tgt Blosc2::Blosc2) - if (Blosc2_VERSION VERSION_GREATER_EQUAL 2.10.1) - if (Blosc2_shlib_available AND ADIOS2_Blosc2_PREFER_SHARED) - set(adios2_blosc2_tgt Blosc2::blosc2_shared) - else() - set(adios2_blosc2_tgt Blosc2::blosc2_static) - endif() + if(TARGET Blosc2::blosc2_static) + set(blosc2_slib_available ON) + endif() + + if (blosc2_shlib_available AND (NOT blosc2_slib_available OR ADIOS2_Blosc2_PREFER_SHARED)) + set(adios2_blosc2_tgt Blosc2::blosc2_shared) + elseif(blosc2_slib_available) + set(adios2_blosc2_tgt Blosc2::blosc2_static) + else() + message(FATAL_ERROR "Blosc2 cmake package found but no targets exists inside it.") endif() add_library(adios2_blosc2 ALIAS ${adios2_blosc2_tgt}) diff --git a/cmake/FindBlosc2.cmake b/cmake/FindBlosc2.cmake deleted file mode 100644 index 10762297cc..0000000000 --- a/cmake/FindBlosc2.cmake +++ /dev/null @@ -1,91 +0,0 @@ -#------------------------------------------------------------------------------# -# Distributed under the OSI-approved Apache License, Version 2.0. See -# accompanying file Copyright.txt for details. -#------------------------------------------------------------------------------# -# -# FindBLOSC22 -# ----------- -# -# Try to find the BLOSC2 library -# -# This module defines the following variables: -# -# BLOSC2_FOUND - System has BLOSC2 -# BLOSC2_INCLUDE_DIRS - The BLOSC2 include directory -# BLOSC2_LIBRARIES - Link these to use BLOSC2 -# BLOSC2_VERSION - Version of the BLOSC2 library to support -# -# and the following imported targets: -# Blosc2::Blosc2 - The core BLOSC2 library -# -# You can also set the following variable to help guide the search: -# BLOSC2_ROOT - The install prefix for BLOSC2 containing the -# include and lib folders -# Note: this can be set as a CMake variable or an -# environment variable. If specified as a CMake -# variable, it will override any setting specified -# as an environment variable. - -if(NOT BLOSC2_FOUND) - if((NOT BLOSC2_ROOT) AND (DEFINED ENV{BLOSC2_ROOT})) - set(BLOSC2_ROOT "$ENV{BLOSC2_ROOT}") - endif() - if(BLOSC2_ROOT) - set(BLOSC2_INCLUDE_OPTS HINTS ${BLOSC2_ROOT}/include NO_DEFAULT_PATHS) - set(BLOSC2_LIBRARY_OPTS - HINTS ${BLOSC2_ROOT}/lib ${BLOSC2_ROOT}/lib64 - NO_DEFAULT_PATHS - ) - endif() - if(WIN32) # uses a Unix-like library prefix on Windows - set(BLOSC2_LIBRARY_OPTS - libblosc2 ${BLOSC2_LIBRARY_OPTS} - ) - endif() - - find_path(BLOSC2_INCLUDE_DIR blosc2.h ${BLOSC2_INCLUDE_OPTS}) - find_library(BLOSC2_LIBRARY NAMES blosc2 ${BLOSC2_LIBRARY_OPTS}) - if(BLOSC2_INCLUDE_DIR) - file(STRINGS ${BLOSC2_INCLUDE_DIR}/blosc2.h _ver_string - REGEX [=[BLOSC2_VERSION_STRING +"[^"]*"]=] - ) - if(_ver_string MATCHES [=[BLOSC2_VERSION_STRING +"([0-9]+.[0-9]+.[0-9]+)]=]) - set(BLOSC2_VERSION ${CMAKE_MATCH_1}) - endif() - endif() - - # Blosc2 depends on pthreads - set(THREADS_PREFER_PTHREAD_FLAG TRUE) - if(WIN32) - # try to use the system library - find_package(Threads) - if(NOT Threads_FOUND) - message(STATUS "Blosc2: used the internal pthread library for win32 systems.") - set(BLOSC2_LIBRARIES) - else() - set(BLOSC2_LIBRARIES Threads::Threads) - endif() - else() - find_package(Threads REQUIRED) - set(BLOSC2_LIBRARIES Threads::Threads) - endif() - - include(FindPackageHandleStandardArgs) - find_package_handle_standard_args(Blosc2 - FOUND_VAR BLOSC2_FOUND - VERSION_VAR BLOSC2_VERSION - REQUIRED_VARS BLOSC2_LIBRARY BLOSC2_INCLUDE_DIR - ) - if(BLOSC2_FOUND) - set(BLOSC2_INCLUDE_DIRS ${BLOSC2_INCLUDE_DIR}) - set(BLOSC2_LIBRARIES ${BLOSC2_LIBRARY}) - if(BLOSC2_FOUND AND NOT TARGET Blosc2::Blosc2) - add_library(Blosc2::Blosc2 UNKNOWN IMPORTED) - set_target_properties(Blosc2::Blosc2 PROPERTIES - IMPORTED_LOCATION "${BLOSC2_LIBRARY}" - INTERFACE_INCLUDE_DIRECTORIES "${BLOSC2_INCLUDE_DIR}" - INTERFACE_LINK_LIBRARIES "${BLOSC2_LIBRARIES}" - ) - endif() - endif() -endif() diff --git a/cmake/adios2-config-common.cmake.in b/cmake/adios2-config-common.cmake.in index 4be4ff7ff0..d3f931f128 100644 --- a/cmake/adios2-config-common.cmake.in +++ b/cmake/adios2-config-common.cmake.in @@ -81,6 +81,11 @@ else() endif() if(NOT @BUILD_SHARED_LIBS@) + set(ADIOS2_HAVE_Blosc2 @ADIOS2_HAVE_Blosc2@) + if(ADIOS2_HAVE_Blosc2) + find_dependency(Blosc2 2.10.1) + endif() + set(ADIOS2_HAVE_BZip2 @ADIOS2_HAVE_BZip2@) if(ADIOS2_HAVE_BZip2) find_dependency(BZip2) diff --git a/scripts/ci/cmake/ci-ascent-xl.cmake b/scripts/ci/cmake/ci-ascent-xl.cmake index 6be12224bb..f9cf68a629 100644 --- a/scripts/ci/cmake/ci-ascent-xl.cmake +++ b/scripts/ci/cmake/ci-ascent-xl.cmake @@ -18,7 +18,7 @@ ADIOS2_USE_SST:BOOL=ON ADIOS2_USE_ZeroMQ:STRING=OFF ADIOS2_USE_ZFP:BOOL=OFF ADIOS2_USE_SZ:BOOL=OFF -ADIOS2_USE_Blosc:BOOL=OFF +ADIOS2_USE_Blosc2:BOOL=OFF CMAKE_C_COMPILER_LAUNCHER=ccache CMAKE_CXX_COMPILER_LAUNCHER=ccache diff --git a/scripts/ci/cmake/ci-crusher-cray.cmake b/scripts/ci/cmake/ci-crusher-cray.cmake index b48dba9768..abd710c7cf 100644 --- a/scripts/ci/cmake/ci-crusher-cray.cmake +++ b/scripts/ci/cmake/ci-crusher-cray.cmake @@ -18,7 +18,7 @@ ADIOS2_USE_SST:BOOL=ON ADIOS2_USE_ZeroMQ:STRING=OFF ADIOS2_USE_ZFP:BOOL=OFF ADIOS2_USE_SZ:BOOL=OFF -ADIOS2_USE_Blosc:BOOL=OFF +ADIOS2_USE_Blosc2:BOOL=OFF CMAKE_C_COMPILER_LAUNCHER=ccache CMAKE_CXX_COMPILER_LAUNCHER=ccache diff --git a/scripts/ci/cmake/ci-el7-spack.cmake b/scripts/ci/cmake/ci-el7-spack.cmake index 025ad80e77..0a0c378c21 100644 --- a/scripts/ci/cmake/ci-el7-spack.cmake +++ b/scripts/ci/cmake/ci-el7-spack.cmake @@ -13,7 +13,7 @@ BUILD_TESTING:BOOL=ON ADIOS2_BUILD_EXAMPLES:BOOL=ON ADIOS2_USE_BZip2:BOOL=ON -ADIOS2_USE_Blosc:BOOL=ON +ADIOS2_USE_Blosc2:BOOL=ON ADIOS2_USE_DataMan:BOOL=ON ADIOS2_USE_Fortran:BOOL=ON ADIOS2_USE_HDF5:BOOL=ON @@ -22,7 +22,7 @@ ADIOS2_USE_Python:BOOL=ON ADIOS2_USE_SZ:BOOL=ON ADIOS2_USE_ZeroMQ:STRING=ON ADIOS2_USE_ZFP:BOOL=ON -ADIOS2_USE_Blosc:BOOL=ON +ADIOS2_USE_Blosc2:BOOL=ON ADIOS2_USE_DataSpaces:BOOL=OFF ADIOS2_USE_EXTERNAL_DEPENDENCIES:BOOL=ON diff --git a/scripts/ci/cmake/ci-el8-icc-mpich.cmake b/scripts/ci/cmake/ci-el8-icc-mpich.cmake index 8f461e4e3b..2c0809cba8 100644 --- a/scripts/ci/cmake/ci-el8-icc-mpich.cmake +++ b/scripts/ci/cmake/ci-el8-icc-mpich.cmake @@ -11,7 +11,7 @@ BUILD_TESTING:BOOL=ON ADIOS2_BUILD_EXAMPLES:BOOL=ON ADIOS2_USE_BZip2:BOOL=ON -ADIOS2_USE_Blosc:BOOL=ON +ADIOS2_USE_Blosc2:BOOL=OFF ADIOS2_USE_DataMan:BOOL=ON ADIOS2_USE_DataSpaces:BOOL=OFF ADIOS2_USE_Fortran:BOOL=OFF diff --git a/scripts/ci/cmake/ci-el8-icc-ompi.cmake b/scripts/ci/cmake/ci-el8-icc-ompi.cmake index 785ea2ea3b..d3e9fbbf81 100644 --- a/scripts/ci/cmake/ci-el8-icc-ompi.cmake +++ b/scripts/ci/cmake/ci-el8-icc-ompi.cmake @@ -7,7 +7,7 @@ BUILD_TESTING:BOOL=ON ADIOS2_BUILD_EXAMPLES:BOOL=ON ADIOS2_USE_BZip2:BOOL=ON -ADIOS2_USE_Blosc:BOOL=ON +ADIOS2_USE_Blosc2:BOOL=ON ADIOS2_USE_DataMan:BOOL=ON ADIOS2_USE_DataSpaces:BOOL=OFF ADIOS2_USE_Fortran:BOOL=OFF diff --git a/scripts/ci/cmake/ci-el8-icc-serial.cmake b/scripts/ci/cmake/ci-el8-icc-serial.cmake index a769ffbbef..af80b436f0 100644 --- a/scripts/ci/cmake/ci-el8-icc-serial.cmake +++ b/scripts/ci/cmake/ci-el8-icc-serial.cmake @@ -7,7 +7,7 @@ BUILD_TESTING:BOOL=ON ADIOS2_BUILD_EXAMPLES:BOOL=ON ADIOS2_USE_BZip2:BOOL=ON -ADIOS2_USE_Blosc:BOOL=OFF +ADIOS2_USE_Blosc2:BOOL=OFF ADIOS2_USE_DataMan:BOOL=ON ADIOS2_USE_DataSpaces:BOOL=OFF ADIOS2_USE_Fortran:BOOL=OFF diff --git a/scripts/ci/cmake/ci-el8-oneapi-mpich.cmake b/scripts/ci/cmake/ci-el8-oneapi-mpich.cmake index 49ed45643c..fe41c3e616 100644 --- a/scripts/ci/cmake/ci-el8-oneapi-mpich.cmake +++ b/scripts/ci/cmake/ci-el8-oneapi-mpich.cmake @@ -11,7 +11,7 @@ BUILD_TESTING:BOOL=ON ADIOS2_BUILD_EXAMPLES:BOOL=ON ADIOS2_USE_BZip2:BOOL=ON -ADIOS2_USE_Blosc:BOOL=OFF +ADIOS2_USE_Blosc2:BOOL=OFF ADIOS2_USE_DataMan:BOOL=ON ADIOS2_USE_DataSpaces:BOOL=OFF ADIOS2_USE_Fortran:BOOL=OFF diff --git a/scripts/ci/cmake/ci-el8-oneapi-ompi.cmake b/scripts/ci/cmake/ci-el8-oneapi-ompi.cmake index bf9ffa5f74..a2a7ee38fe 100644 --- a/scripts/ci/cmake/ci-el8-oneapi-ompi.cmake +++ b/scripts/ci/cmake/ci-el8-oneapi-ompi.cmake @@ -7,7 +7,7 @@ BUILD_TESTING:BOOL=ON ADIOS2_BUILD_EXAMPLES:BOOL=ON ADIOS2_USE_BZip2:BOOL=ON -ADIOS2_USE_Blosc:BOOL=OFF +ADIOS2_USE_Blosc2:BOOL=OFF ADIOS2_USE_DataMan:BOOL=ON ADIOS2_USE_DataSpaces:BOOL=OFF ADIOS2_USE_Fortran:BOOL=OFF diff --git a/scripts/ci/cmake/ci-el8-oneapi-serial.cmake b/scripts/ci/cmake/ci-el8-oneapi-serial.cmake index 0b7a95acec..db27e74dfd 100644 --- a/scripts/ci/cmake/ci-el8-oneapi-serial.cmake +++ b/scripts/ci/cmake/ci-el8-oneapi-serial.cmake @@ -7,7 +7,7 @@ BUILD_TESTING:BOOL=ON ADIOS2_BUILD_EXAMPLES:BOOL=ON ADIOS2_USE_BZip2:BOOL=ON -ADIOS2_USE_Blosc:BOOL=OFF +ADIOS2_USE_Blosc2:BOOL=OFF ADIOS2_USE_DataMan:BOOL=ON ADIOS2_USE_DataSpaces:BOOL=OFF ADIOS2_USE_Fortran:BOOL=OFF diff --git a/scripts/ci/cmake/ci-macos-12-xcode13_4_1-static-serial.cmake b/scripts/ci/cmake/ci-macos-12-xcode13_4_1-static-serial.cmake index 1c10c3a157..68e43b31b3 100644 --- a/scripts/ci/cmake/ci-macos-12-xcode13_4_1-static-serial.cmake +++ b/scripts/ci/cmake/ci-macos-12-xcode13_4_1-static-serial.cmake @@ -11,7 +11,7 @@ ADIOS2_BUILD_EXAMPLES:BOOL=OFF ADIOS2_USE_AWSSDK:STRING=OFF ADIOS2_USE_Blosc2:STRING=OFF -ADIOS2_USE_Blosc:BOOL=OFF +ADIOS2_USE_Blosc2:BOOL=OFF ADIOS2_USE_BZip2:BOOL=OFF ADIOS2_USE_Catalyst:STRING=OFF ADIOS2_USE_CUDA:STRING=OFF diff --git a/scripts/ci/cmake/ci-ubuntu20.04-clang10-mpich.cmake b/scripts/ci/cmake/ci-ubuntu20.04-clang10-mpich.cmake index fd1ab9be47..f9a882f25d 100644 --- a/scripts/ci/cmake/ci-ubuntu20.04-clang10-mpich.cmake +++ b/scripts/ci/cmake/ci-ubuntu20.04-clang10-mpich.cmake @@ -15,7 +15,7 @@ set(dashboard_cache " BUILD_TESTING:BOOL=ON ADIOS2_BUILD_EXAMPLES:BOOL=ON -ADIOS2_USE_Blosc:BOOL=ON +ADIOS2_USE_Blosc2:BOOL=ON ADIOS2_USE_BZip2:BOOL=ON ADIOS2_USE_DataMan:BOOL=ON ADIOS2_USE_Fortran:BOOL=ON diff --git a/scripts/ci/cmake/ci-ubuntu20.04-clang10-ompi.cmake b/scripts/ci/cmake/ci-ubuntu20.04-clang10-ompi.cmake index 447c304316..18a517e147 100644 --- a/scripts/ci/cmake/ci-ubuntu20.04-clang10-ompi.cmake +++ b/scripts/ci/cmake/ci-ubuntu20.04-clang10-ompi.cmake @@ -11,7 +11,7 @@ set(dashboard_cache " BUILD_TESTING:BOOL=ON ADIOS2_BUILD_EXAMPLES:BOOL=ON -ADIOS2_USE_Blosc:BOOL=ON +ADIOS2_USE_Blosc2:BOOL=ON ADIOS2_USE_BZip2:BOOL=ON ADIOS2_USE_DataMan:BOOL=ON ADIOS2_USE_Fortran:BOOL=ON diff --git a/scripts/ci/cmake/ci-ubuntu20.04-clang10-serial.cmake b/scripts/ci/cmake/ci-ubuntu20.04-clang10-serial.cmake index 8ac28ae6a3..c0987abea8 100644 --- a/scripts/ci/cmake/ci-ubuntu20.04-clang10-serial.cmake +++ b/scripts/ci/cmake/ci-ubuntu20.04-clang10-serial.cmake @@ -11,7 +11,7 @@ set(dashboard_cache " BUILD_TESTING:BOOL=ON ADIOS2_BUILD_EXAMPLES:BOOL=ON -ADIOS2_USE_Blosc:BOOL=ON +ADIOS2_USE_Blosc2:BOOL=ON ADIOS2_USE_BZip2:BOOL=ON ADIOS2_USE_DataMan:BOOL=ON ADIOS2_USE_Fortran:BOOL=ON diff --git a/scripts/ci/cmake/ci-ubuntu20.04-clang6-mpich.cmake b/scripts/ci/cmake/ci-ubuntu20.04-clang6-mpich.cmake index cb2977eeb6..572e01d848 100644 --- a/scripts/ci/cmake/ci-ubuntu20.04-clang6-mpich.cmake +++ b/scripts/ci/cmake/ci-ubuntu20.04-clang6-mpich.cmake @@ -15,7 +15,7 @@ set(dashboard_cache " BUILD_TESTING:BOOL=ON ADIOS2_BUILD_EXAMPLES:BOOL=ON -ADIOS2_USE_Blosc:BOOL=ON +ADIOS2_USE_Blosc2:BOOL=ON ADIOS2_USE_BZip2:BOOL=ON ADIOS2_USE_DataMan:BOOL=ON ADIOS2_USE_Fortran:BOOL=ON diff --git a/scripts/ci/cmake/ci-ubuntu20.04-clang6-ompi.cmake b/scripts/ci/cmake/ci-ubuntu20.04-clang6-ompi.cmake index bfe89a699c..b728fe9493 100644 --- a/scripts/ci/cmake/ci-ubuntu20.04-clang6-ompi.cmake +++ b/scripts/ci/cmake/ci-ubuntu20.04-clang6-ompi.cmake @@ -11,7 +11,7 @@ set(dashboard_cache " BUILD_TESTING:BOOL=ON ADIOS2_BUILD_EXAMPLES:BOOL=ON -ADIOS2_USE_Blosc:BOOL=ON +ADIOS2_USE_Blosc2:BOOL=ON ADIOS2_USE_BZip2:BOOL=ON ADIOS2_USE_DataMan:BOOL=ON ADIOS2_USE_Fortran:BOOL=ON diff --git a/scripts/ci/cmake/ci-ubuntu20.04-clang6-serial.cmake b/scripts/ci/cmake/ci-ubuntu20.04-clang6-serial.cmake index 3ec1112997..ce99c7550a 100644 --- a/scripts/ci/cmake/ci-ubuntu20.04-clang6-serial.cmake +++ b/scripts/ci/cmake/ci-ubuntu20.04-clang6-serial.cmake @@ -11,7 +11,7 @@ set(dashboard_cache " BUILD_TESTING:BOOL=ON ADIOS2_BUILD_EXAMPLES:BOOL=ON -ADIOS2_USE_Blosc:BOOL=ON +ADIOS2_USE_Blosc2:BOOL=ON ADIOS2_USE_BZip2:BOOL=ON ADIOS2_USE_DataMan:BOOL=ON ADIOS2_USE_Fortran:BOOL=ON diff --git a/scripts/ci/cmake/ci-ubuntu20.04-clang6-static-ompi.cmake b/scripts/ci/cmake/ci-ubuntu20.04-clang6-static-ompi.cmake index 71c0f9710e..d6f5595f0a 100644 --- a/scripts/ci/cmake/ci-ubuntu20.04-clang6-static-ompi.cmake +++ b/scripts/ci/cmake/ci-ubuntu20.04-clang6-static-ompi.cmake @@ -13,7 +13,7 @@ BUILD_TESTING:BOOL=OFF ADIOS2_BUILD_EXAMPLES:BOOL=ON ADIOS2_USE_BZip2:BOOL=ON -ADIOS2_USE_Blosc:BOOL=ON +ADIOS2_USE_Blosc2:BOOL=ON ADIOS2_USE_DataMan:BOOL=ON ADIOS2_USE_Fortran:BOOL=ON ADIOS2_USE_HDF5:BOOL=ON diff --git a/scripts/ci/cmake/ci-ubuntu20.04-gcc10-mpich.cmake b/scripts/ci/cmake/ci-ubuntu20.04-gcc10-mpich.cmake index f4c486289f..dde765bf4d 100644 --- a/scripts/ci/cmake/ci-ubuntu20.04-gcc10-mpich.cmake +++ b/scripts/ci/cmake/ci-ubuntu20.04-gcc10-mpich.cmake @@ -16,7 +16,7 @@ BUILD_TESTING:BOOL=ON ADIOS2_BUILD_EXAMPLES:BOOL=ON ADIOS2_USE_BZip2:BOOL=ON -ADIOS2_USE_Blosc:BOOL=ON +ADIOS2_USE_Blosc2:BOOL=ON ADIOS2_USE_DataMan:BOOL=ON ADIOS2_USE_Fortran:BOOL=ON ADIOS2_USE_HDF5:BOOL=ON diff --git a/scripts/ci/cmake/ci-ubuntu20.04-gcc10-ompi.cmake b/scripts/ci/cmake/ci-ubuntu20.04-gcc10-ompi.cmake index 6463d8f72a..d1acb886a8 100644 --- a/scripts/ci/cmake/ci-ubuntu20.04-gcc10-ompi.cmake +++ b/scripts/ci/cmake/ci-ubuntu20.04-gcc10-ompi.cmake @@ -12,7 +12,7 @@ BUILD_TESTING:BOOL=ON ADIOS2_BUILD_EXAMPLES:BOOL=ON ADIOS2_USE_BZip2:BOOL=ON -ADIOS2_USE_Blosc:BOOL=ON +ADIOS2_USE_Blosc2:BOOL=ON ADIOS2_USE_DataMan:BOOL=ON ADIOS2_USE_Fortran:BOOL=ON ADIOS2_USE_HDF5:BOOL=ON diff --git a/scripts/ci/cmake/ci-ubuntu20.04-gcc10-serial.cmake b/scripts/ci/cmake/ci-ubuntu20.04-gcc10-serial.cmake index 82f5f6c37d..e459b5f536 100644 --- a/scripts/ci/cmake/ci-ubuntu20.04-gcc10-serial.cmake +++ b/scripts/ci/cmake/ci-ubuntu20.04-gcc10-serial.cmake @@ -12,7 +12,7 @@ BUILD_TESTING:BOOL=ON ADIOS2_BUILD_EXAMPLES:BOOL=ON ADIOS2_USE_BZip2:BOOL=ON -ADIOS2_USE_Blosc:BOOL=ON +ADIOS2_USE_Blosc2:BOOL=ON ADIOS2_USE_DataMan:BOOL=ON ADIOS2_USE_Fortran:BOOL=ON ADIOS2_USE_HDF5:BOOL=ON diff --git a/scripts/ci/cmake/ci-ubuntu20.04-gcc11-mpich.cmake b/scripts/ci/cmake/ci-ubuntu20.04-gcc11-mpich.cmake index e83c946898..5a8181281c 100644 --- a/scripts/ci/cmake/ci-ubuntu20.04-gcc11-mpich.cmake +++ b/scripts/ci/cmake/ci-ubuntu20.04-gcc11-mpich.cmake @@ -16,7 +16,7 @@ BUILD_TESTING:BOOL=ON ADIOS2_BUILD_EXAMPLES:BOOL=ON ADIOS2_USE_BZip2:BOOL=ON -ADIOS2_USE_Blosc:BOOL=ON +ADIOS2_USE_Blosc2:BOOL=ON ADIOS2_USE_DataMan:BOOL=ON ADIOS2_USE_Fortran:BOOL=ON ADIOS2_USE_HDF5:BOOL=ON diff --git a/scripts/ci/cmake/ci-ubuntu20.04-gcc11-ompi.cmake b/scripts/ci/cmake/ci-ubuntu20.04-gcc11-ompi.cmake index a686bb0c87..722776a8cd 100644 --- a/scripts/ci/cmake/ci-ubuntu20.04-gcc11-ompi.cmake +++ b/scripts/ci/cmake/ci-ubuntu20.04-gcc11-ompi.cmake @@ -12,7 +12,7 @@ BUILD_TESTING:BOOL=ON ADIOS2_BUILD_EXAMPLES:BOOL=ON ADIOS2_USE_BZip2:BOOL=ON -ADIOS2_USE_Blosc:BOOL=ON +ADIOS2_USE_Blosc2:BOOL=ON ADIOS2_USE_DataMan:BOOL=ON ADIOS2_USE_Fortran:BOOL=ON ADIOS2_USE_HDF5:BOOL=ON diff --git a/scripts/ci/cmake/ci-ubuntu20.04-gcc11-serial.cmake b/scripts/ci/cmake/ci-ubuntu20.04-gcc11-serial.cmake index 82f5f6c37d..e459b5f536 100644 --- a/scripts/ci/cmake/ci-ubuntu20.04-gcc11-serial.cmake +++ b/scripts/ci/cmake/ci-ubuntu20.04-gcc11-serial.cmake @@ -12,7 +12,7 @@ BUILD_TESTING:BOOL=ON ADIOS2_BUILD_EXAMPLES:BOOL=ON ADIOS2_USE_BZip2:BOOL=ON -ADIOS2_USE_Blosc:BOOL=ON +ADIOS2_USE_Blosc2:BOOL=ON ADIOS2_USE_DataMan:BOOL=ON ADIOS2_USE_Fortran:BOOL=ON ADIOS2_USE_HDF5:BOOL=ON diff --git a/scripts/ci/cmake/ci-ubuntu20.04-gcc8-mpich.cmake b/scripts/ci/cmake/ci-ubuntu20.04-gcc8-mpich.cmake index 4824d437ba..546cc82ef2 100644 --- a/scripts/ci/cmake/ci-ubuntu20.04-gcc8-mpich.cmake +++ b/scripts/ci/cmake/ci-ubuntu20.04-gcc8-mpich.cmake @@ -16,7 +16,7 @@ BUILD_TESTING:BOOL=ON ADIOS2_BUILD_EXAMPLES:BOOL=ON ADIOS2_USE_BZip2:BOOL=ON -ADIOS2_USE_Blosc:BOOL=ON +ADIOS2_USE_Blosc2:BOOL=ON ADIOS2_USE_DataMan:BOOL=ON ADIOS2_USE_Fortran:BOOL=ON ADIOS2_USE_HDF5:BOOL=ON diff --git a/scripts/ci/cmake/ci-ubuntu20.04-gcc8-ompi.cmake b/scripts/ci/cmake/ci-ubuntu20.04-gcc8-ompi.cmake index 6463d8f72a..d1acb886a8 100644 --- a/scripts/ci/cmake/ci-ubuntu20.04-gcc8-ompi.cmake +++ b/scripts/ci/cmake/ci-ubuntu20.04-gcc8-ompi.cmake @@ -12,7 +12,7 @@ BUILD_TESTING:BOOL=ON ADIOS2_BUILD_EXAMPLES:BOOL=ON ADIOS2_USE_BZip2:BOOL=ON -ADIOS2_USE_Blosc:BOOL=ON +ADIOS2_USE_Blosc2:BOOL=ON ADIOS2_USE_DataMan:BOOL=ON ADIOS2_USE_Fortran:BOOL=ON ADIOS2_USE_HDF5:BOOL=ON diff --git a/scripts/ci/cmake/ci-ubuntu20.04-gcc8-serial-codeql.cmake b/scripts/ci/cmake/ci-ubuntu20.04-gcc8-serial-codeql.cmake index 82f5f6c37d..e459b5f536 100644 --- a/scripts/ci/cmake/ci-ubuntu20.04-gcc8-serial-codeql.cmake +++ b/scripts/ci/cmake/ci-ubuntu20.04-gcc8-serial-codeql.cmake @@ -12,7 +12,7 @@ BUILD_TESTING:BOOL=ON ADIOS2_BUILD_EXAMPLES:BOOL=ON ADIOS2_USE_BZip2:BOOL=ON -ADIOS2_USE_Blosc:BOOL=ON +ADIOS2_USE_Blosc2:BOOL=ON ADIOS2_USE_DataMan:BOOL=ON ADIOS2_USE_Fortran:BOOL=ON ADIOS2_USE_HDF5:BOOL=ON diff --git a/scripts/ci/cmake/ci-ubuntu20.04-gcc8-serial.cmake b/scripts/ci/cmake/ci-ubuntu20.04-gcc8-serial.cmake index 82f5f6c37d..e459b5f536 100644 --- a/scripts/ci/cmake/ci-ubuntu20.04-gcc8-serial.cmake +++ b/scripts/ci/cmake/ci-ubuntu20.04-gcc8-serial.cmake @@ -12,7 +12,7 @@ BUILD_TESTING:BOOL=ON ADIOS2_BUILD_EXAMPLES:BOOL=ON ADIOS2_USE_BZip2:BOOL=ON -ADIOS2_USE_Blosc:BOOL=ON +ADIOS2_USE_Blosc2:BOOL=ON ADIOS2_USE_DataMan:BOOL=ON ADIOS2_USE_Fortran:BOOL=ON ADIOS2_USE_HDF5:BOOL=ON diff --git a/scripts/ci/cmake/ci-ubuntu20.04-gcc8-static-ompi.cmake b/scripts/ci/cmake/ci-ubuntu20.04-gcc8-static-ompi.cmake index 88ed69ff7a..ba344c87be 100644 --- a/scripts/ci/cmake/ci-ubuntu20.04-gcc8-static-ompi.cmake +++ b/scripts/ci/cmake/ci-ubuntu20.04-gcc8-static-ompi.cmake @@ -13,7 +13,7 @@ BUILD_TESTING:BOOL=OFF ADIOS2_BUILD_EXAMPLES:BOOL=ON ADIOS2_USE_BZip2:BOOL=ON -ADIOS2_USE_Blosc:BOOL=ON +ADIOS2_USE_Blosc2:BOOL=ON ADIOS2_USE_DataMan:BOOL=ON ADIOS2_USE_Fortran:BOOL=ON ADIOS2_USE_HDF5:BOOL=ON diff --git a/scripts/ci/cmake/ci-ubuntu20.04-gcc8-static-serial.cmake b/scripts/ci/cmake/ci-ubuntu20.04-gcc8-static-serial.cmake index d0e0826fa0..10046da165 100644 --- a/scripts/ci/cmake/ci-ubuntu20.04-gcc8-static-serial.cmake +++ b/scripts/ci/cmake/ci-ubuntu20.04-gcc8-static-serial.cmake @@ -9,7 +9,7 @@ ADIOS2_BUILD_EXAMPLES:BOOL=OFF ADIOS2_USE_AWSSDK:STRING=OFF ADIOS2_USE_Blosc2:STRING=OFF -ADIOS2_USE_Blosc:BOOL=OFF +ADIOS2_USE_Blosc2:BOOL=OFF ADIOS2_USE_BZip2:BOOL=OFF ADIOS2_USE_Catalyst:STRING=OFF ADIOS2_USE_CUDA:STRING=OFF diff --git a/scripts/ci/cmake/ci-ubuntu20.04-gcc9-mpich.cmake b/scripts/ci/cmake/ci-ubuntu20.04-gcc9-mpich.cmake index e83c946898..5a8181281c 100644 --- a/scripts/ci/cmake/ci-ubuntu20.04-gcc9-mpich.cmake +++ b/scripts/ci/cmake/ci-ubuntu20.04-gcc9-mpich.cmake @@ -16,7 +16,7 @@ BUILD_TESTING:BOOL=ON ADIOS2_BUILD_EXAMPLES:BOOL=ON ADIOS2_USE_BZip2:BOOL=ON -ADIOS2_USE_Blosc:BOOL=ON +ADIOS2_USE_Blosc2:BOOL=ON ADIOS2_USE_DataMan:BOOL=ON ADIOS2_USE_Fortran:BOOL=ON ADIOS2_USE_HDF5:BOOL=ON diff --git a/scripts/ci/cmake/ci-ubuntu20.04-gcc9-ompi.cmake b/scripts/ci/cmake/ci-ubuntu20.04-gcc9-ompi.cmake index a686bb0c87..722776a8cd 100644 --- a/scripts/ci/cmake/ci-ubuntu20.04-gcc9-ompi.cmake +++ b/scripts/ci/cmake/ci-ubuntu20.04-gcc9-ompi.cmake @@ -12,7 +12,7 @@ BUILD_TESTING:BOOL=ON ADIOS2_BUILD_EXAMPLES:BOOL=ON ADIOS2_USE_BZip2:BOOL=ON -ADIOS2_USE_Blosc:BOOL=ON +ADIOS2_USE_Blosc2:BOOL=ON ADIOS2_USE_DataMan:BOOL=ON ADIOS2_USE_Fortran:BOOL=ON ADIOS2_USE_HDF5:BOOL=ON diff --git a/scripts/ci/cmake/ci-ubuntu20.04-gcc9-serial.cmake b/scripts/ci/cmake/ci-ubuntu20.04-gcc9-serial.cmake index 82f5f6c37d..e459b5f536 100644 --- a/scripts/ci/cmake/ci-ubuntu20.04-gcc9-serial.cmake +++ b/scripts/ci/cmake/ci-ubuntu20.04-gcc9-serial.cmake @@ -12,7 +12,7 @@ BUILD_TESTING:BOOL=ON ADIOS2_BUILD_EXAMPLES:BOOL=ON ADIOS2_USE_BZip2:BOOL=ON -ADIOS2_USE_Blosc:BOOL=ON +ADIOS2_USE_Blosc2:BOOL=ON ADIOS2_USE_DataMan:BOOL=ON ADIOS2_USE_Fortran:BOOL=ON ADIOS2_USE_HDF5:BOOL=ON diff --git a/scripts/ci/cmake/ci-win2019-vs2019-ninja.cmake b/scripts/ci/cmake/ci-win2019-vs2019-ninja.cmake index 4b30ac7598..f8b8ea75da 100644 --- a/scripts/ci/cmake/ci-win2019-vs2019-ninja.cmake +++ b/scripts/ci/cmake/ci-win2019-vs2019-ninja.cmake @@ -9,6 +9,7 @@ set(dashboard_cache " BUILD_TESTING:BOOL=ON ADIOS2_BUILD_EXAMPLES:BOOL=ON +ADIOS2_USE_BZip2:BOOL=OFF ADIOS2_USE_Fortran:BOOL=OFF ADIOS2_USE_MPI:BOOL=OFF ") diff --git a/scripts/ci/cmake/ci-win2019-vs2019-ompi.cmake b/scripts/ci/cmake/ci-win2019-vs2019-ompi.cmake index 6707f95d90..3561a44160 100644 --- a/scripts/ci/cmake/ci-win2019-vs2019-ompi.cmake +++ b/scripts/ci/cmake/ci-win2019-vs2019-ompi.cmake @@ -7,6 +7,7 @@ set(dashboard_cache " BUILD_TESTING:BOOL=ON ADIOS2_BUILD_EXAMPLES:BOOL=ON +ADIOS2_USE_BZip2:BOOL=OFF ADIOS2_USE_Fortran:BOOL=OFF ADIOS2_USE_MPI:BOOL=ON ") diff --git a/scripts/ci/cmake/ci-win2019-vs2019-serial.cmake b/scripts/ci/cmake/ci-win2019-vs2019-serial.cmake index 700c13b4b6..1fcf2afbcf 100644 --- a/scripts/ci/cmake/ci-win2019-vs2019-serial.cmake +++ b/scripts/ci/cmake/ci-win2019-vs2019-serial.cmake @@ -7,6 +7,7 @@ set(dashboard_cache " BUILD_TESTING:BOOL=ON ADIOS2_BUILD_EXAMPLES:BOOL=ON +ADIOS2_USE_BZip2:BOOL=OFF ADIOS2_USE_Fortran:BOOL=OFF ADIOS2_USE_Python:BOOL=OFF ADIOS2_USE_MPI:BOOL=OFF diff --git a/scripts/ci/cmake/ci-win2022-vs2022-ompi.cmake b/scripts/ci/cmake/ci-win2022-vs2022-ompi.cmake index b1f6f29101..0d4a61ff6d 100644 --- a/scripts/ci/cmake/ci-win2022-vs2022-ompi.cmake +++ b/scripts/ci/cmake/ci-win2022-vs2022-ompi.cmake @@ -10,6 +10,7 @@ set(dashboard_cache " BUILD_TESTING:BOOL=ON ADIOS2_BUILD_EXAMPLES:BOOL=ON +ADIOS2_USE_BZip2:BOOL=OFF ADIOS2_USE_Fortran:BOOL=OFF ADIOS2_USE_MPI:BOOL=ON ADIOS2_USE_HDF5:STRING=ON diff --git a/scripts/ci/cmake/ci-win2022-vs2022-serial.cmake b/scripts/ci/cmake/ci-win2022-vs2022-serial.cmake index 6fcee2ed46..7bc000ff36 100644 --- a/scripts/ci/cmake/ci-win2022-vs2022-serial.cmake +++ b/scripts/ci/cmake/ci-win2022-vs2022-serial.cmake @@ -10,6 +10,7 @@ set(dashboard_cache " BUILD_TESTING:BOOL=ON ADIOS2_BUILD_EXAMPLES:BOOL=ON +ADIOS2_USE_BZip2:BOOL=OFF ADIOS2_USE_Fortran:BOOL=OFF ADIOS2_USE_MPI:BOOL=OFF ADIOS2_USE_HDF5:STRING=ON diff --git a/scripts/ci/cmake/ci-win2022-vs2022-static-serial.cmake b/scripts/ci/cmake/ci-win2022-vs2022-static-serial.cmake index 605be8e69b..f637c688af 100644 --- a/scripts/ci/cmake/ci-win2022-vs2022-static-serial.cmake +++ b/scripts/ci/cmake/ci-win2022-vs2022-static-serial.cmake @@ -14,6 +14,8 @@ ADIOS2_USE_EXTERNAL_GTEST=OFF BUILD_TESTING:BOOL=ON ADIOS2_BUILD_EXAMPLES:BOOL=ON +ADIOS2_USE_Blosc2:BOOL=ON +ADIOS2_USE_BZip2:BOOL=OFF ADIOS2_USE_Fortran:BOOL=OFF ADIOS2_USE_MPI:BOOL=OFF ADIOS2_USE_HDF5:STRING=OFF diff --git a/scripts/ci/gh-actions/windows-setup.ps1 b/scripts/ci/gh-actions/windows-setup.ps1 index 4bcae1d35a..3be2f86fea 100644 --- a/scripts/ci/gh-actions/windows-setup.ps1 +++ b/scripts/ci/gh-actions/windows-setup.ps1 @@ -8,6 +8,16 @@ Write-Host "::group::Installing NumPy" pip install "numpy>=1.19" Write-Host "::endgroup::" +Write-Host "::group::Setup CONDA" +$Env:Path += ";$Env:CONDA\condabin" +conda.bat init powershell +conda.bat init bash +Write-Host "::endgroup::" + +Write-Host "::group::Installing c-blosc2" +conda.bat install -y conda-forge::c-blosc2 +Write-Host "::endgroup::" + if($Env:GH_YML_MATRIX_PARALLEL -eq "ompi") { # This is taken from the MSMPI VCPKG diff --git a/scripts/ci/setup-run/ci-Windows.sh b/scripts/ci/setup-run/ci-Windows.sh new file mode 100755 index 0000000000..1965fc4990 --- /dev/null +++ b/scripts/ci/setup-run/ci-Windows.sh @@ -0,0 +1,5 @@ +#!/bin/bash +set +u +# We do not use conda cmake, thus we have to hint cmake where is +# conda root dir. +export CMAKE_PREFIX_PATH="C:/Miniconda/Library;$CMAKE_PREFIX_PATH" diff --git a/source/adios2/operator/compress/CompressBlosc.cpp b/source/adios2/operator/compress/CompressBlosc.cpp index c4bb6d53ae..97c51f0109 100644 --- a/source/adios2/operator/compress/CompressBlosc.cpp +++ b/source/adios2/operator/compress/CompressBlosc.cpp @@ -145,7 +145,7 @@ size_t CompressBlosc::Operate(const char *dataIn, const Dims &blockStart, const *headerPtr = DataHeader{}; bufferOutOffset += sizeof(DataHeader); - int32_t typesize = helper::GetDataTypeSize(type); + int32_t typesize = static_cast(helper::GetDataTypeSize(type)); if (typesize > BLOSC_MAX_TYPESIZE) typesize = 1; @@ -166,7 +166,7 @@ size_t CompressBlosc::Operate(const char *dataIn, const Dims &blockStart, const "Operator", "CompressBlosc", "Operate", "blosc library linked does not support compressor " + compressor); } - blosc2_set_nthreads(threads); + blosc2_set_nthreads(static_cast(threads)); blosc1_set_blocksize(blockSize); uint32_t chunk = 0; @@ -327,7 +327,7 @@ size_t CompressBlosc::DecompressChunkedFormat(const char *bufferIn, const size_t helper::StringTo(value, "when setting Blosc nthreads parameter\n")); } } - blosc2_set_nthreads(threads); + blosc2_set_nthreads(static_cast(threads)); while (inputOffset < inputDataSize) { @@ -398,8 +398,9 @@ size_t CompressBlosc::DecompressOldFormat(const char *bufferIn, const size_t siz helper::StringTo(value, "when setting Blosc nthreads parameter\n")); } } - blosc2_set_nthreads(threads); - const int decompressedSize = blosc2_decompress(bufferIn, sizeIn, dataOut, sizeOut); + blosc2_set_nthreads(static_cast(threads)); + const int decompressedSize = blosc2_decompress(bufferIn, static_cast(sizeIn), dataOut, + static_cast(sizeOut)); blosc2_destroy(); return static_cast(decompressedSize); } From a7fe26164345fd054047e6fa3526184def796886 Mon Sep 17 00:00:00 2001 From: Greg Eisenhauer Date: Mon, 25 Mar 2024 14:33:39 -0400 Subject: [PATCH 094/124] Don't run derived test in MPI mode, it's not written for that (#4104) --- testing/adios2/derived/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/adios2/derived/CMakeLists.txt b/testing/adios2/derived/CMakeLists.txt index 2df2938853..097e7cc7d7 100644 --- a/testing/adios2/derived/CMakeLists.txt +++ b/testing/adios2/derived/CMakeLists.txt @@ -3,4 +3,4 @@ #accompanying file Copyright.txt for details. #------------------------------------------------------------------------------# -gtest_add_tests_helper(DerivedCorrectness MPI_ALLOW BP Derived. "") +gtest_add_tests_helper(DerivedCorrectness MPI_NONE BP Derived. "") From f857343dea9bf67b8a4f824eabe6bed823708ff8 Mon Sep 17 00:00:00 2001 From: Greg Eisenhauer Date: Mon, 25 Mar 2024 15:34:56 -0400 Subject: [PATCH 095/124] dill 2024-03-12 (ebc98c4d) (#4091) Upstream Shortlog --- thirdparty/dill/dill/CMakeLists.txt | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/thirdparty/dill/dill/CMakeLists.txt b/thirdparty/dill/dill/CMakeLists.txt index 0ac720b7fd..8d6bfe8b24 100644 --- a/thirdparty/dill/dill/CMakeLists.txt +++ b/thirdparty/dill/dill/CMakeLists.txt @@ -186,12 +186,17 @@ set(ARM_HARD_FLOAT ${HARDFP_AVAILABLE}) option(DILL_IGNORE_NATIVE "Build to do emulation, regardless of architecture" OFF) +option(DILL_NATIVE_ONLY "Build to native code only" ON) if(DILL_IGNORE_NATIVE) set(NATIVE_CG FALSE) set(NATIVE_ARCH UNSUPPORTED) endif() set(LIBFFI_INTERNAL OFF) -find_package(LibFFI) +if (DILL_NATIVE_ONLY) + set(LIBFFI_FOUND FALSE) +else() + find_package(LibFFI) +endif() if(LIBFFI_FOUND) message(STATUS "Enabling emulation") set(EMULATION_POSSIBLE TRUE) @@ -531,8 +536,10 @@ if(NOT DILL_QUIET) message(STATUS "CMAKE_BUILD_TYPE = ${CMAKE_BUILD_TYPE}" ) message(STATUS "DILL_ENABLE_DISASSEMBLY = ${DILL_ENABLE_DISASSEMBLY}" ) message(STATUS "DILL_MULTI_TARGET = ${DILL_MULTI_TARGET}" ) - message(STATUS "BUILD_SHARED_LIBS = ${BUILD_SHARED_LIBS}" ) + message(STATUS "DILL_IGNORE_NATIVE = ${DILL_IGNORE_NATIVE}" ) + message(STATUS "DILL_NATIVE_ONLY = ${DILL_NATIVE_ONLY}" ) message(STATUS "BUILD_TESTING = ${BUILD_TESTING}" ) + message(STATUS "BUILD_SHARED_LIBS = ${BUILD_SHARED_LIBS}" ) message(STATUS "Change a value with: cmake -D=" ) message(STATUS "-----------------------------------------------------------------------------") endif() From c4ebbc500562a3b824fb23be27f912dfa1efd64c Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Tue, 26 Mar 2024 12:31:09 -0400 Subject: [PATCH 096/124] Update campaign manager script to handle config file, time in nanosecond format, and avoiding conflict when updating database --- source/adios2/helper/adiosNetwork.cpp | 2 +- .../adios2_campaign_manager.py | 400 +++++++++++------- 2 files changed, 256 insertions(+), 146 deletions(-) diff --git a/source/adios2/helper/adiosNetwork.cpp b/source/adios2/helper/adiosNetwork.cpp index a29abd462b..309260fcdf 100644 --- a/source/adios2/helper/adiosNetwork.cpp +++ b/source/adios2/helper/adiosNetwork.cpp @@ -79,7 +79,7 @@ std::string GetFQDN() noexcept { for (p = info; p != NULL; p = p->ai_next) { - printf("hostname: %s\n", p->ai_canonname); + // printf("hostname: %s\n", p->ai_canonname); if (strchr(p->ai_canonname, '.') != NULL) { strncpy(hostname, p->ai_canonname, sizeof(hostname)); diff --git a/source/utils/adios_campaign_manager/adios2_campaign_manager.py b/source/utils/adios_campaign_manager/adios2_campaign_manager.py index c752d95aac..51e3bf5ce6 100755 --- a/source/utils/adios_campaign_manager/adios2_campaign_manager.py +++ b/source/utils/adios_campaign_manager/adios2_campaign_manager.py @@ -6,61 +6,89 @@ import zlib from datetime import datetime from os import chdir, getcwd, remove, stat -from os.path import exists, isdir +from os.path import exists, isdir, expanduser from re import sub from socket import getfqdn -from time import time +from time import time_ns # from adios2.adios2_campaign_manager import * ADIOS_ACA_VERSION = "1.0" + +def ReadConfig(): + path = expanduser("~/.config/adios2/campaign.cfg") + try: + with open(path) as f: + lines = f.readlines() + for line in lines: + lst = line.split() + if lst[0] == "campaignstorepath": + adios_campaign_store = expanduser(lst[1]) + except FileNotFoundError: + adios_campaign_store = None + return adios_campaign_store + + def SetupArgs(): parser = argparse.ArgumentParser() parser.add_argument( - "command", help="Command: create/update/delete", - choices=['create', 'update', 'delete', 'info']) - parser.add_argument("--verbose", "-v", - help="More verbosity", action="count") - parser.add_argument("--project", "-p", - help="Project name", - required=True) - parser.add_argument("--app", "-a", - help="Application name", - required=True) - parser.add_argument("--shot", "-s", - help="Shot name", - required=True) - parser.add_argument("--campaign_store", "-c", - help="Path to local campaign store", - required=True) - parser.add_argument("--hostname", "-n", - help="Host name unique for hosts in a campaign", - required=False) + "command", + help="Command: create/update/delete/info/list", + choices=["create", "update", "delete", "info", "list"], + ) + parser.add_argument( + "campaign", help="Campaign name or path, with .aca or without", default=None, nargs="?" + ) + parser.add_argument("--verbose", "-v", help="More verbosity", action="count", default=0) + parser.add_argument( + "--campaign_store", "-s", help="Path to local campaign store", default=None + ) + parser.add_argument( + "--hostname", "-n", help="Host name unique for hosts in a campaign", required=False + ) args = parser.parse_args() # default values args.update = False - args.CampaignFileName = args.campaign_store + "/" + \ - args.project + "_" + args.app + "_" + args.shot + ".aca" + if args.campaign_store is None: + args.campaign_store = ReadConfig() + + if args.campaign_store is not None: + while args.campaign_store[-1] == "/": + args.campaign_store = args.campaign_store[:-1] + + args.CampaignFileName = args.campaign + if args.campaign is not None: + if not args.campaign.endswith(".aca"): + args.CampaignFileName += ".aca" + if args.campaign_store is not None: + args.CampaignFileName = args.campaign_store + "/" + args.CampaignFileName + args.LocalCampaignDir = "adios-campaign/" - # print("Verbosity = {0}".format(args.verbose)) - print(f"Campaign File Name = {args.CampaignFileName}") + if args.verbose > 0: + print(f"# Verbosity = {args.verbose}") + print(f"# Command = {args.command}") + print(f"# Campaign File Name = {args.CampaignFileName}") + print(f"# Campaign Store = {args.campaign_store}") return args def CheckCampaignStore(args): - if not isdir(args.campaign_store): - print("ERROR: Campaign directory " + args.campaign_store + - " does not exist", flush=True) + if args.campaign_store is not None and not isdir(args.campaign_store): + print("ERROR: Campaign directory " + args.campaign_store + " does not exist", flush=True) exit(1) def CheckLocalCampaignDir(args): if not isdir(args.LocalCampaignDir): - print("ERROR: Shot campaign data '" + args.LocalCampaignDir + - "' does not exist. Run this command where the code was executed.", flush=True) + print( + "ERROR: Shot campaign data '" + + args.LocalCampaignDir + + "' does not exist. Run this command where the code was executed.", + flush=True, + ) exit(1) @@ -102,7 +130,7 @@ def decompressBuffer(buf: bytearray): def AddFileToArchive(args: dict, filename: str, cur: sqlite3.Cursor, dsID: int): compressed = 1 try: - f = open(filename, 'rb') + f = open(filename, "rb") compressed_data, len_orig, len_compressed = compressFile(f) except IOError: @@ -110,38 +138,57 @@ def AddFileToArchive(args: dict, filename: str, cur: sqlite3.Cursor, dsID: int): return statres = stat(filename) - ct = statres.st_ctime - - cur.execute('insert into bpfile values (?, ?, ?, ?, ?, ?, ?)', - (dsID, filename, compressed, len_orig, len_compressed, ct, compressed_data)) - con.commit() - - # test - # if (filename == "dataAll.bp/md.0"): - # data = decompressBuffer(compressed_data) - # of = open("dataAll.bp-md.0", "wb") - # of.write(data) - # of.close() - + ct = statres.st_ctime_ns -def AddDatasetToArchive(args: dict, dataset: str, cur: sqlite3.Cursor, hostID: int, dirID: int): - if (IsADIOSDataset(dataset)): - print(f"Add dataset {dataset} to archive") - statres = stat(dataset) - ct = statres.st_ctime - curDS = cur.execute('insert into bpdataset values (?, ?, ?, ?)', - (hostID, dirID, dataset, ct)) - dsID = curDS.lastrowid - cwd = getcwd() - chdir(dataset) - mdFileList = glob.glob('*md.*') - profileList = glob.glob('profiling.json') - files = mdFileList + profileList - for f in files: - AddFileToArchive(args, f, cur, dsID) - chdir(cwd) + cur.execute( + "insert into bpfile " + "(bpdatasetid, name, compression, lenorig, lencompressed, ctime, data) " + "values (?, ?, ?, ?, ?, ?, ?) " + "on conflict (bpdatasetid, name) do update " + "set compression = ?, lenorig = ?, lencompressed = ?, ctime = ?, data = ?", + ( + dsID, + filename, + compressed, + len_orig, + len_compressed, + ct, + compressed_data, + compressed, + len_orig, + len_compressed, + ct, + compressed_data, + ), + ) + + +def AddDatasetToArchive(hostID: int, dirID: int, dataset: str, cur: sqlite3.Cursor) -> int: + statres = stat(dataset) + ct = statres.st_ctime_ns + select_cmd = ( + "select rowid from bpdataset " + f"where hostid = {hostID} and dirid = {dirID} and name = '{dataset}'" + ) + res = cur.execute(select_cmd) + row = res.fetchone() + if row is not None: + rowID = row[0] + print( + f"Found dataset {dataset} in database on host {hostID} " + f"in dir {dirID}, rowid = {rowID}" + ) else: - print(f"WARNING: Dataset {dataset} is not an ADIOS dataset. Skip") + print(f"Add dataset {dataset} to archive") + curDS = cur.execute( + "insert into bpdataset (hostid, dirid, name, ctime) values (?, ?, ?, ?)", + (hostID, dirID, dataset, ct), + ) + rowID = curDS.lastrowid + # print( + # f"Inserted bpdataset {dataset} in database on host {hostID} in dir {dirID}, rowid = {rowID}" + # ) + return rowID def ProcessDBFile(args: dict, jsonlist: list, cur: sqlite3.Cursor, hostID: int, dirID: int): @@ -149,8 +196,20 @@ def ProcessDBFile(args: dict, jsonlist: list, cur: sqlite3.Cursor, hostID: int, # print(f"Process entry {entry}:") if isinstance(entry, dict): if "name" in entry: - AddDatasetToArchive( - args, entry['name'], cur, hostID, dirID) + dsID = 0 + dataset = entry["name"] + if IsADIOSDataset(dataset): + dsID = AddDatasetToArchive(hostID, dirID, dataset, cur) + cwd = getcwd() + chdir(dataset) + mdFileList = glob.glob("*md.*") + profileList = glob.glob("profiling.json") + files = mdFileList + profileList + for f in files: + AddFileToArchive(args, f, cur, dsID) + chdir(cwd) + else: + print(f"WARNING: Dataset {dataset} is not an ADIOS dataset. Skip") else: print(f"WARNING: your object is not a dictionary, skip : {entry}") @@ -158,50 +217,60 @@ def ProcessDBFile(args: dict, jsonlist: list, cur: sqlite3.Cursor, hostID: int, def GetHostName(): host = getfqdn() if host.startswith("login"): - host = sub('^login[0-9]*\\.', '', host) + host = sub("^login[0-9]*\\.", "", host) if host.startswith("batch"): - host = sub('^batch[0-9]*\\.', '', host) - shorthost = host.split('.')[0] + host = sub("^batch[0-9]*\\.", "", host) + shorthost = host.split(".")[0] return host, shorthost def AddHostName(longHostName, shortHostName): - res = cur.execute( - 'select rowid from host where hostname = "' + shortHostName + '"') + res = cur.execute('select rowid from host where hostname = "' + shortHostName + '"') row = res.fetchone() if row is not None: hostID = row[0] print(f"Found host {shortHostName} in database, rowid = {hostID}") else: - curHost = cur.execute('insert into host values (?, ?)', - (shortHostName, longHostName)) + curHost = cur.execute("insert into host values (?, ?)", (shortHostName, longHostName)) hostID = curHost.lastrowid print(f"Inserted host {shortHostName} into database, rowid = {hostID}") return hostID -def Info(args: dict, cur: sqlite3.Cursor): - res = cur.execute('select id, name, version, ctime from info') - info = res.fetchone() - t = datetime.fromtimestamp(float(info[3])) - print(f"{info[1]}, version {info[2]}, created on {t}") +def MergeDBFiles(dbfiles: list): + # read db files here + result = list() + for f1 in dbfiles: + try: + con = sqlite3.connect(f1) + except sqlite3.Error as e: + print(e) - res = cur.execute('select rowid, hostname, longhostname from host') - hosts = res.fetchall() - for host in hosts: - print(f"hostname = {host[1]} longhostname = {host[2]}") - res2 = cur.execute( - 'select rowid, name from directory where hostid = "' + str(host[0]) + '"') - dirs = res2.fetchall() - for dir in dirs: - print(f" dir = {dir[1]}") - res3 = cur.execute( - 'select rowid, name, ctime from bpdataset where hostid = "' + str(host[0]) + - '" and dirid = "' + str(dir[0]) + '"') - bpdatasets = res3.fetchall() - for bpdataset in bpdatasets: - t = datetime.fromtimestamp(float(bpdataset[2])) - print(f" dataset = {bpdataset[1]} created on {t}") + cur = con.cursor() + try: + cur.execute("select * from bpfiles") + except sqlite3.Error as e: + print(e) + record = cur.fetchall() + for item in record: + result.append({"name": item[0]}) + cur.close() + return result + + +def AddDirectory(hostID: int, path: str) -> int: + res = cur.execute( + "select rowid from directory where hostid = " + str(hostID) + ' and name = "' + path + '"' + ) + row = res.fetchone() + if row is not None: + dirID = row[0] + print(f"Found directory {path} with hostID {hostID} in database, rowid = {dirID}") + else: + curDirectory = cur.execute("insert into directory values (?, ?)", (hostID, path)) + dirID = curDirectory.lastrowid + print(f"Inserted directory {path} into database, rowid = {dirID}") + return dirID def Update(args: dict, cur: sqlite3.Cursor): @@ -212,13 +281,8 @@ def Update(args: dict, cur: sqlite3.Cursor): hostID = AddHostName(longHostName, shortHostName) rootdir = getcwd() - # curHost = cur.execute('insert into host values (?, ?)', - # (shortHostName, longHostName)) - # hostID = curHost.lastrowid + dirID = AddDirectory(hostID, rootdir) - curDir = cur.execute('insert or replace into directory values (?, ?)', - (hostID, rootdir)) - dirID = curDir.lastrowid con.commit() db_list = MergeDBFiles(dbFileList) @@ -230,66 +294,112 @@ def Update(args: dict, cur: sqlite3.Cursor): def Create(args: dict, cur: sqlite3.Cursor): - epoch = int(time()) + epoch = time_ns() + cur.execute("create table info(id TEXT, name TEXT, version TEXT, ctime INT)") cur.execute( - "create table info(id TEXT, name TEXT, version TEXT, ctime INT)") - cur.execute('insert into info values (?, ?, ?, ?)', - ("ACA", "ADIOS Campaign Archive", ADIOS_ACA_VERSION, epoch)) - cur.execute("create table host" + - "(hostname TEXT PRIMARY KEY, longhostname TEXT)") - cur.execute("create table directory" + - "(hostid INT, name TEXT, PRIMARY KEY (hostid, name))") - cur.execute("create table bpdataset" + - "(hostid INT, dirid INT, name TEXT, ctime INT" + - ", PRIMARY KEY (hostid, dirid, name))") - cur.execute("create table bpfile" + - "(bpdatasetid INT, name TEXT, compression INT, lenorig INT" + - ", lencompressed INT, ctime INT, data BLOB" + - ", PRIMARY KEY (bpdatasetid, name))") + "insert into info values (?, ?, ?, ?)", + ("ACA", "ADIOS Campaign Archive", ADIOS_ACA_VERSION, epoch), + ) + cur.execute("create table host" + "(hostname TEXT PRIMARY KEY, longhostname TEXT)") + cur.execute("create table directory" + "(hostid INT, name TEXT, PRIMARY KEY (hostid, name))") + cur.execute( + "create table bpdataset" + + "(hostid INT, dirid INT, name TEXT, ctime INT" + + ", PRIMARY KEY (hostid, dirid, name))" + ) + cur.execute( + "create table bpfile" + + "(bpdatasetid INT, name TEXT, compression INT, lenorig INT" + + ", lencompressed INT, ctime INT, data BLOB" + + ", PRIMARY KEY (bpdatasetid, name))" + ) Update(args, cur) -def MergeDBFiles(dbfiles: list): - # read db files here - result = list() - for f1 in dbfiles: - try: - con = sqlite3.connect(f1) - except sqlite3.Error as e: - print(e) +def timestamp_to_datetime(timestamp: int) -> datetime: + digits = len(str(int(timestamp))) + t = float(timestamp) + if digits > 18: + t = t / 1000000000 + elif digits > 15: + t = t / 1000000 + elif digits > 12: + t = t / 1000 + return datetime.fromtimestamp(t) - cur = con.cursor() - try: - cur.execute("select * from bpfiles") - except sqlite3.Error as e: - print(e) - record = cur.fetchall() - for item in record: - result.append({"name": item[0]}) - cur.close() - return result +def Info(args: dict, cur: sqlite3.Cursor): + res = cur.execute("select id, name, version, ctime from info") + info = res.fetchone() + t = timestamp_to_datetime(info[3]) + print(f"{info[1]}, version {info[2]}, created on {t}") + + res = cur.execute("select rowid, hostname, longhostname from host") + hosts = res.fetchall() + for host in hosts: + print(f"hostname = {host[1]} longhostname = {host[2]}") + res2 = cur.execute( + 'select rowid, name from directory where hostid = "' + str(host[0]) + '"' + ) + dirs = res2.fetchall() + for dir in dirs: + print(f" dir = {dir[1]}") + res3 = cur.execute( + 'select rowid, name, ctime from bpdataset where hostid = "' + + str(host[0]) + + '" and dirid = "' + + str(dir[0]) + + '"' + ) + bpdatasets = res3.fetchall() + for bpdataset in bpdatasets: + t = timestamp_to_datetime(bpdataset[2]) + print(f" dataset = {bpdataset[1]} created on {t}") -if __name__ == "__main__": +def List(): + if args.campaign_store is None: + print("ERROR: Set --campaign_store for this command") + return 1 + else: + # List the local campaign store + acaList = glob.glob(args.campaign_store + "/**/*.aca", recursive=True) + if len(acaList) == 0: + print("There are no campaign archives in " + args.campaign_store) + return 2 + else: + startCharPos = len(args.campaign_store) + 1 + for f in acaList: + print(f[startCharPos:]) + return 0 + + +def Delete(): + if exists(args.CampaignFileName): + print(f"Delete archive {args.CampaignFileName}") + remove(args.CampaignFileName) + return 0 + else: + print(f"ERROR: archive {args.CampaignFileName} does not exist") + return 1 + + +if __name__ == "__main__": args = SetupArgs() CheckCampaignStore(args) - if (args.command == "delete"): - if exists(args.CampaignFileName): - print(f"Delete archive {args.CampaignFileName}") - remove(args.CampaignFileName) - exit(0) - else: - print(f"ERROR: archive {args.CampaignFileName} does not exist") - exit(1) + if args.command == "list": + exit(List()) + + if args.command == "delete": + exit(Delete()) - if (args.command == "create"): + if args.command == "create": print("Create archive") if exists(args.CampaignFileName): print(f"ERROR: archive {args.CampaignFileName} already exist") exit(1) - elif (args.command == "update" or args.command == 'info'): + elif args.command == "update" or args.command == "info": print(f"{args.command} archive") if not exists(args.CampaignFileName): print(f"ERROR: archive {args.CampaignFileName} does not exist") @@ -298,19 +408,19 @@ def MergeDBFiles(dbfiles: list): con = sqlite3.connect(args.CampaignFileName) cur = con.cursor() - if (args.command == "info"): + if args.command == "info": Info(args, cur) else: CheckLocalCampaignDir(args) # List the local campaign directory - dbFileList = glob.glob(args.LocalCampaignDir + '/*.db') + dbFileList = glob.glob(args.LocalCampaignDir + "/*.db") if len(dbFileList) == 0: print("There are no campaign data files in " + args.LocalCampaignDir) exit(2) - if (args.command == "create"): + if args.command == "create": Create(args, cur) - elif (args.command == "update"): + elif args.command == "update": Update(args, cur) cur.close() From e9d1fb44470fad76cb155aca67c64c1f6d8362f5 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Tue, 26 Mar 2024 12:59:12 -0400 Subject: [PATCH 097/124] flake8 fixes --- .../adios2_campaign_manager.py | 33 ++++++++++--------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/source/utils/adios_campaign_manager/adios2_campaign_manager.py b/source/utils/adios_campaign_manager/adios2_campaign_manager.py index 51e3bf5ce6..18f8b78771 100755 --- a/source/utils/adios_campaign_manager/adios2_campaign_manager.py +++ b/source/utils/adios_campaign_manager/adios2_campaign_manager.py @@ -84,9 +84,9 @@ def CheckCampaignStore(args): def CheckLocalCampaignDir(args): if not isdir(args.LocalCampaignDir): print( - "ERROR: Shot campaign data '" - + args.LocalCampaignDir - + "' does not exist. Run this command where the code was executed.", + "ERROR: Shot campaign data '" + + args.LocalCampaignDir + + "' does not exist. Run this command where the code was executed.", flush=True, ) exit(1) @@ -186,7 +186,8 @@ def AddDatasetToArchive(hostID: int, dirID: int, dataset: str, cur: sqlite3.Curs ) rowID = curDS.lastrowid # print( - # f"Inserted bpdataset {dataset} in database on host {hostID} in dir {dirID}, rowid = {rowID}" + # f"Inserted bpdataset {dataset} in database on host {hostID}" + # f" in dir {dirID}, rowid = {rowID}" # ) return rowID @@ -303,15 +304,15 @@ def Create(args: dict, cur: sqlite3.Cursor): cur.execute("create table host" + "(hostname TEXT PRIMARY KEY, longhostname TEXT)") cur.execute("create table directory" + "(hostid INT, name TEXT, PRIMARY KEY (hostid, name))") cur.execute( - "create table bpdataset" - + "(hostid INT, dirid INT, name TEXT, ctime INT" - + ", PRIMARY KEY (hostid, dirid, name))" + "create table bpdataset" + + "(hostid INT, dirid INT, name TEXT, ctime INT" + + ", PRIMARY KEY (hostid, dirid, name))" ) cur.execute( - "create table bpfile" - + "(bpdatasetid INT, name TEXT, compression INT, lenorig INT" - + ", lencompressed INT, ctime INT, data BLOB" - + ", PRIMARY KEY (bpdatasetid, name))" + "create table bpfile" + + "(bpdatasetid INT, name TEXT, compression INT, lenorig INT" + + ", lencompressed INT, ctime INT, data BLOB" + + ", PRIMARY KEY (bpdatasetid, name))" ) Update(args, cur) @@ -345,11 +346,11 @@ def Info(args: dict, cur: sqlite3.Cursor): for dir in dirs: print(f" dir = {dir[1]}") res3 = cur.execute( - 'select rowid, name, ctime from bpdataset where hostid = "' - + str(host[0]) - + '" and dirid = "' - + str(dir[0]) - + '"' + 'select rowid, name, ctime from bpdataset where hostid = "' + + str(host[0]) + + '" and dirid = "' + + str(dir[0]) + + '"' ) bpdatasets = res3.fetchall() for bpdataset in bpdatasets: From 23fa9cd34ab0538cd081fa11223f36185c010e71 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Tue, 26 Mar 2024 13:50:37 -0400 Subject: [PATCH 098/124] - Only add campaign store to file name if that is not absolute path - list command supports second argument as path --- .../adios2_campaign_manager.py | 34 ++++++++++++------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/source/utils/adios_campaign_manager/adios2_campaign_manager.py b/source/utils/adios_campaign_manager/adios2_campaign_manager.py index 18f8b78771..171af0b4cc 100755 --- a/source/utils/adios_campaign_manager/adios2_campaign_manager.py +++ b/source/utils/adios_campaign_manager/adios2_campaign_manager.py @@ -62,7 +62,9 @@ def SetupArgs(): if args.campaign is not None: if not args.campaign.endswith(".aca"): args.CampaignFileName += ".aca" - if args.campaign_store is not None: + if (not exists(args.CampaignFileName) and + not args.CampaignFileName.startswith("/") and + args.campaign_store is not None): args.CampaignFileName = args.campaign_store + "/" + args.CampaignFileName args.LocalCampaignDir = "adios-campaign/" @@ -359,19 +361,25 @@ def Info(args: dict, cur: sqlite3.Cursor): def List(): - if args.campaign_store is None: - print("ERROR: Set --campaign_store for this command") - return 1 + path = args.campaign + if path is None: + if args.campaign_store is None: + print("ERROR: Set --campaign_store for this command") + return 1 + path = args.campaign_store else: - # List the local campaign store - acaList = glob.glob(args.campaign_store + "/**/*.aca", recursive=True) - if len(acaList) == 0: - print("There are no campaign archives in " + args.campaign_store) - return 2 - else: - startCharPos = len(args.campaign_store) + 1 - for f in acaList: - print(f[startCharPos:]) + while path[-1] == "/": + path = path[:-1] + + # List the local campaign store + acaList = glob.glob(path + "/**/*.aca", recursive=True) + if len(acaList) == 0: + print("There are no campaign archives in " + path) + return 2 + else: + startCharPos = len(path) + 1 + for f in acaList: + print(f[startCharPos:]) return 0 From 1533a07194675d3333adf502d2bdbc7c369a5173 Mon Sep 17 00:00:00 2001 From: Greg Eisenhauer Date: Wed, 27 Mar 2024 11:16:21 -0400 Subject: [PATCH 099/124] Don't use assert() in tests (#4108) --- testing/adios2/engine/bp/TestBPAccuracyDefaults.cpp | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/testing/adios2/engine/bp/TestBPAccuracyDefaults.cpp b/testing/adios2/engine/bp/TestBPAccuracyDefaults.cpp index d0493cdc22..96eb8e61d1 100644 --- a/testing/adios2/engine/bp/TestBPAccuracyDefaults.cpp +++ b/testing/adios2/engine/bp/TestBPAccuracyDefaults.cpp @@ -73,9 +73,9 @@ TEST_F(AccuracyTests, DefaultAccuracy) bpWriter.Close(); auto accuracyGot = var.GetAccuracy(); - assert(accuracyGot.error == 0.0); // no error whatsoever - assert(accuracyGot.norm == accuracyRequested.norm); - assert(accuracyGot.relative == accuracyRequested.relative); + EXPECT_EQ(accuracyGot.error, 0.0); // no error whatsoever + EXPECT_EQ(accuracyGot.norm, accuracyRequested.norm); + EXPECT_EQ(accuracyGot.relative, accuracyRequested.relative); } // Reader { @@ -100,9 +100,10 @@ TEST_F(AccuracyTests, DefaultAccuracy) bpReader.PerformGets(); auto accuracyGot = varRange.GetAccuracy(); - assert(accuracyGot.error == 0.0); // no error whatsoever - assert(accuracyGot.norm == accuracyRequested.norm); - assert(accuracyGot.relative == accuracyRequested.relative); + EXPECT_EQ(accuracyGot.error, 0.0); // no error whatsoever + EXPECT_EQ(accuracyGot.error, 0.0); // no error whatsoever + EXPECT_EQ(accuracyGot.norm, accuracyRequested.norm); + EXPECT_EQ(accuracyGot.relative, accuracyRequested.relative); std::vector iStartEndData; iStartEndData.reserve(gNx); // maximum possible From 445e5faf6c21f21035a43bee5e490839fb369e98 Mon Sep 17 00:00:00 2001 From: Greg Eisenhauer Date: Thu, 28 Mar 2024 17:20:59 -0400 Subject: [PATCH 100/124] Warnings (#4113) --- bindings/Fortran/f2c/adios2_f2c_io.cpp | 2 +- source/adios2/helper/adiosNetwork.cpp | 6 ++---- source/adios2/toolkit/format/bp5/BP5Deserializer.cpp | 2 +- source/h5vol/H5Vol_attr.c | 2 +- testing/utils/cwriter/TestUtilsCWriter.c | 2 +- 5 files changed, 6 insertions(+), 8 deletions(-) diff --git a/bindings/Fortran/f2c/adios2_f2c_io.cpp b/bindings/Fortran/f2c/adios2_f2c_io.cpp index da477de384..b1acb6079e 100644 --- a/bindings/Fortran/f2c/adios2_f2c_io.cpp +++ b/bindings/Fortran/f2c/adios2_f2c_io.cpp @@ -288,7 +288,7 @@ void FC_GLOBAL(adios2_retrieve_namelist_f2c, len = static_cast(namelist_len); } // copy C string without '\0' - strncpy(fs, info->names[i], len); + memcpy(fs, info->names[i], len); // pad with spaces memset(fs + len, ' ', namelist_len - len); } diff --git a/source/adios2/helper/adiosNetwork.cpp b/source/adios2/helper/adiosNetwork.cpp index 309260fcdf..11837376ae 100644 --- a/source/adios2/helper/adiosNetwork.cpp +++ b/source/adios2/helper/adiosNetwork.cpp @@ -4,8 +4,6 @@ * * adiosNetwork.cpp implementation of adiosNetwork.h functions * - * Created on: March 22, 2019 - * Author: William F Godoy godoywf@ornl.gov */ #include "adiosNetwork.h" @@ -82,7 +80,7 @@ std::string GetFQDN() noexcept // printf("hostname: %s\n", p->ai_canonname); if (strchr(p->ai_canonname, '.') != NULL) { - strncpy(hostname, p->ai_canonname, sizeof(hostname)); + strncpy(hostname, p->ai_canonname, sizeof(hostname) - 1); break; } } @@ -138,7 +136,7 @@ AvailableIpAddresses() noexcept for (struct if_nameindex *p = head; !(p->if_index == 0 && p->if_name == NULL); ++p) { struct ifreq req; - strncpy(req.ifr_name, p->if_name, IFNAMSIZ); + strncpy(req.ifr_name, p->if_name, IFNAMSIZ - 1); if (ioctl(socket_handler, SIOCGIFADDR, &req) < 0) { if (errno == EADDRNOTAVAIL) diff --git a/source/adios2/toolkit/format/bp5/BP5Deserializer.cpp b/source/adios2/toolkit/format/bp5/BP5Deserializer.cpp index 9c1a127408..f78f554c9d 100644 --- a/source/adios2/toolkit/format/bp5/BP5Deserializer.cpp +++ b/source/adios2/toolkit/format/bp5/BP5Deserializer.cpp @@ -366,7 +366,7 @@ BP5Deserializer::ControlInfo *BP5Deserializer::BuildControl(FMFormat Format) size_t VarIndex = 0; while (FieldList[i].field_name) { - size_t HeaderSkip; + size_t HeaderSkip = 0; char *ExprStr = NULL; int Derived = 0; ret = (ControlInfo *)realloc(ret, sizeof(*ret) + ControlCount * sizeof(struct ControlInfo)); diff --git a/source/h5vol/H5Vol_attr.c b/source/h5vol/H5Vol_attr.c index 17a89cb0d5..8a4dc84081 100644 --- a/source/h5vol/H5Vol_attr.c +++ b/source/h5vol/H5Vol_attr.c @@ -183,7 +183,7 @@ herr_t H5VL_adios2_attr_get(void *obj, H5VL_attr_get_args_t *args, hid_t dxpl_id *ret_val = strlen(attrDef->m_Name); if (buf) { - strncpy(buf, attrDef->m_Name, *ret_val); + memcpy(buf, attrDef->m_Name, *ret_val); } } else if (H5VL_OBJECT_BY_IDX == loc_params->type) diff --git a/testing/utils/cwriter/TestUtilsCWriter.c b/testing/utils/cwriter/TestUtilsCWriter.c index 04db7d8d6e..ed5acc3ac7 100644 --- a/testing/utils/cwriter/TestUtilsCWriter.c +++ b/testing/utils/cwriter/TestUtilsCWriter.c @@ -38,7 +38,7 @@ int main(int argc, char *argv[]) char engineName[32] = "BPFile"; if (argc > 1) { - strncpy(engineName, argv[1], sizeof(engineName)); + strncpy(engineName, argv[1], sizeof(engineName) - 1); } // IO From c1d25b7d4fcfdb69280b38f6ecc536913ec43a8f Mon Sep 17 00:00:00 2001 From: Greg Eisenhauer Date: Fri, 29 Mar 2024 11:16:35 -0400 Subject: [PATCH 101/124] EVpath upstream to make NO_RDMA more robust (#4116) --- testing/adios2/engine/bp/TestBPWriteReadMultiblock.cpp | 3 ++- thirdparty/EVPath/EVPath/CMakeLists.txt | 10 +++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/testing/adios2/engine/bp/TestBPWriteReadMultiblock.cpp b/testing/adios2/engine/bp/TestBPWriteReadMultiblock.cpp index a43c0e20df..5014ba6335 100644 --- a/testing/adios2/engine/bp/TestBPWriteReadMultiblock.cpp +++ b/testing/adios2/engine/bp/TestBPWriteReadMultiblock.cpp @@ -1971,7 +1971,6 @@ TEST_F(BPWriteReadMultiblockTest, MultiblockPerformDataWrite) } // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("MultiblockPerformDataWrite.bp"); int mpiRank = 0, mpiSize = 1; // Number of elements per blocks (blocksize) @@ -1990,8 +1989,10 @@ TEST_F(BPWriteReadMultiblockTest, MultiblockPerformDataWrite) #if ADIOS2_USE_MPI adios2::ADIOS adios(MPI_COMM_WORLD); + const std::string fname("MultiblockPerformDataWrite_MPI.bp"); #else adios2::ADIOS adios; + const std::string fname("MultiblockPerformDataWrite.bp"); #endif /* Write output */ { diff --git a/thirdparty/EVPath/EVPath/CMakeLists.txt b/thirdparty/EVPath/EVPath/CMakeLists.txt index 132c650619..384160c512 100644 --- a/thirdparty/EVPath/EVPath/CMakeLists.txt +++ b/thirdparty/EVPath/EVPath/CMakeLists.txt @@ -376,7 +376,7 @@ if (MSVC) set(EVPATH_USE_ZPL_ENET FALSE) endif() if(NOT (DEFINED EVPATH_USE_ZPL_ENET)) - option(EVPATH_USE_ZPL_ENET "Build the enet transport" "OFF") + option(EVPATH_USE_ZPL_ENET "Build the zplenet transport" "OFF") endif() if(EVPATH_USE_ZPL_ENET) set(RUN_ZPL_ENET_TESTS TRUE) @@ -447,7 +447,7 @@ else() option(EVPATH_USE_LIBFABRIC "Build the libfabric transport" ${LIBFABRIC_FOUND}) endif() -if(LIBFABRIC_FOUND) +if(LIBFABRIC_FOUND AND NOT EVPATH_NO_RDMA) if(EVPATH_TRANSPORT_MODULES) add_library(cmfabric MODULE cmfabric.c ip_config.c) set_target_properties(cmfabric PROPERTIES @@ -485,7 +485,7 @@ else() option(EVPATH_USE_IBVERBS "Build the libfabric transport" ${IBVERBS_FOUND}) endif() set(HAVE_IBVERBS ${IBVERBS_FOUND}) -if(IBVERBS_FOUND) +if(IBVERBS_FOUND AND NOT EVPATH_NO_RDMA) if(BUILD_TESTING) if(NOT CMAKE_CROSSCOMPILING) message(STATUS "Check MEMLOCK rlimit for IB tests") @@ -548,7 +548,7 @@ if(IBVERBS_FOUND) find_package(NNTI) option(EVPATH_USE_NNTI "Build the nnti transport" ${NNTI_FOUND}) endif() - if(NNTI_FOUND) + if(NNTI_FOUND AND NOT EVPATH_NO_RDMA) if(CercsArch STREQUAL "ppc64") set(RUN_NNTI_TESTS FALSE) endif() @@ -571,7 +571,7 @@ if(IBVERBS_FOUND) endif() endif() else() - set(EVPATH_USE_NNTI FALSE "Build the nnti transport" FORCE) + set(EVPATH_USE_NNTI FALSE CACHE INTERNAL "Build the nnti transport" FORCE) endif() # Install extra find module dependencies From 5e3a2e71ac24ead13d5a0a2fea2f084f7cd41329 Mon Sep 17 00:00:00 2001 From: Greg Eisenhauer Date: Sat, 30 Mar 2024 07:12:01 -0400 Subject: [PATCH 102/124] Different names for MPI and Serial tests (#4118) * Different names for MPI and Serial tests --- .../C/TestBPAvailableVariablesAttribites.cpp | 10 +++- .../adios2/bindings/C/TestBPMemorySpace.cpp | 4 ++ .../C/TestBPWriteAggregateReadLocal.cpp | 10 +++- .../bindings/C/TestBPWriteReadMultiblock.cpp | 9 +++- .../adios2/bindings/C/TestBPWriteTypes.cpp | 8 +++ .../adios2/bindings/C/TestNullWriteRead.cpp | 3 +- .../bindings/fortran/TestBPWriteTypes.F90 | 14 +++++- .../engine/bp/TestBPAccuracyDefaults.cpp | 4 +- testing/adios2/engine/bp/TestBPBufferSize.cpp | 11 +++-- .../adios2/engine/bp/TestBPChangingShape.cpp | 3 +- .../bp/TestBPChangingShapeWithinStep.cpp | 3 +- testing/adios2/engine/bp/TestBPDirectIO.cpp | 4 +- .../bp/TestBPFStreamWriteReadHighLevelAPI.cpp | 16 ++++-- .../engine/bp/TestBPFortranToCppReader.cpp | 4 +- .../engine/bp/TestBPFortranToCppWriter.F90 | 5 ++ .../adios2/engine/bp/TestBPInquireDefine.cpp | 4 +- .../bp/TestBPInquireVariableException.cpp | 4 +- .../adios2/engine/bp/TestBPLargeMetadata.cpp | 7 ++- .../engine/bp/TestBPTimeAggregation.cpp | 8 ++- .../engine/bp/TestBPWriteAppendReadADIOS2.cpp | 10 ++-- .../adios2/engine/bp/TestBPWriteFlushRead.cpp | 43 ++++++++++++++-- .../bp/TestBPWriteMemorySelectionRead.cpp | 12 +++-- .../engine/bp/TestBPWriteMultiblockRead.cpp | 16 ++++-- testing/adios2/engine/bp/TestBPWriteNull.cpp | 8 ++- .../engine/bp/TestBPWriteProfilingJSON.cpp | 4 +- .../engine/bp/TestBPWriteReadADIOS2.cpp | 40 +++++++++++---- .../bp/TestBPWriteReadADIOS2fstream.cpp | 20 ++++++-- .../engine/bp/TestBPWriteReadADIOS2stdio.cpp | 20 ++++++-- .../bp/TestBPWriteReadAsStreamADIOS2.cpp | 19 +++++-- .../TestBPWriteReadAsStreamADIOS2_Threads.cpp | 12 +++-- .../engine/bp/TestBPWriteReadAttributes.cpp | 40 ++++++++++----- .../adios2/engine/bp/TestBPWriteReadCuda.cpp | 4 +- .../bp/TestBPWriteReadLocalVariables.cpp | 25 +++++++--- .../bp/TestBPWriteReadLocalVariablesSel.cpp | 16 ++++-- ...tBPWriteReadLocalVariablesSelHighLevel.cpp | 12 +++-- .../engine/bp/TestBPWriteReadMultiblock.cpp | 16 ++++-- .../engine/bp/TestBPWriteReadVariableSpan.cpp | 24 ++++++--- .../engine/bp/TestBPWriteReadVector.cpp | 16 ++++-- .../bp/operations/TestBPWriteReadBZIP2.cpp | 28 ++++++++--- .../bp/operations/TestBPWriteReadBlosc.cpp | 42 +++++++++++----- .../bp/operations/TestBPWriteReadBlosc2.cpp | 49 +++++++++++++------ .../TestBPWriteReadLocalVariables.cpp | 20 ++++++-- .../bp/operations/TestBPWriteReadMGARD.cpp | 28 ++++++++--- .../operations/TestBPWriteReadMGARDCuda.cpp | 8 ++- .../bp/operations/TestBPWriteReadMGARDMDR.cpp | 4 +- .../operations/TestBPWriteReadMGARDPlus.cpp | 28 ++++++++--- .../bp/operations/TestBPWriteReadPNG.cpp | 8 ++- .../bp/operations/TestBPWriteReadSZ.cpp | 28 ++++++++--- .../operations/TestBPWriteReadSzComplex.cpp | 1 + .../bp/operations/TestBPWriteReadZfp.cpp | 28 ++++++++--- .../operations/TestBPWriteReadZfpConfig.cpp | 28 ++++++++--- .../bp/operations/TestBPWriteReadZfpCuda.cpp | 4 +- .../TestBPWriteReadZfpRemoveOperations.cpp | 12 +++-- .../engine/hdf5/TestHDF5WriteReadAsStream.cpp | 20 +++++--- .../TestHDF5WriteReadAttributesADIOS2.cpp | 33 ++++++++----- .../hierarchy/TestBPHierarchicalReading.cpp | 6 ++- .../performance/manyvars/TestManyVars.cpp | 5 ++ .../adios2/performance/query/TestBPQuery.cpp | 7 ++- testing/h5vol/TestH5VolWriteReadBPFile.cpp | 11 ++++- 59 files changed, 670 insertions(+), 216 deletions(-) diff --git a/testing/adios2/bindings/C/TestBPAvailableVariablesAttribites.cpp b/testing/adios2/bindings/C/TestBPAvailableVariablesAttribites.cpp index c4ad9f62bd..8399de3c50 100644 --- a/testing/adios2/bindings/C/TestBPAvailableVariablesAttribites.cpp +++ b/testing/adios2/bindings/C/TestBPAvailableVariablesAttribites.cpp @@ -99,8 +99,11 @@ TEST_F(BPAvailableVariablesAttributes, AvailableVariablesAttributes) adios2_variable *varR64 = adios2_define_variable( ioH, "varR64", adios2_type_double, 1, shape, start, count, adios2_constant_dims_false); +#if ADIOS2_USE_MPI + adios2_engine *engineH = adios2_open(ioH, "available_MPI.bp", adios2_mode_write); +#else adios2_engine *engineH = adios2_open(ioH, "available.bp", adios2_mode_write); - +#endif for (size_t i = 0; i < steps; ++i) { adios2_begin_step(engineH, adios2_step_mode_append, -1., &status); @@ -177,8 +180,11 @@ TEST_F(BPAvailableVariablesAttributes, AvailableVariablesAttributes) std::vector inR64(data_Nx / 2); adios2_io *ioH = adios2_declare_io(adiosH, "Reader"); +#if ADIOS2_USE_MPI + adios2_engine *engineH = adios2_open(ioH, "available_MPI.bp", adios2_mode_read); +#else adios2_engine *engineH = adios2_open(ioH, "available.bp", adios2_mode_read); - +#endif size_t nsteps; adios2_steps(&nsteps, engineH); EXPECT_EQ(nsteps, steps); diff --git a/testing/adios2/bindings/C/TestBPMemorySpace.cpp b/testing/adios2/bindings/C/TestBPMemorySpace.cpp index 65c2381969..d947ccad2d 100644 --- a/testing/adios2/bindings/C/TestBPMemorySpace.cpp +++ b/testing/adios2/bindings/C/TestBPMemorySpace.cpp @@ -52,7 +52,11 @@ TEST_F(ADIOS2_C_API, ADIOS2BPMemorySpaceGPU) TEST_F(ADIOS2_C_API, ADIOS2BPMemorySpaceShape) { +#if ADIOS2_USE_MPI + const char fname[] = "ADIOS2_C_API.ADIOS2BPMemorySpace_MPI.bp"; +#else const char fname[] = "ADIOS2_C_API.ADIOS2BPMemorySpace.bp"; +#endif // write { adios2_io *ioH = adios2_declare_io(adiosH, "CMemSpace"); diff --git a/testing/adios2/bindings/C/TestBPWriteAggregateReadLocal.cpp b/testing/adios2/bindings/C/TestBPWriteAggregateReadLocal.cpp index 941d70e5cb..e8d616405e 100644 --- a/testing/adios2/bindings/C/TestBPWriteAggregateReadLocal.cpp +++ b/testing/adios2/bindings/C/TestBPWriteAggregateReadLocal.cpp @@ -23,8 +23,11 @@ void LocalAggregate1D(const std::string substreams) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array +#if ADIOS2_USE_MPI + const std::string fname("LocalAggregate1D_" + substreams + "_MPI.bp"); +#else const std::string fname("LocalAggregate1D_" + substreams + ".bp"); - +#endif int mpiRank = 0, mpiSize = 1; // Number of steps constexpr size_t NSteps = 5; @@ -151,8 +154,11 @@ void LocalAggregate1DBlock0(const std::string substreams) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array +#if ADIOS2_USE_MPI + const std::string fname("LocalAggregate1DSubFile_" + substreams + "_MPI.bp"); +#else const std::string fname("LocalAggregate1DSubFile_" + substreams + ".bp"); - +#endif int mpiRank = 0, mpiSize = 1; // Number of steps constexpr size_t NSteps = 5; diff --git a/testing/adios2/bindings/C/TestBPWriteReadMultiblock.cpp b/testing/adios2/bindings/C/TestBPWriteReadMultiblock.cpp index 439790103b..4f34f971f2 100644 --- a/testing/adios2/bindings/C/TestBPWriteReadMultiblock.cpp +++ b/testing/adios2/bindings/C/TestBPWriteReadMultiblock.cpp @@ -94,7 +94,11 @@ TEST_F(BPWriteReadMultiblockCC, ZeroSizeBlocks) adios2_variable *varR64 = adios2_define_variable( ioH, "varR64", adios2_type_double, 1, shape, start, count, adios2_constant_dims_false); +#if ADIOS2_USE_MPI + adios2_engine *engineH = adios2_open(ioH, "cmblocks_MPI.bp", adios2_mode_write); +#else adios2_engine *engineH = adios2_open(ioH, "cmblocks.bp", adios2_mode_write); +#endif for (size_t i = 0; i < steps; ++i) { @@ -172,8 +176,11 @@ TEST_F(BPWriteReadMultiblockCC, ZeroSizeBlocks) std::vector inR64(data_Nx / 2); adios2_io *ioH = adios2_declare_io(adiosH, "Reader"); +#if ADIOS2_USE_MPI + adios2_engine *engineH = adios2_open(ioH, "cmblocks_MPI.bp", adios2_mode_read); +#else adios2_engine *engineH = adios2_open(ioH, "cmblocks.bp", adios2_mode_read); - +#endif size_t nsteps; adios2_steps(&nsteps, engineH); EXPECT_EQ(nsteps, steps); diff --git a/testing/adios2/bindings/C/TestBPWriteTypes.cpp b/testing/adios2/bindings/C/TestBPWriteTypes.cpp index 94b1214b51..c3b6e4b872 100644 --- a/testing/adios2/bindings/C/TestBPWriteTypes.cpp +++ b/testing/adios2/bindings/C/TestBPWriteTypes.cpp @@ -45,7 +45,11 @@ class ADIOS2_C_API : public ::testing::Test TEST_F(ADIOS2_C_API, ADIOS2BPWriteTypes) { +#if ADIOS2_USE_MPI + const char fname[] = "ADIOS2_C_API.ADIOS2BPWriteTypes_MPI.bp"; +#else const char fname[] = "ADIOS2_C_API.ADIOS2BPWriteTypes.bp"; +#endif // write { // IO @@ -408,7 +412,11 @@ std::string adios2_engine_name_as_string(adios2_engine *engineH) TEST_F(ADIOS2_C_API_IO, Engine) { +#if ADIOS2_USE_MPI + const char fname[] = "ADIOS2_C_API_IO.engine_MPI.bp"; +#else const char fname[] = "ADIOS2_C_API_IO.engine.bp"; +#endif int ierr; ierr = adios2_set_engine(ioH, "bpfile"); diff --git a/testing/adios2/bindings/C/TestNullWriteRead.cpp b/testing/adios2/bindings/C/TestNullWriteRead.cpp index 3ea6322da1..dcf9d0708e 100644 --- a/testing/adios2/bindings/C/TestNullWriteRead.cpp +++ b/testing/adios2/bindings/C/TestNullWriteRead.cpp @@ -27,7 +27,6 @@ TEST_F(NullWriteReadTests_C_API, NullWriteRead1D8) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("NullWriteRead1D8_c.bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -45,8 +44,10 @@ TEST_F(NullWriteReadTests_C_API, NullWriteRead1D8) #if ADIOS2_USE_MPI adios2_adios *adios = adios2_init_mpi(MPI_COMM_WORLD); + const std::string fname("NullWriteRead1D8_c_MPI.bp"); #else adios2_adios *adios = adios2_init_serial(); + const std::string fname("NullWriteRead1D8_c.bp"); #endif { adios2_io *io = adios2_declare_io(adios, "WriteNull"); diff --git a/testing/adios2/bindings/fortran/TestBPWriteTypes.F90 b/testing/adios2/bindings/fortran/TestBPWriteTypes.F90 index 2145baedf8..d5086631ff 100644 --- a/testing/adios2/bindings/fortran/TestBPWriteTypes.F90 +++ b/testing/adios2/bindings/fortran/TestBPWriteTypes.F90 @@ -240,13 +240,20 @@ program TestBPWriteTypes stop 1 end if +#if ADIOS2_USE_MPI + call adios2_open(bpWriter, ioWrite, "ftypes_mpi.bp", adios2_mode_write, ierr) +#else call adios2_open(bpWriter, ioWrite, "ftypes.bp", adios2_mode_write, ierr) - +#endif if( bpWriter%valid .eqv. .false. ) then write(*,*) 'Invalid adios2_engine post-open' stop 1 end if +#if ADIOS2_USE_MPI + if( TRIM(bpWriter%name) /= "ftypes_mpi.bp") then +#else if( TRIM(bpWriter%name) /= "ftypes.bp") then +#endif write(*,*) 'Invalid adios2_engine name' stop 1 end if @@ -307,8 +314,11 @@ program TestBPWriteTypes ! Declare io reader call adios2_declare_io(ioRead, adios, "ioRead", ierr) ! Open bpReader engine +#if ADIOS2_USE_MPI + call adios2_open(bpReader, ioRead, "ftypes_mpi.bp", adios2_mode_readRandomAccess, ierr) +#else call adios2_open(bpReader, ioRead, "ftypes.bp", adios2_mode_readRandomAccess, ierr) - +#endif call adios2_steps(nsteps, bpReader, ierr) if(nsteps /= 3) then write(*,*) 'ftypes.bp must have 3 steps' diff --git a/testing/adios2/engine/bp/TestBPAccuracyDefaults.cpp b/testing/adios2/engine/bp/TestBPAccuracyDefaults.cpp index 96eb8e61d1..ea8562ea27 100644 --- a/testing/adios2/engine/bp/TestBPAccuracyDefaults.cpp +++ b/testing/adios2/engine/bp/TestBPAccuracyDefaults.cpp @@ -31,7 +31,6 @@ class AccuracyTests : public ::testing::Test // Check if SetAccuracy/GetAccuracy default behavior works TEST_F(AccuracyTests, DefaultAccuracy) { - const std::string fname("DefaultAccuracy.bp"); int mpiRank = 0, mpiSize = 1; @@ -40,6 +39,9 @@ TEST_F(AccuracyTests, DefaultAccuracy) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("DefaultAccuracy_MPI.bp"); +#else + const std::string fname("DefaultAccuracy.bp"); #endif std::vector localData(Nx); diff --git a/testing/adios2/engine/bp/TestBPBufferSize.cpp b/testing/adios2/engine/bp/TestBPBufferSize.cpp index 82764552f5..9127a22f53 100644 --- a/testing/adios2/engine/bp/TestBPBufferSize.cpp +++ b/testing/adios2/engine/bp/TestBPBufferSize.cpp @@ -79,9 +79,6 @@ size_t GetAndPrintBufferSize(adios2::Engine &engine, const std::string &info, // Put(Sync) and Put(Deferred) should have the same buffer consumption TEST_F(BPBufferSizeTest, SyncDeferredIdenticalUsage) { - std::string fnameSync = "ADIOS2BPBufferSizeSync.bp"; - std::string fnameDeferred = "ADIOS2BPBufferSizeDeferred.bp"; - std::string fnameDeferredPP = "ADIOS2BPBufferSizeDeferredPP.bp"; int mpiRank = 0, mpiSize = 1; // Number of rows const std::size_t Nx = 10485760; // 10M elements, 80MB variable @@ -94,6 +91,14 @@ TEST_F(BPBufferSizeTest, SyncDeferredIdenticalUsage) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + std::string fnameSync = "ADIOS2BPBufferSizeSync_MPI.bp"; + std::string fnameDeferred = "ADIOS2BPBufferSizeDeferred_MPI.bp"; + std::string fnameDeferredPP = "ADIOS2BPBufferSizeDeferredPP_MPI.bp"; +#else + std::string fnameSync = "ADIOS2BPBufferSizeSync.bp"; + std::string fnameDeferred = "ADIOS2BPBufferSizeDeferred.bp"; + std::string fnameDeferredPP = "ADIOS2BPBufferSizeDeferredPP.bp"; + #endif // Write test data using BP diff --git a/testing/adios2/engine/bp/TestBPChangingShape.cpp b/testing/adios2/engine/bp/TestBPChangingShape.cpp index 9beceadb8b..6fdadb6248 100644 --- a/testing/adios2/engine/bp/TestBPChangingShape.cpp +++ b/testing/adios2/engine/bp/TestBPChangingShape.cpp @@ -31,15 +31,16 @@ TEST_F(BPChangingShape, BPWriteReadShape2D) // Each process would write a 4x2 array and all processes would // form a 2D 4 * (NumberOfProcess * Nx) matrix where Nx is 2 here - const std::string fname("BPChangingShape.bp"); const int nsteps = 10; int rank = 0, nproc = 1; #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &nproc); + const std::string fname("BPChangingShape_MPI.bp"); adios2::ADIOS adios(MPI_COMM_WORLD); #else + const std::string fname("BPChangingShape.bp"); adios2::ADIOS adios; #endif // Writer diff --git a/testing/adios2/engine/bp/TestBPChangingShapeWithinStep.cpp b/testing/adios2/engine/bp/TestBPChangingShapeWithinStep.cpp index b8392cc92f..1ee438d0cb 100644 --- a/testing/adios2/engine/bp/TestBPChangingShapeWithinStep.cpp +++ b/testing/adios2/engine/bp/TestBPChangingShapeWithinStep.cpp @@ -38,7 +38,6 @@ TEST_P(BPChangingShapeWithinStep, MultiBlock) auto params = std::get<1>(GetParam()); double epsilon = std::get<2>(GetParam()); - const std::string fname("BPChangingShapeMultiblock_" + operatorName + ".bp"); const int nsteps = 2; const std::vector nblocks = {2, 3}; const int N = 16384; // size of one block (should be big enough to compress) @@ -48,8 +47,10 @@ TEST_P(BPChangingShapeWithinStep, MultiBlock) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &nproc); + const std::string fname("BPChangingShapeMultiblock_" + operatorName + "_MPI.bp"); adios2::ADIOS adios(MPI_COMM_WORLD); #else + const std::string fname("BPChangingShapeMultiblock_" + operatorName + ".bp"); adios2::ADIOS adios; #endif diff --git a/testing/adios2/engine/bp/TestBPDirectIO.cpp b/testing/adios2/engine/bp/TestBPDirectIO.cpp index 67e3884b5c..a0edb30d89 100644 --- a/testing/adios2/engine/bp/TestBPDirectIO.cpp +++ b/testing/adios2/engine/bp/TestBPDirectIO.cpp @@ -27,13 +27,15 @@ TEST_F(ADIOSReadDirectIOTest, BufferResize) and the last chunck is resized back. It should be properly aligned to not cause any problems at writing that chunk. */ - std::string filename = "ADIOSDirectIO.bp"; int mpiRank = 0, mpiSize = 1; #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + std::string filename = "ADIOSDirectIO_MPI.bp"; +#else + std::string filename = "ADIOSDirectIO.bp"; #endif // Write test data using BP diff --git a/testing/adios2/engine/bp/TestBPFStreamWriteReadHighLevelAPI.cpp b/testing/adios2/engine/bp/TestBPFStreamWriteReadHighLevelAPI.cpp index f3c3e2b6b2..6a6c71adb8 100644 --- a/testing/adios2/engine/bp/TestBPFStreamWriteReadHighLevelAPI.cpp +++ b/testing/adios2/engine/bp/TestBPFStreamWriteReadHighLevelAPI.cpp @@ -32,7 +32,6 @@ TEST_F(StreamWriteReadHighLevelAPI, ADIOS2BPWriteRead1D8) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("ADIOS2BPWriteRead1D8_hl.bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -44,6 +43,9 @@ TEST_F(StreamWriteReadHighLevelAPI, ADIOS2BPWriteRead1D8) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("ADIOS2BPWriteRead1D8_hl_MPI.bp"); +#else + const std::string fname("ADIOS2BPWriteRead1D8_hl.bp"); #endif // write test data using BP @@ -414,7 +416,6 @@ TEST_F(StreamWriteReadHighLevelAPI, ADIOS2BPwriteRead2D2x4) { // Each process would write a 2x4 array and all processes would // form a 2D 2 * (numberOfProcess*Nx) matrix where Nx is 4 here - const std::string fname("ADIOS2BPwriteRead2D2x4Test_hl.bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -429,6 +430,9 @@ TEST_F(StreamWriteReadHighLevelAPI, ADIOS2BPwriteRead2D2x4) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("ADIOS2BPwriteRead2D2x4Test_hl_MPI.bp"); +#else + const std::string fname("ADIOS2BPwriteRead2D2x4Test_hl.bp"); #endif // write test data using ADIOS2 @@ -534,7 +538,6 @@ TEST_F(StreamWriteReadHighLevelAPI, ADIOS2BPwriteRead2D4x2) { // Each process would write a 4x2 array and all processes would // form a 2D 4 * (NumberOfProcess * Nx) matrix where Nx is 2 here - const std::string fname("ADIOS2BPwriteRead2D4x2Test_hl.bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -548,6 +551,9 @@ TEST_F(StreamWriteReadHighLevelAPI, ADIOS2BPwriteRead2D4x2) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("ADIOS2BPwriteRead2D4x2Test_hl_MPI.bp"); +#else + const std::string fname("ADIOS2BPwriteRead2D4x2Test_hl.bp"); #endif // write test data using ADIOS2 @@ -652,16 +658,18 @@ TEST_F(StreamWriteReadHighLevelAPI, DoubleOpenException) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("ADIOS2BP_hl_exception.bp"); { #if ADIOS2_USE_MPI + const std::string fname("ADIOS2BP_hl_exception_MPI.bp"); adios2::fstream oStream(fname, adios2::fstream::out, MPI_COMM_WORLD, engineName); EXPECT_THROW(oStream.open("second", adios2::fstream::out, MPI_COMM_WORLD, engineName), std::invalid_argument); #else + const std::string fname("ADIOS2BP_hl_exception.bp"); + adios2::fstream oStream(fname, adios2::fstream::out); EXPECT_THROW(oStream.open("second", adios2::fstream::out, engineName), std::invalid_argument); diff --git a/testing/adios2/engine/bp/TestBPFortranToCppReader.cpp b/testing/adios2/engine/bp/TestBPFortranToCppReader.cpp index ec81cd9e0b..f79ed890cb 100644 --- a/testing/adios2/engine/bp/TestBPFortranToCppReader.cpp +++ b/testing/adios2/engine/bp/TestBPFortranToCppReader.cpp @@ -27,13 +27,15 @@ class BPFortranToCppRead : public ::testing::Test TEST_F(BPFortranToCppRead, ADIOS2BPFortranToCppRead) { - const std::string fname("FortranToCpp.bp"); int mpiRank = 0, mpiSize = 1; #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("FortranToCpp_MPI.bp"); +#else + const std::string fname("FortranToCpp.bp"); #endif #if ADIOS2_USE_MPI diff --git a/testing/adios2/engine/bp/TestBPFortranToCppWriter.F90 b/testing/adios2/engine/bp/TestBPFortranToCppWriter.F90 index 2fc0a46e9c..0c5de0df8e 100644 --- a/testing/adios2/engine/bp/TestBPFortranToCppWriter.F90 +++ b/testing/adios2/engine/bp/TestBPFortranToCppWriter.F90 @@ -76,8 +76,13 @@ end function iargc if (irank == 0) print *,"engine type :",trim(engine_type) +#if ADIOS2_USE_MPI + call adios2_open(bpWriter, ioWrite, "FortranToCpp_MPI.bp", & + adios2_mode_write, ierr) +#else call adios2_open(bpWriter, ioWrite, "FortranToCpp.bp", & adios2_mode_write, ierr) +#endif do s = 1, 3 call adios2_begin_step(bpWriter, ierr) diff --git a/testing/adios2/engine/bp/TestBPInquireDefine.cpp b/testing/adios2/engine/bp/TestBPInquireDefine.cpp index 05c827ef47..47c12df9f1 100644 --- a/testing/adios2/engine/bp/TestBPInquireDefine.cpp +++ b/testing/adios2/engine/bp/TestBPInquireDefine.cpp @@ -24,7 +24,6 @@ class ADIOSInquireDefineTest : public ::testing::Test TEST_F(ADIOSInquireDefineTest, Read) { - std::string filename = "ADIOSInquireDefine.bp"; // Number of steps const int32_t NSteps = 5; @@ -33,6 +32,9 @@ TEST_F(ADIOSInquireDefineTest, Read) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + std::string filename = "ADIOSInquireDefine_MPI.bp"; +#else + std::string filename = "ADIOSInquireDefine.bp"; #endif // Write test data using BP diff --git a/testing/adios2/engine/bp/TestBPInquireVariableException.cpp b/testing/adios2/engine/bp/TestBPInquireVariableException.cpp index c0d251f651..f798dae94d 100644 --- a/testing/adios2/engine/bp/TestBPInquireVariableException.cpp +++ b/testing/adios2/engine/bp/TestBPInquireVariableException.cpp @@ -18,7 +18,6 @@ class ADIOSInquireVariableException : public ::testing::Test TEST_F(ADIOSInquireVariableException, Read) { - std::string filename = "ADIOSInquireVariableException"; // Number of steps const std::size_t NSteps = 5; @@ -28,6 +27,9 @@ TEST_F(ADIOSInquireVariableException, Read) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &size); + std::string filename = "ADIOSInquireVariableException"; +#else + std::string filename = "ADIOSInquireVariableException"; #endif // Write test data using BP diff --git a/testing/adios2/engine/bp/TestBPLargeMetadata.cpp b/testing/adios2/engine/bp/TestBPLargeMetadata.cpp index e190d40c71..598ea2ecc9 100644 --- a/testing/adios2/engine/bp/TestBPLargeMetadata.cpp +++ b/testing/adios2/engine/bp/TestBPLargeMetadata.cpp @@ -34,7 +34,6 @@ TEST_F(BPLargeMetadata, BPWrite1D_LargeMetadata) { // Each process would write a 4x2 array and all processes would // form a 2D 4 * (NumberOfProcess * Nx) matrix where Nx is 2 here - const std::string fname("BPWrite1D_LargeMetadata.bp"); int mpiRank = 0, mpiSize = 1; @@ -45,6 +44,9 @@ TEST_F(BPLargeMetadata, BPWrite1D_LargeMetadata) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWrite1D_LargeMetadata_MPI.bp"); +#else + const std::string fname("BPWrite1D_LargeMetadata.bp"); #endif // Write test data using ADIOS2 @@ -99,7 +101,6 @@ TEST_F(BPLargeMetadata, BPWrite1D_LargeMetadata) TEST_F(BPLargeMetadata, ManyLongStrings) { - const std::string fname("BPWrite1D_LargeMetadataStrings.bp"); const std::string longString = "test_string " "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" @@ -110,8 +111,10 @@ TEST_F(BPLargeMetadata, ManyLongStrings) #if ADIOS2_USE_MPI adios2::ADIOS adios(MPI_COMM_WORLD); + const std::string fname("BPWrite1D_LargeMetadataStrings_MPI.bp"); #else adios2::ADIOS adios; + const std::string fname("BPWrite1D_LargeMetadataStrings.bp"); #endif adios2::IO io = adios.DeclareIO("myIO"); diff --git a/testing/adios2/engine/bp/TestBPTimeAggregation.cpp b/testing/adios2/engine/bp/TestBPTimeAggregation.cpp index e567b6ff1f..1d65e0569c 100644 --- a/testing/adios2/engine/bp/TestBPTimeAggregation.cpp +++ b/testing/adios2/engine/bp/TestBPTimeAggregation.cpp @@ -21,7 +21,6 @@ void TimeAggregation1D8(const std::string flushstepscount) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname = "BPTimeAggregation1D8_" + flushstepscount + ".bp"; int mpiRank = 0, mpiSize = 1; // Number of rows @@ -33,6 +32,9 @@ void TimeAggregation1D8(const std::string flushstepscount) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname = "BPTimeAggregation1D8_" + flushstepscount + "_MPI.bp"; +#else + const std::string fname = "BPTimeAggregation1D8_" + flushstepscount + ".bp"; #endif // Write test data using ADIOS2 @@ -343,7 +345,6 @@ void TimeAggregation2D4x2(const std::string flushstepscount) { // Each process would write a 2x4 array and all processes would // form a 2D 2 * (numberOfProcess*Nx) matrix where Nx is 4 here - const std::string fname = "BPTimeAggregation2D2x4_" + flushstepscount + ".bp"; int mpiRank = 0, mpiSize = 1; // Number of rows @@ -358,6 +359,9 @@ void TimeAggregation2D4x2(const std::string flushstepscount) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname = "BPTimeAggregation2D2x4_" + flushstepscount + "_MPI.bp"; +#else + const std::string fname = "BPTimeAggregation2D2x4_" + flushstepscount + ".bp"; #endif // Write test data using ADIOS2 diff --git a/testing/adios2/engine/bp/TestBPWriteAppendReadADIOS2.cpp b/testing/adios2/engine/bp/TestBPWriteAppendReadADIOS2.cpp index d6bf3776fa..6d3a9d0381 100644 --- a/testing/adios2/engine/bp/TestBPWriteAppendReadADIOS2.cpp +++ b/testing/adios2/engine/bp/TestBPWriteAppendReadADIOS2.cpp @@ -34,7 +34,6 @@ TEST_F(BPWriteAppendReadTestADIOS2, ADIOS2BPWriteAppendRead2D2x4) { // Each process would write a 2x4 array and all processes would // form a 2D 2 * (numberOfProcess*Nx) matrix where Nx is 4 here - const std::string fname("ADIOS2BPWriteAppendRead2D2x4Test.bp"); const std::string zero = std::to_string(0); const std::string s1_Single = std::string("s1_Single_") + zero; @@ -70,6 +69,9 @@ TEST_F(BPWriteAppendReadTestADIOS2, ADIOS2BPWriteAppendRead2D2x4) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("ADIOS2BPWriteAppendRead2D2x4Test_MPI.bp"); +#else + const std::string fname("ADIOS2BPWriteAppendRead2D2x4Test.bp"); #endif // Write test data using ADIOS2 @@ -643,7 +645,6 @@ TEST_F(BPWriteAppendReadTestADIOS2, ADIOS2BPWriteAppendRead2D2x4) // Write with append combined with aggregation, same aggregation ratio TEST_F(BPWriteAppendReadTestADIOS2, ADIOS2BPWriteAppendReadAggregate) { - const std::string fname("ADIOS2BPWriteAppendReadAggregate.bp"); int mpiRank = 0, mpiSize = 1; const std::size_t Nx = 4; const std::size_t Ny = 2; @@ -652,8 +653,10 @@ TEST_F(BPWriteAppendReadTestADIOS2, ADIOS2BPWriteAppendReadAggregate) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("ADIOS2BPWriteAppendReadAggregate_MPI.bp"); adios2::ADIOS adios(MPI_COMM_WORLD); #else + const std::string fname("ADIOS2BPWriteAppendReadAggregate.bp"); adios2::ADIOS adios; #endif { @@ -741,7 +744,6 @@ TEST_F(BPWriteAppendReadTestADIOS2, ADIOS2BPWriteAppendReadAggregate) // Write with append combined with aggregation, same aggregation ratio TEST_F(BPWriteAppendReadTestADIOS2, ADIOS2BPWriteAppendReadVaryingAggregation) { - const std::string fname("ADIOS2BPWriteAppendReadVaryingAggregate.bp"); int mpiRank = 0, mpiSize = 1; const std::size_t Nx = 4; const std::size_t Ny = 2; @@ -750,8 +752,10 @@ TEST_F(BPWriteAppendReadTestADIOS2, ADIOS2BPWriteAppendReadVaryingAggregation) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("ADIOS2BPWriteAppendReadVaryingAggregate_MPI.bp"); adios2::ADIOS adios(MPI_COMM_WORLD); #else + const std::string fname("ADIOS2BPWriteAppendReadVaryingAggregate.bp"); adios2::ADIOS adios; #endif { diff --git a/testing/adios2/engine/bp/TestBPWriteFlushRead.cpp b/testing/adios2/engine/bp/TestBPWriteFlushRead.cpp index 0a769fff0b..3d84552918 100644 --- a/testing/adios2/engine/bp/TestBPWriteFlushRead.cpp +++ b/testing/adios2/engine/bp/TestBPWriteFlushRead.cpp @@ -111,9 +111,13 @@ TEST_F(BPWriteFlushRead, ADIOS2BPWrite1D2D) io2D.DefineVariable("r64", shape, start, count, adios2::ConstantDims); } +#if ADIOS2_USE_MPI + adios2::Engine bpWriter1D = io1D.Open("Flush1D_MPI.bp", adios2::Mode::Write); + adios2::Engine bpWriter2D = io2D.Open("Flush2D_MPI.bp", adios2::Mode::Write); +#else adios2::Engine bpWriter1D = io1D.Open("Flush1D.bp", adios2::Mode::Write); adios2::Engine bpWriter2D = io2D.Open("Flush2D.bp", adios2::Mode::Write); - +#endif for (size_t step = 0; step < NSteps / 2; ++step) { SmallTestData currentTestData = @@ -160,8 +164,11 @@ TEST_F(BPWriteFlushRead, ADIOS2BPWrite1D2D) io.SetEngine(engineName); } +#if ADIOS2_USE_MPI + adios2::Engine bpReader = io.Open("Flush1D_MPI.bp", adios2::Mode::Read); +#else adios2::Engine bpReader = io.Open("Flush1D.bp", adios2::Mode::Read); - +#endif unsigned int t = 0; while (bpReader.BeginStep(adios2::StepMode::Read, 0.0) == adios2::StepStatus::OK) @@ -312,8 +319,11 @@ TEST_F(BPWriteFlushRead, ADIOS2BPWrite1D2D) io.SetEngine(engineName); } +#if ADIOS2_USE_MPI + adios2::Engine bpReader = io.Open("Flush2D_MPI.bp", adios2::Mode::Read); +#else adios2::Engine bpReader = io.Open("Flush2D.bp", adios2::Mode::Read); - +#endif unsigned int t = 0; while (bpReader.BeginStep(adios2::StepMode::Read, 0.0) == adios2::StepStatus::OK) @@ -553,8 +563,13 @@ TEST_F(BPWriteFlushRead, ADIOS2BPWrite1D2Dstdio) io2D.DefineVariable("r64", shape, start, count, adios2::ConstantDims); } +#if ADIOS2_USE_MPI + adios2::Engine bpWriter1D = io1D.Open("Flush1Dstdio_MPI.bp", adios2::Mode::Write); + adios2::Engine bpWriter2D = io2D.Open("Flush2Dstdio_MPI.bp", adios2::Mode::Write); +#else adios2::Engine bpWriter1D = io1D.Open("Flush1Dstdio.bp", adios2::Mode::Write); adios2::Engine bpWriter2D = io2D.Open("Flush2Dstdio.bp", adios2::Mode::Write); +#endif for (size_t step = 0; step < NSteps / 2; ++step) { @@ -602,8 +617,11 @@ TEST_F(BPWriteFlushRead, ADIOS2BPWrite1D2Dstdio) io.SetEngine(engineName); } +#if ADIOS2_USE_MPI + adios2::Engine bpReader = io.Open("Flush1Dstdio_MPI.bp", adios2::Mode::Read); +#else adios2::Engine bpReader = io.Open("Flush1Dstdio.bp", adios2::Mode::Read); - +#endif unsigned int t = 0; while (bpReader.BeginStep(adios2::StepMode::Read, 0.0) == adios2::StepStatus::OK) @@ -754,7 +772,11 @@ TEST_F(BPWriteFlushRead, ADIOS2BPWrite1D2Dstdio) io.SetEngine(engineName); } +#if ADIOS2_USE_MPI + adios2::Engine bpReader = io.Open("Flush2Dstdio_MPI.bp", adios2::Mode::Read); +#else adios2::Engine bpReader = io.Open("Flush2Dstdio.bp", adios2::Mode::Read); +#endif unsigned int t = 0; @@ -995,8 +1017,13 @@ TEST_F(BPWriteFlushRead, ADIOS2BPWrite1D2Dfstream) io2D.DefineVariable("r64", shape, start, count, adios2::ConstantDims); } +#if ADIOS2_USE_MPI + adios2::Engine bpWriter1D = io1D.Open("Flush1Dfstream_MPI.bp", adios2::Mode::Write); + adios2::Engine bpWriter2D = io2D.Open("Flush2Dfstream_MPI.bp", adios2::Mode::Write); +#else adios2::Engine bpWriter1D = io1D.Open("Flush1Dfstream.bp", adios2::Mode::Write); adios2::Engine bpWriter2D = io2D.Open("Flush2Dfstream.bp", adios2::Mode::Write); +#endif for (size_t step = 0; step < NSteps / 2; ++step) { @@ -1044,7 +1071,11 @@ TEST_F(BPWriteFlushRead, ADIOS2BPWrite1D2Dfstream) io.SetEngine(engineName); } +#if ADIOS2_USE_MPI + adios2::Engine bpReader = io.Open("Flush1Dfstream_MPI.bp", adios2::Mode::Read); +#else adios2::Engine bpReader = io.Open("Flush1Dfstream.bp", adios2::Mode::Read); +#endif unsigned int t = 0; @@ -1196,7 +1227,11 @@ TEST_F(BPWriteFlushRead, ADIOS2BPWrite1D2Dfstream) io.SetEngine(engineName); } +#if ADIOS2_USE_MPI + adios2::Engine bpReader = io.Open("Flush2Dfstream_MPI.bp", adios2::Mode::Read); +#else adios2::Engine bpReader = io.Open("Flush2Dfstream.bp", adios2::Mode::Read); +#endif unsigned int t = 0; diff --git a/testing/adios2/engine/bp/TestBPWriteMemorySelectionRead.cpp b/testing/adios2/engine/bp/TestBPWriteMemorySelectionRead.cpp index aa8d516b21..fa70fd610e 100644 --- a/testing/adios2/engine/bp/TestBPWriteMemorySelectionRead.cpp +++ b/testing/adios2/engine/bp/TestBPWriteMemorySelectionRead.cpp @@ -174,7 +174,6 @@ MPI_Comm testComm; void BPSteps1D(const size_t ghostCells) { - const std::string fname("BPSteps1D_" + std::to_string(ghostCells)); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -186,6 +185,9 @@ void BPSteps1D(const size_t ghostCells) #if ADIOS2_USE_MPI MPI_Comm_rank(testComm, &mpiRank); MPI_Comm_size(testComm, &mpiSize); + const std::string fname("BPSteps1D_" + std::to_string(ghostCells) + "_MPI"); +#else + const std::string fname("BPSteps1D_" + std::to_string(ghostCells)); #endif #if ADIOS2_USE_MPI @@ -386,7 +388,6 @@ void BPSteps1D(const size_t ghostCells) void BPSteps2D4x2(const size_t ghostCells) { - const std::string fname("BPSteps2D4x2_" + std::to_string(ghostCells)); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -402,6 +403,9 @@ void BPSteps2D4x2(const size_t ghostCells) #if ADIOS2_USE_MPI MPI_Comm_rank(testComm, &mpiRank); MPI_Comm_size(testComm, &mpiSize); + const std::string fname("BPSteps2D4x2_" + std::to_string(ghostCells) + "_MPI"); +#else + const std::string fname("BPSteps2D4x2_" + std::to_string(ghostCells)); #endif #if ADIOS2_USE_MPI @@ -612,7 +616,6 @@ void BPSteps2D4x2(const size_t ghostCells) void BPSteps3D8x2x4(const size_t ghostCells) { - const std::string fname("BPSteps3D8x2x4_" + std::to_string(ghostCells)); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -630,6 +633,9 @@ void BPSteps3D8x2x4(const size_t ghostCells) #if ADIOS2_USE_MPI MPI_Comm_rank(testComm, &mpiRank); MPI_Comm_size(testComm, &mpiSize); + const std::string fname("BPSteps3D8x2x4_" + std::to_string(ghostCells) + "_MPI"); +#else + const std::string fname("BPSteps3D8x2x4_" + std::to_string(ghostCells)); #endif #if ADIOS2_USE_MPI diff --git a/testing/adios2/engine/bp/TestBPWriteMultiblockRead.cpp b/testing/adios2/engine/bp/TestBPWriteMultiblockRead.cpp index 4edc0b8566..79b325bdfc 100644 --- a/testing/adios2/engine/bp/TestBPWriteMultiblockRead.cpp +++ b/testing/adios2/engine/bp/TestBPWriteMultiblockRead.cpp @@ -32,7 +32,6 @@ TEST_F(BPWriteMultiblockReadTest, ADIOS2BPWriteMultiblockRead1D8) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("ADIOS2BPWriteMultiblockRead1D8.bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -44,6 +43,9 @@ TEST_F(BPWriteMultiblockReadTest, ADIOS2BPWriteMultiblockRead1D8) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("ADIOS2BPWriteMultiblockRead1D8_MPI.bp"); +#else + const std::string fname("ADIOS2BPWriteMultiblockRead1D8.bp"); #endif // Write test data using BP @@ -369,7 +371,6 @@ TEST_F(BPWriteMultiblockReadTest, ADIOS2BPWriteMultiblockRead2D2x4) { // Each process would write a 2x4 array and all processes would // form a 2D 2 * (numberOfProcess*Nx) matrix where Nx is 4 here - const std::string fname("ADIOS2BPWriteMultiblockRead2D2x4Test.bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -384,6 +385,9 @@ TEST_F(BPWriteMultiblockReadTest, ADIOS2BPWriteMultiblockRead2D2x4) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("ADIOS2BPWriteMultiblockRead2D2x4Test_MPI.bp"); +#else + const std::string fname("ADIOS2BPWriteMultiblockRead2D2x4Test.bp"); #endif // Write test data using ADIOS2 @@ -711,7 +715,6 @@ TEST_F(BPWriteMultiblockReadTest, ADIOS2BPWriteMultiblockRead2D4x2) { // Each process would write a 4x2 array and all processes would // form a 2D 4 * (NumberOfProcess * Nx) matrix where Nx is 2 here - const std::string fname("ADIOS2BPWriteMultiblockRead2D4x2Test.bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -725,6 +728,9 @@ TEST_F(BPWriteMultiblockReadTest, ADIOS2BPWriteMultiblockRead2D4x2) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("ADIOS2BPWriteMultiblockRead2D4x2Test_MPI.bp"); +#else + const std::string fname("ADIOS2BPWriteMultiblockRead2D4x2Test.bp"); #endif // Write test data using ADIOS2 @@ -1039,7 +1045,6 @@ TEST_F(BPWriteMultiblockReadTest, ADIOS2BPWriteRead1D8ZeroBlock) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("ADIOS2BPWriteRead1DZeroBlock.bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -1051,6 +1056,9 @@ TEST_F(BPWriteMultiblockReadTest, ADIOS2BPWriteRead1D8ZeroBlock) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("ADIOS2BPWriteRead1DZeroBlock_MPI.bp"); +#else + const std::string fname("ADIOS2BPWriteRead1DZeroBlock.bp"); #endif // Write test data using BP diff --git a/testing/adios2/engine/bp/TestBPWriteNull.cpp b/testing/adios2/engine/bp/TestBPWriteNull.cpp index 43f16a59ac..ab20533216 100644 --- a/testing/adios2/engine/bp/TestBPWriteNull.cpp +++ b/testing/adios2/engine/bp/TestBPWriteNull.cpp @@ -33,7 +33,6 @@ TEST_F(BPWriteNullTest, BPWrite1D1x8) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("BPWriteNull.bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -45,6 +44,9 @@ TEST_F(BPWriteNullTest, BPWrite1D1x8) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWriteNull_MPI.bp"); +#else + const std::string fname("BPWriteNull.bp"); #endif // Write test data using BP @@ -317,7 +319,6 @@ TEST_F(BPWriteNullTest, BPWrite2D4x2) { // Each process would write a 4x2 array and all processes would // form a 2D 4 * (NumberOfProcess * Nx) matrix where Nx is 2 here - const std::string fname("BPWrite2D4x2TestNull.bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -331,6 +332,9 @@ TEST_F(BPWriteNullTest, BPWrite2D4x2) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWrite2D4x2TestNull_MPI.bp"); +#else + const std::string fname("BPWrite2D4x2TestNull.bp"); #endif // Write test data using ADIOS2 diff --git a/testing/adios2/engine/bp/TestBPWriteProfilingJSON.cpp b/testing/adios2/engine/bp/TestBPWriteProfilingJSON.cpp index 1c2ef91ee3..959106b3b1 100644 --- a/testing/adios2/engine/bp/TestBPWriteProfilingJSON.cpp +++ b/testing/adios2/engine/bp/TestBPWriteProfilingJSON.cpp @@ -43,7 +43,6 @@ TEST_F(BPWriteProfilingJSONTest, DISABLED_ADIOS2BPWriteProfilingJSON) { // Use a relative path + file name to test path in file name capability std::string fname; - fname = "foo/ADIOS2BPWriteProfilingJSON.bp"; int mpiRank = 0, mpiSize = 1; // Number of rows @@ -55,6 +54,9 @@ TEST_F(BPWriteProfilingJSONTest, DISABLED_ADIOS2BPWriteProfilingJSON) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + fname = "foo/ADIOS2BPWriteProfilingJSON_MPI.bp"; +#else + fname = "foo/ADIOS2BPWriteProfilingJSON.bp"; #endif // Write test data and profiling.json using ADIOS2 diff --git a/testing/adios2/engine/bp/TestBPWriteReadADIOS2.cpp b/testing/adios2/engine/bp/TestBPWriteReadADIOS2.cpp index ef44278fdb..5bcb64af4b 100644 --- a/testing/adios2/engine/bp/TestBPWriteReadADIOS2.cpp +++ b/testing/adios2/engine/bp/TestBPWriteReadADIOS2.cpp @@ -35,7 +35,6 @@ TEST_F(BPWriteReadTestADIOS2, ADIOS2BPWriteRead1D8) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("ADIOS2BPWriteRead1D8.bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -47,6 +46,9 @@ TEST_F(BPWriteReadTestADIOS2, ADIOS2BPWriteRead1D8) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("ADIOS2BPWriteRead1D8_MPI.bp"); +#else + const std::string fname("ADIOS2BPWriteRead1D8.bp"); #endif // Write test data using BP @@ -419,7 +421,6 @@ TEST_F(BPWriteReadTestADIOS2, ADIOS2BPWriteRead2D2x4) { // Each process would write a 2x4 array and all processes would // form a 2D 2 * (numberOfProcess*Nx) matrix where Nx is 4 here - const std::string fname("ADIOS2BPWriteRead2D2x4Test.bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -434,6 +435,9 @@ TEST_F(BPWriteReadTestADIOS2, ADIOS2BPWriteRead2D2x4) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("ADIOS2BPWriteRead2D2x4Test_MPI.bp"); +#else + const std::string fname("ADIOS2BPWriteRead2D2x4Test.bp"); #endif // Write test data using ADIOS2 @@ -739,7 +743,6 @@ TEST_F(BPWriteReadTestADIOS2, ADIOS2BPWriteRead2D4x2) { // Each process would write a 4x2 array and all processes would // form a 2D 4 * (NumberOfProcess * Nx) matrix where Nx is 2 here - const std::string fname("ADIOS2BPWriteRead2D4x2Test.bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -753,6 +756,9 @@ TEST_F(BPWriteReadTestADIOS2, ADIOS2BPWriteRead2D4x2) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("ADIOS2BPWriteRead2D4x2Test_MPI.bp"); +#else + const std::string fname("ADIOS2BPWriteRead2D4x2Test.bp"); #endif // Write test data using ADIOS2 @@ -1045,7 +1051,6 @@ TEST_F(BPWriteReadTestADIOS2, ADIOS2BPWriteRead10D2x2) { // Each process would write a 2x2x...x2 9D array and all processes would // form a 10D NumberOfProcess x 2 x ... x 2) array - const std::string fname("ADIOS2BPWriteRead10D2x2Test.bp"); int mpiRank = 0, mpiSize = 1; // Number of steps @@ -1054,6 +1059,9 @@ TEST_F(BPWriteReadTestADIOS2, ADIOS2BPWriteRead10D2x2) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("ADIOS2BPWriteRead10D2x2Test_MPI.bp"); +#else + const std::string fname("ADIOS2BPWriteRead10D2x2Test.bp"); #endif size_t NX = static_cast(mpiSize); @@ -1227,7 +1235,6 @@ TEST_F(BPWriteReadTestADIOS2, ADIOS2BPWriteRead2D4x2_ReadMultiSteps) { // Each process would write a 4x2 array and all processes would // form a 2D 4 * (NumberOfProcess * Nx) matrix where Nx is 2 here - const std::string fname("ADIOS2BPWriteRead2D4x2Test_ReadMultiSteps.bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -1242,6 +1249,9 @@ TEST_F(BPWriteReadTestADIOS2, ADIOS2BPWriteRead2D4x2_ReadMultiSteps) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("ADIOS2BPWriteRead2D4x2Test_ReadMultiSteps_MPI.bp"); +#else + const std::string fname("ADIOS2BPWriteRead2D4x2Test_ReadMultiSteps.bp"); #endif // Write test data using ADIOS2 @@ -1534,7 +1544,6 @@ TEST_F(BPWriteReadTestADIOS2, ADIOS2BPWriteRead2D4x2_MultiStepsOverflow) { // Each process would write a 4x2 array and all processes would // form a 2D 4 * (NumberOfProcess * Nx) matrix where Nx is 2 here - const std::string fname("ADIOS2BPWriteRead2D4x2Test_Overflow.bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -1549,6 +1558,9 @@ TEST_F(BPWriteReadTestADIOS2, ADIOS2BPWriteRead2D4x2_MultiStepsOverflow) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("ADIOS2BPWriteRead2D4x2Test_Overflow_MPI.bp"); +#else + const std::string fname("ADIOS2BPWriteRead2D4x2Test_Overflow.bp"); #endif // Write test data using ADIOS2 @@ -1790,7 +1802,6 @@ TEST_F(BPWriteReadTestADIOS2, ReadStartCount) { // Each process would write a 4x2 array and all processes would // form a 2D 4 * (NumberOfProcess * Nx) matrix where Nx is 2 here - const std::string fname("ReadStartCount.bp"); int mpiRank = 0, mpiSize = 1; @@ -1799,6 +1810,9 @@ TEST_F(BPWriteReadTestADIOS2, ReadStartCount) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("ReadStartCount_MPI.bp"); +#else + const std::string fname("ReadStartCount.bp"); #endif std::vector localData(Nx); @@ -2027,7 +2041,6 @@ TEST_F(BPWriteReadTestADIOS2, ADIOS2BPWriteReadEmptyProcess) TEST_F(BPWriteReadTestADIOS2, GetDeferredInClose) { // Test if Get() will retrieve data in Close() - const std::string fname("GetDeferredInClose.bp"); int mpiRank = 0, mpiSize = 1; @@ -2036,6 +2049,9 @@ TEST_F(BPWriteReadTestADIOS2, GetDeferredInClose) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("GetDeferredInClose_MPI.bp"); +#else + const std::string fname("GetDeferredInClose.bp"); #endif std::vector localData(Nx); @@ -2095,7 +2111,6 @@ TEST_F(BPWriteReadTestADIOS2, GetDeferredInClose) TEST_F(BPWriteReadTestADIOS2, GetDeferredInEndStep) { // Test if Get() will retrieve data in EndStep() - const std::string fname("GetDeferredInEndStep.bp"); int mpiRank = 0, mpiSize = 1; @@ -2104,6 +2119,9 @@ TEST_F(BPWriteReadTestADIOS2, GetDeferredInEndStep) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("GetDeferredInEndStep_MPI.bp"); +#else + const std::string fname("GetDeferredInEndStep.bp"); #endif std::vector localData(Nx); @@ -2164,7 +2182,6 @@ TEST_F(BPWriteReadTestADIOS2, GetDeferredInEndStep) TEST_F(BPWriteReadTestADIOS2, GetDeferredWithoutEndStep) { // Test if Get() will retrieve data in Close() when EndStep() is not called - const std::string fname("GetDeferredWithoutEndStep.bp"); int mpiRank = 0, mpiSize = 1; @@ -2173,6 +2190,9 @@ TEST_F(BPWriteReadTestADIOS2, GetDeferredWithoutEndStep) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("GetDeferredWithoutEndStep_MPI.bp"); +#else + const std::string fname("GetDeferredWithoutEndStep.bp"); #endif std::vector localData(Nx); diff --git a/testing/adios2/engine/bp/TestBPWriteReadADIOS2fstream.cpp b/testing/adios2/engine/bp/TestBPWriteReadADIOS2fstream.cpp index 9eaab05c5a..db8116021e 100644 --- a/testing/adios2/engine/bp/TestBPWriteReadADIOS2fstream.cpp +++ b/testing/adios2/engine/bp/TestBPWriteReadADIOS2fstream.cpp @@ -33,7 +33,6 @@ TEST_F(BPWriteReadTestADIOS2fstream, ADIOS2BPWriteRead1D8) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("ADIOS2BPWriteRead1D8fstream.bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -45,6 +44,9 @@ TEST_F(BPWriteReadTestADIOS2fstream, ADIOS2BPWriteRead1D8) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("ADIOS2BPWriteRead1D8fstream_MPI.bp"); +#else + const std::string fname("ADIOS2BPWriteRead1D8fstream.bp"); #endif // Write test data using BP @@ -370,7 +372,6 @@ TEST_F(BPWriteReadTestADIOS2fstream, ADIOS2BPWriteRead2D2x4) { // Each process would write a 2x4 array and all processes would // form a 2D 2 * (numberOfProcess*Nx) matrix where Nx is 4 here - const std::string fname("ADIOS2BPWriteRead2D2x4Testfstream.bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -385,6 +386,9 @@ TEST_F(BPWriteReadTestADIOS2fstream, ADIOS2BPWriteRead2D2x4) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("ADIOS2BPWriteRead2D2x4Testfstream_MPI.bp"); +#else + const std::string fname("ADIOS2BPWriteRead2D2x4Testfstream.bp"); #endif // Write test data using ADIOS2 @@ -711,7 +715,6 @@ TEST_F(BPWriteReadTestADIOS2fstream, ADIOS2BPWriteRead2D4x2) { // Each process would write a 4x2 array and all processes would // form a 2D 4 * (NumberOfProcess * Nx) matrix where Nx is 2 here - const std::string fname("ADIOS2BPWriteRead2D4x2Testfstream.bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -725,6 +728,9 @@ TEST_F(BPWriteReadTestADIOS2fstream, ADIOS2BPWriteRead2D4x2) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("ADIOS2BPWriteRead2D4x2Testfstream_MPI.bp"); +#else + const std::string fname("ADIOS2BPWriteRead2D4x2Testfstream.bp"); #endif // Write test data using ADIOS2 @@ -1037,7 +1043,6 @@ TEST_F(BPWriteReadTestADIOS2fstream, ADIOS2BPWriteRead2D4x2_ReadMultiSteps) { // Each process would write a 4x2 array and all processes would // form a 2D 4 * (NumberOfProcess * Nx) matrix where Nx is 2 here - const std::string fname("ADIOS2BPWriteRead2D4x2Test_ReadMultiStepsfstream.bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -1052,6 +1057,9 @@ TEST_F(BPWriteReadTestADIOS2fstream, ADIOS2BPWriteRead2D4x2_ReadMultiSteps) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("ADIOS2BPWriteRead2D4x2Test_ReadMultiStepsfstream_MPI.bp"); +#else + const std::string fname("ADIOS2BPWriteRead2D4x2Test_ReadMultiStepsfstream.bp"); #endif // Write test data using ADIOS2 @@ -1364,7 +1372,6 @@ TEST_F(BPWriteReadTestADIOS2fstream, ADIOS2BPWriteRead2D4x2_MultiStepsOverflow) { // Each process would write a 4x2 array and all processes would // form a 2D 4 * (NumberOfProcess * Nx) matrix where Nx is 2 here - const std::string fname("ADIOS2BPWriteRead2D4x2Test_Overflowfstream.bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -1379,6 +1386,9 @@ TEST_F(BPWriteReadTestADIOS2fstream, ADIOS2BPWriteRead2D4x2_MultiStepsOverflow) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("ADIOS2BPWriteRead2D4x2Test_Overflowfstream_MPI.bp"); +#else + const std::string fname("ADIOS2BPWriteRead2D4x2Test_Overflowfstream.bp"); #endif // Write test data using ADIOS2 diff --git a/testing/adios2/engine/bp/TestBPWriteReadADIOS2stdio.cpp b/testing/adios2/engine/bp/TestBPWriteReadADIOS2stdio.cpp index af8b80c411..c4b259f12a 100644 --- a/testing/adios2/engine/bp/TestBPWriteReadADIOS2stdio.cpp +++ b/testing/adios2/engine/bp/TestBPWriteReadADIOS2stdio.cpp @@ -33,7 +33,6 @@ TEST_F(BPWriteReadTestADIOS2stdio, ADIOS2BPWriteRead1D8) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("ADIOS2BPWriteRead1D8stdio.bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -45,6 +44,9 @@ TEST_F(BPWriteReadTestADIOS2stdio, ADIOS2BPWriteRead1D8) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("ADIOS2BPWriteRead1D8stdio_MPI.bp"); +#else + const std::string fname("ADIOS2BPWriteRead1D8stdio.bp"); #endif // Write test data using BP @@ -369,7 +371,6 @@ TEST_F(BPWriteReadTestADIOS2stdio, ADIOS2BPWriteRead2D2x4) { // Each process would write a 2x4 array and all processes would // form a 2D 2 * (numberOfProcess*Nx) matrix where Nx is 4 here - const std::string fname("ADIOS2BPWriteRead2D2x4Teststdio.bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -384,6 +385,9 @@ TEST_F(BPWriteReadTestADIOS2stdio, ADIOS2BPWriteRead2D2x4) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("ADIOS2BPWriteRead2D2x4Teststdio_MPI.bp"); +#else + const std::string fname("ADIOS2BPWriteRead2D2x4Teststdio.bp"); #endif // Write test data using ADIOS2 @@ -710,7 +714,6 @@ TEST_F(BPWriteReadTestADIOS2stdio, ADIOS2BPWriteRead2D4x2) { // Each process would write a 4x2 array and all processes would // form a 2D 4 * (NumberOfProcess * Nx) matrix where Nx is 2 here - const std::string fname("ADIOS2BPWriteRead2D4x2Teststdio.bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -724,6 +727,9 @@ TEST_F(BPWriteReadTestADIOS2stdio, ADIOS2BPWriteRead2D4x2) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("ADIOS2BPWriteRead2D4x2Teststdio_MPI.bp"); +#else + const std::string fname("ADIOS2BPWriteRead2D4x2Teststdio.bp"); #endif // Write test data using ADIOS2 @@ -1036,7 +1042,6 @@ TEST_F(BPWriteReadTestADIOS2stdio, ADIOS2BPWriteRead2D4x2_ReadMultiSteps) { // Each process would write a 4x2 array and all processes would // form a 2D 4 * (NumberOfProcess * Nx) matrix where Nx is 2 here - const std::string fname("ADIOS2BPWriteRead2D4x2Test_ReadMultiStepsstdio.bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -1051,6 +1056,9 @@ TEST_F(BPWriteReadTestADIOS2stdio, ADIOS2BPWriteRead2D4x2_ReadMultiSteps) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("ADIOS2BPWriteRead2D4x2Test_ReadMultiStepsstdio_MPI.bp"); +#else + const std::string fname("ADIOS2BPWriteRead2D4x2Test_ReadMultiStepsstdio.bp"); #endif // Write test data using ADIOS2 @@ -1363,7 +1371,6 @@ TEST_F(BPWriteReadTestADIOS2stdio, ADIOS2BPWriteRead2D4x2_MultiStepsOverflow) { // Each process would write a 4x2 array and all processes would // form a 2D 4 * (NumberOfProcess * Nx) matrix where Nx is 2 here - const std::string fname("ADIOS2BPWriteRead2D4x2Test_Overflowstdio.bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -1378,6 +1385,9 @@ TEST_F(BPWriteReadTestADIOS2stdio, ADIOS2BPWriteRead2D4x2_MultiStepsOverflow) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("ADIOS2BPWriteRead2D4x2Test_Overflowstdio_MPI.bp"); +#else + const std::string fname("ADIOS2BPWriteRead2D4x2Test_Overflowstdio.bp"); #endif // Write test data using ADIOS2 diff --git a/testing/adios2/engine/bp/TestBPWriteReadAsStreamADIOS2.cpp b/testing/adios2/engine/bp/TestBPWriteReadAsStreamADIOS2.cpp index 2089eff466..e6e4c03347 100644 --- a/testing/adios2/engine/bp/TestBPWriteReadAsStreamADIOS2.cpp +++ b/testing/adios2/engine/bp/TestBPWriteReadAsStreamADIOS2.cpp @@ -28,7 +28,6 @@ TEST_F(BPWriteReadAsStreamTestADIOS2, ADIOS2BPWriteRead1D8) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("ADIOS2BPWriteReadAsStream1D8.bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -38,8 +37,11 @@ TEST_F(BPWriteReadAsStreamTestADIOS2, ADIOS2BPWriteRead1D8) const size_t NSteps = 5; #if ADIOS2_USE_MPI + const std::string fname("ADIOS2BPWriteReadAsStream1D8_MPI.bp"); MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); +#else + const std::string fname("ADIOS2BPWriteReadAsStream1D8.bp"); #endif // Write test data using BP @@ -383,7 +385,6 @@ TEST_F(BPWriteReadAsStreamTestADIOS2, ADIOS2BPWriteRead2D2x4) { // Each process would write a 2x4 array and all processes would // form a 2D 2 * (numberOfProcess*Nx) matrix where Nx is 4 here - const std::string fname("ADIOS2BPWriteReadAsStream2D2x4Test.bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -396,8 +397,11 @@ TEST_F(BPWriteReadAsStreamTestADIOS2, ADIOS2BPWriteRead2D2x4) const std::size_t NSteps = 3; #if ADIOS2_USE_MPI + const std::string fname("ADIOS2BPWriteReadAsStream2D2x4Test_MPI.bp"); MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); +#else + const std::string fname("ADIOS2BPWriteReadAsStream2D2x4Test.bp"); #endif // Write test data using ADIOS2 @@ -624,7 +628,6 @@ TEST_F(BPWriteReadAsStreamTestADIOS2, ADIOS2BPWriteRead2D4x2) { // Each process would write a 4x2 array and all processes would // form a 2D 4 * (NumberOfProcess * Nx) matrix where Nx is 2 here - const std::string fname("ADIOS2BPWriteReadAsStream2D4x2Test.bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -636,8 +639,11 @@ TEST_F(BPWriteReadAsStreamTestADIOS2, ADIOS2BPWriteRead2D4x2) const std::size_t NSteps = 3; #if ADIOS2_USE_MPI + const std::string fname("ADIOS2BPWriteReadAsStream2D4x2Test_MPI.bp"); MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); +#else + const std::string fname("ADIOS2BPWriteReadAsStream2D4x2Test.bp"); #endif // Write test data using ADIOS2 @@ -869,8 +875,6 @@ TEST_F(BPWriteReadAsStreamTestADIOS2, ADIOS2BPWriteRead2D4x2) TEST_F(BPWriteReadAsStreamTestADIOS2, ReaderWriterDefineVariable) { - const std::string fnameFloat("BPReaderWriterDefineVariable_float.bp"); - const std::string fname("BPReaderWriterDefineVariable_all.bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -882,8 +886,13 @@ TEST_F(BPWriteReadAsStreamTestADIOS2, ReaderWriterDefineVariable) const std::size_t NSteps = 3; #if ADIOS2_USE_MPI + const std::string fnameFloat("BPReaderWriterDefineVariable_float_MPI.bp"); + const std::string fname("BPReaderWriterDefineVariable_all_MPI.bp"); MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); +#else + const std::string fnameFloat("BPReaderWriterDefineVariable_float.bp"); + const std::string fname("BPReaderWriterDefineVariable_all.bp"); #endif // Write test data using ADIOS2 diff --git a/testing/adios2/engine/bp/TestBPWriteReadAsStreamADIOS2_Threads.cpp b/testing/adios2/engine/bp/TestBPWriteReadAsStreamADIOS2_Threads.cpp index 87aea696d8..50744630fc 100644 --- a/testing/adios2/engine/bp/TestBPWriteReadAsStreamADIOS2_Threads.cpp +++ b/testing/adios2/engine/bp/TestBPWriteReadAsStreamADIOS2_Threads.cpp @@ -32,7 +32,6 @@ TEST_F(BPWriteReadAsStreamTestADIOS2_Threads, ADIOS2BPWriteRead1D8) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("ADIOS2BPWriteReadAsStream_Threads1D8.bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -44,6 +43,9 @@ TEST_F(BPWriteReadAsStreamTestADIOS2_Threads, ADIOS2BPWriteRead1D8) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("ADIOS2BPWriteReadAsStream_Threads1D8_MPI.bp"); +#else + const std::string fname("ADIOS2BPWriteReadAsStream_Threads1D8.bp"); #endif // Write test data using BP @@ -266,7 +268,6 @@ TEST_F(BPWriteReadAsStreamTestADIOS2_Threads, ADIOS2BPWriteRead2D2x4) { // Each process would write a 2x4 array and all processes would // form a 2D 2 * (numberOfProcess*Nx) matrix where Nx is 4 here - const std::string fname("ADIOS2BPWriteReadAsStream_Threads2D2x4Test.bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -281,6 +282,9 @@ TEST_F(BPWriteReadAsStreamTestADIOS2_Threads, ADIOS2BPWriteRead2D2x4) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("ADIOS2BPWriteReadAsStream_Threads2D2x4Test_MPI.bp"); +#else + const std::string fname("ADIOS2BPWriteReadAsStream_Threads2D2x4Test.bp"); #endif // Write test data using ADIOS2 @@ -513,7 +517,6 @@ TEST_F(BPWriteReadAsStreamTestADIOS2_Threads, ADIOS2BPWriteRead2D4x2) { // Each process would write a 4x2 array and all processes would // form a 2D 4 * (NumberOfProcess * Nx) matrix where Nx is 2 here - const std::string fname("ADIOS2BPWriteReadAsStream_Threads2D4x2Test.bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -527,6 +530,9 @@ TEST_F(BPWriteReadAsStreamTestADIOS2_Threads, ADIOS2BPWriteRead2D4x2) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("ADIOS2BPWriteReadAsStream_Threads2D4x2Test_MPI.bp"); +#else + const std::string fname("ADIOS2BPWriteReadAsStream_Threads2D4x2Test.bp"); #endif // Write test data using ADIOS2 diff --git a/testing/adios2/engine/bp/TestBPWriteReadAttributes.cpp b/testing/adios2/engine/bp/TestBPWriteReadAttributes.cpp index 5195fdb902..70da65f48c 100644 --- a/testing/adios2/engine/bp/TestBPWriteReadAttributes.cpp +++ b/testing/adios2/engine/bp/TestBPWriteReadAttributes.cpp @@ -28,8 +28,6 @@ class BPWriteReadAttributes : public ::testing::Test // ADIOS2 write, read for single value attributes TEST_F(BPWriteReadAttributes, WriteReadSingleTypes) { - const std::string fName = - "foo" + std::string(&adios2::PathSeparator, 1) + "WriteAttributeReadSingleTypes.bp"; const std::string zero = std::to_string(0); const std::string s1_Single = std::string("s1_Single_") + zero; @@ -56,7 +54,11 @@ TEST_F(BPWriteReadAttributes, WriteReadSingleTypes) // Write test data using BP #if ADIOS2_USE_MPI adios2::ADIOS adios(MPI_COMM_WORLD); + const std::string fName = + "foo" + std::string(&adios2::PathSeparator, 1) + "WriteAttributeReadSingleTypes_MPI.bp"; #else + const std::string fName = + "foo" + std::string(&adios2::PathSeparator, 1) + "WriteAttributeReadSingleTypes.bp"; adios2::ADIOS adios; #endif { @@ -227,13 +229,16 @@ TEST_F(BPWriteReadAttributes, WriteReadSingleTypes) // ADIOS2 write read for array attributes TEST_F(BPWriteReadAttributes, WriteReadArrayTypes) { - const std::string fName = - "foo" + std::string(&adios2::PathSeparator, 1) + "WriteAttributeReadArrayTypes.bp"; #if ADIOS2_USE_MPI int mpiRank = 0, mpiSize = 1; MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fName = + "foo" + std::string(&adios2::PathSeparator, 1) + "WriteAttributeReadArrayTypes_MPI.bp"; +#else + const std::string fName = + "foo" + std::string(&adios2::PathSeparator, 1) + "WriteAttributeReadArrayTypes.bp"; #endif const std::string zero = std::to_string(0); @@ -443,8 +448,6 @@ TEST_F(BPWriteReadAttributes, WriteReadArrayTypes) TEST_F(BPWriteReadAttributes, BPWriteReadSingleTypesVar) { - const std::string fName = - "foo" + std::string(&adios2::PathSeparator, 1) + "BPWriteAttributeReadSingleTypesVar.bp"; const std::string zero = std::to_string(0); const std::string s1_Single = std::string("s1_Single_") + zero; @@ -472,7 +475,11 @@ TEST_F(BPWriteReadAttributes, BPWriteReadSingleTypesVar) // Write test data using BP #if ADIOS2_USE_MPI adios2::ADIOS adios(MPI_COMM_WORLD); + const std::string fName = "foo" + std::string(&adios2::PathSeparator, 1) + + "BPWriteAttributeReadSingleTypesVar_MPI.bp"; #else + const std::string fName = + "foo" + std::string(&adios2::PathSeparator, 1) + "BPWriteAttributeReadSingleTypesVar.bp"; adios2::ADIOS adios; #endif { @@ -630,13 +637,16 @@ TEST_F(BPWriteReadAttributes, BPWriteReadSingleTypesVar) // ADIOS2 write read for array attributes TEST_F(BPWriteReadAttributes, WriteReadArrayTypesVar) { - const std::string fName = - "foo" + std::string(&adios2::PathSeparator, 1) + "BPWriteAttributeReadArrayTypesVar.bp"; #if ADIOS2_USE_MPI int mpiRank = 0, mpiSize = 1; MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fName = + "foo" + std::string(&adios2::PathSeparator, 1) + "BPWriteAttributeReadArrayTypesVar_MPI.bp"; +#else + const std::string fName = + "foo" + std::string(&adios2::PathSeparator, 1) + "BPWriteAttributeReadArrayTypesVar.bp"; #endif const std::string zero = std::to_string(0); @@ -849,8 +859,6 @@ TEST_F(BPWriteReadAttributes, WriteReadArrayTypesVar) TEST_F(BPWriteReadAttributes, WriteReadStreamVarp) { - const std::string fName = - "foo" + std::string(&adios2::PathSeparator, 1) + "AttributesWriteReadVar.bp"; const std::string separator = "\\"; @@ -864,6 +872,11 @@ TEST_F(BPWriteReadAttributes, WriteReadStreamVarp) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fName = + "foo" + std::string(&adios2::PathSeparator, 1) + "AttributesWriteReadVar_MPI.bp"; +#else + const std::string fName = + "foo" + std::string(&adios2::PathSeparator, 1) + "AttributesWriteReadVar.bp"; #endif SmallTestData currentTestData = generateNewSmallTestData(m_TestData, 0, 0, 0); @@ -1003,8 +1016,6 @@ TEST_F(BPWriteReadAttributes, WriteReadStreamVarp) TEST_F(BPWriteReadAttributes, WriteReadStreamModifiable) { - const std::string fName = - "foo" + std::string(&adios2::PathSeparator, 1) + "AttributesWriteReadModifiable.bp"; const std::string separator = "\\"; @@ -1018,6 +1029,11 @@ TEST_F(BPWriteReadAttributes, WriteReadStreamModifiable) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fName = + "foo" + std::string(&adios2::PathSeparator, 1) + "AttributesWriteReadModifiable_MPI.bp"; +#else + const std::string fName = + "foo" + std::string(&adios2::PathSeparator, 1) + "AttributesWriteReadModifiable.bp"; #endif const double d3[3] = {-1.1, -1.2, -1.3}; diff --git a/testing/adios2/engine/bp/TestBPWriteReadCuda.cpp b/testing/adios2/engine/bp/TestBPWriteReadCuda.cpp index a3f1797e34..880c55de35 100644 --- a/testing/adios2/engine/bp/TestBPWriteReadCuda.cpp +++ b/testing/adios2/engine/bp/TestBPWriteReadCuda.cpp @@ -292,7 +292,6 @@ void CUDAWriteReadMemorySelection() void CUDAWriteReadMPI1D(const std::string mode) { - const std::string fname("BPWRCU1D_" + mode + ".bp"); adios2::Mode ioMode = adios2::Mode::Deferred; if (mode == "Sync") ioMode = adios2::Mode::Sync; @@ -306,6 +305,9 @@ void CUDAWriteReadMPI1D(const std::string mode) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWRCU1D_" + mode + "_MPI.bp"); +#else + const std::string fname("BPWRCU1D_" + mode + ".bp"); #endif #if ADIOS2_USE_MPI diff --git a/testing/adios2/engine/bp/TestBPWriteReadLocalVariables.cpp b/testing/adios2/engine/bp/TestBPWriteReadLocalVariables.cpp index 9fd6c2d71a..05c23ccbfc 100644 --- a/testing/adios2/engine/bp/TestBPWriteReadLocalVariables.cpp +++ b/testing/adios2/engine/bp/TestBPWriteReadLocalVariables.cpp @@ -29,7 +29,6 @@ TEST_F(BPWriteReadLocalVariables, ADIOS2BPWriteReadLocal1D) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("ADIOS2BPWriteReadLocal1D.bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -41,6 +40,9 @@ TEST_F(BPWriteReadLocalVariables, ADIOS2BPWriteReadLocal1D) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("ADIOS2BPWriteReadLocal1D_MPI.bp"); +#else + const std::string fname("ADIOS2BPWriteReadLocal1D.bp"); #endif // Write test data using BP @@ -418,7 +420,6 @@ TEST_F(BPWriteReadLocalVariables, ADIOS2BPWriteReadLocal2D2x4) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("ADIOS2BPWriteReadLocal2D2x4.bp"); int mpiRank = 0, mpiSize = 1; const size_t Nx = 4; @@ -430,6 +431,9 @@ TEST_F(BPWriteReadLocalVariables, ADIOS2BPWriteReadLocal2D2x4) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("ADIOS2BPWriteReadLocal2D2x4_MPI.bp"); +#else + const std::string fname("ADIOS2BPWriteReadLocal2D2x4.bp"); #endif // Write test data using BP @@ -793,7 +797,6 @@ TEST_F(BPWriteReadLocalVariables, ADIOS2BPWriteReadLocal2D4x2) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("ADIOS2BPWriteReadLocal2D4x2.bp"); int mpiRank = 0, mpiSize = 1; const size_t Nx = 2; @@ -805,6 +808,9 @@ TEST_F(BPWriteReadLocalVariables, ADIOS2BPWriteReadLocal2D4x2) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("ADIOS2BPWriteReadLocal2D4x2_MPI.bp"); +#else + const std::string fname("ADIOS2BPWriteReadLocal2D4x2.bp"); #endif // Write test data using BP @@ -1169,7 +1175,6 @@ TEST_F(BPWriteReadLocalVariables, ADIOS2BPWriteReadLocal1DAllSteps) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("ADIOS2BPWriteReadLocal1DAllSteps.bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -1181,6 +1186,9 @@ TEST_F(BPWriteReadLocalVariables, ADIOS2BPWriteReadLocal1DAllSteps) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("ADIOS2BPWriteReadLocal1DAllSteps_MPI.bp"); +#else + const std::string fname("ADIOS2BPWriteReadLocal1DAllSteps.bp"); #endif // Write test data using BP @@ -1443,7 +1451,6 @@ TEST_F(BPWriteReadLocalVariables, ADIOS2BPWriteReadLocal1DBlockInfo) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("ADIOS2BPWriteReadLocal1DBlockInfo.bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -1455,6 +1462,9 @@ TEST_F(BPWriteReadLocalVariables, ADIOS2BPWriteReadLocal1DBlockInfo) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("ADIOS2BPWriteReadLocal1DBlockInfo_MPI.bp"); +#else + const std::string fname("ADIOS2BPWriteReadLocal1DBlockInfo.bp"); #endif // Write test data using BP @@ -1721,13 +1731,14 @@ TEST_F(BPWriteReadLocalVariables, ADIOS2BPWriteReadLocal1DSubFile) TEST_F(BPWriteReadLocalVariables, ADIOS2BPWriteReadLocal2DChangeCount) { - const std::string fname("BPWRLocal2DChangeCount_" + engineName + ".bp"); - int mpiRank = 0; int mpiSize = 1; #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWRLocal2DChangeCount_" + engineName + "_MPI.bp"); +#else + const std::string fname("BPWRLocal2DChangeCount_" + engineName + ".bp"); #endif const size_t Nx0 = static_cast(std::pow(2 - mpiRank, 2)) + 1; diff --git a/testing/adios2/engine/bp/TestBPWriteReadLocalVariablesSel.cpp b/testing/adios2/engine/bp/TestBPWriteReadLocalVariablesSel.cpp index 558e55014a..399b1b6fd2 100644 --- a/testing/adios2/engine/bp/TestBPWriteReadLocalVariablesSel.cpp +++ b/testing/adios2/engine/bp/TestBPWriteReadLocalVariablesSel.cpp @@ -28,7 +28,6 @@ TEST_F(BPWriteReadLocalVariablesSel, BPWriteReadLocal1DSel) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("BPWriteReadLocal1DSel.bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -40,6 +39,9 @@ TEST_F(BPWriteReadLocalVariablesSel, BPWriteReadLocal1DSel) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWriteReadLocal1DSel_MPI.bp"); +#else + const std::string fname("BPWriteReadLocal1DSel.bp"); #endif // Write test data using BP @@ -440,7 +442,6 @@ TEST_F(BPWriteReadLocalVariablesSel, BPWriteReadLocal2D2x4Sel) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("BPWriteReadLocal2D2x4Sel.bp"); int mpiRank = 0, mpiSize = 1; const size_t Nx = 4; @@ -452,6 +453,9 @@ TEST_F(BPWriteReadLocalVariablesSel, BPWriteReadLocal2D2x4Sel) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWriteReadLocal2D2x4Sel_MPI.bp"); +#else + const std::string fname("BPWriteReadLocal2D2x4Sel.bp"); #endif // Write test data using BP @@ -899,7 +903,6 @@ TEST_F(BPWriteReadLocalVariablesSel, BPWriteReadLocal2D4x2Sel) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("BPWriteReadLocal2D4x2Sel.bp"); int mpiRank = 0, mpiSize = 1; const size_t Nx = 2; @@ -911,6 +914,9 @@ TEST_F(BPWriteReadLocalVariablesSel, BPWriteReadLocal2D4x2Sel) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWriteReadLocal2D4x2Sel_MPI.bp"); +#else + const std::string fname("BPWriteReadLocal2D4x2Sel.bp"); #endif // Write test data using BP @@ -1357,7 +1363,6 @@ TEST_F(BPWriteReadLocalVariablesSel, BPWriteReadLocal1DAllStepsSel) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("BPWriteReadLocal1DAllStepsSel.bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -1369,6 +1374,9 @@ TEST_F(BPWriteReadLocalVariablesSel, BPWriteReadLocal1DAllStepsSel) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWriteReadLocal1DAllStepsSel_MPI.bp"); +#else + const std::string fname("BPWriteReadLocal1DAllStepsSel.bp"); #endif // Write test data using BP diff --git a/testing/adios2/engine/bp/TestBPWriteReadLocalVariablesSelHighLevel.cpp b/testing/adios2/engine/bp/TestBPWriteReadLocalVariablesSelHighLevel.cpp index eaa9d3fdfb..525a955ccc 100644 --- a/testing/adios2/engine/bp/TestBPWriteReadLocalVariablesSelHighLevel.cpp +++ b/testing/adios2/engine/bp/TestBPWriteReadLocalVariablesSelHighLevel.cpp @@ -28,7 +28,6 @@ TEST_F(BPWriteReadLocalVariablesSelHighLevel, BPWriteReadLocal1DSel) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("BPWriteReadLocal1DSelHighLevel.bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -40,6 +39,9 @@ TEST_F(BPWriteReadLocalVariablesSelHighLevel, BPWriteReadLocal1DSel) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWriteReadLocal1DSelHighLevel_MPI.bp"); +#else + const std::string fname("BPWriteReadLocal1DSelHighLevel.bp"); #endif // Write test data using BP @@ -168,7 +170,6 @@ TEST_F(BPWriteReadLocalVariablesSelHighLevel, BPWriteReadLocal1DSel) TEST_F(BPWriteReadLocalVariablesSelHighLevel, BPWriteReadLocal2D2x4Sel) { - const std::string fname("BPWriteReadLocal2D2x4SelHighLevel.bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -181,6 +182,9 @@ TEST_F(BPWriteReadLocalVariablesSelHighLevel, BPWriteReadLocal2D2x4Sel) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWriteReadLocal2D2x4SelHighLevel_MPI.bp"); +#else + const std::string fname("BPWriteReadLocal2D2x4SelHighLevel.bp"); #endif // Write test data using BP @@ -332,7 +336,6 @@ TEST_F(BPWriteReadLocalVariablesSelHighLevel, BPWriteReadLocal1DAllStepsSel) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("BPWriteReadLocal1DAllStepsSelHighLevel.bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -344,6 +347,9 @@ TEST_F(BPWriteReadLocalVariablesSelHighLevel, BPWriteReadLocal1DAllStepsSel) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWriteReadLocal1DAllStepsSelHighLevel_MPI.bp"); +#else + const std::string fname("BPWriteReadLocal1DAllStepsSelHighLevel.bp"); #endif // Write test data using BP diff --git a/testing/adios2/engine/bp/TestBPWriteReadMultiblock.cpp b/testing/adios2/engine/bp/TestBPWriteReadMultiblock.cpp index 5014ba6335..0edfffc738 100644 --- a/testing/adios2/engine/bp/TestBPWriteReadMultiblock.cpp +++ b/testing/adios2/engine/bp/TestBPWriteReadMultiblock.cpp @@ -33,7 +33,6 @@ TEST_F(BPWriteReadMultiblockTest, ADIOS2BPWriteReadMultiblock1D8) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("ADIOS2BPWriteReadMultiblock1D8.bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -45,6 +44,9 @@ TEST_F(BPWriteReadMultiblockTest, ADIOS2BPWriteReadMultiblock1D8) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("ADIOS2BPWriteReadMultiblock1D8_MPI.bp"); +#else + const std::string fname("ADIOS2BPWriteReadMultiblock1D8.bp"); #endif // Write test data using BP @@ -773,7 +775,6 @@ TEST_F(BPWriteReadMultiblockTest, ADIOS2BPWriteReadMultiblock2D2x4) { // Each process would write a 2x4 array and all processes would // form a 2D 2 * (numberOfProcess*Nx) matrix where Nx is 4 here - const std::string fname("ADIOS2BPWriteReadMultiblock2D2x4Test.bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -788,6 +789,9 @@ TEST_F(BPWriteReadMultiblockTest, ADIOS2BPWriteReadMultiblock2D2x4) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("ADIOS2BPWriteReadMultiblock2D2x4Test_MPI.bp"); +#else + const std::string fname("ADIOS2BPWriteReadMultiblock2D2x4Test.bp"); #endif // Write test data using ADIOS2 @@ -1182,7 +1186,6 @@ TEST_F(BPWriteReadMultiblockTest, ADIOS2BPWriteReadMultiblock2D4x2) { // Each process would write a 4x2 array and all processes would // form a 2D 4 * (NumberOfProcess * Nx) matrix where Nx is 2 here - const std::string fname("ADIOS2BPWriteReadMultiblock2D4x2Test.bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -1196,6 +1199,9 @@ TEST_F(BPWriteReadMultiblockTest, ADIOS2BPWriteReadMultiblock2D4x2) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("ADIOS2BPWriteReadMultiblock2D4x2Test_MPI.bp"); +#else + const std::string fname("ADIOS2BPWriteReadMultiblock2D4x2Test.bp"); #endif // Write test data using ADIOS2 @@ -2103,7 +2109,6 @@ TEST_F(BPWriteReadMultiblockTest, MultiblockNullBlocks) { // Each process would write a 2x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("MultiblockNullBlocks.bp"); int mpiRank = 0, mpiSize = 1; // Number of elements per blocks (blocksize) @@ -2116,6 +2121,9 @@ TEST_F(BPWriteReadMultiblockTest, MultiblockNullBlocks) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("MultiblockNullBlocks_MPI.bp"); +#else + const std::string fname("MultiblockNullBlocks.bp"); #endif #if ADIOS2_USE_MPI diff --git a/testing/adios2/engine/bp/TestBPWriteReadVariableSpan.cpp b/testing/adios2/engine/bp/TestBPWriteReadVariableSpan.cpp index 57fe702338..4043a4c3d2 100644 --- a/testing/adios2/engine/bp/TestBPWriteReadVariableSpan.cpp +++ b/testing/adios2/engine/bp/TestBPWriteReadVariableSpan.cpp @@ -28,7 +28,6 @@ TEST_F(BPWriteReadSpan, BPWriteRead1D8) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("BPWriteReadSpan1D8.bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -40,6 +39,9 @@ TEST_F(BPWriteReadSpan, BPWriteRead1D8) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWriteReadSpan1D8_MPI.bp"); +#else + const std::string fname("BPWriteReadSpan1D8.bp"); #endif #if ADIOS2_USE_MPI @@ -317,7 +319,6 @@ TEST_F(BPWriteReadSpan, BPWriteRead2D2x4) { // Each process would write a 2x4 array and all processes would // form a 2D 2 * (numberOfProcess*Nx) matrix where Nx is 4 here - const std::string fname("BPWriteReadSpan2D2x4.bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -332,6 +333,9 @@ TEST_F(BPWriteReadSpan, BPWriteRead2D2x4) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWriteReadSpan2D2x4_MPI.bp"); +#else + const std::string fname("BPWriteReadSpan2D2x4.bp"); #endif // Write test data using ADIOS2 @@ -628,7 +632,6 @@ TEST_F(BPWriteReadSpan, BPWriteRead2D2x4) TEST_F(BPWriteReadSpan, BPWriteRead1D8Local) { - const std::string fname("BPWriteReadSpan1D8Local.bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -640,6 +643,9 @@ TEST_F(BPWriteReadSpan, BPWriteRead1D8Local) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWriteReadSpan1D8Local_MPI.bp"); +#else + const std::string fname("BPWriteReadSpan1D8Local.bp"); #endif // Write test data using BP @@ -877,7 +883,6 @@ TEST_F(BPWriteReadSpan, BPWriteRead2D2x4Local) { // Each process would write a 2x4 array and all processes would // form a 2D 2 * (numberOfProcess*Nx) matrix where Nx is 4 here - const std::string fname("BPWriteReadSpan2D2x4Local.bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -892,6 +897,9 @@ TEST_F(BPWriteReadSpan, BPWriteRead2D2x4Local) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWriteReadSpan2D2x4Local_MPI.bp"); +#else + const std::string fname("BPWriteReadSpan2D2x4Local.bp"); #endif // Write test data using ADIOS2 @@ -1156,7 +1164,6 @@ TEST_F(BPWriteReadSpan, BPWriteRead1D8FillValue) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("BPWriteReadSpan1D8FillValue.bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -1168,6 +1175,9 @@ TEST_F(BPWriteReadSpan, BPWriteRead1D8FillValue) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWriteReadSpan1D8FillValue_MPI.bp"); +#else + const std::string fname("BPWriteReadSpan1D8FillValue.bp"); #endif // Write test data using BP @@ -1469,7 +1479,6 @@ TEST_F(BPWriteReadSpan, BPWriteRead1D8FillValue) #ifdef ADIOS2_HAVE_BZIP2 TEST_F(BPWriteReadSpan, BPWriteSpanOperatorException) { - const std::string fname("BPWriteSpanOperatorException.bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -1481,6 +1490,9 @@ TEST_F(BPWriteReadSpan, BPWriteSpanOperatorException) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWriteSpanOperatorException_MPI.bp"); +#else + const std::string fname("BPWriteSpanOperatorException.bp"); #endif // Write test data using BP diff --git a/testing/adios2/engine/bp/TestBPWriteReadVector.cpp b/testing/adios2/engine/bp/TestBPWriteReadVector.cpp index 595a6c5578..9baeb7369e 100644 --- a/testing/adios2/engine/bp/TestBPWriteReadVector.cpp +++ b/testing/adios2/engine/bp/TestBPWriteReadVector.cpp @@ -32,7 +32,6 @@ TEST_F(BPWriteReadVector, ADIOS2BPWriteRead1D8) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("ADIOS2BPWriteReadVector1D8.bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -44,6 +43,9 @@ TEST_F(BPWriteReadVector, ADIOS2BPWriteRead1D8) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("ADIOS2BPWriteReadVector1D8_MPI.bp"); +#else + const std::string fname("ADIOS2BPWriteReadVector1D8.bp"); #endif // Write test data using BP @@ -342,7 +344,6 @@ TEST_F(BPWriteReadVector, ADIOS2BPWriteRead2D2x4) { // Each process would write a 2x4 array and all processes would // form a 2D 2 * (numberOfProcess*Nx) matrix where Nx is 4 here - const std::string fname("ADIOS2BPWriteReadVector2D2x4Test.bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -357,6 +358,9 @@ TEST_F(BPWriteReadVector, ADIOS2BPWriteRead2D2x4) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("ADIOS2BPWriteReadVector2D2x4Test_MPI.bp"); +#else + const std::string fname("ADIOS2BPWriteReadVector2D2x4Test.bp"); #endif // Write test data using ADIOS2 @@ -655,7 +659,6 @@ TEST_F(BPWriteReadVector, ADIOS2BPWriteRead2D4x2) { // Each process would write a 4x2 array and all processes would // form a 2D 4 * (NumberOfProcess * Nx) matrix where Nx is 2 here - const std::string fname("ADIOS2BPWriteReadVector2D4x2Test.bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -669,6 +672,9 @@ TEST_F(BPWriteReadVector, ADIOS2BPWriteRead2D4x2) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("ADIOS2BPWriteReadVector2D4x2Test_MPI.bp"); +#else + const std::string fname("ADIOS2BPWriteReadVector2D4x2Test.bp"); #endif // Write test data using ADIOS2 @@ -952,7 +958,6 @@ TEST_F(BPWriteReadVector, ADIOS2BPWriteReadVector2D4x2_MultiSteps) { // Each process would write a 4x2 array and all processes would // form a 2D 4 * (NumberOfProcess * Nx) matrix where Nx is 2 here - const std::string fname("ADIOS2BPWriteReadVector2D4x2_MultiSteps.bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -967,6 +972,9 @@ TEST_F(BPWriteReadVector, ADIOS2BPWriteReadVector2D4x2_MultiSteps) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("ADIOS2BPWriteReadVector2D4x2_MultiSteps_MPI.bp"); +#else + const std::string fname("ADIOS2BPWriteReadVector2D4x2_MultiSteps.bp"); #endif // Write test data using ADIOS2 diff --git a/testing/adios2/engine/bp/operations/TestBPWriteReadBZIP2.cpp b/testing/adios2/engine/bp/operations/TestBPWriteReadBZIP2.cpp index dcbafcccf0..651531ff22 100644 --- a/testing/adios2/engine/bp/operations/TestBPWriteReadBZIP2.cpp +++ b/testing/adios2/engine/bp/operations/TestBPWriteReadBZIP2.cpp @@ -19,7 +19,6 @@ void BZIP2Accuracy1D(const std::string accuracy) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("BPWR_BZIP2_1D_" + accuracy + ".bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -38,6 +37,9 @@ void BZIP2Accuracy1D(const std::string accuracy) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWR_BZIP2_1D_" + accuracy + "_MPI.bp"); +#else + const std::string fname("BPWR_BZIP2_1D_" + accuracy + ".bp"); #endif #if ADIOS2_USE_MPI @@ -152,7 +154,6 @@ void BZIP2Accuracy1DLocal(const std::string accuracy) { // Each process would write a 1x8 array and all processes would // write a Nx 1D array - const std::string fname("BPWR_BZIP2_1D_Local_" + accuracy + ".bp"); int mpiRank = 0; // Number of rows @@ -170,6 +171,9 @@ void BZIP2Accuracy1DLocal(const std::string accuracy) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); + const std::string fname("BPWR_BZIP2_1D_Local_" + accuracy + "_MPI.bp"); +#else + const std::string fname("BPWR_BZIP2_1D_Local_" + accuracy + ".bp"); #endif #if ADIOS2_USE_MPI @@ -280,7 +284,6 @@ void BZIP2Accuracy2D(const std::string accuracy) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("BPWRBZIP22D_" + accuracy + ".bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -300,6 +303,9 @@ void BZIP2Accuracy2D(const std::string accuracy) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWRBZIP22D_" + accuracy + "_MPI.bp"); +#else + const std::string fname("BPWRBZIP22D_" + accuracy + ".bp"); #endif #if ADIOS2_USE_MPI @@ -414,7 +420,6 @@ void BZIP2Accuracy3D(const std::string accuracy) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("BPWRBZIP23D_" + accuracy + ".bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -435,6 +440,9 @@ void BZIP2Accuracy3D(const std::string accuracy) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWRBZIP23D_" + accuracy + "_MPI.bp"); +#else + const std::string fname("BPWRBZIP23D_" + accuracy + ".bp"); #endif #if ADIOS2_USE_MPI @@ -551,7 +559,6 @@ void BZIP2Accuracy1DSel(const std::string accuracy) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("BPWRBZIP21DSel_" + accuracy + ".bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -570,6 +577,9 @@ void BZIP2Accuracy1DSel(const std::string accuracy) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWRBZIP21DSel_" + accuracy + "_MPI.bp"); +#else + const std::string fname("BPWRBZIP21DSel_" + accuracy + ".bp"); #endif #if ADIOS2_USE_MPI @@ -682,7 +692,6 @@ void BZIP2Accuracy2DSel(const std::string accuracy) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("BPWRBZIP22DSel_" + accuracy + ".bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -702,6 +711,9 @@ void BZIP2Accuracy2DSel(const std::string accuracy) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWRBZIP22DSel_" + accuracy + "_MPI.bp"); +#else + const std::string fname("BPWRBZIP22DSel_" + accuracy + ".bp"); #endif #if ADIOS2_USE_MPI @@ -816,7 +828,6 @@ void BZIP2Accuracy3DSel(const std::string accuracy) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("BPWRBZIP23DSel_" + accuracy + ".bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -837,6 +848,9 @@ void BZIP2Accuracy3DSel(const std::string accuracy) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWRBZIP23DSel_" + accuracy + "_MPI.bp"); +#else + const std::string fname("BPWRBZIP23DSel_" + accuracy + ".bp"); #endif #if ADIOS2_USE_MPI diff --git a/testing/adios2/engine/bp/operations/TestBPWriteReadBlosc.cpp b/testing/adios2/engine/bp/operations/TestBPWriteReadBlosc.cpp index 4f711437d8..09908075cb 100644 --- a/testing/adios2/engine/bp/operations/TestBPWriteReadBlosc.cpp +++ b/testing/adios2/engine/bp/operations/TestBPWriteReadBlosc.cpp @@ -21,8 +21,6 @@ void BloscAccuracy1D(const std::string accuracy, const std::string threshold, { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("BPWR_Blosc_1D_" + accuracy + "_" + threshold + "_" + doshuffle + - ".bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -41,6 +39,11 @@ void BloscAccuracy1D(const std::string accuracy, const std::string threshold, #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWR_Blosc_1D_" + accuracy + "_" + threshold + "_" + doshuffle + + "_MPI.bp"); +#else + const std::string fname("BPWR_Blosc_1D_" + accuracy + "_" + threshold + "_" + doshuffle + + ".bp"); #endif #if ADIOS2_USE_MPI @@ -160,8 +163,6 @@ void BloscAccuracy2D(const std::string accuracy, const std::string threshold, { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("BPWRBlosc2D_" + accuracy + "_" + threshold + threshold + "_" + - doshuffle + ".bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -181,6 +182,11 @@ void BloscAccuracy2D(const std::string accuracy, const std::string threshold, #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWRBlosc2D_" + accuracy + "_" + threshold + threshold + "_" + + doshuffle + "_MPI.bp"); +#else + const std::string fname("BPWRBlosc2D_" + accuracy + "_" + threshold + threshold + "_" + + doshuffle + ".bp"); #endif #if ADIOS2_USE_MPI @@ -300,8 +306,6 @@ void BloscAccuracy3D(const std::string accuracy, const std::string threshold, { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("BPWRBlosc3D_" + accuracy + "_" + threshold + threshold + "_" + - doshuffle + ".bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -322,6 +326,11 @@ void BloscAccuracy3D(const std::string accuracy, const std::string threshold, #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWRBlosc3D_" + accuracy + "_" + threshold + threshold + "_" + + doshuffle + "_MPI.bp"); +#else + const std::string fname("BPWRBlosc3D_" + accuracy + "_" + threshold + threshold + "_" + + doshuffle + ".bp"); #endif #if ADIOS2_USE_MPI @@ -443,8 +452,6 @@ void BloscAccuracy1DSel(const std::string accuracy, const std::string threshold, { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("BPWRBlosc1DSel_" + accuracy + "_" + threshold + threshold + "_" + - doshuffle + ".bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -463,6 +470,11 @@ void BloscAccuracy1DSel(const std::string accuracy, const std::string threshold, #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWRBlosc1DSel_" + accuracy + "_" + threshold + threshold + "_" + + doshuffle + "_MPI.bp"); +#else + const std::string fname("BPWRBlosc1DSel_" + accuracy + "_" + threshold + threshold + "_" + + doshuffle + ".bp"); #endif #if ADIOS2_USE_MPI @@ -580,8 +592,6 @@ void BloscAccuracy2DSel(const std::string accuracy, const std::string threshold, { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("BPWRBlosc2DSel_" + accuracy + "_" + threshold + threshold + "_" + - doshuffle + ".bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -601,6 +611,11 @@ void BloscAccuracy2DSel(const std::string accuracy, const std::string threshold, #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWRBlosc2DSel_" + accuracy + "_" + threshold + threshold + "_" + + doshuffle + "_MPI.bp"); +#else + const std::string fname("BPWRBlosc2DSel_" + accuracy + "_" + threshold + threshold + "_" + + doshuffle + ".bp"); #endif #if ADIOS2_USE_MPI @@ -720,8 +735,6 @@ void BloscAccuracy3DSel(const std::string accuracy, const std::string threshold, { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("BPWRBlosc3DSel_" + accuracy + "_" + threshold + threshold + "_" + - doshuffle + ".bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -742,6 +755,11 @@ void BloscAccuracy3DSel(const std::string accuracy, const std::string threshold, #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWRBlosc3DSel_" + accuracy + "_" + threshold + threshold + "_" + + doshuffle + "_MPI.bp"); +#else + const std::string fname("BPWRBlosc3DSel_" + accuracy + "_" + threshold + threshold + "_" + + doshuffle + ".bp"); #endif #if ADIOS2_USE_MPI diff --git a/testing/adios2/engine/bp/operations/TestBPWriteReadBlosc2.cpp b/testing/adios2/engine/bp/operations/TestBPWriteReadBlosc2.cpp index 9e72a351b5..e6852d3471 100644 --- a/testing/adios2/engine/bp/operations/TestBPWriteReadBlosc2.cpp +++ b/testing/adios2/engine/bp/operations/TestBPWriteReadBlosc2.cpp @@ -21,8 +21,6 @@ void Blosc2Accuracy1D(const std::string accuracy, const std::string threshold, { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("BPWR_Blosc2_1D_" + accuracy + "_" + threshold + "_" + doshuffle + - ".bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -41,6 +39,11 @@ void Blosc2Accuracy1D(const std::string accuracy, const std::string threshold, #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWR_Blosc2_1D_" + accuracy + "_" + threshold + "_" + doshuffle + + "_MPI.bp"); +#else + const std::string fname("BPWR_Blosc2_1D_" + accuracy + "_" + threshold + "_" + doshuffle + + ".bp"); #endif #if ADIOS2_USE_MPI @@ -160,8 +163,6 @@ void Blosc2Accuracy2D(const std::string accuracy, const std::string threshold, { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("BPWRBlosc22D_" + accuracy + "_" + threshold + threshold + "_" + - doshuffle + ".bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -181,6 +182,11 @@ void Blosc2Accuracy2D(const std::string accuracy, const std::string threshold, #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWRBlosc22D_" + accuracy + "_" + threshold + threshold + "_" + + doshuffle + "_MPI.bp"); +#else + const std::string fname("BPWRBlosc22D_" + accuracy + "_" + threshold + threshold + "_" + + doshuffle + ".bp"); #endif #if ADIOS2_USE_MPI @@ -300,8 +306,6 @@ void Blosc2Accuracy3D(const std::string accuracy, const std::string threshold, { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("BPWRBlosc23D_" + accuracy + "_" + threshold + threshold + "_" + - doshuffle + ".bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -322,6 +326,11 @@ void Blosc2Accuracy3D(const std::string accuracy, const std::string threshold, #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWRBlosc23D_" + accuracy + "_" + threshold + threshold + "_" + + doshuffle + "_MPI.bp"); +#else + const std::string fname("BPWRBlosc23D_" + accuracy + "_" + threshold + threshold + "_" + + doshuffle + ".bp"); #endif #if ADIOS2_USE_MPI @@ -443,8 +452,6 @@ void Blosc2Accuracy1DSel(const std::string accuracy, const std::string threshold { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("BPWRBlosc21DSel_" + accuracy + "_" + threshold + threshold + "_" + - doshuffle + ".bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -463,6 +470,11 @@ void Blosc2Accuracy1DSel(const std::string accuracy, const std::string threshold #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWRBlosc21DSel_" + accuracy + "_" + threshold + threshold + "_" + + doshuffle + "_MPI.bp"); +#else + const std::string fname("BPWRBlosc21DSel_" + accuracy + "_" + threshold + threshold + "_" + + doshuffle + ".bp"); #endif #if ADIOS2_USE_MPI @@ -580,8 +592,6 @@ void Blosc2Accuracy2DSel(const std::string accuracy, const std::string threshold { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("BPWRBlosc22DSel_" + accuracy + "_" + threshold + threshold + "_" + - doshuffle + ".bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -601,6 +611,11 @@ void Blosc2Accuracy2DSel(const std::string accuracy, const std::string threshold #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWRBlosc22DSel_" + accuracy + "_" + threshold + threshold + "_" + + doshuffle + "_MPI.bp"); +#else + const std::string fname("BPWRBlosc22DSel_" + accuracy + "_" + threshold + threshold + "_" + + doshuffle + ".bp"); #endif #if ADIOS2_USE_MPI @@ -720,8 +735,6 @@ void Blosc2Accuracy3DSel(const std::string accuracy, const std::string threshold { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("BPWRBlosc23DSel_" + accuracy + "_" + threshold + threshold + "_" + - doshuffle + ".bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -742,6 +755,11 @@ void Blosc2Accuracy3DSel(const std::string accuracy, const std::string threshold #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWRBlosc23DSel_" + accuracy + "_" + threshold + threshold + "_" + + doshuffle + "_MPI.bp"); +#else + const std::string fname("BPWRBlosc23DSel_" + accuracy + "_" + threshold + threshold + "_" + + doshuffle + ".bp"); #endif #if ADIOS2_USE_MPI @@ -867,8 +885,6 @@ void Blosc2NullBlocks(const std::string accuracy, const std::string threshold, // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("BPWRBlosc2NullBlock_" + accuracy + "_" + threshold + threshold + "_" + - doshuffle + ".bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -883,6 +899,11 @@ void Blosc2NullBlocks(const std::string accuracy, const std::string threshold, #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWRBlosc2NullBlock_" + accuracy + "_" + threshold + threshold + "_" + + doshuffle + "_MPI.bp"); +#else + const std::string fname("BPWRBlosc2NullBlock_" + accuracy + "_" + threshold + threshold + "_" + + doshuffle + ".bp"); #endif #if ADIOS2_USE_MPI diff --git a/testing/adios2/engine/bp/operations/TestBPWriteReadLocalVariables.cpp b/testing/adios2/engine/bp/operations/TestBPWriteReadLocalVariables.cpp index ec36175659..a725ddf620 100644 --- a/testing/adios2/engine/bp/operations/TestBPWriteReadLocalVariables.cpp +++ b/testing/adios2/engine/bp/operations/TestBPWriteReadLocalVariables.cpp @@ -26,7 +26,6 @@ TEST_F(BPWriteReadLocalVariables, ADIOS2BPWriteReadLocal1D) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("ADIOS2BPWriteReadLocal1D.bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -38,6 +37,9 @@ TEST_F(BPWriteReadLocalVariables, ADIOS2BPWriteReadLocal1D) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("ADIOS2BPWriteReadLocal1D_MPI.bp"); +#else + const std::string fname("ADIOS2BPWriteReadLocal1D.bp"); #endif // Write test data using BP @@ -375,7 +377,6 @@ TEST_F(BPWriteReadLocalVariables, ADIOS2BPWriteReadLocal2D2x4) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("ADIOS2BPWriteReadLocal2D2x4.bp"); int mpiRank = 0, mpiSize = 1; const size_t Nx = 4; @@ -387,6 +388,9 @@ TEST_F(BPWriteReadLocalVariables, ADIOS2BPWriteReadLocal2D2x4) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("ADIOS2BPWriteReadLocal2D2x4_MPI.bp"); +#else + const std::string fname("ADIOS2BPWriteReadLocal2D2x4.bp"); #endif // Write test data using BP @@ -730,7 +734,6 @@ TEST_F(BPWriteReadLocalVariables, ADIOS2BPWriteReadLocal2D4x2) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("ADIOS2BPWriteReadLocal2D4x2.bp"); int mpiRank = 0, mpiSize = 1; const size_t Nx = 2; @@ -742,6 +745,9 @@ TEST_F(BPWriteReadLocalVariables, ADIOS2BPWriteReadLocal2D4x2) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("ADIOS2BPWriteReadLocal2D4x2_MPI.bp"); +#else + const std::string fname("ADIOS2BPWriteReadLocal2D4x2.bp"); #endif // Write test data using BP @@ -1086,7 +1092,6 @@ TEST_F(BPWriteReadLocalVariables, ADIOS2BPWriteReadLocal1DAllSteps) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("ADIOS2BPWriteReadLocal1DAllSteps.bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -1098,6 +1103,9 @@ TEST_F(BPWriteReadLocalVariables, ADIOS2BPWriteReadLocal1DAllSteps) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("ADIOS2BPWriteReadLocal1DAllSteps_MPI.bp"); +#else + const std::string fname("ADIOS2BPWriteReadLocal1DAllSteps.bp"); #endif // Write test data using BP @@ -1324,7 +1332,6 @@ TEST_F(BPWriteReadLocalVariables, ADIOS2BPWriteReadLocal1DBlockInfo) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("ADIOS2BPWriteReadLocal1DBlockInfo.bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -1336,6 +1343,9 @@ TEST_F(BPWriteReadLocalVariables, ADIOS2BPWriteReadLocal1DBlockInfo) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("ADIOS2BPWriteReadLocal1DBlockInfo_MPI.bp"); +#else + const std::string fname("ADIOS2BPWriteReadLocal1DBlockInfo.bp"); #endif // Write test data using BP diff --git a/testing/adios2/engine/bp/operations/TestBPWriteReadMGARD.cpp b/testing/adios2/engine/bp/operations/TestBPWriteReadMGARD.cpp index c5f98e4a90..282347cc4d 100644 --- a/testing/adios2/engine/bp/operations/TestBPWriteReadMGARD.cpp +++ b/testing/adios2/engine/bp/operations/TestBPWriteReadMGARD.cpp @@ -20,7 +20,6 @@ void MGARDAccuracy1D(const std::string tolerance) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("BPWRMGARD1D_" + tolerance + ".bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -39,6 +38,9 @@ void MGARDAccuracy1D(const std::string tolerance) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWRMGARD1D_" + tolerance + "_MPI.bp"); +#else + const std::string fname("BPWRMGARD1D_" + tolerance + ".bp"); #endif #if ADIOS2_USE_MPI @@ -175,7 +177,6 @@ void MGARDAccuracy2D(const std::string tolerance) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("BPWRMGARD2D_" + tolerance + ".bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -195,6 +196,9 @@ void MGARDAccuracy2D(const std::string tolerance) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWRMGARD2D_" + tolerance + "_MPI.bp"); +#else + const std::string fname("BPWRMGARD2D_" + tolerance + ".bp"); #endif #if ADIOS2_USE_MPI @@ -336,7 +340,6 @@ void MGARDAccuracy3D(const std::string tolerance) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("BPWRMGARD3D_" + tolerance + ".bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -357,6 +360,9 @@ void MGARDAccuracy3D(const std::string tolerance) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWRMGARD3D_" + tolerance + "_MPI.bp"); +#else + const std::string fname("BPWRMGARD3D_" + tolerance + ".bp"); #endif #if ADIOS2_USE_MPI @@ -498,7 +504,6 @@ void MGARDAccuracy1DSel(const std::string tolerance) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("BPWRMGARD1DSel_" + tolerance + ".bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -517,6 +522,9 @@ void MGARDAccuracy1DSel(const std::string tolerance) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWRMGARD1DSel_" + tolerance + "_MPI.bp"); +#else + const std::string fname("BPWRMGARD1DSel_" + tolerance + ".bp"); #endif #if ADIOS2_USE_MPI @@ -627,7 +635,6 @@ void MGARDAccuracy2DSel(const std::string tolerance) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("BPWRMGARD2DSel_" + tolerance + ".bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -647,6 +654,9 @@ void MGARDAccuracy2DSel(const std::string tolerance) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWRMGARD2DSel_" + tolerance + "_MPI.bp"); +#else + const std::string fname("BPWRMGARD2DSel_" + tolerance + ".bp"); #endif #if ADIOS2_USE_MPI @@ -759,7 +769,6 @@ void MGARDAccuracy3DSel(const std::string tolerance) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("BPWRMGARD3DSel_" + tolerance + ".bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -780,6 +789,9 @@ void MGARDAccuracy3DSel(const std::string tolerance) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWRMGARD3DSel_" + tolerance + "_MPI.bp"); +#else + const std::string fname("BPWRMGARD3DSel_" + tolerance + ".bp"); #endif #if ADIOS2_USE_MPI @@ -899,7 +911,6 @@ void MGARDNullBlocks(const std::string tolerance) // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("BPWRMGARDNull_" + tolerance + ".bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -915,6 +926,9 @@ void MGARDNullBlocks(const std::string tolerance) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWRMGARDNull_" + tolerance + "_MPI.bp"); +#else + const std::string fname("BPWRMGARDNull_" + tolerance + ".bp"); #endif #if ADIOS2_USE_MPI diff --git a/testing/adios2/engine/bp/operations/TestBPWriteReadMGARDCuda.cpp b/testing/adios2/engine/bp/operations/TestBPWriteReadMGARDCuda.cpp index a93adfcbbe..24b4c2dffe 100644 --- a/testing/adios2/engine/bp/operations/TestBPWriteReadMGARDCuda.cpp +++ b/testing/adios2/engine/bp/operations/TestBPWriteReadMGARDCuda.cpp @@ -20,7 +20,6 @@ std::string engineName; // comes from command line void MGARDAccuracy2D(const std::string tolerance) { - const std::string fname("BPWRMGARDCU2D_" + tolerance + ".bp"); int mpiRank = 0, mpiSize = 1; const size_t Nx = 100; @@ -33,6 +32,9 @@ void MGARDAccuracy2D(const std::string tolerance) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWRMGARDCU2D_" + tolerance + "_MPI.bp"); +#else + const std::string fname("BPWRMGARDCU2D_" + tolerance + ".bp"); #endif #if ADIOS2_USE_MPI @@ -140,7 +142,6 @@ void MGARDAccuracySmall(const std::string tolerance) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("BPWRMGARD1D_" + tolerance + ".bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -155,6 +156,9 @@ void MGARDAccuracySmall(const std::string tolerance) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWRMGARD1D_" + tolerance + "_MPI.bp"); +#else + const std::string fname("BPWRMGARD1D_" + tolerance + ".bp"); #endif #if ADIOS2_USE_MPI diff --git a/testing/adios2/engine/bp/operations/TestBPWriteReadMGARDMDR.cpp b/testing/adios2/engine/bp/operations/TestBPWriteReadMGARDMDR.cpp index 084e21631b..253c71da24 100644 --- a/testing/adios2/engine/bp/operations/TestBPWriteReadMGARDMDR.cpp +++ b/testing/adios2/engine/bp/operations/TestBPWriteReadMGARDMDR.cpp @@ -28,7 +28,6 @@ TEST_F(BPWriteReadMGARDMDR, BPWRMGARD1D) { // Refactor a dataset with MDR, then // read back with various accuracies - const std::string fname("BPWRMGARDMDR1D.bp"); int mpiRank = 0, mpiSize = 1; const size_t Nx = 30000; // 100k minimum data size for MDR @@ -49,6 +48,9 @@ TEST_F(BPWriteReadMGARDMDR, BPWRMGARD1D) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWRMGARDMDR1D_MPI.bp"); +#else + const std::string fname("BPWRMGARDMDR1D.bp"); #endif #if ADIOS2_USE_MPI diff --git a/testing/adios2/engine/bp/operations/TestBPWriteReadMGARDPlus.cpp b/testing/adios2/engine/bp/operations/TestBPWriteReadMGARDPlus.cpp index 3897361960..a1f95eaa08 100644 --- a/testing/adios2/engine/bp/operations/TestBPWriteReadMGARDPlus.cpp +++ b/testing/adios2/engine/bp/operations/TestBPWriteReadMGARDPlus.cpp @@ -20,7 +20,6 @@ void MGARDAccuracy1D(const std::string tolerance) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("BPWRMGARD1D_" + tolerance + ".bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -39,6 +38,9 @@ void MGARDAccuracy1D(const std::string tolerance) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWRMGARD1D_" + tolerance + "_MPI.bp"); +#else + const std::string fname("BPWRMGARD1D_" + tolerance + ".bp"); #endif #if ADIOS2_USE_MPI @@ -151,7 +153,6 @@ void MGARDAccuracy2D(const std::string tolerance) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("BPWRMGARD2D_" + tolerance + ".bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -171,6 +172,9 @@ void MGARDAccuracy2D(const std::string tolerance) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWRMGARD2D_" + tolerance + "_MPI.bp"); +#else + const std::string fname("BPWRMGARD2D_" + tolerance + ".bp"); #endif #if ADIOS2_USE_MPI @@ -284,7 +288,6 @@ void MGARDAccuracy3D(const std::string tolerance) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("BPWRMGARD3D_" + tolerance + ".bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -305,6 +308,9 @@ void MGARDAccuracy3D(const std::string tolerance) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWRMGARD3D_" + tolerance + "_MPI.bp"); +#else + const std::string fname("BPWRMGARD3D_" + tolerance + ".bp"); #endif #if ADIOS2_USE_MPI @@ -420,7 +426,6 @@ void MGARDAccuracy1DSel(const std::string tolerance) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("BPWRMGARD1DSel_" + tolerance + ".bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -439,6 +444,9 @@ void MGARDAccuracy1DSel(const std::string tolerance) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWRMGARD1DSel_" + tolerance + "_MPI.bp"); +#else + const std::string fname("BPWRMGARD1DSel_" + tolerance + ".bp"); #endif #if ADIOS2_USE_MPI @@ -541,7 +549,6 @@ void MGARDAccuracy2DSel(const std::string tolerance) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("BPWRMGARD2DSel_" + tolerance + ".bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -561,6 +568,9 @@ void MGARDAccuracy2DSel(const std::string tolerance) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWRMGARD2DSel_" + tolerance + "_MPI.bp"); +#else + const std::string fname("BPWRMGARD2DSel_" + tolerance + ".bp"); #endif #if ADIOS2_USE_MPI @@ -666,7 +676,6 @@ void MGARDAccuracy3DSel(const std::string tolerance) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("BPWRMGARD3DSel_" + tolerance + ".bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -687,6 +696,9 @@ void MGARDAccuracy3DSel(const std::string tolerance) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWRMGARD3DSel_" + tolerance + "_MPI.bp"); +#else + const std::string fname("BPWRMGARD3DSel_" + tolerance + ".bp"); #endif #if ADIOS2_USE_MPI @@ -804,7 +816,6 @@ void MGARDAccuracy2DSmallSel(const std::string tolerance) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("BPWRMGARD2DSmallSel_" + tolerance + ".bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -823,6 +834,9 @@ void MGARDAccuracy2DSmallSel(const std::string tolerance) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWRMGARD2DSmallSel_" + tolerance + "_MPI.bp"); +#else + const std::string fname("BPWRMGARD2DSmallSel_" + tolerance + ".bp"); #endif #if ADIOS2_USE_MPI diff --git a/testing/adios2/engine/bp/operations/TestBPWriteReadPNG.cpp b/testing/adios2/engine/bp/operations/TestBPWriteReadPNG.cpp index 84924f9de6..9437764eb4 100644 --- a/testing/adios2/engine/bp/operations/TestBPWriteReadPNG.cpp +++ b/testing/adios2/engine/bp/operations/TestBPWriteReadPNG.cpp @@ -26,7 +26,6 @@ void PNGAccuracy2D(const std::string compressionLevel) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("BPWRPNG2D_" + compressionLevel + ".bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -59,6 +58,9 @@ void PNGAccuracy2D(const std::string compressionLevel) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWRPNG2D_" + compressionLevel + "_MPI.bp"); +#else + const std::string fname("BPWRPNG2D_" + compressionLevel + ".bp"); #endif #if ADIOS2_USE_MPI @@ -275,7 +277,6 @@ void PNGAccuracy2DSel(const std::string accuracy) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("BPWRPNG2DSel_" + accuracy + ".bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -295,6 +296,9 @@ void PNGAccuracy2DSel(const std::string accuracy) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWRPNG2DSel_" + accuracy + "_MPI.bp"); +#else + const std::string fname("BPWRPNG2DSel_" + accuracy + ".bp"); #endif #if ADIOS2_USE_MPI diff --git a/testing/adios2/engine/bp/operations/TestBPWriteReadSZ.cpp b/testing/adios2/engine/bp/operations/TestBPWriteReadSZ.cpp index 3f95fa0d22..dc1045ca02 100644 --- a/testing/adios2/engine/bp/operations/TestBPWriteReadSZ.cpp +++ b/testing/adios2/engine/bp/operations/TestBPWriteReadSZ.cpp @@ -19,7 +19,6 @@ void SZAccuracy1D(const std::string accuracy) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("BPWRSZ1D_" + accuracy + ".bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -38,6 +37,9 @@ void SZAccuracy1D(const std::string accuracy) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWRSZ1D_" + accuracy + "_MPI.bp"); +#else + const std::string fname("BPWRSZ1D_" + accuracy + ".bp"); #endif #if ADIOS2_USE_MPI @@ -151,7 +153,6 @@ void SZAccuracy2D(const std::string accuracy) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("BPWRSZ2D_" + accuracy + ".bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -171,6 +172,9 @@ void SZAccuracy2D(const std::string accuracy) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWRSZ2D_" + accuracy + "_MPI.bp"); +#else + const std::string fname("BPWRSZ2D_" + accuracy + ".bp"); #endif #if ADIOS2_USE_MPI @@ -284,7 +288,6 @@ void SZAccuracy3D(const std::string accuracy) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("BPWRSZ3D_" + accuracy + ".bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -305,6 +308,9 @@ void SZAccuracy3D(const std::string accuracy) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWRSZ3D_" + accuracy + "_MPI.bp"); +#else + const std::string fname("BPWRSZ3D_" + accuracy + ".bp"); #endif #if ADIOS2_USE_MPI @@ -420,7 +426,6 @@ void SZAccuracy1DSel(const std::string accuracy) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("BPWRSZ1DSel_" + accuracy + ".bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -439,6 +444,9 @@ void SZAccuracy1DSel(const std::string accuracy) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWRSZ1DSel_" + accuracy + "_MPI.bp"); +#else + const std::string fname("BPWRSZ1DSel_" + accuracy + ".bp"); #endif #if ADIOS2_USE_MPI @@ -552,7 +560,6 @@ void SZAccuracy2DSel(const std::string accuracy) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("BPWRSZ2DSel_" + accuracy + ".bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -572,6 +579,9 @@ void SZAccuracy2DSel(const std::string accuracy) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWRSZ2DSel_" + accuracy + "_MPI.bp"); +#else + const std::string fname("BPWRSZ2DSel_" + accuracy + ".bp"); #endif #if ADIOS2_USE_MPI @@ -689,7 +699,6 @@ void SZAccuracy3DSel(const std::string accuracy) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("BPWRSZ3DSel_" + accuracy + ".bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -710,6 +719,9 @@ void SZAccuracy3DSel(const std::string accuracy) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWRSZ3DSel_" + accuracy + "_MPI.bp"); +#else + const std::string fname("BPWRSZ3DSel_" + accuracy + ".bp"); #endif #if ADIOS2_USE_MPI @@ -829,7 +841,6 @@ void SZAccuracy2DSmallSel(const std::string accuracy) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("BPWRSZ2DSmallSel_" + accuracy + ".bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -848,6 +859,9 @@ void SZAccuracy2DSmallSel(const std::string accuracy) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWRSZ2DSmallSel_" + accuracy + "_MPI.bp"); +#else + const std::string fname("BPWRSZ2DSmallSel_" + accuracy + ".bp"); #endif #if ADIOS2_USE_MPI diff --git a/testing/adios2/engine/bp/operations/TestBPWriteReadSzComplex.cpp b/testing/adios2/engine/bp/operations/TestBPWriteReadSzComplex.cpp index b0c526b379..b46c33d612 100644 --- a/testing/adios2/engine/bp/operations/TestBPWriteReadSzComplex.cpp +++ b/testing/adios2/engine/bp/operations/TestBPWriteReadSzComplex.cpp @@ -131,6 +131,7 @@ void Writer(const Dims &shape, const Dims &start, const Dims &count, const size_ size_t datasize = std::accumulate(count.begin(), count.end(), 1, std::multiplies()); #if ADIOS2_USE_MPI adios2::ADIOS adios(MPI_COMM_WORLD); + fileName = "TestBPWriteReadSzComplex_MPI"; #else adios2::ADIOS adios; #endif diff --git a/testing/adios2/engine/bp/operations/TestBPWriteReadZfp.cpp b/testing/adios2/engine/bp/operations/TestBPWriteReadZfp.cpp index 12e2208306..c1c4acb6d4 100644 --- a/testing/adios2/engine/bp/operations/TestBPWriteReadZfp.cpp +++ b/testing/adios2/engine/bp/operations/TestBPWriteReadZfp.cpp @@ -20,7 +20,6 @@ void ZFPRate1D(const std::string rate) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("BPWRZFP1D_" + rate + ".bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -39,6 +38,9 @@ void ZFPRate1D(const std::string rate) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWRZFP1D_" + rate + "_MPI.bp"); +#else + const std::string fname("BPWRZFP1D_" + rate + ".bp"); #endif #if ADIOS2_USE_MPI @@ -151,7 +153,6 @@ void ZFPRate2D(const std::string rate) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("BPWRZFP2D_" + rate + ".bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -171,6 +172,9 @@ void ZFPRate2D(const std::string rate) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWRZFP2D_" + rate + "_MPI.bp"); +#else + const std::string fname("BPWRZFP2D_" + rate + ".bp"); #endif #if ADIOS2_USE_MPI @@ -281,7 +285,6 @@ void ZFPRate3D(const std::string rate) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("BPWRZFP3D_" + rate + ".bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -302,6 +305,9 @@ void ZFPRate3D(const std::string rate) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWRZFP3D_" + rate + "_MPI.bp"); +#else + const std::string fname("BPWRZFP3D_" + rate + ".bp"); #endif #if ADIOS2_USE_MPI @@ -414,7 +420,6 @@ void ZFPRate1DSel(const std::string rate) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("BPWRZFP1DSel_" + rate + ".bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -433,6 +438,9 @@ void ZFPRate1DSel(const std::string rate) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWRZFP1DSel_" + rate + "_MPI.bp"); +#else + const std::string fname("BPWRZFP1DSel_" + rate + ".bp"); #endif #if ADIOS2_USE_MPI @@ -540,7 +548,6 @@ void ZFPRate2DSel(const std::string rate) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("BPWRZFP2DSel_" + rate + ".bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -560,6 +567,9 @@ void ZFPRate2DSel(const std::string rate) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWRZFP2DSel_" + rate + "_MPI.bp"); +#else + const std::string fname("BPWRZFP2DSel_" + rate + ".bp"); #endif #if ADIOS2_USE_MPI @@ -664,7 +674,6 @@ void ZFPRate3DSel(const std::string rate) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("BPWRZFP3DSel_" + rate + ".bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -685,6 +694,9 @@ void ZFPRate3DSel(const std::string rate) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWRZFP3DSel_" + rate + "_MPI.bp"); +#else + const std::string fname("BPWRZFP3DSel_" + rate + ".bp"); #endif #if ADIOS2_USE_MPI @@ -790,7 +802,6 @@ void ZFPRate2DSmallSel(const std::string rate) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("BPWRZFP2DSmallSel_" + rate + ".bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -809,6 +820,9 @@ void ZFPRate2DSmallSel(const std::string rate) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWRZFP2DSmallSel_" + rate + "_MPI.bp"); +#else + const std::string fname("BPWRZFP2DSmallSel_" + rate + ".bp"); #endif #if ADIOS2_USE_MPI diff --git a/testing/adios2/engine/bp/operations/TestBPWriteReadZfpConfig.cpp b/testing/adios2/engine/bp/operations/TestBPWriteReadZfpConfig.cpp index 7379963ea6..76ea948c4e 100644 --- a/testing/adios2/engine/bp/operations/TestBPWriteReadZfpConfig.cpp +++ b/testing/adios2/engine/bp/operations/TestBPWriteReadZfpConfig.cpp @@ -23,7 +23,6 @@ void ZfpRate1D(const std::string configFile) const auto begin = configFile.find("_rate") + 5; const auto end = configFile.find_last_of(".") - begin; const std::string rate = configFile.substr(begin, end); - const std::string fname("BPWriteReadZfpConfig1D_" + rate + ".bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -42,6 +41,9 @@ void ZfpRate1D(const std::string configFile) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWriteReadZfpConfig1D_" + rate + "_MPI.bp"); +#else + const std::string fname("BPWriteReadZfpConfig1D_" + rate + ".bp"); #endif #if ADIOS2_USE_MPI @@ -137,7 +139,6 @@ void ZfpRate2D(const std::string configFile) const auto begin = configFile.find("_rate") + 5; const auto end = configFile.find_last_of(".") - begin; const std::string rate = configFile.substr(begin, end); - const std::string fname("BPWriteReadZfpConfig2D_" + rate + ".bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -157,6 +158,9 @@ void ZfpRate2D(const std::string configFile) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWriteReadZfpConfig2D_" + rate + "_MPI.bp"); +#else + const std::string fname("BPWriteReadZfpConfig2D_" + rate + ".bp"); #endif #if ADIOS2_USE_MPI @@ -251,7 +255,6 @@ void ZfpRate3D(const std::string configFile) const auto begin = configFile.find("_rate") + 5; const auto end = configFile.find_last_of(".") - begin; const std::string rate = configFile.substr(begin, end); - const std::string fname("BPWriteReadZfpConfig3D_" + rate + ".bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -272,6 +275,9 @@ void ZfpRate3D(const std::string configFile) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWriteReadZfpConfig3D_" + rate + "_MPI.bp"); +#else + const std::string fname("BPWriteReadZfpConfig3D_" + rate + ".bp"); #endif #if ADIOS2_USE_MPI @@ -368,7 +374,6 @@ void ZfpRate1DSel(const std::string configFile) const auto begin = configFile.find("_rate") + 5; const auto end = configFile.find_last_of(".") - begin; const std::string rate = configFile.substr(begin, end); - const std::string fname("BPWriteReadZfpConfig1DSel_" + rate + ".bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -387,6 +392,9 @@ void ZfpRate1DSel(const std::string configFile) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWriteReadZfpConfig1DSel_" + rate + "_MPI.bp"); +#else + const std::string fname("BPWriteReadZfpConfig1DSel_" + rate + ".bp"); #endif #if ADIOS2_USE_MPI @@ -482,7 +490,6 @@ void ZfpRate2DSel(const std::string configFile) const auto begin = configFile.find("_rate") + 5; const auto end = configFile.find_last_of(".") - begin; const std::string rate = configFile.substr(begin, end); - const std::string fname("BPWriteReadZfpConfig2DSel_" + rate + ".bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -502,6 +509,9 @@ void ZfpRate2DSel(const std::string configFile) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWriteReadZfpConfig2DSel_" + rate + "_MPI.bp"); +#else + const std::string fname("BPWriteReadZfpConfig2DSel_" + rate + ".bp"); #endif #if ADIOS2_USE_MPI @@ -596,7 +606,6 @@ void ZfpRate3DSel(const std::string configFile) const auto begin = configFile.find("_rate") + 5; const auto end = configFile.find_last_of(".") - begin; const std::string rate = configFile.substr(begin, end); - const std::string fname("BPWriteReadZfpConfig3DSel_" + rate + ".bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -617,6 +626,9 @@ void ZfpRate3DSel(const std::string configFile) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWriteReadZfpConfig3DSel_" + rate + "_MPI.bp"); +#else + const std::string fname("BPWriteReadZfpConfig3DSel_" + rate + ".bp"); #endif #if ADIOS2_USE_MPI @@ -713,7 +725,6 @@ void ZfpRate2DSmallSel(const std::string configFile) const auto begin = configFile.find("_rate") + 5; const auto end = configFile.find_last_of(".") - begin; const std::string rate = configFile.substr(begin, end); - const std::string fname("BPWriteReadZfpConfig2DSmallSel_" + rate + ".bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -732,6 +743,9 @@ void ZfpRate2DSmallSel(const std::string configFile) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWriteReadZfpConfig2DSmallSel_" + rate + "_MPI.bp"); +#else + const std::string fname("BPWriteReadZfpConfig2DSmallSel_" + rate + ".bp"); #endif #if ADIOS2_USE_MPI diff --git a/testing/adios2/engine/bp/operations/TestBPWriteReadZfpCuda.cpp b/testing/adios2/engine/bp/operations/TestBPWriteReadZfpCuda.cpp index 299fa10f25..c0edb583b0 100644 --- a/testing/adios2/engine/bp/operations/TestBPWriteReadZfpCuda.cpp +++ b/testing/adios2/engine/bp/operations/TestBPWriteReadZfpCuda.cpp @@ -22,7 +22,6 @@ void ZFPRateCUDA(const std::string rate) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("BPWRZFP1D_" + rate + ".bp"); // Number of rows const size_t Nx = 100; @@ -34,6 +33,9 @@ void ZFPRateCUDA(const std::string rate) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWRZFP1D_" + rate + "_MPI.bp"); +#else + const std::string fname("BPWRZFP1D_" + rate + ".bp"); #endif #if ADIOS2_USE_MPI diff --git a/testing/adios2/engine/bp/operations/TestBPWriteReadZfpRemoveOperations.cpp b/testing/adios2/engine/bp/operations/TestBPWriteReadZfpRemoveOperations.cpp index 7aaf7c5c8a..c02f03da21 100644 --- a/testing/adios2/engine/bp/operations/TestBPWriteReadZfpRemoveOperations.cpp +++ b/testing/adios2/engine/bp/operations/TestBPWriteReadZfpRemoveOperations.cpp @@ -23,7 +23,6 @@ void ZFPRate1D(const std::string rate) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("BPWRZFPOdd1D_" + rate + ".bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -42,6 +41,9 @@ void ZFPRate1D(const std::string rate) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWRZFPOdd1D_" + rate + "_MPI.bp"); +#else + const std::string fname("BPWRZFPOdd1D_" + rate + ".bp"); #endif #if ADIOS2_USE_MPI @@ -169,7 +171,6 @@ void ZFPRate2D(const std::string rate) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("BPWRZFPOdd2D_" + rate + ".bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -189,6 +190,9 @@ void ZFPRate2D(const std::string rate) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWRZFPOdd2D_" + rate + "_MPI.bp"); +#else + const std::string fname("BPWRZFPOdd2D_" + rate + ".bp"); #endif #if ADIOS2_USE_MPI @@ -318,7 +322,6 @@ void ZFPRate3D(const std::string rate) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("BPWRZFPOdd3D_" + rate + ".bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -339,6 +342,9 @@ void ZFPRate3D(const std::string rate) #if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("BPWRZFPOdd3D_" + rate + "_MPI.bp"); +#else + const std::string fname("BPWRZFPOdd3D_" + rate + ".bp"); #endif #if ADIOS2_USE_MPI diff --git a/testing/adios2/engine/hdf5/TestHDF5WriteReadAsStream.cpp b/testing/adios2/engine/hdf5/TestHDF5WriteReadAsStream.cpp index 57b30e32b8..604ca9604f 100644 --- a/testing/adios2/engine/hdf5/TestHDF5WriteReadAsStream.cpp +++ b/testing/adios2/engine/hdf5/TestHDF5WriteReadAsStream.cpp @@ -26,7 +26,6 @@ TEST_F(HDF5WriteReadAsStreamTestADIOS2, ADIOS2HDF5WriteRead1D8) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("ADIOS2HDF5WriteReadAsStream1D8.h5"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -38,6 +37,9 @@ TEST_F(HDF5WriteReadAsStreamTestADIOS2, ADIOS2HDF5WriteRead1D8) #ifdef TEST_HDF5_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("ADIOS2HDF5WriteReadAsStream1D8_MPI.h5"); +#else + const std::string fname("ADIOS2HDF5WriteReadAsStream1D8.h5"); #endif // Write test data using HDF5 @@ -395,7 +397,6 @@ TEST_F(HDF5WriteReadAsStreamTestADIOS2, ADIOS2HDF5WriteRead2D2x4) { // Each process would write a 2x4 array and all processes would // form a 2D 2 * (numberOfProcess*Nx) matrix where Nx is 4 here - const std::string fname("ADIOS2HDF5WriteReadAsStream2D2x4Test.h5"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -410,6 +411,9 @@ TEST_F(HDF5WriteReadAsStreamTestADIOS2, ADIOS2HDF5WriteRead2D2x4) #ifdef TEST_HDF5_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("ADIOS2HDF5WriteReadAsStream2D2x4Test_MPI.h5"); +#else + const std::string fname("ADIOS2HDF5WriteReadAsStream2D2x4Test.h5"); #endif // Write test data using ADIOS2 @@ -634,7 +638,6 @@ TEST_F(HDF5WriteReadAsStreamTestADIOS2, ADIOS2HDF5WriteRead2D4x2) { // Each process would write a 4x2 array and all processes would // form a 2D 4 * (NumberOfProcess * Nx) matrix where Nx is 2 here - const std::string fname("ADIOS2HDF5WriteReadAsStream2D4x2Test.h5"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -648,6 +651,9 @@ TEST_F(HDF5WriteReadAsStreamTestADIOS2, ADIOS2HDF5WriteRead2D4x2) #ifdef TEST_HDF5_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("ADIOS2HDF5WriteReadAsStream2D4x2Test_MPI.h5"); +#else + const std::string fname("ADIOS2HDF5WriteReadAsStream2D4x2Test.h5"); #endif // Write test data using ADIOS2 @@ -874,9 +880,6 @@ TEST_F(HDF5WriteReadAsStreamTestADIOS2, ADIOS2HDF5WriteRead2D4x2) TEST_F(HDF5WriteReadAsStreamTestADIOS2, ReaderWriterDefineVariable) { - const std::string fnameFloat("HDF5ReaderWriterDefineVariable_float.h5"); - const std::string fname("HDF5ReaderWriterDefineVariable_all.h5"); - int mpiRank = 0, mpiSize = 1; // Number of rows const std::size_t Nx = 2; @@ -889,6 +892,11 @@ TEST_F(HDF5WriteReadAsStreamTestADIOS2, ReaderWriterDefineVariable) #ifdef TEST_HDF5_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fnameFloat("HDF5ReaderWriterDefineVariable_float.h5"); + const std::string fname("HDF5ReaderWriterDefineVariable_all.h5"); +#else + const std::string fnameFloat("HDF5ReaderWriterDefineVariable_float.h5"); + const std::string fname("HDF5ReaderWriterDefineVariable_all.h5"); #endif // Write test data using ADIOS2 diff --git a/testing/adios2/engine/hdf5/TestHDF5WriteReadAttributesADIOS2.cpp b/testing/adios2/engine/hdf5/TestHDF5WriteReadAttributesADIOS2.cpp index ac746724d4..a49f3352f8 100644 --- a/testing/adios2/engine/hdf5/TestHDF5WriteReadAttributesADIOS2.cpp +++ b/testing/adios2/engine/hdf5/TestHDF5WriteReadAttributesADIOS2.cpp @@ -25,9 +25,6 @@ class BPWriteReadAttributeTestADIOS2 : public ::testing::Test // ADIOS2 write, read for single value attributes TEST_F(BPWriteReadAttributeTestADIOS2, ADIOS2BPWriteReadSingleTypes) { - const std::string fName = - "." + std::string(&adios2::PathSeparator, 1) + "ADIOS2BPWriteAttributeReadSingleTypes.h5"; - const std::string zero = std::to_string(0); const std::string s1_Single = std::string("s1_Single_") + zero; const std::string s1_Array = std::string("s1_Array_") + zero; @@ -50,8 +47,14 @@ TEST_F(BPWriteReadAttributeTestADIOS2, ADIOS2BPWriteReadSingleTypes) // Write test data using BP #ifdef TEST_HDF5_MPI adios2::ADIOS adios(MPI_COMM_WORLD); + const std::string fName = "." + std::string(&adios2::PathSeparator, 1) + + "ADIOS2BPWriteAttributeReadSingleTypes_MPI.h5"; + #else adios2::ADIOS adios; + const std::string fName = + "." + std::string(&adios2::PathSeparator, 1) + "ADIOS2BPWriteAttributeReadSingleTypes.h5"; + #endif { adios2::IO io = adios.DeclareIO("TestIO"); @@ -185,13 +188,16 @@ TEST_F(BPWriteReadAttributeTestADIOS2, ADIOS2BPWriteReadSingleTypes) // ADIOS2 write read for array attributes TEST_F(BPWriteReadAttributeTestADIOS2, ADIOS2BPWriteReadArrayTypes) { - const std::string fName = - "." + std::string(&adios2::PathSeparator, 1) + "ADIOS2BPWriteAttributeReadArrayTypes.h5"; #ifdef TEST_HDF5_MPI int mpiRank = 0, mpiSize = 1; MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fName = "." + std::string(&adios2::PathSeparator, 1) + + "ADIOS2BPWriteAttributeReadArrayTypes_MPI.h5"; +#else + const std::string fName = + "." + std::string(&adios2::PathSeparator, 1) + "ADIOS2BPWriteAttributeReadArrayTypes.h5"; #endif const std::string zero = std::to_string(0); @@ -361,9 +367,6 @@ TEST_F(BPWriteReadAttributeTestADIOS2, ADIOS2BPWriteReadArrayTypes) TEST_F(BPWriteReadAttributeTestADIOS2, BPWriteReadSingleTypesVar) { - const std::string fName = - "." + std::string(&adios2::PathSeparator, 1) + "BPWriteAttributeReadSingleTypesVar.h5"; - const std::string zero = std::to_string(0); const std::string s1_Single = std::string("s1_Single_") + zero; const std::string i8_Single = std::string("i8_Single_") + zero; @@ -387,8 +390,14 @@ TEST_F(BPWriteReadAttributeTestADIOS2, BPWriteReadSingleTypesVar) // Write test data using BP #ifdef TEST_HDF5_MPI adios2::ADIOS adios(MPI_COMM_WORLD); + const std::string fName = + "." + std::string(&adios2::PathSeparator, 1) + "BPWriteAttributeReadSingleTypesVar_MPI.h5"; + #else adios2::ADIOS adios; + const std::string fName = + "." + std::string(&adios2::PathSeparator, 1) + "BPWriteAttributeReadSingleTypesVar.h5"; + #endif { adios2::IO io = adios.DeclareIO("TestIO"); @@ -511,13 +520,15 @@ TEST_F(BPWriteReadAttributeTestADIOS2, BPWriteReadSingleTypesVar) // ADIOS2 write read for array attributes TEST_F(BPWriteReadAttributeTestADIOS2, ADIOS2BPWriteReadArrayTypesVar) { - const std::string fName = - "." + std::string(&adios2::PathSeparator, 1) + "BPWriteAttributeReadArrayTypesVar.h5"; - #ifdef TEST_HDF5_MPI int mpiRank = 0, mpiSize = 1; MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fName = + "." + std::string(&adios2::PathSeparator, 1) + "BPWriteAttributeReadArrayTypesVar_MPI.h5"; +#else + const std::string fName = + "." + std::string(&adios2::PathSeparator, 1) + "BPWriteAttributeReadArrayTypesVar.h5"; #endif const std::string zero = std::to_string(0); diff --git a/testing/adios2/hierarchy/TestBPHierarchicalReading.cpp b/testing/adios2/hierarchy/TestBPHierarchicalReading.cpp index c4039dbe19..1d39c84c39 100644 --- a/testing/adios2/hierarchy/TestBPHierarchicalReading.cpp +++ b/testing/adios2/hierarchy/TestBPHierarchicalReading.cpp @@ -23,17 +23,19 @@ class ADIOSHierarchicalReadVariableTest : public ::testing::Test TEST_F(ADIOSHierarchicalReadVariableTest, Read) { - std::string filename = "ADIOSHierarchicalReadVariable." + engineName + ".bp"; - // Number of steps const std::size_t NSteps = 2; long unsigned int rank, size; #if ADIOS2_USE_MPI + std::string filename = "ADIOSHierarchicalReadVariable." + engineName + "_MPI.bp"; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &size); #else + std::string filename = "ADIOSHierarchicalReadVariable." + engineName + ".bp"; + rank = 0; size = 1; #endif diff --git a/testing/adios2/performance/manyvars/TestManyVars.cpp b/testing/adios2/performance/manyvars/TestManyVars.cpp index f17cedd5ec..923b6df054 100644 --- a/testing/adios2/performance/manyvars/TestManyVars.cpp +++ b/testing/adios2/performance/manyvars/TestManyVars.cpp @@ -235,8 +235,13 @@ class TestManyVars : public ::testing::TestWithParam NBLOCKS = p.nblocks; NSTEPS = p.nsteps; REDEFINE = redefineVars; +#if ADIOS2_USE_MPI + snprintf(FILENAME, sizeof(FILENAME), "manyVars.%zu_%zu_%zu%s_MPI.bp", NVARS, NBLOCKS, + NSTEPS, REDEFINE ? "_redefine" : ""); +#else snprintf(FILENAME, sizeof(FILENAME), "manyVars.%zu_%zu_%zu%s.bp", NVARS, NBLOCKS, NSTEPS, REDEFINE ? "_redefine" : ""); +#endif alloc_vars(); #if ADIOS2_USE_MPI diff --git a/testing/adios2/performance/query/TestBPQuery.cpp b/testing/adios2/performance/query/TestBPQuery.cpp index 6e0e918a10..7ded0040ec 100644 --- a/testing/adios2/performance/query/TestBPQuery.cpp +++ b/testing/adios2/performance/query/TestBPQuery.cpp @@ -81,7 +81,11 @@ class BPQueryTest : public ::testing::Test void BPQueryTest::QueryIntVar(const std::string &fname, adios2::ADIOS &adios, const std::string &engineName) { +#if ADIOS2_USE_MPI + std::string ioName = "IOQueryTestInt_MPI" + engineName; +#else std::string ioName = "IOQueryTestInt" + engineName; +#endif adios2::IO io = adios.DeclareIO(ioName.c_str()); if (!engineName.empty()) @@ -235,12 +239,13 @@ TEST_F(BPQueryTest, BP5) std::string engineName = "BP5"; // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname(engineName + "Query1D.bp"); #if ADIOS2_USE_MPI adios2::ADIOS adios(MPI_COMM_WORLD); + const std::string fname(engineName + "Query1D_MPI.bp"); #else adios2::ADIOS adios; + const std::string fname(engineName + "Query1D.bp"); #endif WriteFile(fname, adios, engineName); diff --git a/testing/h5vol/TestH5VolWriteReadBPFile.cpp b/testing/h5vol/TestH5VolWriteReadBPFile.cpp index 5fdd23b679..ddc202f3f5 100644 --- a/testing/h5vol/TestH5VolWriteReadBPFile.cpp +++ b/testing/h5vol/TestH5VolWriteReadBPFile.cpp @@ -462,8 +462,12 @@ TEST_F(H5VolWriteReadTest, H5VolWriteHDF5Read1D8) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname = "H5VolTest1D8.bp"; +#ifdef TEST_HDF5_MPI + const std::string fname = "H5VolTest1D8_MPI.bp"; +#else + const std::string fname = "H5VolTest1D8.bp"; +#endif int mpiRank = 0, mpiSize = 1; // Number of rows const std::size_t Nx = 8; @@ -649,8 +653,11 @@ TEST_F(H5VolWriteReadTest, H5VolWriteHDF5Read2D2x4) { // Each process would write a 2x4 array and all processes would // form a 2D 2 * (numberOfProcess*Nx) matrix where Nx is 4 here +#ifdef TEST_HDF5_MPI + std::string fname = "H5VolTest2D2x4_MPI.bp"; +#else std::string fname = "H5VolTest2D2x4.bp"; - +#endif int mpiRank = 0, mpiSize = 1; // Number of rows const std::size_t Nx = 4; From 0a563afec91bd7cf26dc19cdf11b0284c65920dc Mon Sep 17 00:00:00 2001 From: Vicente Adolfo Bolea Sanchez Date: Sat, 30 Mar 2024 16:50:06 -0400 Subject: [PATCH 103/124] cmake: add sqlite3 and zlib dep in adios2 cmake pkg --- cmake/adios2-config-common.cmake.in | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cmake/adios2-config-common.cmake.in b/cmake/adios2-config-common.cmake.in index d3f931f128..ceb2314868 100644 --- a/cmake/adios2-config-common.cmake.in +++ b/cmake/adios2-config-common.cmake.in @@ -147,6 +147,12 @@ if(NOT @BUILD_SHARED_LIBS@) find_dependency(AWSSDK) endif() + set(ADIOS2_HAVE_Campaign @ADIOS2_HAVE_Campaign@) + if(ADIOS2_HAVE_Campaign) + find_dependency(ZLIB) + find_dependency(SQLite3) + endif() + adios2_add_thirdparty_target(pugixml) set(ADIOS2_USE_EXTERNAL_PUGIXML @ADIOS2_USE_EXTERNAL_PUGIXML@) From 1a0b94349ede29dc02e99237bd5f4f7498b35cfd Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Thu, 28 Mar 2024 16:38:27 -0400 Subject: [PATCH 104/124] Add support for user options in ~/.config/adios2/adios2.yaml Currently supported options: General: verbose: 0 Campaign: active: true hostname: LAP131864 campaignstorepath: ~/dropbox/adios-campaign-store cachepath: /tmp/campaign verbose: 0 SST: verbose: 0 ~ is processed to $HOME (%HOMEPATH% on Windows) (but only in select path options) Usage: every engine has "const UserOptions &m_UserOption" member available and can go from there. Other parts of the code can call "static const UserOptions ADIOS::GetUserOptions()" and go from there. Make Campaign AUTO (on when sqlite3 and zlib are found) - still the campaign manager can be deactivated in adios2.yaml --- CMakeLists.txt | 2 +- source/adios2/common/ADIOSTypes.h | 27 ++++ source/adios2/core/ADIOS.cpp | 39 +++++- source/adios2/core/ADIOS.h | 5 + source/adios2/core/Engine.cpp | 3 +- source/adios2/core/Engine.h | 3 + .../engine/campaign/CampaignManager.cpp | 32 ++--- .../adios2/engine/campaign/CampaignManager.h | 7 +- .../adios2/engine/campaign/CampaignReader.cpp | 110 ++++++++-------- .../adios2/engine/campaign/CampaignReader.h | 6 +- source/adios2/engine/sst/SstParamParser.cpp | 5 +- source/adios2/engine/sst/SstParamParser.h | 4 +- source/adios2/engine/sst/SstReader.cpp | 2 +- source/adios2/engine/sst/SstWriter.cpp | 2 +- source/adios2/helper/adiosUserOptions.cpp | 121 ++++++++++++++++++ source/adios2/helper/adiosUserOptions.h | 30 +++++ source/adios2/helper/adiosYAML.cpp | 108 ++++++++++++++++ source/adios2/helper/adiosYAML.h | 4 + source/utils/bpls/bpls.cpp | 51 ++++++-- source/utils/bpls/bpls.h | 3 +- 20 files changed, 455 insertions(+), 109 deletions(-) create mode 100644 source/adios2/helper/adiosUserOptions.cpp create mode 100644 source/adios2/helper/adiosUserOptions.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 8511e152bf..37e4b6016b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -183,7 +183,7 @@ adios_option(Profiling "Enable support for profiling" AUTO) adios_option(Endian_Reverse "Enable support for Little/Big Endian Interoperability" AUTO) adios_option(Sodium "Enable support for Sodium for encryption" AUTO) adios_option(Catalyst "Enable support for in situ visualization plugin using ParaView Catalyst" AUTO) -adios_option(Campaign "Enable support for Campaigns (requires SQLite3 and ZLIB)" OFF) +adios_option(Campaign "Enable support for Campaigns (requires SQLite3 and ZLIB)" AUTO) adios_option(AWSSDK "Enable support for S3 compatible storage using AWS SDK's S3 module" OFF) adios_option(Derived_Variable "Enable support for derived variables" OFF) adios_option(PIP "Enable support for pip packaging" OFF) diff --git a/source/adios2/common/ADIOSTypes.h b/source/adios2/common/ADIOSTypes.h index 5d9eeb2624..fad51e8fd9 100644 --- a/source/adios2/common/ADIOSTypes.h +++ b/source/adios2/common/ADIOSTypes.h @@ -357,6 +357,33 @@ std::string ToString(const Dims &dims); std::string ToString(const Box &box); std::string ToString(const MemorySpace value); +/** UserOptions holds all user options from ~/.config/adios2/adios2.yaml */ +struct UserOptions +{ + struct General + { + int verbose; + }; + + struct Campaign + { + bool active; + int verbose; + std::string hostname; + std::string campaignstorepath; + std::string cachepath; + }; + + struct SST + { + int verbose; + }; + + General general; + Campaign campaign; + SST sst; +}; + /** * os << [adios2_type] enables output of adios2 enums/classes directly * to output streams (e.g. std::cout), if ToString() can handle [adios2_type]. diff --git a/source/adios2/core/ADIOS.cpp b/source/adios2/core/ADIOS.cpp index 6504474d81..a77ad1d0fc 100644 --- a/source/adios2/core/ADIOS.cpp +++ b/source/adios2/core/ADIOS.cpp @@ -19,6 +19,7 @@ #include "adios2/core/IO.h" #include "adios2/helper/adiosCommDummy.h" #include "adios2/helper/adiosFunctions.h" //InquireKey, BroadcastFile +#include "adios2/helper/adiosYAML.h" #include "adios2/operator/OperatorFactory.h" #include @@ -106,6 +107,10 @@ std::mutex PerfStubsMutex; static std::atomic_uint adios_refcount(0); // adios objects at the same time static std::atomic_uint adios_count(0); // total adios objects during runtime +/** User defined options from ~/.config/adios2/adios2.yaml if it exists */ +static adios2::UserOptions UserOptions; +const adios2::UserOptions &ADIOS::GetUserOptions() { return UserOptions; }; + ADIOS::ADIOS(const std::string configFile, helper::Comm comm, const std::string hostLanguage) : m_HostLanguage(hostLanguage), m_Comm(std::move(comm)), m_ConfigFile(configFile), m_CampaignManager(m_Comm) @@ -124,6 +129,7 @@ ADIOS::ADIOS(const std::string configFile, helper::Comm comm, const std::string } } #endif + ProcessUserConfig(); if (!configFile.empty()) { if (!adios2sys::SystemTools::FileExists(configFile)) @@ -143,8 +149,11 @@ ADIOS::ADIOS(const std::string configFile, helper::Comm comm, const std::string #ifdef ADIOS2_HAVE_KOKKOS m_GlobalServices.Init_Kokkos_API(); #endif - std::string campaignName = "campaign_" + std::to_string(adios_count); - m_CampaignManager.Open(campaignName); + if (UserOptions.campaign.active) + { + std::string campaignName = "campaign_" + std::to_string(adios_count); + m_CampaignManager.Open(campaignName, UserOptions); + } } ADIOS::ADIOS(const std::string configFile, const std::string hostLanguage) @@ -166,7 +175,26 @@ ADIOS::~ADIOS() { m_GlobalServices.Finalize(); } - m_CampaignManager.Close(); + if (UserOptions.campaign.active) + { + m_CampaignManager.Close(); + } +} + +void ADIOS::ProcessUserConfig() +{ + // read config parameters from config file + std::string homePath; +#ifdef _WIN32 + homePath = getenv("HOMEPATH"); +#else + homePath = getenv("HOME"); +#endif + const std::string cfgFile = homePath + "/.config/adios2/adios2.yaml"; + if (adios2sys::SystemTools::FileExists(cfgFile)) + { + helper::ParseUserOptionsFile(m_Comm, cfgFile, UserOptions, homePath); + } } IO &ADIOS::DeclareIO(const std::string name, const ArrayOrdering ArrayOrder) @@ -330,7 +358,10 @@ void ADIOS::YAMLInitIO(const std::string &configFileYAML, const std::string &con void ADIOS::RecordOutputStep(const std::string &name, const size_t step, const double time) { - m_CampaignManager.Record(name, step, time); + if (UserOptions.campaign.active) + { + m_CampaignManager.Record(name, step, time); + } } void ADIOS::Global_init_AWS_API() diff --git a/source/adios2/core/ADIOS.h b/source/adios2/core/ADIOS.h index bf534a960e..4dd14e17d7 100644 --- a/source/adios2/core/ADIOS.h +++ b/source/adios2/core/ADIOS.h @@ -164,6 +164,9 @@ class ADIOS void RecordOutputStep(const std::string &name, const size_t step = UnknownStep, const double time = UnknownTime); + /** A constant reference to the user options from ~/.config/adios2/adios2.yaml */ + static const adios2::UserOptions &GetUserOptions(); + private: /** Communicator given to parallel constructor. */ helper::Comm m_Comm; @@ -204,6 +207,8 @@ class ADIOS void YAMLInitIO(const std::string &configFileYAML, const std::string &configFileContents, core::IO &io); + void ProcessUserConfig(); + private: /* Global services that we want to initialize at most once and shutdown automatically when the ADIOS object is destructed. This only works diff --git a/source/adios2/core/Engine.cpp b/source/adios2/core/Engine.cpp index ab0a5e1761..72ab54d4ac 100644 --- a/source/adios2/core/Engine.cpp +++ b/source/adios2/core/Engine.cpp @@ -23,7 +23,8 @@ namespace core Engine::Engine(const std::string engineType, IO &io, const std::string &name, const Mode openMode, helper::Comm comm) -: m_EngineType(engineType), m_IO(io), m_Name(name), m_OpenMode(openMode), m_Comm(std::move(comm)) +: m_EngineType(engineType), m_IO(io), m_Name(name), m_OpenMode(openMode), m_Comm(std::move(comm)), + m_UserOptions(io.m_ADIOS.GetUserOptions()) { m_FailVerbose = (m_Comm.Rank() == 0); } diff --git a/source/adios2/core/Engine.h b/source/adios2/core/Engine.h index 5e2a721cae..0ab6ed4a00 100644 --- a/source/adios2/core/Engine.h +++ b/source/adios2/core/Engine.h @@ -505,6 +505,9 @@ class Engine * if no communicator is passed */ helper::Comm m_Comm; + /** User options parsed by the ADIOS object, here just for easy reference */ + const UserOptions &m_UserOptions; + /** keeps track of current advance status */ StepStatus m_AdvanceStatus = StepStatus::OK; diff --git a/source/adios2/engine/campaign/CampaignManager.cpp b/source/adios2/engine/campaign/CampaignManager.cpp index bd511ce687..3375464d6b 100644 --- a/source/adios2/engine/campaign/CampaignManager.cpp +++ b/source/adios2/engine/campaign/CampaignManager.cpp @@ -43,7 +43,7 @@ int CMapToSqlite(const CampaignRecordMap &cmap, const int rank, std::string name "SQL error on writing records:"); sqlite3_free(zErrMsg); } - sqlcmd = "CREATE TABLE if not exists bpfiles (name);"; + sqlcmd = "CREATE TABLE if not exists bpfiles (name PRIMARY KEY);"; rc = sqlite3_exec(db, sqlcmd.c_str(), 0, 0, &zErrMsg); if (rc != SQLITE_OK) { @@ -56,7 +56,7 @@ int CMapToSqlite(const CampaignRecordMap &cmap, const int rank, std::string name for (auto &r : cmap) { - sqlcmd = "INSERT INTO bpfiles (name)\n"; + sqlcmd = "INSERT OR IGNORE INTO bpfiles (name)\n"; sqlcmd += "VALUES('" + r.first + "');"; rc = sqlite3_exec(db, sqlcmd.c_str(), 0, 0, &zErrMsg); if (rc != SQLITE_OK) @@ -74,31 +74,27 @@ int CMapToSqlite(const CampaignRecordMap &cmap, const int rank, std::string name return 0; } -CampaignManager::CampaignManager(adios2::helper::Comm &comm) -{ - m_WriterRank = comm.Rank(); - if (m_Verbosity == 5) - { - std::cout << "Campaign Manager " << m_WriterRank << " constructor called" << std::endl; - } -} +CampaignManager::CampaignManager(adios2::helper::Comm &comm) { m_WriterRank = comm.Rank(); } CampaignManager::~CampaignManager() { - if (m_Verbosity == 5) - { - std::cout << "Campaign Manager " << m_WriterRank << " desctructor called\n"; - } if (m_Opened) { Close(); } } -void CampaignManager::Open(const std::string &name) +void CampaignManager::Open(const std::string &name, const UserOptions &options) { - m_Name = m_CampaignDir + "/" + name + "_" + std::to_string(m_WriterRank); - if (m_Verbosity == 5) + const UserOptions::Campaign &opts = options.campaign; + m_Options.active = opts.active; + m_Options.hostname = opts.hostname; + m_Options.campaignstorepath = opts.campaignstorepath; + m_Options.cachepath = opts.cachepath; + m_Options.verbose = opts.verbose; + + m_Name = m_CampaignDir + PathSeparator + name + "_" + std::to_string(m_WriterRank); + if (m_Options.verbose > 0) { std::cout << "Campaign Manager " << m_WriterRank << " Open(" << m_Name << ")\n"; } @@ -107,7 +103,7 @@ void CampaignManager::Open(const std::string &name) void CampaignManager::Record(const std::string &name, const size_t step, const double time) { - if (m_Verbosity == 5) + if (m_Options.verbose > 0) { std::cout << "Campaign Manager " << m_WriterRank << " Record name = " << name << " step = " << step << " time = " << time << "\n"; diff --git a/source/adios2/engine/campaign/CampaignManager.h b/source/adios2/engine/campaign/CampaignManager.h index 7a313f5627..498de83caa 100644 --- a/source/adios2/engine/campaign/CampaignManager.h +++ b/source/adios2/engine/campaign/CampaignManager.h @@ -16,6 +16,7 @@ #define ADIOS2_ENGINE_CAMPAIGNMANAGER_H_ #include "adios2/common/ADIOSConfig.h" +#include "adios2/common/ADIOSTypes.h" #include "adios2/helper/adiosComm.h" #ifdef ADIOS2_HAVE_CAMPAIGN @@ -39,15 +40,15 @@ class CampaignManager CampaignManager(helper::Comm &comm); ~CampaignManager(); - void Open(const std::string &name); + void Open(const std::string &name, const UserOptions &options); void Record(const std::string &name, const size_t step, const double time); void Close(); private: + UserOptions::Campaign m_Options; bool m_Opened = false; std::string m_Name; int m_WriterRank; - int m_Verbosity = 0; CampaignRecordMap cmap; std::ofstream m_Output; const std::string m_CampaignDir = "adios-campaign"; @@ -56,7 +57,7 @@ class CampaignManager public: CampaignManager(helper::Comm &comm){}; ~CampaignManager() = default; - void Open(const std::string &name){}; + void Open(const std::string &name, const INIOptions &options){}; void Record(const std::string &name, const size_t step, const double time){}; void Close(){}; diff --git a/source/adios2/engine/campaign/CampaignReader.cpp b/source/adios2/engine/campaign/CampaignReader.cpp index 29642d64dc..2bec32bf22 100644 --- a/source/adios2/engine/campaign/CampaignReader.cpp +++ b/source/adios2/engine/campaign/CampaignReader.cpp @@ -15,6 +15,7 @@ #include "adios2/helper/adiosNetwork.h" // GetFQDN #include "adios2/helper/adiosSystem.h" // CreateDirectory #include +#include #include #include @@ -33,7 +34,7 @@ CampaignReader::CampaignReader(IO &io, const std::string &name, const Mode mode, { m_ReaderRank = m_Comm.Rank(); Init(); - if (m_Verbosity == 5) + if (m_Options.verbose == 5) { std::cout << "Campaign Reader " << m_ReaderRank << " Open(" << m_Name << ") in constructor." << std::endl; @@ -44,7 +45,7 @@ CampaignReader::CampaignReader(IO &io, const std::string &name, const Mode mode, CampaignReader::~CampaignReader() { /* CampaignReader destructor does close and finalize */ - if (m_Verbosity == 5) + if (m_Options.verbose == 5) { std::cout << "Campaign Reader " << m_ReaderRank << " destructor on " << m_Name << "\n"; } @@ -61,7 +62,7 @@ StepStatus CampaignReader::BeginStep(const StepMode mode, const float timeoutSec // so this forced increase should not be here ++m_CurrentStep; - if (m_Verbosity == 5) + if (m_Options.verbose == 5) { std::cout << "Campaign Reader " << m_ReaderRank << " BeginStep() new step " << m_CurrentStep << "\n"; @@ -88,7 +89,7 @@ StepStatus CampaignReader::BeginStep(const StepMode mode, const float timeoutSec void CampaignReader::PerformGets() { - if (m_Verbosity == 5) + if (m_Options.verbose == 5) { std::cout << "Campaign Reader " << m_ReaderRank << " PerformGets()\n"; } @@ -106,7 +107,7 @@ void CampaignReader::EndStep() PerformGets(); } - if (m_Verbosity == 5) + if (m_Options.verbose == 5) { std::cout << "Campaign Reader " << m_ReaderRank << " EndStep()\n"; } @@ -116,49 +117,18 @@ void CampaignReader::EndStep() void CampaignReader::Init() { - // read config parameters from config file - const std::string cfgFile = "/.config/adios2/campaign.cfg"; - std::string homePath; -#ifdef _WIN32 - homePath = getenv("HOMEPATH"); -#else - homePath = getenv("HOME"); -#endif - ReadConfig(std::string(homePath + cfgFile)); InitParameters(); InitTransports(); } -void CampaignReader::ReadConfig(std::string configPath) -{ - std::ifstream fileStream(configPath); - - if (!fileStream) - { - return; - } - - std::ostringstream fileSS; - fileSS << fileStream.rdbuf(); - fileStream.close(); - size_t posEndline = 0; - size_t posSpace = 0; - std::string token; - std::string endline = "\n"; - std::string fileString = fileSS.str(); - while ((posEndline = fileString.find(endline)) != std::string::npos) - { - std::string line = fileString.substr(0, posEndline); - posSpace = fileString.find(" "); - std::string token1 = line.substr(0, posSpace); - std::string token2 = line.substr(posSpace + 1, line.size()); - // trim? - fileString.erase(0, posEndline + endline.length()); - } - return; -} void CampaignReader::InitParameters() { + const UserOptions::Campaign &opts = m_UserOptions.campaign; + m_Options.active = true; // this is really just for Recording + m_Options.hostname = opts.hostname; + m_Options.campaignstorepath = opts.campaignstorepath; + m_Options.cachepath = opts.cachepath; + m_Options.verbose = opts.verbose; for (const auto &pair : m_IO.m_Parameters) { std::string key(pair.first); @@ -169,8 +139,8 @@ void CampaignReader::InitParameters() if (key == "verbose") { - m_Verbosity = std::stoi(value); - if (m_Verbosity < 0 || m_Verbosity > 5) + m_Options.verbose = std::stoi(value); + if (m_Options.verbose < 0 || m_Options.verbose > 5) helper::Throw("Engine", "CampaignReader", "InitParameters", "Method verbose argument must be an " "integer in the range [0,5], in call to " @@ -178,37 +148,59 @@ void CampaignReader::InitParameters() } if (key == "hostname") { - m_Hostname = pair.second; + m_Options.hostname = pair.second; + } + if (key == "campaignstorepath") + { + m_Options.campaignstorepath = pair.second; } if (key == "cachepath") { - m_CachePath = pair.second; + m_Options.cachepath = pair.second; } } - if (m_Hostname.empty()) + if (m_Options.hostname.empty()) { - m_Hostname = helper::GetClusterName(); + m_Options.hostname = helper::GetClusterName(); + } + + if (m_Options.verbose > 0) + { + std::cout << "CampaignReader: \n"; + std::cout << " Hostname = " << m_Options.hostname << std::endl; + std::cout << " Campaign Store Path = " << m_Options.campaignstorepath << std::endl; + std::cout << " Cache Path = " << m_Options.cachepath << std::endl; } - // std::cout << "My Hostname is " << m_Hostname << std::endl; } void CampaignReader::InitTransports() { - int rc = sqlite3_open(m_Name.c_str(), &m_DB); + std::string path = m_Name; + if (!adios2sys::SystemTools::FileExists(path) && path[0] != '/' && path[0] != '\\' && + !m_Options.campaignstorepath.empty()) + { + std::string path2 = m_Options.campaignstorepath + PathSeparator + m_Name; + if (adios2sys::SystemTools::FileExists(path2)) + { + path = path2; + } + } + + int rc = sqlite3_open(path.c_str(), &m_DB); if (rc) { std::string dbmsg(sqlite3_errmsg(m_DB)); sqlite3_close(m_DB); helper::Throw("Engine", "CampaignReader", "Open", - "Cannot open database" + m_Name + ": " + dbmsg); + "Cannot open database" + path + ": " + dbmsg); } ReadCampaignData(m_DB, m_CampaignData); - if (m_Verbosity == 1) + if (m_Options.verbose == 1) { - std::cout << "Local hostname = " << m_Hostname << "\n"; + std::cout << "Local hostname = " << m_Options.hostname << "\n"; std::cout << "Database result:\n version = " << m_CampaignData.version << "\n hosts:\n"; for (size_t hostidx = 0; hostidx < m_CampaignData.hosts.size(); ++hostidx) @@ -243,18 +235,18 @@ void CampaignReader::InitTransports() { adios2::core::IO &io = m_IO.m_ADIOS.DeclareIO("CampaignReader" + std::to_string(i)); std::string localPath; - if (m_CampaignData.hosts[ds.hostIdx].hostname != m_Hostname) + if (m_CampaignData.hosts[ds.hostIdx].hostname != m_Options.hostname) { const std::string remotePath = m_CampaignData.hosts[ds.hostIdx].directory[ds.dirIdx] + PathSeparator + ds.name; const std::string remoteURL = m_CampaignData.hosts[ds.hostIdx].hostname + ":" + remotePath; - if (m_Verbosity == 1) + if (m_Options.verbose == 1) { std::cout << "Open remote file " << remoteURL << "\n"; } - localPath = m_CachePath + PathSeparator + m_CampaignData.hosts[ds.hostIdx].hostname + - PathSeparator + ds.name; + localPath = m_Options.cachepath + PathSeparator + + m_CampaignData.hosts[ds.hostIdx].hostname + PathSeparator + ds.name; helper::CreateDirectory(localPath); for (auto &bpf : ds.files) { @@ -269,7 +261,7 @@ void CampaignReader::InitTransports() { localPath = m_CampaignData.hosts[ds.hostIdx].directory[ds.dirIdx] + PathSeparator + ds.name; - if (m_Verbosity == 1) + if (m_Options.verbose == 1) { std::cout << "Open local file " << localPath << "\n"; } @@ -311,7 +303,7 @@ void CampaignReader::InitTransports() void CampaignReader::DoClose(const int transportIndex) { - if (m_Verbosity == 5) + if (m_Options.verbose == 5) { std::cout << "Campaign Reader " << m_ReaderRank << " Close(" << m_Name << ")\n"; } diff --git a/source/adios2/engine/campaign/CampaignReader.h b/source/adios2/engine/campaign/CampaignReader.h index 3678318f16..4747f7d875 100644 --- a/source/adios2/engine/campaign/CampaignReader.h +++ b/source/adios2/engine/campaign/CampaignReader.h @@ -54,10 +54,8 @@ class CampaignReader : public Engine bool VariableMinMax(const VariableBase &, const size_t Step, MinMaxStruct &MinMax); private: - int m_Verbosity = 0; // runtime parameter Verbose - std::string m_Hostname; // runtime parameter Hostname - std::string m_CachePath = "/tmp"; // runtime parameter CachePath - int m_ReaderRank; // my rank in the readers' comm + UserOptions::Campaign m_Options; + int m_ReaderRank; // my rank in the readers' comm int m_CurrentStep = 0; diff --git a/source/adios2/engine/sst/SstParamParser.cpp b/source/adios2/engine/sst/SstParamParser.cpp index ee8bd9a8cc..2b7041afa6 100644 --- a/source/adios2/engine/sst/SstParamParser.cpp +++ b/source/adios2/engine/sst/SstParamParser.cpp @@ -9,7 +9,8 @@ using namespace adios2; using namespace adios2::core; -void SstParamParser::ParseParams(IO &io, struct _SstParams &Params) +void SstParamParser::ParseParams(IO &io, struct _SstParams &Params, + const adios2::UserOptions &userOptions) { std::memset(&Params, 0, sizeof(Params)); @@ -254,6 +255,8 @@ void SstParamParser::ParseParams(IO &io, struct _SstParams &Params) return false; }; + Params.verbose = userOptions.sst.verbose; + #define get_params(Param, Type, Typedecl, Default) \ Params.Param = Default; \ lf_Set##Type##Parameter(#Param, Params.Param); diff --git a/source/adios2/engine/sst/SstParamParser.h b/source/adios2/engine/sst/SstParamParser.h index 0502cfcd5d..251c74cfa9 100644 --- a/source/adios2/engine/sst/SstParamParser.h +++ b/source/adios2/engine/sst/SstParamParser.h @@ -2,6 +2,7 @@ #define ADIOS2_ENGINE_SST_SSTPARAMPARSER_H_ #include "adios2/common/ADIOSConfig.h" +#include "adios2/common/ADIOSTypes.h" #include "adios2/core/IO.h" #include #include @@ -13,7 +14,8 @@ using namespace adios2::core; class SstParamParser { public: - void ParseParams(adios2::core::IO &io, struct _SstParams &Params); + void ParseParams(adios2::core::IO &io, struct _SstParams &Params, + const adios2::UserOptions &userOptions); }; #endif diff --git a/source/adios2/engine/sst/SstReader.cpp b/source/adios2/engine/sst/SstReader.cpp index 2754e75f6e..03bfced452 100644 --- a/source/adios2/engine/sst/SstReader.cpp +++ b/source/adios2/engine/sst/SstReader.cpp @@ -509,7 +509,7 @@ void SstReader::Init() { SstParamParser Parser; - Parser.ParseParams(m_IO, Params); + Parser.ParseParams(m_IO, Params, m_UserOptions); } #define declare_gets(T) \ diff --git a/source/adios2/engine/sst/SstWriter.cpp b/source/adios2/engine/sst/SstWriter.cpp index 305578ef5c..146ff0f735 100644 --- a/source/adios2/engine/sst/SstWriter.cpp +++ b/source/adios2/engine/sst/SstWriter.cpp @@ -389,7 +389,7 @@ void SstWriter::Init() { SstParamParser Parser; - Parser.ParseParams(m_IO, Params); + Parser.ParseParams(m_IO, Params, m_UserOptions); if (Params.verbose < 0 || Params.verbose > 5) { diff --git a/source/adios2/helper/adiosUserOptions.cpp b/source/adios2/helper/adiosUserOptions.cpp new file mode 100644 index 0000000000..34db1778ff --- /dev/null +++ b/source/adios2/helper/adiosUserOptions.cpp @@ -0,0 +1,121 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + */ + +#include "adiosUserOptions.h" + +#include "adios2/helper/adiosString.h" + +#include + +namespace adios2 +{ +namespace helper +{ + +namespace +{ + +inline void FixHomePath(std::string &path, std::string &homePath) +{ + if (!path.empty() && path[0] == '~') + { + path = homePath + path.substr(1); + } +} + +constexpr bool isMandatory = true; +constexpr bool isNotMandatory = false; + +YAML::Node YAMLNode(const std::string nodeName, const YAML::Node &upperNode, + const std::string &hint, const bool isMandatory, + const YAML::NodeType::value nodeType) +{ + const YAML::Node node = upperNode[nodeName]; + + if (isMandatory && !node) + { + helper::Throw( + "Helper", "adiosYAML", "YAMLNode", + "no " + nodeName + " node found, (is your node key lower case?), " + hint); + } + if (node && node.Type() != nodeType) + { + helper::Throw("Helper", "adiosYAML", "YAMLNode", + "node " + nodeName + + " is the wrong type, review adios2 " + "config YAML specs for the node, " + + hint); + } + return node; +} + +} // end empty namespace + +void ParseUserOptionsFile(Comm &comm, const std::string &configFileYAML, UserOptions &options, + std::string &homePath) +{ + const std::string hint = + "when parsing user config file " + configFileYAML + " in call to ADIOS constructor"; + + const std::string configFileContents = comm.BroadcastFile(configFileYAML, hint); + const YAML::Node document = YAML::Load(configFileContents); + + if (!document) + { + helper::Throw( + "Helper", "adiosUserOptions", "ParseUserOptionsFile", + "parser error in file " + configFileYAML + + " invalid format. Check with any YAML editor if format is ill-formed, " + hint); + } + + /* + * This code section below determines what options we recognize at all from the + * ~/.config/adios2/adios2.yaml file + */ + + for (auto itNode = document.begin(); itNode != document.end(); ++itNode) + { + std::cout << itNode->const YAML::Node general = + YAMLNode("IO", *itNode, hint, isNotMandatory, YAML::NodeType::Scalar); + if (ioScalar) + { + const std::string ioName = ioScalar.as(); + // Build the IO object + auto itCurrentIO = + ios.emplace(std::piecewise_construct, std::forward_as_tuple(ioName), + std::forward_as_tuple(adios, ioName, true, adios.m_HostLanguage)); + core::IO ¤tIO = itCurrentIO.first->second; + IOYAML(adios, *itNode, currentIO, hint); + } + } + + /* + if (iniReader.HasSection("General")) + { + options.general.verbose = static_cast(iniReader.GetInteger("General", "verbose", + 0)); + } + + if (iniReader.HasSection("Campaign")) + { + options.campaign.active = iniReader.GetBoolean("Campaign", "active", true); + options.campaign.verbose = static_cast(iniReader.GetInteger("Campaign", "verbose", + 0)); options.campaign.hostname = iniReader.Get("Campaign", "hostname", ""); + options.campaign.campaignstorepath = iniReader.Get("Campaign", "campaignstorepath", ""); + FixHomePath(options.campaign.campaignstorepath, homePath); + options.campaign.cachepath = iniReader.Get("Campaign", "cachepath", + "/tmp/adios2-cache"); FixHomePath(options.campaign.cachepath, homePath); + } + + if (iniReader.HasSection("SST")) + { + options.sst.verbose = static_cast(iniReader.GetInteger("SST", "verbose", 0)); + } + */ +} + +} // end namespace helper +} // end namespace adios2 diff --git a/source/adios2/helper/adiosUserOptions.h b/source/adios2/helper/adiosUserOptions.h new file mode 100644 index 0000000000..aec207f076 --- /dev/null +++ b/source/adios2/helper/adiosUserOptions.h @@ -0,0 +1,30 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * adiosUserOptions.h YAML file parsing functionality for ~/.config/adios2/adios2.yaml + */ + +#ifndef ADIOS2_HELPER_ADIOSUSEROPTIONS_H_ +#define ADIOS2_HELPER_ADIOSUSEROPTIONS_H_ + +/// \cond EXCLUDE_FROM_DOXYGEN +#include +#include +/// \endcond + +#include "adios2/common/ADIOSTypes.h" +#include "adios2/helper/adiosComm.h" + +namespace adios2 +{ +namespace helper +{ + +void ParseUserOptionsFile(Comm &comm, const std::string &configFileYAML, UserOptions &options, + std::string &homePath); + +} // end namespace helper +} // end namespace adios2 + +#endif /* ADIOS2_HELPER_ADIOSUSEROPTIONS_H_ */ diff --git a/source/adios2/helper/adiosYAML.cpp b/source/adios2/helper/adiosYAML.cpp index ec835ce26c..4283e8b676 100644 --- a/source/adios2/helper/adiosYAML.cpp +++ b/source/adios2/helper/adiosYAML.cpp @@ -65,6 +65,44 @@ Params YAMLNodeMapToParams(const YAML::Node &node, const std::string &hint) constexpr bool isMandatory = true; constexpr bool isNotMandatory = false; + +inline void FixHomePath(std::string &path, std::string &homePath) +{ + if (!path.empty() && path[0] == '~') + { + path = homePath + path.substr(1); + } +} + +/*std::string NodeType(const YAML::Node &node) +{ + switch (node.Type()) + { + case YAML::NodeType::Null: + return "Null"; + case YAML::NodeType::Scalar: + return "Scalar"; + case YAML::NodeType::Sequence: + return "Sequence"; + case YAML::NodeType::Map: + return "Map"; + case YAML::NodeType::Undefined: + return "Undefined"; + } + return "NoIdeaWhatThisIs"; +}*/ + +template +void SetOption(T &value, const std::string nodeName, const YAML::Node &upperNode, + const std::string &hint) +{ + auto node = YAMLNode(nodeName, upperNode, hint, isNotMandatory, YAML::NodeType::Scalar); + if (node) + { + value = node.as(); + } +} + } // end empty namespace void IOVariableYAML(const YAML::Node &variableMap, core::IO ¤tIO, const std::string &hint) @@ -220,5 +258,75 @@ std::string ParseConfigYAML(core::ADIOS &adios, const std::string &configFileYAM return configFileContents; } +void ParseUserOptionsFile(Comm &comm, const std::string &configFileYAML, UserOptions &options, + std::string &homePath) +{ + const std::string hint = + "when parsing user config file " + configFileYAML + " in call to ADIOS constructor"; + + const std::string configFileContents = comm.BroadcastFile(configFileYAML, hint); + + /* + * Set defaults first + */ + options.general.verbose = 0; + + options.campaign.active = true; + options.campaign.verbose = 0; + options.campaign.hostname = ""; + options.campaign.campaignstorepath = ""; + options.campaign.cachepath = "/tmp/adios2-cache"; + + options.sst.verbose = 0; + + const YAML::Node document = YAML::Load(configFileContents); + if (!document) + { + helper::Throw( + "Helper", "adiosUserOptions", "ParseUserOptionsFile", + "parser error in file " + configFileYAML + + " invalid format. Check with any YAML editor if format is ill-formed, " + hint); + } + + /* + * This code section below determines what options we recognize at all from the + * ~/.config/adios2/adios2.yaml file + */ + { + UserOptions::General &opts = options.general; + const YAML::Node general = + YAMLNode("General", document, hint, isNotMandatory, YAML::NodeType::Map); + if (general) + { + SetOption(opts.verbose, "verbose", general, hint); + } + } + + { + UserOptions::Campaign &opts = options.campaign; + const YAML::Node campaign = + YAMLNode("Campaign", document, hint, isNotMandatory, YAML::NodeType::Map); + if (campaign) + { + SetOption(opts.verbose, "verbose", campaign, hint); + SetOption(opts.active, "active", campaign, hint); + SetOption(opts.hostname, "hostname", campaign, hint); + SetOption(opts.campaignstorepath, "campaignstorepath", campaign, hint); + FixHomePath(opts.campaignstorepath, homePath); + SetOption(opts.cachepath, "cachepath", campaign, hint); + FixHomePath(opts.cachepath, homePath); + } + } + + { + UserOptions::SST &opts = options.sst; + const YAML::Node sst = YAMLNode("SST", document, hint, isNotMandatory, YAML::NodeType::Map); + if (sst) + { + SetOption(opts.verbose, "verbose", sst, hint); + } + } +} + } // end namespace helper } // end namespace adios2 diff --git a/source/adios2/helper/adiosYAML.h b/source/adios2/helper/adiosYAML.h index 8014d0cf06..8bb9df19d5 100644 --- a/source/adios2/helper/adiosYAML.h +++ b/source/adios2/helper/adiosYAML.h @@ -18,6 +18,7 @@ #include //std::pair /// \endcond +#include "adios2/common/ADIOSTypes.h" // UserOptions #include "adios2/core/ADIOS.h" #include "adios2/core/IO.h" @@ -31,6 +32,9 @@ void ParseConfigYAMLIO(core::ADIOS &adios, const std::string &configFileYAML, std::string ParseConfigYAML(core::ADIOS &adios, const std::string &configFileYAML, std::map &ios); +void ParseUserOptionsFile(Comm &comm, const std::string &configFileYAML, UserOptions &options, + std::string &homePath); + } // end namespace helper } // end namespace adios2 diff --git a/source/utils/bpls/bpls.cpp b/source/utils/bpls/bpls.cpp index 0d3ce8e73c..52e8e01b2b 100644 --- a/source/utils/bpls/bpls.cpp +++ b/source/utils/bpls/bpls.cpp @@ -1555,13 +1555,25 @@ std::vector getEnginesList(const std::string path) return list; } -int doList(const char *path) +int doList(std::string path) { char init_params[128]; int adios_verbose = 2; if (verbose > 1) - printf("\nADIOS Open: read header info from %s\n", path); + printf("\nADIOS Open: read header info from %s\n", path.c_str()); + + // initialize BP reader + if (verbose > 1) + adios_verbose = 3; // print info lines + if (verbose > 2) + adios_verbose = 4; // print debug lines + snprintf(init_params, sizeof(init_params), "verbose=%d", adios_verbose); + if (hidden_attrs) + strcat(init_params, ";show_hidden_attrs"); + + core::ADIOS adios("C++"); + const adios2::UserOptions userOptions = adios.GetUserOptions(); std::string tpl = helper::LowerCase(transport_params); bool remoteFile = @@ -1579,23 +1591,34 @@ int doList(const char *path) } else { - if (!adios2sys::SystemTools::FileExists(path)) + bool exists = adios2sys::SystemTools::FileExists(path); + if (!exists && !userOptions.campaign.campaignstorepath.empty() && path[0] != PathSeparator) { - fprintf(stderr, "\nError: input path %s does not exist\n", path); + std::string path2 = userOptions.campaign.campaignstorepath + PathSeparator + path; + exists = adios2sys::SystemTools::FileExists(path2); + if (exists) + { + path = path2.c_str(); + } + else + { + std::string path3 = + userOptions.campaign.campaignstorepath + PathSeparator + path + ".aca"; + exists = adios2sys::SystemTools::FileExists(path3); + if (exists) + { + path = path3.c_str(); + } + } + } + + if (!exists) + { + fprintf(stderr, "\nError: input path %s does not exist\n", path.c_str()); return 4; } } - // initialize BP reader - if (verbose > 1) - adios_verbose = 3; // print info lines - if (verbose > 2) - adios_verbose = 4; // print debug lines - snprintf(init_params, sizeof(init_params), "verbose=%d", adios_verbose); - if (hidden_attrs) - strcat(init_params, ";show_hidden_attrs"); - - core::ADIOS adios("C++"); core::IO &io = adios.DeclareIO("bpls"); if (timestep) { diff --git a/source/utils/bpls/bpls.h b/source/utils/bpls/bpls.h index 683c582b4d..8d5c9be035 100644 --- a/source/utils/bpls/bpls.h +++ b/source/utils/bpls/bpls.h @@ -17,6 +17,7 @@ #include "adios2/core/Info.h" #include +#include namespace adios2 { @@ -60,7 +61,7 @@ void parseDimSpec(const std::string &str, int64_t *dims); int parseAccuracy(); int compile_regexp_masks(void); void printSettings(void); -int doList(const char *path); +int doList(std::string path); void mergeLists(int nV, char **listV, int nA, char **listA, char **mlist, bool *isVar); template From 8f01e8704fef611d4eef94481569c97b79016500 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Sat, 30 Mar 2024 09:57:18 -0400 Subject: [PATCH 105/124] Fix compiler error Remove extra file not needed --- .../adios2/engine/campaign/CampaignManager.h | 2 +- source/adios2/helper/adiosUserOptions.cpp | 121 ------------------ source/adios2/helper/adiosUserOptions.h | 30 ----- 3 files changed, 1 insertion(+), 152 deletions(-) delete mode 100644 source/adios2/helper/adiosUserOptions.cpp delete mode 100644 source/adios2/helper/adiosUserOptions.h diff --git a/source/adios2/engine/campaign/CampaignManager.h b/source/adios2/engine/campaign/CampaignManager.h index 498de83caa..2a23064cd6 100644 --- a/source/adios2/engine/campaign/CampaignManager.h +++ b/source/adios2/engine/campaign/CampaignManager.h @@ -57,7 +57,7 @@ class CampaignManager public: CampaignManager(helper::Comm &comm){}; ~CampaignManager() = default; - void Open(const std::string &name, const INIOptions &options){}; + void Open(const std::string &name, const UserOptions &options){}; void Record(const std::string &name, const size_t step, const double time){}; void Close(){}; diff --git a/source/adios2/helper/adiosUserOptions.cpp b/source/adios2/helper/adiosUserOptions.cpp deleted file mode 100644 index 34db1778ff..0000000000 --- a/source/adios2/helper/adiosUserOptions.cpp +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - */ - -#include "adiosUserOptions.h" - -#include "adios2/helper/adiosString.h" - -#include - -namespace adios2 -{ -namespace helper -{ - -namespace -{ - -inline void FixHomePath(std::string &path, std::string &homePath) -{ - if (!path.empty() && path[0] == '~') - { - path = homePath + path.substr(1); - } -} - -constexpr bool isMandatory = true; -constexpr bool isNotMandatory = false; - -YAML::Node YAMLNode(const std::string nodeName, const YAML::Node &upperNode, - const std::string &hint, const bool isMandatory, - const YAML::NodeType::value nodeType) -{ - const YAML::Node node = upperNode[nodeName]; - - if (isMandatory && !node) - { - helper::Throw( - "Helper", "adiosYAML", "YAMLNode", - "no " + nodeName + " node found, (is your node key lower case?), " + hint); - } - if (node && node.Type() != nodeType) - { - helper::Throw("Helper", "adiosYAML", "YAMLNode", - "node " + nodeName + - " is the wrong type, review adios2 " - "config YAML specs for the node, " + - hint); - } - return node; -} - -} // end empty namespace - -void ParseUserOptionsFile(Comm &comm, const std::string &configFileYAML, UserOptions &options, - std::string &homePath) -{ - const std::string hint = - "when parsing user config file " + configFileYAML + " in call to ADIOS constructor"; - - const std::string configFileContents = comm.BroadcastFile(configFileYAML, hint); - const YAML::Node document = YAML::Load(configFileContents); - - if (!document) - { - helper::Throw( - "Helper", "adiosUserOptions", "ParseUserOptionsFile", - "parser error in file " + configFileYAML + - " invalid format. Check with any YAML editor if format is ill-formed, " + hint); - } - - /* - * This code section below determines what options we recognize at all from the - * ~/.config/adios2/adios2.yaml file - */ - - for (auto itNode = document.begin(); itNode != document.end(); ++itNode) - { - std::cout << itNode->const YAML::Node general = - YAMLNode("IO", *itNode, hint, isNotMandatory, YAML::NodeType::Scalar); - if (ioScalar) - { - const std::string ioName = ioScalar.as(); - // Build the IO object - auto itCurrentIO = - ios.emplace(std::piecewise_construct, std::forward_as_tuple(ioName), - std::forward_as_tuple(adios, ioName, true, adios.m_HostLanguage)); - core::IO ¤tIO = itCurrentIO.first->second; - IOYAML(adios, *itNode, currentIO, hint); - } - } - - /* - if (iniReader.HasSection("General")) - { - options.general.verbose = static_cast(iniReader.GetInteger("General", "verbose", - 0)); - } - - if (iniReader.HasSection("Campaign")) - { - options.campaign.active = iniReader.GetBoolean("Campaign", "active", true); - options.campaign.verbose = static_cast(iniReader.GetInteger("Campaign", "verbose", - 0)); options.campaign.hostname = iniReader.Get("Campaign", "hostname", ""); - options.campaign.campaignstorepath = iniReader.Get("Campaign", "campaignstorepath", ""); - FixHomePath(options.campaign.campaignstorepath, homePath); - options.campaign.cachepath = iniReader.Get("Campaign", "cachepath", - "/tmp/adios2-cache"); FixHomePath(options.campaign.cachepath, homePath); - } - - if (iniReader.HasSection("SST")) - { - options.sst.verbose = static_cast(iniReader.GetInteger("SST", "verbose", 0)); - } - */ -} - -} // end namespace helper -} // end namespace adios2 diff --git a/source/adios2/helper/adiosUserOptions.h b/source/adios2/helper/adiosUserOptions.h deleted file mode 100644 index aec207f076..0000000000 --- a/source/adios2/helper/adiosUserOptions.h +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * adiosUserOptions.h YAML file parsing functionality for ~/.config/adios2/adios2.yaml - */ - -#ifndef ADIOS2_HELPER_ADIOSUSEROPTIONS_H_ -#define ADIOS2_HELPER_ADIOSUSEROPTIONS_H_ - -/// \cond EXCLUDE_FROM_DOXYGEN -#include -#include -/// \endcond - -#include "adios2/common/ADIOSTypes.h" -#include "adios2/helper/adiosComm.h" - -namespace adios2 -{ -namespace helper -{ - -void ParseUserOptionsFile(Comm &comm, const std::string &configFileYAML, UserOptions &options, - std::string &homePath); - -} // end namespace helper -} // end namespace adios2 - -#endif /* ADIOS2_HELPER_ADIOSUSEROPTIONS_H_ */ From c78c4316f936c6decbaaa644356a1570a2132228 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Sat, 30 Mar 2024 10:48:33 -0400 Subject: [PATCH 106/124] do not include unistd.h --- source/adios2/engine/campaign/CampaignData.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/source/adios2/engine/campaign/CampaignData.cpp b/source/adios2/engine/campaign/CampaignData.cpp index fa6dda7770..9a6cefc8a1 100644 --- a/source/adios2/engine/campaign/CampaignData.cpp +++ b/source/adios2/engine/campaign/CampaignData.cpp @@ -17,7 +17,6 @@ #include #include #include -#include #include namespace adios2 From fdcf3081022ef3121bf053b7a8202ad15974156a Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Sat, 30 Mar 2024 11:21:32 -0400 Subject: [PATCH 107/124] change a long variable to int64_t to avoid size confusion on windows --- source/adios2/engine/campaign/CampaignData.cpp | 17 ++++++++--------- source/adios2/engine/campaign/CampaignData.h | 2 +- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/source/adios2/engine/campaign/CampaignData.cpp b/source/adios2/engine/campaign/CampaignData.cpp index 9a6cefc8a1..edce699fb6 100644 --- a/source/adios2/engine/campaign/CampaignData.cpp +++ b/source/adios2/engine/campaign/CampaignData.cpp @@ -91,8 +91,7 @@ static int sqlcb_bpfile(void *p, int argc, char **argv, char **azColName) helper::StringToSizeT(std::string(argv[3]), "SQL callback convert text to int"); cf.lengthCompressed = helper::StringToSizeT(std::string(argv[4]), "SQL callback convert text to int"); - cf.ctime = static_cast( - helper::StringTo(std::string(argv[5]), "SQL callback convert ctime to int")); + cf.ctime = helper::StringTo(std::string(argv[5]), "SQL callback convert ctime to int"); CampaignBPDataset &cds = cdp->bpdatasets[cf.bpDatasetIdx]; cds.files.push_back(cf); @@ -232,9 +231,9 @@ int inflateToFile(const unsigned char *source, const size_t blobsize, std::ofstr return ret == Z_STREAM_END ? Z_OK : Z_DATA_ERROR; } -static long timeToSec(long ct) +static int64_t timeToSec(int64_t ct) { - long t; + int64_t t; if (ct > 99999999999999999) { /* nanosec to sec */ @@ -257,7 +256,7 @@ static long timeToSec(long ct) return t; } -static bool isFileNewer(const std::string path, long ctime) +static bool isFileNewer(const std::string path, int64_t ctime) { int result; #ifdef _WIN32 @@ -272,9 +271,9 @@ static bool isFileNewer(const std::string path, long ctime) return false; } - long ct = s.st_ctime; - long ctSec = timeToSec(ct); - long ctimeSec = timeToSec(ctime); + int64_t ct = static_cast(s.st_ctime); + int64_t ctSec = timeToSec(ct); + int64_t ctimeSec = timeToSec(ctime); /*std::cout << " Stat(" << path << "): size = " << s.st_size << " ct = " << ctSec << " ctime = " << ctimeSec << "\n";*/ @@ -297,7 +296,7 @@ void SaveToFile(sqlite3 *db, const std::string &path, const CampaignBPFile &bpfi sqlcmd = "SELECT data FROM bpfile WHERE bpdatasetid = " + id + " AND name = '" + bpfile.name + "'"; // std::cout << "SQL statement: " << sqlcmd << "\n"; - rc = sqlite3_prepare_v2(db, sqlcmd.c_str(), sqlcmd.size(), &statement, NULL); + rc = sqlite3_prepare_v2(db, sqlcmd.c_str(), static_cast(sqlcmd.size()), &statement, NULL); if (rc != SQLITE_OK) { std::cout << "SQL error: " << zErrMsg << std::endl; diff --git a/source/adios2/engine/campaign/CampaignData.h b/source/adios2/engine/campaign/CampaignData.h index 8d360abfa4..d177ad30a6 100644 --- a/source/adios2/engine/campaign/CampaignData.h +++ b/source/adios2/engine/campaign/CampaignData.h @@ -41,7 +41,7 @@ struct CampaignBPFile bool compressed; size_t lengthOriginal; size_t lengthCompressed; - long ctime; + int64_t ctime; }; struct CampaignBPDataset From 5b33af1cd0ba5d55630ccbfedf6ab01162ee9047 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Sat, 30 Mar 2024 13:00:38 -0400 Subject: [PATCH 108/124] Use yaml parser in campaign manager python script --- .../adios2_campaign_manager.py | 49 +++++++++++++------ 1 file changed, 35 insertions(+), 14 deletions(-) diff --git a/source/utils/adios_campaign_manager/adios2_campaign_manager.py b/source/utils/adios_campaign_manager/adios2_campaign_manager.py index 171af0b4cc..88036afdd4 100755 --- a/source/utils/adios_campaign_manager/adios2_campaign_manager.py +++ b/source/utils/adios_campaign_manager/adios2_campaign_manager.py @@ -4,6 +4,8 @@ import glob import sqlite3 import zlib +import yaml +from dataclasses import dataclass from datetime import datetime from os import chdir, getcwd, remove, stat from os.path import exists, isdir, expanduser @@ -15,19 +17,32 @@ ADIOS_ACA_VERSION = "1.0" +@dataclass +class UserOption: + adios_campaign_store: str = None + hostname: str = None + verbose: int = 0 -def ReadConfig(): - path = expanduser("~/.config/adios2/campaign.cfg") + +def ReadUserConfig(): + path = expanduser("~/.config/adios2/adios2.yaml") + opts = UserOption() try: + doc = {} with open(path) as f: - lines = f.readlines() - for line in lines: - lst = line.split() - if lst[0] == "campaignstorepath": - adios_campaign_store = expanduser(lst[1]) + doc = yaml.safe_load(f) + camp = doc.get("Campaign") + if isinstance(camp, dict): + for key, value in camp.items(): + if key == "campaignstorepath": + opts.adios_campaign_store = expanduser(value) + if key == "hostname": + opts.hostname = value + if key == "verbose": + opts.verbose = value except FileNotFoundError: - adios_campaign_store = None - return adios_campaign_store + None + return opts def SetupArgs(): @@ -50,9 +65,13 @@ def SetupArgs(): args = parser.parse_args() # default values - args.update = False + args.user_options = ReadUserConfig() + + if args.verbose == 0: + args.verbose = args.user_options.verbose + if args.campaign_store is None: - args.campaign_store = ReadConfig() + args.campaign_store = args.user_options.adios_campaign_store if args.campaign_store is not None: while args.campaign_store[-1] == "/": @@ -223,7 +242,10 @@ def GetHostName(): host = sub("^login[0-9]*\\.", "", host) if host.startswith("batch"): host = sub("^batch[0-9]*\\.", "", host) - shorthost = host.split(".")[0] + if args.user_options.hostname is None: + shorthost = host.split(".")[0] + else: + shorthost = args.user_options.hostname return host, shorthost @@ -278,8 +300,6 @@ def AddDirectory(hostID: int, path: str) -> int: def Update(args: dict, cur: sqlite3.Cursor): longHostName, shortHostName = GetHostName() - if args.hostname is not None: - shortHostName = args.hostname hostID = AddHostName(longHostName, shortHostName) @@ -355,6 +375,7 @@ def Info(args: dict, cur: sqlite3.Cursor): '"' ) bpdatasets = res3.fetchall() + print(f"BPDATASETS type = {type(bpdatasets)}") for bpdataset in bpdatasets: t = timestamp_to_datetime(bpdataset[2]) print(f" dataset = {bpdataset[1]} created on {t}") From e1d699e7a2568b73bfd10d18698ffee77720e125 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Sat, 30 Mar 2024 13:01:02 -0400 Subject: [PATCH 109/124] bug fix: the order of entries in bpdataset table is undefined but the campaign data reader relied on calculating the index as if it was sorted by the insertion order. Use a map instead to store the rowid and use that as index for the bpfile elements. --- source/adios2/engine/campaign/CampaignData.cpp | 15 ++++++++------- source/adios2/engine/campaign/CampaignData.h | 6 +++--- source/adios2/engine/campaign/CampaignReader.cpp | 6 ++++-- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/source/adios2/engine/campaign/CampaignData.cpp b/source/adios2/engine/campaign/CampaignData.cpp index edce699fb6..1f003052a6 100644 --- a/source/adios2/engine/campaign/CampaignData.cpp +++ b/source/adios2/engine/campaign/CampaignData.cpp @@ -69,12 +69,13 @@ static int sqlcb_bpdataset(void *p, int argc, char **argv, char **azColName) { CampaignData *cdp = reinterpret_cast(p); CampaignBPDataset cds; - size_t hostid = helper::StringToSizeT(std::string(argv[0]), "SQL callback convert text to int"); - size_t dirid = helper::StringToSizeT(std::string(argv[1]), "SQL callback convert text to int"); + size_t dsid = helper::StringToSizeT(std::string(argv[0]), "SQL callback convert text to int"); + size_t hostid = helper::StringToSizeT(std::string(argv[1]), "SQL callback convert text to int"); + size_t dirid = helper::StringToSizeT(std::string(argv[2]), "SQL callback convert text to int"); cds.hostIdx = hostid - 1; // SQL rows start from 1, vector idx start from 0 cds.dirIdx = dirid - 1; - cds.name = argv[2]; - cdp->bpdatasets.push_back(cds); + cds.name = argv[3]; + cdp->bpdatasets[dsid] = cds; return 0; }; @@ -83,7 +84,7 @@ static int sqlcb_bpfile(void *p, int argc, char **argv, char **azColName) CampaignData *cdp = reinterpret_cast(p); CampaignBPFile cf; size_t dsid = helper::StringToSizeT(std::string(argv[0]), "SQL callback convert text to int"); - cf.bpDatasetIdx = dsid - 1; + cf.bpDatasetIdx = dsid; cf.name = std::string(argv[1]); int comp = helper::StringTo(std::string(argv[2]), "SQL callback convert text to int"); cf.compressed = (bool)comp; @@ -137,7 +138,7 @@ void ReadCampaignData(sqlite3 *db, CampaignData &cd) sqlite3_free(zErrMsg); } - sqlcmd = "SELECT hostid, dirid, name FROM bpdataset"; + sqlcmd = "SELECT rowid, hostid, dirid, name FROM bpdataset"; rc = sqlite3_exec(db, sqlcmd.c_str(), sqlcb_bpdataset, &cd, &zErrMsg); if (rc != SQLITE_OK) { @@ -290,7 +291,7 @@ void SaveToFile(sqlite3 *db, const std::string &path, const CampaignBPFile &bpfi int rc; char *zErrMsg = 0; std::string sqlcmd; - std::string id = std::to_string(bpfile.bpDatasetIdx + 1); + std::string id = std::to_string(bpfile.bpDatasetIdx); sqlite3_stmt *statement; sqlcmd = diff --git a/source/adios2/engine/campaign/CampaignData.h b/source/adios2/engine/campaign/CampaignData.h index d177ad30a6..140cfb89cf 100644 --- a/source/adios2/engine/campaign/CampaignData.h +++ b/source/adios2/engine/campaign/CampaignData.h @@ -14,8 +14,8 @@ #include #include +#include #include -#include #include #include @@ -37,7 +37,7 @@ struct CampaignHost struct CampaignBPFile { std::string name; - size_t bpDatasetIdx; + size_t bpDatasetIdx; // index of parent CampaignBPDataset in the map bool compressed; size_t lengthOriginal; size_t lengthCompressed; @@ -56,7 +56,7 @@ struct CampaignData { std::string version; std::vector hosts; - std::vector bpdatasets; + std::map bpdatasets; }; void ReadCampaignData(sqlite3 *db, CampaignData &cd); diff --git a/source/adios2/engine/campaign/CampaignReader.cpp b/source/adios2/engine/campaign/CampaignReader.cpp index 2bec32bf22..79cf87ba22 100644 --- a/source/adios2/engine/campaign/CampaignReader.cpp +++ b/source/adios2/engine/campaign/CampaignReader.cpp @@ -214,8 +214,9 @@ void CampaignReader::InitTransports() } } std::cout << " datasets:\n"; - for (auto &ds : m_CampaignData.bpdatasets) + for (auto &it : m_CampaignData.bpdatasets) { + CampaignBPDataset &ds = it.second; std::cout << " " << m_CampaignData.hosts[ds.hostIdx].hostname << ":" << m_CampaignData.hosts[ds.hostIdx].directory[ds.dirIdx] << PathSeparator << ds.name << "\n"; @@ -231,8 +232,9 @@ void CampaignReader::InitTransports() // std::cout << "JSON rank " << m_ReaderRank << ": " << js.size() << // std::endl; int i = 0; - for (auto &ds : m_CampaignData.bpdatasets) + for (auto &it : m_CampaignData.bpdatasets) { + CampaignBPDataset &ds = it.second; adios2::core::IO &io = m_IO.m_ADIOS.DeclareIO("CampaignReader" + std::to_string(i)); std::string localPath; if (m_CampaignData.hosts[ds.hostIdx].hostname != m_Options.hostname) From 00180737c5ba65dd85186372cee7bf0546a905b4 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Sat, 30 Mar 2024 13:16:26 -0400 Subject: [PATCH 110/124] rename remote_server to adios2_remote_server --- source/adios2/toolkit/remote/CMakeLists.txt | 10 +++++----- source/adios2/toolkit/remote/remote_server.cpp | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/source/adios2/toolkit/remote/CMakeLists.txt b/source/adios2/toolkit/remote/CMakeLists.txt index 9b613d4b45..a739e1ad63 100644 --- a/source/adios2/toolkit/remote/CMakeLists.txt +++ b/source/adios2/toolkit/remote/CMakeLists.txt @@ -4,9 +4,9 @@ #------------------------------------------------------------------------------# if (NOT ADIOS2_USE_PIP) - add_executable(remote_server ./remote_server.cpp remote_common.cpp) + add_executable(adios2_remote_server ./remote_server.cpp remote_common.cpp) - target_link_libraries(remote_server PUBLIC EVPath::EVPath adios2_core adios2sys + target_link_libraries(adios2_remote_server PUBLIC EVPath::EVPath adios2_core adios2sys PRIVATE $<$:shlwapi>) get_property(pugixml_headers_path @@ -14,10 +14,10 @@ if (NOT ADIOS2_USE_PIP) PROPERTY INTERFACE_INCLUDE_DIRECTORIES ) - target_include_directories(remote_server PRIVATE ${PROJECT_BINARY_DIR} ${pugixml_headers_path}) + target_include_directories(adios2_remote_server PRIVATE ${PROJECT_BINARY_DIR} ${pugixml_headers_path}) - set_property(TARGET remote_server PROPERTY OUTPUT_NAME remote_server${ADIOS2_EXECUTABLE_SUFFIX}) - install(TARGETS remote_server EXPORT adios2 + set_property(TARGET adios2_remote_server PROPERTY OUTPUT_NAME adios2_remote_server${ADIOS2_EXECUTABLE_SUFFIX}) + install(TARGETS adios2_remote_server EXPORT adios2 RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT adios2_tools-runtime ) endif () diff --git a/source/adios2/toolkit/remote/remote_server.cpp b/source/adios2/toolkit/remote/remote_server.cpp index e46e06bca0..1dedea3d09 100644 --- a/source/adios2/toolkit/remote/remote_server.cpp +++ b/source/adios2/toolkit/remote/remote_server.cpp @@ -543,7 +543,7 @@ int main(int argc, char **argv) else { fprintf(stderr, "Unknown argument \"%s\"\n", argv[i]); - fprintf(stderr, "Usage: remote_server [-background] [-kill_server] [-no_timeout] " + fprintf(stderr, "Usage: adios2_remote_server [-background] [-kill_server] [-no_timeout] " "[-status] [-v] [-q]\n"); exit(1); } From 89c53fcbb0b34f7030e29d8afa78b5f05a5bb1c1 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Sat, 30 Mar 2024 13:33:09 -0400 Subject: [PATCH 111/124] Use the name of the campaign in the cache path to avoid name collision --- .../adios2/engine/campaign/CampaignReader.cpp | 28 ++++++++++--------- source/utils/bpls/bpls.cpp | 4 +-- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/source/adios2/engine/campaign/CampaignReader.cpp b/source/adios2/engine/campaign/CampaignReader.cpp index 79cf87ba22..a8c0cc0fde 100644 --- a/source/adios2/engine/campaign/CampaignReader.cpp +++ b/source/adios2/engine/campaign/CampaignReader.cpp @@ -33,19 +33,19 @@ CampaignReader::CampaignReader(IO &io, const std::string &name, const Mode mode, : Engine("CampaignReader", io, name, mode, std::move(comm)) { m_ReaderRank = m_Comm.Rank(); - Init(); - if (m_Options.verbose == 5) + if (m_Options.verbose > 1) { std::cout << "Campaign Reader " << m_ReaderRank << " Open(" << m_Name << ") in constructor." << std::endl; } + Init(); m_IsOpen = true; } CampaignReader::~CampaignReader() { /* CampaignReader destructor does close and finalize */ - if (m_Options.verbose == 5) + if (m_Options.verbose > 1) { std::cout << "Campaign Reader " << m_ReaderRank << " destructor on " << m_Name << "\n"; } @@ -62,7 +62,7 @@ StepStatus CampaignReader::BeginStep(const StepMode mode, const float timeoutSec // so this forced increase should not be here ++m_CurrentStep; - if (m_Options.verbose == 5) + if (m_Options.verbose > 1) { std::cout << "Campaign Reader " << m_ReaderRank << " BeginStep() new step " << m_CurrentStep << "\n"; @@ -89,7 +89,7 @@ StepStatus CampaignReader::BeginStep(const StepMode mode, const float timeoutSec void CampaignReader::PerformGets() { - if (m_Options.verbose == 5) + if (m_Options.verbose > 1) { std::cout << "Campaign Reader " << m_ReaderRank << " PerformGets()\n"; } @@ -107,7 +107,7 @@ void CampaignReader::EndStep() PerformGets(); } - if (m_Options.verbose == 5) + if (m_Options.verbose > 1) { std::cout << "Campaign Reader " << m_ReaderRank << " EndStep()\n"; } @@ -198,7 +198,7 @@ void CampaignReader::InitTransports() ReadCampaignData(m_DB, m_CampaignData); - if (m_Options.verbose == 1) + if (m_Options.verbose > 0) { std::cout << "Local hostname = " << m_Options.hostname << "\n"; std::cout << "Database result:\n version = " << m_CampaignData.version << "\n hosts:\n"; @@ -243,12 +243,14 @@ void CampaignReader::InitTransports() m_CampaignData.hosts[ds.hostIdx].directory[ds.dirIdx] + PathSeparator + ds.name; const std::string remoteURL = m_CampaignData.hosts[ds.hostIdx].hostname + ":" + remotePath; - if (m_Options.verbose == 1) + localPath = m_Options.cachepath + PathSeparator + + m_CampaignData.hosts[ds.hostIdx].hostname + PathSeparator + m_Name + + PathSeparator + ds.name; + if (m_Options.verbose > 0) { - std::cout << "Open remote file " << remoteURL << "\n"; + std::cout << "Open remote file " << remoteURL + << "\n and use local cache for metadata at " << localPath << " \n"; } - localPath = m_Options.cachepath + PathSeparator + - m_CampaignData.hosts[ds.hostIdx].hostname + PathSeparator + ds.name; helper::CreateDirectory(localPath); for (auto &bpf : ds.files) { @@ -263,7 +265,7 @@ void CampaignReader::InitTransports() { localPath = m_CampaignData.hosts[ds.hostIdx].directory[ds.dirIdx] + PathSeparator + ds.name; - if (m_Options.verbose == 1) + if (m_Options.verbose > 0) { std::cout << "Open local file " << localPath << "\n"; } @@ -305,7 +307,7 @@ void CampaignReader::InitTransports() void CampaignReader::DoClose(const int transportIndex) { - if (m_Options.verbose == 5) + if (m_Options.verbose > 1) { std::cout << "Campaign Reader " << m_ReaderRank << " Close(" << m_Name << ")\n"; } diff --git a/source/utils/bpls/bpls.cpp b/source/utils/bpls/bpls.cpp index 52e8e01b2b..94dd7805cf 100644 --- a/source/utils/bpls/bpls.cpp +++ b/source/utils/bpls/bpls.cpp @@ -1598,7 +1598,7 @@ int doList(std::string path) exists = adios2sys::SystemTools::FileExists(path2); if (exists) { - path = path2.c_str(); + ; // path = path2.c_str(); } else { @@ -1607,7 +1607,7 @@ int doList(std::string path) exists = adios2sys::SystemTools::FileExists(path3); if (exists) { - path = path3.c_str(); + path += ".aca"; // path = path3.c_str(); } } } From f0a9a680a3ea721e8b4e717acc6570629a015e32 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Sat, 30 Mar 2024 13:38:07 -0400 Subject: [PATCH 112/124] clang-format --- source/adios2/toolkit/remote/remote_server.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/source/adios2/toolkit/remote/remote_server.cpp b/source/adios2/toolkit/remote/remote_server.cpp index 1dedea3d09..5108ed421e 100644 --- a/source/adios2/toolkit/remote/remote_server.cpp +++ b/source/adios2/toolkit/remote/remote_server.cpp @@ -543,8 +543,9 @@ int main(int argc, char **argv) else { fprintf(stderr, "Unknown argument \"%s\"\n", argv[i]); - fprintf(stderr, "Usage: adios2_remote_server [-background] [-kill_server] [-no_timeout] " - "[-status] [-v] [-q]\n"); + fprintf(stderr, + "Usage: adios2_remote_server [-background] [-kill_server] [-no_timeout] " + "[-status] [-v] [-q]\n"); exit(1); } } From be41e19fc9b8c2d771e174c90090471b5c4cb7c9 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Sat, 30 Mar 2024 16:42:24 -0400 Subject: [PATCH 113/124] fix remote server test, use the new binary name --- testing/adios2/engine/bp/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testing/adios2/engine/bp/CMakeLists.txt b/testing/adios2/engine/bp/CMakeLists.txt index 8c5b90606b..bf99106bfa 100644 --- a/testing/adios2/engine/bp/CMakeLists.txt +++ b/testing/adios2/engine/bp/CMakeLists.txt @@ -120,10 +120,10 @@ if ((NOT WIN32) AND ADIOS2_HAVE_SST) set_tests_properties(Remote.BP${testname}.FileRemote PROPERTIES FIXTURES_REQUIRED Server ENVIRONMENT "DoFileRemote=1") endmacro() - add_test(NAME remoteServerSetup COMMAND remote_server -background) + add_test(NAME remoteServerSetup COMMAND adios2_remote_server -background) set_tests_properties(remoteServerSetup PROPERTIES FIXTURES_SETUP Server) - add_test(NAME remoteServerCleanup COMMAND remote_server -kill_server) + add_test(NAME remoteServerCleanup COMMAND adios2_remote_server -kill_server) set_tests_properties(remoteServerCleanup PROPERTIES FIXTURES_CLEANUP Server) ##### add remote tests below this line From 6ba3a67378529ec39a40fba712b6d657cd40474d Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Sat, 30 Mar 2024 17:00:06 -0400 Subject: [PATCH 114/124] used size_t not int for map indexing to avoid type conversion --- source/adios2/engine/campaign/CampaignData.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/adios2/engine/campaign/CampaignData.h b/source/adios2/engine/campaign/CampaignData.h index 140cfb89cf..fa316c1bcb 100644 --- a/source/adios2/engine/campaign/CampaignData.h +++ b/source/adios2/engine/campaign/CampaignData.h @@ -56,7 +56,7 @@ struct CampaignData { std::string version; std::vector hosts; - std::map bpdatasets; + std::map bpdatasets; }; void ReadCampaignData(sqlite3 *db, CampaignData &cd); From a0bbda0f2cf96df39b44c54ab61d48cd278d0dfd Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Mon, 1 Apr 2024 08:24:48 -0400 Subject: [PATCH 115/124] - set ACA version to 0.1 - remove debug prints - add doc on Campaign Management in Advanced section - change static struct of UserOptions to class member of ADIOS class to make it work with gcc 4.8 --- .../source/advanced/campaign_management.rst | 187 ++++++++++++++++++ docs/user_guide/source/index.rst | 1 + source/adios2/core/ADIOS.cpp | 13 +- source/adios2/core/ADIOS.h | 3 +- .../adios2/engine/campaign/CampaignData.cpp | 4 +- .../adios2/engine/campaign/CampaignReader.cpp | 10 - .../adios2_campaign_manager.py | 3 +- 7 files changed, 199 insertions(+), 22 deletions(-) create mode 100644 docs/user_guide/source/advanced/campaign_management.rst diff --git a/docs/user_guide/source/advanced/campaign_management.rst b/docs/user_guide/source/advanced/campaign_management.rst new file mode 100644 index 0000000000..f7a091e4d5 --- /dev/null +++ b/docs/user_guide/source/advanced/campaign_management.rst @@ -0,0 +1,187 @@ +#################### +Campaign Management +#################### + +The campaign management in ADIOS2 is for collecting basic information and metadata about a collection of ADIOS2 output files, from a single application run or multiple runs. The campaign archive is a single file (.ACA) that can be transferred to other locations. The campaign file can be opened by ADIOS2 and all the metadata can be processed (including the values of GlobalValue and LocalValue variables, or min/max of each Arrays at each step and decomposition/min/max of each block in an Array at each step). However, Get() operations will only succeed to read actual data of the arrays, if the data belonging to the campaign is either local or some mechanism for remote data access to the location of the data is set up in advance. + +.. warning:: + + In 2.10, Campaign Management is just a first prototype and is included only for evaluation purposes. It will change substantially in the future and campaign files produced by this version will unlikely to be supported going forward. + +The idea +======== +Applications produce one or more output files in a single run. Subsequent analysis and visualization runs produce more output files. Campaign is a data organization concept one step higher than a file. A campaign archive includes information about multiple files, including the scalar variable's values and the min/max of arrays and the location of the data files (host and directory information). A science project can agree on how to organize their campaigns, i.e., how to name them, what files to include in a single campaign archive, how to distribute them, how to name the hosts where the actual data resides. + +Example +------- + +The Gray-Scott example, that is included with ADIOS2, in `examples/simulation/gray-scott`, has two programs, Gray-Scott and PDF-Calc. The first one produces the main output `gs.bp` which includes the main 3D variables `U` and `V`, and a checkpoint file `ckpt.bp` with a single step in it. PDF-Calc processes the main output and produces histograms on 2D slices of U and V (`U/bins` and `U/pdf`) in `pdf.bp`. A campaign can include all the three output files as they logically belong together. + +.. code-block:: bash + + # run application as usual + $ mpirun -n 4 adios2_simulations_gray-scott settings-files.json + $ ls -d *.bp + ckpt.bp gs.bp + + $ adios2_campaign_manager.py create demoproject/frontier_gray-scott_100 + + $ mpirun -n 3 adios2_simulations_gray-scott_pdf-calc gs.bp pdf.bp 1000 + $ ls -d *.bp + ckpt.bp gs.bp pdf.bp + + $ adios2_campaign_manager.py update demoproject/frontier_gray-scott_100 + + $ adios2_campaign_manager.py info demoproject/frontier_gray-scott_100 + info archive + ADIOS Campaign Archive, version 1.0, created on 2024-04-01 10:44:11.644942 + hostname = OLCF longhostname = frontier.olcf.ornl.gov + dir = /lustre/orion/csc143/proj-shared/demo/gray-scott + dataset = ckpt.bp created on 2024-04-01 10:38:19 + dataset = gs.bp created on 2024-04-01 10:38:17 + dataset = pdf.bp created on 2024-04-01 10:38:08 + + # The campaign archive is small compared to the data it points to + $ du -sh *bp + 7.9M ckpt.bp + 40M gs.bp + 9.9M pdf.bp + + $ du -sh /lustre/orion/csc143/proj-shared/adios-campaign-store/demoproject/frontier_gray-scott_100.aca + 97K /lustre/orion/csc143/proj-shared/adios-campaign-store/demoproject/frontier_gray-scott_100.aca + + # ADIOS can list the content of the campaign archive + $ bpls -l demoproject/frontier_gray-scott_100 + double ckpt.bp/U {4, 34, 34, 66} = 0.171103 / 1 + double ckpt.bp/V {4, 34, 34, 66} = 1.71085e-19 / 0.438921 + int32_t ckpt.bp/step scalar = 700 + double gs.bp/U 10*{64, 64, 64} = 0.090778 / 1 + double gs.bp/V 10*{64, 64, 64} = 8.24719e-63 / 0.515145 + int32_t gs.bp/step 10*scalar = 100 / 1000 + double pdf.bp/U/bins 10*{1000} = 0.0908158 / 0.999938 + double pdf.bp/U/pdf 10*{64, 1000} = 0 / 4096 + double pdf.bp/V/bins 10*{1000} = 8.24719e-63 / 0.514267 + double pdf.bp/V/pdf 10*{64, 1000} = 0 / 4096 + int32_t pdf.bp/step 10*scalar = 100 / 1000 + + # scalar over steps is available in metadata + $ bpls -l demoproject/frontier_gray-scott_100 -d pdf.bp/step -n 10 + int32_t pdf.bp/step 10*scalar = 100 / 1000 + (0) 100 200 300 400 500 600 700 800 900 1000 + + # Array decomposition including min/max are available in metadata + $ bpls -l demoproject/frontier_gray-scott_100 -D gs.bp/V + double gs.bp/V 10*{64, 64, 64} = 8.24719e-63 / 0.515145 + step 0: + block 0: [ 0:63, 0:31, 0:31] = 8.24719e-63 / 0.410653 + block 1: [ 0:63, 32:63, 0:31] = 8.24719e-63 / 0.410652 + block 2: [ 0:63, 0:31, 32:63] = 8.24719e-63 / 0.410653 + block 3: [ 0:63, 32:63, 32:63] = 8.24719e-63 / 0.410653 + ... + step 9: + block 0: [ 0:63, 0:31, 0:31] = 3.99908e-09 / 0.441847 + block 1: [ 0:63, 32:63, 0:31] = 3.99931e-09 / 0.44192 + block 2: [ 0:63, 0:31, 32:63] = 3.99928e-09 / 0.441813 + block 3: [ 0:63, 32:63, 32:63] = 3.99899e-09 / 0.441796 + + # Array data is only available if data is local + $ ./bin/bpls -l demoproject/frontier_gray-scott_100 -d pdf.bp/U/bins + double pdf.bp/U/bins 10*{1000} = 0.0908158 / 0.999938 + (0, 0) 0.93792 0.937982 0.938044 0.938106 0.938168 0.93823 0.938292 0.938354 0.938416 0.938479 + ... + (9,990) 0.990306 0.991157 0.992007 0.992858 0.993708 0.994559 0.995409 0.99626 0.99711 0.997961 + + +Setup +===== + +There are three paths/names important in the campaign setup. + +- `hostname` is the name detected by the adios2_campaign_manager when creating a campaign archive, however, it is better to define a specific name the project agrees upon (e.g. OLCF, NERSC, ALCF) that identifies the generic location of the data and then use that name later to specify the modes of remote data access (not available in this release). + +- `campaignstorepath` is the directory where all the campaign archives are stored. This should be shared between project members in a center, and a private one on every member's laptop. It is up to the project to determine what file sharing / synchronization mechanism to use to sync this directories. `Rclone is a great command-line tool `_ to sync the campaign store with many cloud-based file sharing services and cloud instances. + +- `cachepath` is the directory where ADIOS can unpack metadata from the campaign archive so that ADIOS engines can read them as if they were entirely local datasets. The cache only contains the metadata for now but in the future data that have already been retrieved by previous read requests will be stored here as well. + + +Use `~/.config/adios2/adios2.yaml` to specify these options. + +.. code-block:: bash + + $ cat ~/.config/adios2/adios2.yaml + + Campaign: + active: true + hostname: OLCF + campaignstorepath: /lustre/orion/csc143/proj-shared/adios-campaign-store + cachepath: /lustre/orion/csc143/proj-shared/campaign-cache + verbose: 0 + + $ ls -R ~/dropbox/adios-campaign-store + /lustre/orion/csc143/proj-shared/adios-campaign-store/demoproject: + frontier_gray-scott_100.aca + + $ adios2_campaign_manager.py list + demoproject/frontier_gray-scott_100.aca + + +Remote access +============= +For now, we have one way to access data, through SSH port forwarding and running a remote server program to read in data on the remote host and to send back the data to the local ADIOS program. `adios2_remote_server` is included in the adios installation. You need to use the one built on the host. + +Launch the server by SSH-ing to the remote machine, and specifying the `26200` port for fowarding. For example: + +.. code-block:: bash + + $ ssh -L 26200:dtn.olcf.ornl.gov:26200 -l dtn.olcf.ornl.gov "/bin/adios2_remote_server -v " + +Assuming the campaign archive was synced to a local machine's campaign store under `csc143/demoproject`, now we can retrieve data: + +.. code-block:: bash + + $ adios2_campaign_manager.py list + csc143/demoproject/frontier_gray-scott_100.aca + + $ bpls -l csc143/demoproject/frontier_gray-scott_100 + double ckpt.bp/U {4, 34, 34, 66} = 0.171103 / 1 + ... + double pdf.bp/U/bins 10*{1000} = 0.0908158 / 0.999938 + + # metadata is extracted to the local cachepath + $ du -sh /tmp/campaign/OLCF/csc143/demoproject/frontier_gray-scott_100.aca/* + 20K /tmp/campaign/OLCF/csc143/demoproject/frontier_gray-scott_100.aca/ckpt.bp + 40K /tmp/campaign/OLCF/csc143/demoproject/frontier_gray-scott_100.aca/gs.bp + 32K /tmp/campaign/OLCF/csc143/demoproject/frontier_gray-scott_100.aca/pdf.bp + + # data is requested from the remote server + # read 16 values (4x4x4) from U from last step, from offset 30,30,30 + $ bpls -l csc143/demoproject/frontier_gray-scott_100 -d gs.bp/U -s "-1,30,30,30" -c "1,4,4,4" -n 4 + double gs.bp/U 10*{64, 64, 64} + slice (9:9, 30:33, 30:33, 30:33) + (9,30,30,30) 0.89189 0.899854 0.899854 0.891891 + (9,30,31,30) 0.899851 0.908278 0.908278 0.899852 + (9,30,32,30) 0.899849 0.908276 0.908277 0.899851 + (9,30,33,30) 0.891885 0.899848 0.899849 0.891886 + (9,31,30,30) 0.89985 0.908276 0.908276 0.899849 + (9,31,31,30) 0.908274 0.916977 0.916977 0.908274 + (9,31,32,30) 0.908273 0.916976 0.916976 0.908273 + (9,31,33,30) 0.899844 0.908271 0.908271 0.899844 + (9,32,30,30) 0.89985 0.908276 0.908275 0.899848 + (9,32,31,30) 0.908274 0.916976 0.916976 0.908272 + (9,32,32,30) 0.908272 0.916975 0.916974 0.908271 + (9,32,33,30) 0.899844 0.90827 0.90827 0.899842 + (9,33,30,30) 0.89189 0.899851 0.899851 0.891886 + (9,33,31,30) 0.89985 0.908275 0.908275 0.899847 + (9,33,32,30) 0.899848 0.908274 0.908273 0.899845 + (9,33,33,30) 0.891882 0.899845 0.899844 0.89188 + +Requirements +============ +The Campaign Manager uses SQlite3 and ZLIB for its operations, and Python3 3.8 or higher for the `adios2_campaign_manager` tool. Check `bpls -Vv` to see if `CAMPAIGN` is in the list of "Available features". + +Limitations +=========== + +- The Campaign Reader engine only supports ReadRandomAccess mode, not step-by-step reading. Campaign management will need to change in the future to support sorting the steps from different outputs to a coherent order. +- Attributes are not processed by the Campaign Reader yet +- Updates to moving data for other location is not supported yet diff --git a/docs/user_guide/source/index.rst b/docs/user_guide/source/index.rst index a30daad270..f2bf42b8cd 100644 --- a/docs/user_guide/source/index.rst +++ b/docs/user_guide/source/index.rst @@ -40,6 +40,7 @@ Funded by the `Exascale Computing Project (ECP) (iBlobsize); std::ofstream f; diff --git a/source/adios2/engine/campaign/CampaignReader.cpp b/source/adios2/engine/campaign/CampaignReader.cpp index a8c0cc0fde..37e2f8c7da 100644 --- a/source/adios2/engine/campaign/CampaignReader.cpp +++ b/source/adios2/engine/campaign/CampaignReader.cpp @@ -33,22 +33,12 @@ CampaignReader::CampaignReader(IO &io, const std::string &name, const Mode mode, : Engine("CampaignReader", io, name, mode, std::move(comm)) { m_ReaderRank = m_Comm.Rank(); - if (m_Options.verbose > 1) - { - std::cout << "Campaign Reader " << m_ReaderRank << " Open(" << m_Name << ") in constructor." - << std::endl; - } Init(); m_IsOpen = true; } CampaignReader::~CampaignReader() { - /* CampaignReader destructor does close and finalize */ - if (m_Options.verbose > 1) - { - std::cout << "Campaign Reader " << m_ReaderRank << " destructor on " << m_Name << "\n"; - } if (m_IsOpen) { DestructorClose(m_FailVerbose); diff --git a/source/utils/adios_campaign_manager/adios2_campaign_manager.py b/source/utils/adios_campaign_manager/adios2_campaign_manager.py index 88036afdd4..35a9c20cf4 100755 --- a/source/utils/adios_campaign_manager/adios2_campaign_manager.py +++ b/source/utils/adios_campaign_manager/adios2_campaign_manager.py @@ -15,7 +15,7 @@ # from adios2.adios2_campaign_manager import * -ADIOS_ACA_VERSION = "1.0" +ADIOS_ACA_VERSION = "0.1" @dataclass class UserOption: @@ -375,7 +375,6 @@ def Info(args: dict, cur: sqlite3.Cursor): '"' ) bpdatasets = res3.fetchall() - print(f"BPDATASETS type = {type(bpdatasets)}") for bpdataset in bpdatasets: t = timestamp_to_datetime(bpdataset[2]) print(f" dataset = {bpdataset[1]} created on {t}") From 55d3676e1e2fc4659118659167af1b2ab30801e8 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Mon, 1 Apr 2024 16:16:49 -0400 Subject: [PATCH 116/124] Initialize ADIOS::m_UserOption before tentaively calling ProcessUserConfig() --- source/adios2/core/ADIOS.cpp | 14 ++++++++++++++ source/adios2/core/ADIOS.h | 1 + source/adios2/helper/adiosYAML.cpp | 13 ------------- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/source/adios2/core/ADIOS.cpp b/source/adios2/core/ADIOS.cpp index 2c4e355b7c..40bd5f01b0 100644 --- a/source/adios2/core/ADIOS.cpp +++ b/source/adios2/core/ADIOS.cpp @@ -180,6 +180,19 @@ ADIOS::~ADIOS() } } +void ADIOS::SetUserOptionDefaults() +{ + m_UserOptions.general.verbose = 0; + + m_UserOptions.campaign.active = true; + m_UserOptions.campaign.verbose = 0; + m_UserOptions.campaign.hostname = ""; + m_UserOptions.campaign.campaignstorepath = ""; + m_UserOptions.campaign.cachepath = "/tmp/adios2-cache"; + + m_UserOptions.sst.verbose = 0; +} + void ADIOS::ProcessUserConfig() { // read config parameters from config file @@ -189,6 +202,7 @@ void ADIOS::ProcessUserConfig() #else homePath = getenv("HOME"); #endif + SetUserOptionDefaults(); const std::string cfgFile = homePath + "/.config/adios2/adios2.yaml"; if (adios2sys::SystemTools::FileExists(cfgFile)) { diff --git a/source/adios2/core/ADIOS.h b/source/adios2/core/ADIOS.h index f729c9443e..00bff3f89c 100644 --- a/source/adios2/core/ADIOS.h +++ b/source/adios2/core/ADIOS.h @@ -208,6 +208,7 @@ class ADIOS core::IO &io); adios2::UserOptions m_UserOptions; + void SetUserOptionDefaults(); void ProcessUserConfig(); private: diff --git a/source/adios2/helper/adiosYAML.cpp b/source/adios2/helper/adiosYAML.cpp index 4283e8b676..07f93697e9 100644 --- a/source/adios2/helper/adiosYAML.cpp +++ b/source/adios2/helper/adiosYAML.cpp @@ -266,19 +266,6 @@ void ParseUserOptionsFile(Comm &comm, const std::string &configFileYAML, UserOpt const std::string configFileContents = comm.BroadcastFile(configFileYAML, hint); - /* - * Set defaults first - */ - options.general.verbose = 0; - - options.campaign.active = true; - options.campaign.verbose = 0; - options.campaign.hostname = ""; - options.campaign.campaignstorepath = ""; - options.campaign.cachepath = "/tmp/adios2-cache"; - - options.sst.verbose = 0; - const YAML::Node document = YAML::Load(configFileContents); if (!document) { From e7f6eb588145c2206194c64901035cb163f8d433 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Tue, 2 Apr 2024 09:30:35 -0400 Subject: [PATCH 117/124] Add a random string to each database name to avoid collision when running multiple applications in the same directory at the same time. Fixes issues with CI that runs ctest in parallel --- source/adios2/core/ADIOS.cpp | 3 ++- source/adios2/helper/adiosString.cpp | 19 +++++++++++++++++++ source/adios2/helper/adiosString.h | 7 +++++++ 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/source/adios2/core/ADIOS.cpp b/source/adios2/core/ADIOS.cpp index 40bd5f01b0..a280483ce2 100644 --- a/source/adios2/core/ADIOS.cpp +++ b/source/adios2/core/ADIOS.cpp @@ -150,7 +150,8 @@ ADIOS::ADIOS(const std::string configFile, helper::Comm comm, const std::string #endif if (m_UserOptions.campaign.active) { - std::string campaignName = "campaign_" + std::to_string(adios_count); + std::string campaignName = + "campaign_" + helper::RandomString(8) + "_" + std::to_string(adios_count); m_CampaignManager.Open(campaignName, m_UserOptions); } } diff --git a/source/adios2/helper/adiosString.cpp b/source/adios2/helper/adiosString.cpp index e160899404..895ecd450b 100644 --- a/source/adios2/helper/adiosString.cpp +++ b/source/adios2/helper/adiosString.cpp @@ -16,6 +16,7 @@ /// \cond EXCLUDE_FROM_DOXYGEN #include #include //std::ios_base::failure +#include #include #include // std::invalid_argument /// \endcond @@ -484,5 +485,23 @@ std::string RemoveTrailingSlash(const std::string &name) noexcept return name.substr(0, len); } +std::string RandomString(const size_t length) +{ + size_t len = length; + if (len == 0) + len = 1; + if (len > 64) + len = 64; + + std::string str("0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyzA"); + + std::random_device rd; + std::mt19937 generator(rd()); + + std::shuffle(str.begin(), str.end(), generator); + + return str.substr(0, len); +} + } // end namespace helper } // end namespace adios2 diff --git a/source/adios2/helper/adiosString.h b/source/adios2/helper/adiosString.h index cba94dd2b1..7174da61e4 100644 --- a/source/adios2/helper/adiosString.h +++ b/source/adios2/helper/adiosString.h @@ -193,6 +193,13 @@ std::set PrefixMatches(const std::string &prefix, */ std::string RemoveTrailingSlash(const std::string &name) noexcept; +/** + * Generate a random string of length between 1 and 64 + * This is a dummy string generator, don't use it for uuid or + * for generating truly random unique strings en masse + */ +std::string RandomString(const size_t length); + } // end namespace helper } // end namespace adios2 From 9b89e5d7b4651125f0ca6cd65067fdcbdc2d753d Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Tue, 2 Apr 2024 11:12:32 -0400 Subject: [PATCH 118/124] Add attribute support to campaign --- .../adios2/engine/campaign/CampaignReader.cpp | 22 +++++++++++++++++++ .../adios2/engine/campaign/CampaignReader.h | 7 ++++++ .../adios2/engine/campaign/CampaignReader.tcc | 13 +++++++++++ 3 files changed, 42 insertions(+) diff --git a/source/adios2/engine/campaign/CampaignReader.cpp b/source/adios2/engine/campaign/CampaignReader.cpp index 37e2f8c7da..dbf6ea3d43 100644 --- a/source/adios2/engine/campaign/CampaignReader.cpp +++ b/source/adios2/engine/campaign/CampaignReader.cpp @@ -291,6 +291,28 @@ void CampaignReader::InitTransports() ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) #undef declare_type } + + for (auto &ar : amap) + { + auto aname = ar.first; + std::string fname = ds.name; + std::string newname = fname + "/" + aname; + + const DataType type = io.InquireAttributeType(aname); + + if (type == DataType::Struct) + { + } +#define declare_type(T) \ + else if (type == helper::GetDataType()) \ + { \ + Attribute *ai = io.InquireAttribute(aname); \ + Attribute v = DuplicateAttribute(ai, m_IO, newname); \ + } + + ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) +#undef declare_type + } ++i; } } diff --git a/source/adios2/engine/campaign/CampaignReader.h b/source/adios2/engine/campaign/CampaignReader.h index 4747f7d875..d56475bf20 100644 --- a/source/adios2/engine/campaign/CampaignReader.h +++ b/source/adios2/engine/campaign/CampaignReader.h @@ -115,6 +115,13 @@ class CampaignReader : public Engine Variable DuplicateVariable(Variable *variable, IO &io, std::string &name, VarInternalInfo &vii); + /** + * Create a new attribute with name `name` in `io` + * based on an existing attribute. + */ + template + Attribute DuplicateAttribute(Attribute *attribute, IO &io, std::string &name); + /** * Create a new variable with name `name` in `io` * based on an existing variable. diff --git a/source/adios2/engine/campaign/CampaignReader.tcc b/source/adios2/engine/campaign/CampaignReader.tcc index d635a1a976..1a5357a25f 100644 --- a/source/adios2/engine/campaign/CampaignReader.tcc +++ b/source/adios2/engine/campaign/CampaignReader.tcc @@ -52,6 +52,19 @@ inline Variable CampaignReader::DuplicateVariable(Variable *variable, IO & return v; } +template +inline Attribute CampaignReader::DuplicateAttribute(Attribute *attribute, IO &io, + std::string &name) +{ + if (attribute->m_IsSingleValue) + { + auto &a = io.DefineAttribute(name, attribute->m_DataSingleValue); + return a; + } + auto &a = io.DefineAttribute(name, attribute->m_DataArray.data(), attribute->m_Elements); + return a; +} + template inline std::pair *, Engine *> CampaignReader::TranslateToActualVariable(Variable &variable) From 84ea59b5ad5a49dac20c672cd9b1a5db6b452322 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Tue, 2 Apr 2024 11:15:29 -0400 Subject: [PATCH 119/124] update doc --- docs/user_guide/source/advanced/campaign_management.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/user_guide/source/advanced/campaign_management.rst b/docs/user_guide/source/advanced/campaign_management.rst index f7a091e4d5..fd57f703a7 100644 --- a/docs/user_guide/source/advanced/campaign_management.rst +++ b/docs/user_guide/source/advanced/campaign_management.rst @@ -183,5 +183,4 @@ Limitations =========== - The Campaign Reader engine only supports ReadRandomAccess mode, not step-by-step reading. Campaign management will need to change in the future to support sorting the steps from different outputs to a coherent order. -- Attributes are not processed by the Campaign Reader yet - Updates to moving data for other location is not supported yet From 736e3cead8cf8b61647c8275fd27d79c6c768898 Mon Sep 17 00:00:00 2001 From: Greg Eisenhauer Date: Wed, 3 Apr 2024 08:53:06 -0400 Subject: [PATCH 120/124] DataPlane Configuration changes (#4121) * DataPlane Configuration changes * Change name to include HAVE rather than BUILD --- CMakeLists.txt | 18 +++++++++++++++--- cmake/DetectOptions.cmake | 9 +++++++-- source/adios2/toolkit/sst/CMakeLists.txt | 2 +- 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 37e4b6016b..4c2ea67687 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -443,9 +443,21 @@ foreach(opt IN LISTS ADIOS2_CONFIG_OPTS) endif() endforeach() -if (ADIOS2_HAVE_SST AND (ADIOS2_SST_HAVE_LIBFABRIC OR ADIOS2_SST_HAVE_UCX)) -message(" RDMA Transport for Staging: Available") +set (HPCDataPlaneList "") +if (ADIOS2_HAVE_SST) + if (ADIOS2_SST_HAVE_LIBFABRIC) + set (HPCDataPlaneList "${HPCDataPlaneList} fabric") + endif() + if (ADIOS2_SST_HAVE_UCX) + set (HPCDataPlaneList "${HPCDataPlaneList} UCX") + endif() + if (ADIOS2_SST_HAVE_MPI_DP) + set (HPCDataPlaneList "${HPCDataPlaneList} MPI") + endif() +endif() +if ({HPCDataPlaneList} STREQUAL "") + message(" Possible RDMA DataPlanes for SST: ") else() -message(" RDMA Transport for Staging: Unconfigured") + message(" Possible RDMA DataPlanes for SST: ${HPCDataPlaneList}") endif() diff --git a/cmake/DetectOptions.cmake b/cmake/DetectOptions.cmake index 0dedcb61ca..832e32934c 100644 --- a/cmake/DetectOptions.cmake +++ b/cmake/DetectOptions.cmake @@ -460,13 +460,14 @@ if(ADIOS2_USE_SST AND NOT WIN32) "-DLINK_DIRECTORIES=${LIBFABRIC_LIBRARIES}") message(STATUS "Libfabric support for the HPE CXI provider: ${ADIOS2_SST_HAVE_CRAY_CXI}") endif() - if(ADIOS2_HAVE_MPI) + if(ADIOS2_HAVE_MPI AND NOT "${ADIOS2_SST_HAVE_MPI_DP}") set(CMAKE_REQUIRED_LIBRARIES "MPI::MPI_C;Threads::Threads") include(CheckCXXSourceRuns) check_cxx_source_runs([=[ #include #include #include + #include #include #if !defined(MPICH) @@ -491,7 +492,11 @@ if(ADIOS2_USE_SST AND NOT WIN32) ADIOS2_HAVE_MPI_CLIENT_SERVER) unset(CMAKE_REQUIRED_LIBRARIES) if (ADIOS2_HAVE_MPI_CLIENT_SERVER) - set(ADIOS2_SST_HAVE_MPI TRUE) + set(ADIOS2_SST_HAVE_MPI_DP TRUE) + else() + if ("${ADIOS2_SST_EXPECT_MPI_DP}") + message(FATAL_ERROR "Expected MPI to support Client-server connection model, but test failed.") + endif() endif() endif() # UCX diff --git a/source/adios2/toolkit/sst/CMakeLists.txt b/source/adios2/toolkit/sst/CMakeLists.txt index 3c539b16a9..4a8252eeef 100644 --- a/source/adios2/toolkit/sst/CMakeLists.txt +++ b/source/adios2/toolkit/sst/CMakeLists.txt @@ -45,7 +45,7 @@ if(ADIOS2_HAVE_ZFP) target_link_libraries(sst PRIVATE zfp::zfp) endif() -if(ADIOS2_SST_HAVE_MPI) +if(ADIOS2_SST_BUILD_MPI_DP) target_sources(sst PRIVATE dp/mpi_dp.c) target_link_libraries(sst PRIVATE MPI::MPI_C) endif() From aeaab4e27d9865092fdf13396641a09d6555ce98 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Wed, 3 Apr 2024 09:41:37 -0400 Subject: [PATCH 121/124] Add -f file1 [file2...] option to process adios files from a list instead of a campaign recording --- .../adios2_campaign_manager.py | 71 +++++++++---------- 1 file changed, 34 insertions(+), 37 deletions(-) diff --git a/source/utils/adios_campaign_manager/adios2_campaign_manager.py b/source/utils/adios_campaign_manager/adios2_campaign_manager.py index 35a9c20cf4..be9eecfad3 100755 --- a/source/utils/adios_campaign_manager/adios2_campaign_manager.py +++ b/source/utils/adios_campaign_manager/adios2_campaign_manager.py @@ -59,9 +59,8 @@ def SetupArgs(): parser.add_argument( "--campaign_store", "-s", help="Path to local campaign store", default=None ) - parser.add_argument( - "--hostname", "-n", help="Host name unique for hosts in a campaign", required=False - ) + parser.add_argument("--hostname", "-n", help="Host name unique for hosts in a campaign") + parser.add_argument("-f", "--files", nargs="+", help="Add ADIOS files manually") args = parser.parse_args() # default values @@ -77,6 +76,9 @@ def SetupArgs(): while args.campaign_store[-1] == "/": args.campaign_store = args.campaign_store[:-1] + if args.hostname is None: + args.hostname = args.user_options.hostname + args.CampaignFileName = args.campaign if args.campaign is not None: if not args.campaign.endswith(".aca"): @@ -86,7 +88,8 @@ def SetupArgs(): args.campaign_store is not None): args.CampaignFileName = args.campaign_store + "/" + args.CampaignFileName - args.LocalCampaignDir = "adios-campaign/" + if args.files is None: + args.LocalCampaignDir = "adios-campaign/" if args.verbose > 0: print(f"# Verbosity = {args.verbose}") @@ -213,27 +216,23 @@ def AddDatasetToArchive(hostID: int, dirID: int, dataset: str, cur: sqlite3.Curs return rowID -def ProcessDBFile(args: dict, jsonlist: list, cur: sqlite3.Cursor, hostID: int, dirID: int): - for entry in jsonlist: - # print(f"Process entry {entry}:") - if isinstance(entry, dict): - if "name" in entry: - dsID = 0 - dataset = entry["name"] - if IsADIOSDataset(dataset): - dsID = AddDatasetToArchive(hostID, dirID, dataset, cur) - cwd = getcwd() - chdir(dataset) - mdFileList = glob.glob("*md.*") - profileList = glob.glob("profiling.json") - files = mdFileList + profileList - for f in files: - AddFileToArchive(args, f, cur, dsID) - chdir(cwd) - else: - print(f"WARNING: Dataset {dataset} is not an ADIOS dataset. Skip") +def ProcessFiles(args: dict, cur: sqlite3.Cursor, hostID: int, dirID: int): + for entry in args.files: + print(f"Process entry {entry}:") + dsID = 0 + dataset = entry + if IsADIOSDataset(dataset): + dsID = AddDatasetToArchive(hostID, dirID, dataset, cur) + cwd = getcwd() + chdir(dataset) + mdFileList = glob.glob("*md.*") + profileList = glob.glob("profiling.json") + files = mdFileList + profileList + for f in files: + AddFileToArchive(args, f, cur, dsID) + chdir(cwd) else: - print(f"WARNING: your object is not a dictionary, skip : {entry}") + print(f"WARNING: Dataset {dataset} is not an ADIOS dataset. Skip") def GetHostName(): @@ -242,7 +241,7 @@ def GetHostName(): host = sub("^login[0-9]*\\.", "", host) if host.startswith("batch"): host = sub("^batch[0-9]*\\.", "", host) - if args.user_options.hostname is None: + if args.hostname is None: shorthost = host.split(".")[0] else: shorthost = args.user_options.hostname @@ -278,7 +277,7 @@ def MergeDBFiles(dbfiles: list): print(e) record = cur.fetchall() for item in record: - result.append({"name": item[0]}) + result.append(item[0]) cur.close() return result @@ -305,13 +304,9 @@ def Update(args: dict, cur: sqlite3.Cursor): rootdir = getcwd() dirID = AddDirectory(hostID, rootdir) - con.commit() - db_list = MergeDBFiles(dbFileList) - - # print(f"Merged json = {jsonlist}") - ProcessDBFile(args, db_list, cur, hostID, dirID) + ProcessFiles(args, cur, hostID, dirID) con.commit() @@ -440,12 +435,14 @@ def Delete(): if args.command == "info": Info(args, cur) else: - CheckLocalCampaignDir(args) - # List the local campaign directory - dbFileList = glob.glob(args.LocalCampaignDir + "/*.db") - if len(dbFileList) == 0: - print("There are no campaign data files in " + args.LocalCampaignDir) - exit(2) + if args.files is None: + CheckLocalCampaignDir(args) + # List the local campaign directory + dbFileList = glob.glob(args.LocalCampaignDir + "/*.db") + if len(dbFileList) == 0: + print("There are no campaign data files in " + args.LocalCampaignDir) + exit(2) + args.files = MergeDBFiles(dbFileList) if args.command == "create": Create(args, cur) From 433bde308d7b5c5db73a6f4653d7f1abef091f2f Mon Sep 17 00:00:00 2001 From: Liz Dulac <47396187+lizdulac@users.noreply.github.com> Date: Wed, 3 Apr 2024 13:40:27 -0400 Subject: [PATCH 122/124] Add the CURL function to derived variables (#4114) * Derived Expressions Curl function with correctness test * Fixing index bug for internal boundaries * Curl Correctness Test Cleanup * Remove debug statements in Test * Merging Bison3.8 parser into ADIOS (ERRORS - cannot build) * CMake Tweaks * whoops, missed the cmake file. * Updated parser merged with adios, updated error message for missing operation, missing curl implementation * Clean up * Removed 'var' keyword from grammar and correctness test, and minor changes: ensured only one expression in grammar, turned off scanner debugging, resolved shift/reduce conflicts, ASTNode prettyprint can be called without operand. * Set Derived_Variable flag to ON (for CI testing) * Remove unnecessary files * Cleanup curl code * Formatting * Formatting * Restore examples CMakeLists.txt file * Restore cmake compile flags * Unused variable curlV in testing * Implicit type conversion warnings * Formatting * Use float specific math, remove infinity check * Convert size_t iterator to float math * Default to DerivedVariables OFF * Default to DerivedVariables ON * Change the way to check for dimension equality and function rename * format --------- Co-authored-by: lizdulac Co-authored-by: Greg Eisenhauer Co-authored-by: anagainaru --- CMakeLists.txt | 2 +- source/adios2/toolkit/derived/ExprHelper.h | 5 +- source/adios2/toolkit/derived/Function.cpp | 108 ++++++++++++- source/adios2/toolkit/derived/Function.h | 6 + .../derived/parser/pregen-source/lexer.cpp | 1 - .../derived/TestBPDerivedCorrectness.cpp | 152 +++++++++++++++++- 6 files changed, 268 insertions(+), 6 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4c2ea67687..dd48e7f921 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -185,7 +185,7 @@ adios_option(Sodium "Enable support for Sodium for encryption" AUTO) adios_option(Catalyst "Enable support for in situ visualization plugin using ParaView Catalyst" AUTO) adios_option(Campaign "Enable support for Campaigns (requires SQLite3 and ZLIB)" AUTO) adios_option(AWSSDK "Enable support for S3 compatible storage using AWS SDK's S3 module" OFF) -adios_option(Derived_Variable "Enable support for derived variables" OFF) +adios_option(Derived_Variable "Enable support for derived variables" ON) adios_option(PIP "Enable support for pip packaging" OFF) option(ADIOS2_Blosc2_PREFER_SHARED "Prefer shared Blosc2 libraries" ON) diff --git a/source/adios2/toolkit/derived/ExprHelper.h b/source/adios2/toolkit/derived/ExprHelper.h index e1d31b8bff..38672cfec4 100644 --- a/source/adios2/toolkit/derived/ExprHelper.h +++ b/source/adios2/toolkit/derived/ExprHelper.h @@ -21,7 +21,8 @@ enum ExpressionOperator OP_ADD, OP_SQRT, OP_POW, - OP_MAGN + OP_MAGN, + OP_CURL }; struct OperatorProperty @@ -39,6 +40,7 @@ const std::map op_property = { {ExpressionOperator::OP_ADD, {"ADD", true}}, {ExpressionOperator::OP_SQRT, {"SQRT", false}}, {ExpressionOperator::OP_POW, {"POW", false}}, + {ExpressionOperator::OP_CURL, {"CURL", false}}, {ExpressionOperator::OP_MAGN, {"MAGNITUDE", false}}}; const std::map string_to_op = { @@ -49,6 +51,7 @@ const std::map string_to_op = { {"add", ExpressionOperator::OP_ADD}, {"ADD", ExpressionOperator::OP_ADD}, {"SQRT", ExpressionOperator::OP_SQRT}, {"sqrt", ExpressionOperator::OP_SQRT}, {"POW", ExpressionOperator::OP_POW}, {"^", ExpressionOperator::OP_POW}, + {"CURL", ExpressionOperator::OP_CURL}, {"curl", ExpressionOperator::OP_CURL}, {"MAGNITUDE", ExpressionOperator::OP_MAGN}, {"magnitude", ExpressionOperator::OP_MAGN}}; inline std::string get_op_name(ExpressionOperator op) { return op_property.at(op).name; } diff --git a/source/adios2/toolkit/derived/Function.cpp b/source/adios2/toolkit/derived/Function.cpp index cb145d8927..441be6a7c6 100644 --- a/source/adios2/toolkit/derived/Function.cpp +++ b/source/adios2/toolkit/derived/Function.cpp @@ -54,15 +54,119 @@ Dims SameDimsFunc(std::vector input) // check that all dimenstions are the same if (input.size() > 1) { - bool dim_are_equal = std::equal(input.begin() + 1, input.end(), input.begin()); + Dims first_element = input[0]; + bool dim_are_equal = std::all_of(input.begin() + 1, input.end(), + [&first_element](Dims x) { return x == first_element; }); if (!dim_are_equal) - helper::Throw("Derived", "Function", "SameDimFunc", + helper::Throw("Derived", "Function", "SameDimsFunc", "Invalid variable dimensions"); } // return the first dimension return input[0]; } +// Input Dims are the same, output is combination of all inputs +Dims CurlDimsFunc(std::vector input) +{ + // check that all dimenstions are the same + if (input.size() > 1) + { + Dims first_element = input[0]; + bool dim_are_equal = std::all_of(input.begin() + 1, input.end(), + [&first_element](Dims x) { return x == first_element; }); + if (!dim_are_equal) + helper::Throw("Derived", "Function", "CurlDimsFunc", + "Invalid variable dimensions"); + } + // return original dimensions with added dimension of number of inputs + Dims output = input[0]; + output.push_back(input.size()); + return output; +} + +/* + * Linear Interpolation - average difference around point "index" + * can be used to approximate derivatives + * + * Input: + * input - data assumed to be uniform/densely populated + * index - index of point of interest + * dim - in which dimension we are approximating the partial derivative + */ +// template +float linear_interp(DerivedData input, size_t index, size_t dim) +{ + size_t stride = 1; + size_t range; + size_t offset; + float result; + float *data = (float *)input.Data; + + for (size_t i = 0; i < input.Count.size() - (dim + 1); ++i) + { + stride *= input.Count[input.Count.size() - (i + 1)]; + } + size_t ind1 = index - stride; + size_t ind2 = index + stride; + range = stride * input.Count[dim]; + offset = index % range; + + if ((offset < stride) && (range - offset <= stride)) + { + return 0; + } + else if (offset < stride) + { + result = data[ind2] - data[index]; + } + else if (range - offset <= stride) + { + result = data[index] - data[ind1]; + } + else + { + result = (data[ind2] - data[ind1]) / 2; + } + + return result; +} + +/* + * Input: 3D vector field F(x,y,z)= {F1(x,y,z), F2(x,y,z), F3(x,y,z)} + * + * inputData - (3) components of 3D vector field + * + * Computation: + * curl(F(x,y,z)) = (partial(F3,y) - partial(F2,z))i + * + (partial(F1,z) - partial(F3,x))j + * + (partial(F2,x) - partial(F1,y))k + * + * boundaries are calculated only with data in block + * (ex: partial derivatives in x direction at point (0,0,0) + * only use data from (1,0,0), etc ) + */ +DerivedData Curl3DFunc(const std::vector inputData, DataType type) +{ + size_t dataSize = inputData[0].Count[0] * inputData[0].Count[1] * inputData[0].Count[2]; + + DerivedData curl; + // ToDo - template type + float *data = (float *)malloc(dataSize * sizeof(float) * 3); + curl.Start = inputData[0].Start; + curl.Start.push_back(0); + curl.Count = inputData[0].Count; + curl.Count.push_back(3); + + for (size_t i = 0; i < dataSize; ++i) + { + data[3 * i] = linear_interp(inputData[2], i, 1) - linear_interp(inputData[1], i, 2); + data[3 * i + 1] = linear_interp(inputData[0], i, 2) - linear_interp(inputData[2], i, 0); + data[3 * i + 2] = linear_interp(inputData[1], i, 0) - linear_interp(inputData[0], i, 1); + } + curl.Data = data; + return curl; +} + #define declare_template_instantiation(T) \ T *ApplyOneToOne(std::vector, size_t, std::function); diff --git a/source/adios2/toolkit/derived/Function.h b/source/adios2/toolkit/derived/Function.h index 5dfa5aba97..d41aab6581 100644 --- a/source/adios2/toolkit/derived/Function.h +++ b/source/adios2/toolkit/derived/Function.h @@ -26,11 +26,17 @@ struct OperatorFunctions DerivedData AddFunc(std::vector input, DataType type); DerivedData MagnitudeFunc(std::vector input, DataType type); +DerivedData Curl3DFunc(std::vector input, DataType type); + +template +T linear_interp(T *data, size_t index, size_t count, size_t stride = 1); Dims SameDimsFunc(std::vector input); +Dims CurlDimsFunc(std::vector input); const std::map OpFunctions = { {adios2::detail::ExpressionOperator::OP_ADD, {AddFunc, SameDimsFunc}}, + {adios2::detail::ExpressionOperator::OP_CURL, {Curl3DFunc, CurlDimsFunc}}, {adios2::detail::ExpressionOperator::OP_MAGN, {MagnitudeFunc, SameDimsFunc}}}; template diff --git a/source/adios2/toolkit/derived/parser/pregen-source/lexer.cpp b/source/adios2/toolkit/derived/parser/pregen-source/lexer.cpp index 11c2e6b66a..598731e63c 100644 --- a/source/adios2/toolkit/derived/parser/pregen-source/lexer.cpp +++ b/source/adios2/toolkit/derived/parser/pregen-source/lexer.cpp @@ -1903,4 +1903,3 @@ adios2::detail::ASTDriver::parse (const std::string input) parse.set_debug_level (trace_parsing); parse (); } - diff --git a/testing/adios2/derived/TestBPDerivedCorrectness.cpp b/testing/adios2/derived/TestBPDerivedCorrectness.cpp index 886b942db3..9d96cc5f41 100644 --- a/testing/adios2/derived/TestBPDerivedCorrectness.cpp +++ b/testing/adios2/derived/TestBPDerivedCorrectness.cpp @@ -3,7 +3,6 @@ #include #include -#include #include #include #include @@ -169,6 +168,157 @@ TEST(DerivedCorrectness, MagCorrectnessTest) bpFileReader.Close(); } +TEST(DerivedCorrectness, CurlCorrectnessTest) +{ + const size_t Nx = 25, Ny = 70, Nz = 13; + float error_limit = 0.0000001f; + + // Application variable + std::vector simArray1(Nx * Ny * Nz); + std::vector simArray2(Nx * Ny * Nz); + std::vector simArray3(Nx * Ny * Nz); + for (size_t i = 0; i < Nx; ++i) + { + for (size_t j = 0; j < Ny; ++j) + { + for (size_t k = 0; k < Nz; ++k) + { + size_t idx = (i * Ny * Nz) + (j * Nz) + k; + float x = static_cast(i); + float y = static_cast(j); + float z = static_cast(k); + // Linear curl example + simArray1[idx] = (6 * x * y) + (7 * z); + simArray2[idx] = (4 * x * z) + powf(y, 2); + simArray3[idx] = sqrtf(z) + (2 * x * y); + /* Less linear example + simArray1[idx] = sinf(z); + simArray2[idx] = 4 * x; + simArray3[idx] = powf(y, 2) * cosf(x); + */ + /* Nonlinear example + simArray1[idx] = expf(2 * y) * sinf(x); + simArray2[idx] = sqrtf(z + 1) * cosf(x); + simArray3[idx] = powf(x, 2) * sinf(y) + (6 * z); + */ + } + } + } + + adios2::ADIOS adios; + adios2::IO bpOut = adios.DeclareIO("BPWriteExpression"); + std::vector varname = {"sim3/VX", "sim3/VY", "sim3/VZ"}; + std::string derivedname = "derived/curlV"; + + auto VX = bpOut.DefineVariable(varname[0], {Nx, Ny, Nz}, {0, 0, 0}, {Nx, Ny, Nz}); + auto VY = bpOut.DefineVariable(varname[1], {Nx, Ny, Nz}, {0, 0, 0}, {Nx, Ny, Nz}); + auto VZ = bpOut.DefineVariable(varname[2], {Nx, Ny, Nz}, {0, 0, 0}, {Nx, Ny, Nz}); + // clang-format off + bpOut.DefineDerivedVariable(derivedname, + "Vx =" + varname[0] + " \n" + "Vy =" + varname[1] + " \n" + "Vz =" + varname[2] + " \n" + "curl(Vx,Vy,Vz)", + adios2::DerivedVarType::StoreData); + // clang-format on + std::string filename = "expCurl.bp"; + adios2::Engine bpFileWriter = bpOut.Open(filename, adios2::Mode::Write); + + bpFileWriter.BeginStep(); + bpFileWriter.Put(VX, simArray1.data()); + bpFileWriter.Put(VY, simArray2.data()); + bpFileWriter.Put(VZ, simArray3.data()); + bpFileWriter.EndStep(); + bpFileWriter.Close(); + + adios2::IO bpIn = adios.DeclareIO("BPReadCurlExpression"); + adios2::Engine bpFileReader = bpIn.Open(filename, adios2::Mode::Read); + + std::vector readVX; + std::vector readVY; + std::vector readVZ; + // TODO/DEBUG - VERIFY DATATYPE + std::vector readCurl; + + std::vector> calcCurl; + double sum_x = 0; + double sum_y = 0; + double sum_z = 0; + bpFileReader.BeginStep(); + auto varVX = bpIn.InquireVariable(varname[0]); + auto varVY = bpIn.InquireVariable(varname[1]); + auto varVZ = bpIn.InquireVariable(varname[2]); + auto varCurl = bpIn.InquireVariable(derivedname); + + bpFileReader.Get(varVX, readVX); + bpFileReader.Get(varVY, readVY); + bpFileReader.Get(varVZ, readVZ); + bpFileReader.Get(varCurl, readCurl); + bpFileReader.EndStep(); + + float curl_x, curl_y, curl_z; + float err_x, err_y, err_z; + for (size_t i = 0; i < Nx; ++i) + { + for (size_t j = 0; j < Ny; ++j) + { + for (size_t k = 0; k < Nz; ++k) + { + size_t idx = (i * Ny * Nz) + (j * Nz) + k; + float x = static_cast(i); + float y = static_cast(j); + float z = static_cast(k); + // Linear example + curl_x = -(2 * x); + curl_y = 7 - (2 * y); + curl_z = (4 * z) - (6 * x); + /* Less linear + curl_x = 2 * y * cosf(x); + curl_y = cosf(z) + (powf(y, 2) * sinf(x)); + curl_z = 4; + */ + /* Nonlinear example + curl_x = powf(x, 2) * cosf(y) - (cosf(x) / (2 * sqrtf(z + 1))); + curl_y = -2 * x * sinf(y); + curl_z = -sqrtf(z + 1) * sinf(x) - (2 * expf(2 * y) * sinf(x)); + */ + if (fabs(curl_x) < 1) + { + err_x = fabs(curl_x - readCurl[3 * idx]) / (1 + fabs(curl_x)); + } + else + { + err_x = fabs(curl_x - readCurl[3 * idx]) / fabs(curl_x); + } + if (fabs(curl_y) < 1) + { + err_y = fabs(curl_y - readCurl[3 * idx + 1]) / (1 + fabs(curl_y)); + } + else + { + err_y = fabs(curl_y - readCurl[3 * idx + 1]) / fabs(curl_y); + } + if (fabs(curl_z) < 1) + { + err_z = fabs(curl_z - readCurl[3 * idx + 2]) / (1 + fabs(curl_z)); + } + else + { + err_z = fabs(curl_z - readCurl[3 * idx + 2]) / fabs(curl_z); + } + sum_x += err_x; + sum_y += err_y; + sum_z += err_z; + } + } + } + bpFileReader.Close(); + EXPECT_LT((sum_x + sum_y + sum_z) / (3 * Nx * Ny * Nz), error_limit); + EXPECT_LT(sum_x / (Nx * Ny * Nz), error_limit); + EXPECT_LT(sum_y / (Nx * Ny * Nz), error_limit); + EXPECT_LT(sum_z / (Nx * Ny * Nz), error_limit); +} + int main(int argc, char **argv) { int result; From 12184579ee5df9ceb7cc015383d249518f61ea02 Mon Sep 17 00:00:00 2001 From: anagainaru Date: Wed, 3 Apr 2024 13:47:14 -0400 Subject: [PATCH 123/124] Setting the derived variable support OFF by default --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index dd48e7f921..4c2ea67687 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -185,7 +185,7 @@ adios_option(Sodium "Enable support for Sodium for encryption" AUTO) adios_option(Catalyst "Enable support for in situ visualization plugin using ParaView Catalyst" AUTO) adios_option(Campaign "Enable support for Campaigns (requires SQLite3 and ZLIB)" AUTO) adios_option(AWSSDK "Enable support for S3 compatible storage using AWS SDK's S3 module" OFF) -adios_option(Derived_Variable "Enable support for derived variables" ON) +adios_option(Derived_Variable "Enable support for derived variables" OFF) adios_option(PIP "Enable support for pip packaging" OFF) option(ADIOS2_Blosc2_PREFER_SHARED "Prefer shared Blosc2 libraries" ON) From 14dbd0c98a642a62ed4977e1db428a8ca256b230 Mon Sep 17 00:00:00 2001 From: Vicente Adolfo Bolea Sanchez Date: Wed, 3 Apr 2024 17:21:59 -0400 Subject: [PATCH 124/124] Bump version to v2.10.0 --- ReadMe.md | 2 +- docs/user_guide/source/conf.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ReadMe.md b/ReadMe.md index 96fa7c26f0..9eec18d848 100644 --- a/ReadMe.md +++ b/ReadMe.md @@ -71,7 +71,7 @@ Once ADIOS2 is installed refer to: ## Releases -* Latest release: [v2.10.0-rc1](https://github.com/ornladios/ADIOS2/releases/tag/v2.10.0-rc1) +* Latest release: [v2.10.0](https://github.com/ornladios/ADIOS2/releases/tag/v2.10.0) * Previous releases: [https://github.com/ornladios/ADIOS2/releases](https://github.com/ornladios/ADIOS2/releases) diff --git a/docs/user_guide/source/conf.py b/docs/user_guide/source/conf.py index e3cf43f332..ec81cf9a34 100644 --- a/docs/user_guide/source/conf.py +++ b/docs/user_guide/source/conf.py @@ -74,7 +74,7 @@ # The short X.Y version. version = u'2' # The full version, including alpha/beta/rc tags. -release = u'2.10.0-rc1' +release = u'2.10.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages.