From 611cbd260e46106f6f0d1a4d217c541f4b4f4b3a Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Wed, 6 Mar 2024 08:10:17 -0500 Subject: [PATCH 1/5] Python: add the same treatment to attributes as to variables before: return scalars (0-dim ndarray) for single value attributes. --- bindings/Python/py11Attribute.cpp | 6 +++ bindings/Python/py11Attribute.h | 2 + bindings/Python/py11IO.cpp | 11 +++++- bindings/Python/py11glue.cpp | 3 +- examples/hello/bpReader/bpReaderHeatMap2D.py | 29 +++++++++----- python/adios2/attribute.py | 19 ++++++++- testing/adios2/python/TestAttribute.py | 4 +- .../python/TestBPWriteTypesHighLevelAPI.py | 39 +++++++++++-------- .../TestBPWriteTypesHighLevelAPI_HDF5.py | 32 +++++++++------ 9 files changed, 102 insertions(+), 43 deletions(-) diff --git a/bindings/Python/py11Attribute.cpp b/bindings/Python/py11Attribute.cpp index 6eb1c0e44b..e5ea93082a 100644 --- a/bindings/Python/py11Attribute.cpp +++ b/bindings/Python/py11Attribute.cpp @@ -36,6 +36,12 @@ std::string Attribute::Type() const return ToString(m_Attribute->m_Type); } +bool Attribute::SingleValue() const +{ + helper::CheckForNullptr(m_Attribute, "in call to Attribute::SingleValue"); + return m_Attribute->m_IsSingleValue; +} + std::vector Attribute::DataString() { helper::CheckForNullptr(m_Attribute, "in call to Attribute::DataStrings"); diff --git a/bindings/Python/py11Attribute.h b/bindings/Python/py11Attribute.h index 81f2723146..4366923ae6 100644 --- a/bindings/Python/py11Attribute.h +++ b/bindings/Python/py11Attribute.h @@ -37,6 +37,8 @@ class Attribute std::string Type() const; + bool SingleValue() const; + pybind11::array Data(); std::vector DataString(); diff --git a/bindings/Python/py11IO.cpp b/bindings/Python/py11IO.cpp index f44e260dd9..3d8f03782c 100644 --- a/bindings/Python/py11IO.cpp +++ b/bindings/Python/py11IO.cpp @@ -180,8 +180,15 @@ Attribute IO::DefineAttribute(const std::string &name, const pybind11::array &ar else if (pybind11::isinstance>(array)) \ { \ const T *data = reinterpret_cast(array.data()); \ - const size_t size = static_cast(array.size()); \ - attribute = &m_IO->DefineAttribute(name, data, size, variableName, separator); \ + if (array.ndim()) \ + { \ + const size_t size = static_cast(array.size()); \ + attribute = &m_IO->DefineAttribute(name, data, size, variableName, separator); \ + } \ + else \ + { \ + attribute = &m_IO->DefineAttribute(name, data[0], variableName, separator); \ + } \ } ADIOS2_FOREACH_NUMPY_ATTRIBUTE_TYPE_1ARG(declare_type) #undef declare_type diff --git a/bindings/Python/py11glue.cpp b/bindings/Python/py11glue.cpp index 13ca876517..2c40280f99 100644 --- a/bindings/Python/py11glue.cpp +++ b/bindings/Python/py11glue.cpp @@ -395,7 +395,8 @@ PYBIND11_MODULE(ADIOS2_PYTHON_MODULE_NAME, m) .def("Name", &adios2::py11::Attribute::Name) .def("Type", &adios2::py11::Attribute::Type) .def("DataString", &adios2::py11::Attribute::DataString) - .def("Data", &adios2::py11::Attribute::Data); + .def("Data", &adios2::py11::Attribute::Data) + .def("SingleValue", &adios2::py11::Attribute::SingleValue); pybind11::class_(m, "Engine") // Python 2 diff --git a/examples/hello/bpReader/bpReaderHeatMap2D.py b/examples/hello/bpReader/bpReaderHeatMap2D.py index 0f7dfe70ea..0c7c870f63 100644 --- a/examples/hello/bpReader/bpReaderHeatMap2D.py +++ b/examples/hello/bpReader/bpReaderHeatMap2D.py @@ -35,7 +35,6 @@ for j in range(0, Ny): value = iGlobal * shape[1] + j temperatures[i * Nx + j] = value -# print(str(i) + "," + str(j) + " " + str(value)) with Stream("HeatMap2D_py.bp", "w", comm) as obpStream: obpStream.write("temperature2D", temperatures, shape, start, count) @@ -43,19 +42,31 @@ obpStream.write("N", [size, Nx, Ny]) # will be an array in output obpStream.write("Nx", numpy.array(Nx)) # will be a scalar in output obpStream.write("Ny", Ny) # will be a scalar in output + obpStream.write_attribute("nproc", size) # will be a single value attribute in output obpStream.write_attribute("dimensions", [size * Nx, Ny], "temperature2D") if not rank: with FileReader("HeatMap2D_py.bp", MPI.COMM_SELF) as ibpFile: + # scalar variables are read as a numpy array with 0 dimension + in_nx = ibpFile.read("Nx") + in_ny = ibpFile.read("Ny") + print(f"Incoming nx, ny = {in_nx}, {in_ny}") + + # single value attribute is read as a numpy array with 0 dimension + in_nproc = ibpFile.read_attribute("nproc") + print(f"Incoming nproc = {in_nproc}") + # array attribute is read as a numpy array or string list + in_dims = ibpFile.read_attribute("temperature2D/dimensions") + print(f"Incoming diumensions = {in_dims}") + + # On option is to inquire a variable to know its type, shape + # directly, not as strings, and then we can use the variable + # object to set selection and/or set steps to read var_inTemperature = ibpFile.inquire_variable("temperature2D") if var_inTemperature is not None: var_inTemperature.set_selection([[2, 2], [4, 4]]) inTemperatures = ibpFile.read(var_inTemperature) - - in_nx = ibpFile.read("Nx") # scalar is read as a numpy array with 1 element - in_ny = ibpFile.read("Ny") # scalar is read as a numpy array with 1 element - print(f"Incoming nx, ny = {in_nx}, {in_ny}") - - print("Incoming temperature map") - for i in range(0, inTemperatures.shape[1]): - print(str(inTemperatures[i])) + print(f"Incoming temperature map with selection " + f"start = {var_inTemperature.start()}, count = {var_inTemperature.count()}") + for i in range(0, inTemperatures.shape[1]): + print(str(inTemperatures[i])) diff --git a/python/adios2/attribute.py b/python/adios2/attribute.py index 8740a23ae3..ec4dd7618e 100644 --- a/python/adios2/attribute.py +++ b/python/adios2/attribute.py @@ -41,6 +41,15 @@ def type(self): Type of the Attribute as a str. """ return self.impl.Type() + + def single_value(self): + """ + True if the attribute is a single value, False if it is an array + + Returns: + True or False. + """ + return self.impl.SingleValue() def data(self): """ @@ -49,7 +58,10 @@ def data(self): Returns: Content of the Attribute as a non string. """ - return self.impl.Data() + if self.single_value(): + return self.impl.Data()[0] + else: + return self.impl.Data() def data_string(self): """ @@ -58,4 +70,7 @@ def data_string(self): Returns: Content of the Attribute as a str. """ - return self.impl.DataString() + if self.single_value(): + return self.impl.DataString()[0] + else: + return self.impl.DataString() diff --git a/testing/adios2/python/TestAttribute.py b/testing/adios2/python/TestAttribute.py index 24fd79ee0d..b547c71715 100644 --- a/testing/adios2/python/TestAttribute.py +++ b/testing/adios2/python/TestAttribute.py @@ -12,14 +12,14 @@ def test_create_write(self): with adios.declare_io("BPWriter") as io: ts = io.define_attribute("timestamp", "20231122") self.assertEqual(ts.name(), "timestamp") - self.assertEqual(ts.data_string(), ["20231122"]) + self.assertEqual(ts.data_string(), "20231122") def test_create_reader(self): adios = Adios() with adios.declare_io("BPWriter") as io: ts = io.define_attribute("timestamp", "20231122") self.assertEqual(ts.name(), "timestamp") - self.assertEqual(ts.data_string(), ["20231122"]) + self.assertEqual(ts.data_string(), "20231122") def test_create_write_ndarray(self): adios = Adios() diff --git a/testing/adios2/python/TestBPWriteTypesHighLevelAPI.py b/testing/adios2/python/TestBPWriteTypesHighLevelAPI.py index 3b300fca55..cc031e2667 100644 --- a/testing/adios2/python/TestBPWriteTypesHighLevelAPI.py +++ b/testing/adios2/python/TestBPWriteTypesHighLevelAPI.py @@ -58,6 +58,8 @@ # single value attributes with numpy variables s.write_attribute("attrStr", "Testing single string attribute") + print(f"---- type of np.array(data.i8[0]) is {type(np.array(data.i8[0]))}" + f" shape = {np.array(data.i8[0]).shape}") s.write_attribute("attrI8", np.array(data.i8[0])) s.write_attribute("attrI16", np.array(data.i16[0])) s.write_attribute("attrI32", np.array(data.i32[0])) @@ -148,6 +150,7 @@ if rank == 0 and step == 0: inTag = fr_step.read("tag") + print(f"tag = {inTag}") nx = fr_step.read("nx") print(f"nx = {nx}") assert nx == data.Nx @@ -198,9 +201,11 @@ # attributes inTag = fr_step.read_attribute("attrStr") + print(f"attrStr = {inTag}") inNx = fr_step.read_attribute("attrNx") inI8 = fr_step.read_attribute("attrI8") inI16 = fr_step.read_attribute("attrI16") + print(f"attrI16 = {inI16}") inI32 = fr_step.read_attribute("attrI32") inI64 = fr_step.read_attribute("attrI64") inU8 = fr_step.read_attribute("attrU8") @@ -210,40 +215,40 @@ inR32 = fr_step.read_attribute("attrR32") inR64 = fr_step.read_attribute("attrR64") - if inTag[0] != "Testing single string attribute": + if inTag != "Testing single string attribute": raise ValueError("attr string read failed") if inNx != data.Nx: raise ValueError("attrNx read failed") - if inI8[0] != data.i8[0]: + if inI8 != data.i8[0]: raise ValueError("attrI8 read failed") - if inI16[0] != data.i16[0]: + if inI16 != data.i16[0]: raise ValueError("attrI16 read failed") - if inI32[0] != data.i32[0]: + if inI32 != data.i32[0]: raise ValueError("attrI32 read failed") - if inI64[0] != data.i64[0]: + if inI64 != data.i64[0]: raise ValueError("attrI64 read failed") - if inU8[0] != data.u8[0]: + if inU8 != data.u8[0]: raise ValueError("attrU8 read failed") - if inU16[0] != data.u16[0]: + if inU16 != data.u16[0]: raise ValueError("attrU16 read failed") - if inU32[0] != data.u32[0]: + if inU32 != data.u32[0]: raise ValueError("attrU32 read failed") - if inU64[0] != data.u64[0]: + if inU64 != data.u64[0]: raise ValueError("attrU64 read failed") - if inR32[0] != data.r32[0]: + if inR32 != data.r32[0]: raise ValueError("attrR32 read failed") - if inR64[0] != data.r64[0]: + if inR64 != data.r64[0]: raise ValueError("attrR64 read failed") in_an_int_value = fr_step.read("an_int_value") @@ -278,9 +283,11 @@ print(f"a_complex_list = {a_complex_list} of type {type(a_complex_list)}") # Array attribute - inTag = fr_step.read_attribute_string("attrStrArray") + inTag = fr_step.read_attribute("attrStrArray") + print(f"attrStrArray = {inTag}") inI8 = fr_step.read_attribute("attrI8Array") inI16 = fr_step.read_attribute("attrI16Array") + print(f"attrI16Array = {inI16}") inI32 = fr_step.read_attribute("attrI32Array") inI64 = fr_step.read_attribute("attrI64Array") inU8 = fr_step.read_attribute("attrU8Array") @@ -386,19 +393,19 @@ sizeI8 = fr_step.read_attribute("size", "varI8") sizeI16 = fr_step.read_attribute("size", "varI16", "::") - if sizeI8[0] != data.Nx: + if sizeI8 != data.Nx: raise ValueError("attribute varI8/size read failed") - if sizeI16[0] != data.Nx: + if sizeI16 != data.Nx: raise ValueError("attribute varI16::size read failed") sizeI8 = fr_step.read_attribute("varI8/size") sizeI16 = fr_step.read_attribute("varI16::size") - if sizeI8[0] != data.Nx: + if sizeI8 != data.Nx: raise ValueError("attribute varI8/size read failed") - if sizeI16[0] != data.Nx: + if sizeI16 != data.Nx: raise ValueError("attribute varI16::size read failed") step_attrs = fr_step.available_attributes() diff --git a/testing/adios2/python/TestBPWriteTypesHighLevelAPI_HDF5.py b/testing/adios2/python/TestBPWriteTypesHighLevelAPI_HDF5.py index d95c160bc9..e7848e900f 100644 --- a/testing/adios2/python/TestBPWriteTypesHighLevelAPI_HDF5.py +++ b/testing/adios2/python/TestBPWriteTypesHighLevelAPI_HDF5.py @@ -35,6 +35,7 @@ if rank == 0 and i == 0: fw.write("tag", "Testing ADIOS2 high-level API") + fw.write("nx", data.Nx) fw.write("gvarI8", np.array(data.i8[0])) fw.write("gvarI16", np.array(data.i16[0])) fw.write("gvarI32", np.array(data.i32[0])) @@ -48,6 +49,7 @@ # single value attributes fw.write_attribute("attrStr", "Testing single string attribute") + fw.write_attribute("attrNx", data.Nx) fw.write_attribute("attrI8", np.array(data.i8[0])) fw.write_attribute("attrI16", np.array(data.i16[0])) fw.write_attribute("attrI32", np.array(data.i32[0])) @@ -120,6 +122,7 @@ if rank == 0 and step == 0: inTag = fr_step.read("tag") + inNx = fr_step.read("nx") inI8 = fr_step.read("gvarI8") inI16 = fr_step.read("gvarI16") inI32 = fr_step.read("gvarI32") @@ -135,6 +138,9 @@ print("InTag: " + str(inTag)) raise ValueError("tag variable read failed") + if inNx != nx: + raise ValueError("tag variable read failed") + if inI8 != data.i8[0]: raise ValueError("gvarI8 read failed") @@ -167,6 +173,7 @@ # attributes inTag = fr_step.read_attribute("attrStr") + inNx = fr_step.read_attribute("attrNx") inI8 = fr_step.read_attribute("attrI8") inI16 = fr_step.read_attribute("attrI16") inI32 = fr_step.read_attribute("attrI32") @@ -178,37 +185,40 @@ inR32 = fr_step.read_attribute("attrR32") inR64 = fr_step.read_attribute("attrR64") - if inTag[0] != "Testing single string attribute": + if inTag != "Testing single string attribute": raise ValueError("attr string read failed") - if inI8[0] != data.i8[0]: + if inNx != nx: raise ValueError("attrI8 read failed") - if inI16[0] != data.i16[0]: + if inI8 != data.i8[0]: + raise ValueError("attrI8 read failed") + + if inI16 != data.i16[0]: raise ValueError("attrI16 read failed") - if inI32[0] != data.i32[0]: + if inI32 != data.i32[0]: raise ValueError("attrI32 read failed") - if inI64[0] != data.i64[0]: + if inI64 != data.i64[0]: raise ValueError("attrI64 read failed") - if inU8[0] != data.u8[0]: + if inU8 != data.u8[0]: raise ValueError("attrU8 read failed") - if inU16[0] != data.u16[0]: + if inU16 != data.u16[0]: raise ValueError("attrU16 read failed") - if inU32[0] != data.u32[0]: + if inU32 != data.u32[0]: raise ValueError("attrU32 read failed") - if inU64[0] != data.u64[0]: + if inU64 != data.u64[0]: raise ValueError("attrU64 read failed") - if inR32[0] != data.r32[0]: + if inR32 != data.r32[0]: raise ValueError("attrR32 read failed") - if inR64[0] != data.r64[0]: + if inR64 != data.r64[0]: raise ValueError("attrR64 read failed") # Array attribute From 6ed1e9274dfc4791ab05050685d39572ba520777 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Wed, 6 Mar 2024 11:38:01 -0500 Subject: [PATCH 2/5] format --- examples/hello/bpReader/bpReaderHeatMap2D.py | 10 ++++++---- python/adios2/attribute.py | 8 +++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/examples/hello/bpReader/bpReaderHeatMap2D.py b/examples/hello/bpReader/bpReaderHeatMap2D.py index 0c7c870f63..ce31e418d9 100644 --- a/examples/hello/bpReader/bpReaderHeatMap2D.py +++ b/examples/hello/bpReader/bpReaderHeatMap2D.py @@ -42,13 +42,13 @@ obpStream.write("N", [size, Nx, Ny]) # will be an array in output obpStream.write("Nx", numpy.array(Nx)) # will be a scalar in output obpStream.write("Ny", Ny) # will be a scalar in output - obpStream.write_attribute("nproc", size) # will be a single value attribute in output + obpStream.write_attribute("nproc", size) # will be a single value attribute in output obpStream.write_attribute("dimensions", [size * Nx, Ny], "temperature2D") if not rank: with FileReader("HeatMap2D_py.bp", MPI.COMM_SELF) as ibpFile: # scalar variables are read as a numpy array with 0 dimension - in_nx = ibpFile.read("Nx") + in_nx = ibpFile.read("Nx") in_ny = ibpFile.read("Ny") print(f"Incoming nx, ny = {in_nx}, {in_ny}") @@ -66,7 +66,9 @@ if var_inTemperature is not None: var_inTemperature.set_selection([[2, 2], [4, 4]]) inTemperatures = ibpFile.read(var_inTemperature) - print(f"Incoming temperature map with selection " - f"start = {var_inTemperature.start()}, count = {var_inTemperature.count()}") + print( + f"Incoming temperature map with selection " + f"start = {var_inTemperature.start()}, count = {var_inTemperature.count()}" + ) for i in range(0, inTemperatures.shape[1]): print(str(inTemperatures[i])) diff --git a/python/adios2/attribute.py b/python/adios2/attribute.py index ec4dd7618e..ddbaa2eb06 100644 --- a/python/adios2/attribute.py +++ b/python/adios2/attribute.py @@ -41,7 +41,7 @@ def type(self): Type of the Attribute as a str. """ return self.impl.Type() - + def single_value(self): """ True if the attribute is a single value, False if it is an array @@ -60,8 +60,7 @@ def data(self): """ if self.single_value(): return self.impl.Data()[0] - else: - return self.impl.Data() + return self.impl.Data() def data_string(self): """ @@ -72,5 +71,4 @@ def data_string(self): """ if self.single_value(): return self.impl.DataString()[0] - else: - return self.impl.DataString() + return self.impl.DataString() From 0b29dde35e207a9f2c4840c293d4f07eec78d832 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Wed, 6 Mar 2024 12:08:59 -0500 Subject: [PATCH 3/5] format more --- testing/adios2/python/TestBPWriteTypesHighLevelAPI.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/testing/adios2/python/TestBPWriteTypesHighLevelAPI.py b/testing/adios2/python/TestBPWriteTypesHighLevelAPI.py index cc031e2667..dbd9cf6635 100644 --- a/testing/adios2/python/TestBPWriteTypesHighLevelAPI.py +++ b/testing/adios2/python/TestBPWriteTypesHighLevelAPI.py @@ -58,8 +58,10 @@ # single value attributes with numpy variables s.write_attribute("attrStr", "Testing single string attribute") - print(f"---- type of np.array(data.i8[0]) is {type(np.array(data.i8[0]))}" - f" shape = {np.array(data.i8[0]).shape}") + print( + f"---- type of np.array(data.i8[0]) is {type(np.array(data.i8[0]))}" + f" shape = {np.array(data.i8[0]).shape}" + ) s.write_attribute("attrI8", np.array(data.i8[0])) s.write_attribute("attrI16", np.array(data.i16[0])) s.write_attribute("attrI32", np.array(data.i32[0])) From acc1c5398270efa949f4b1019aa15d126521a5bd Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Thu, 7 Mar 2024 14:38:36 -0500 Subject: [PATCH 4/5] - Python: fix for scalar reading. If a global value has 1 step (i.e. always in streaming), read returns a 0-dim numpy array (single value). If the variable has multiple steps (only in ReadRandomAccess mode), read returns a 1-dim numpy array even if the step selection is a single step. This way, read of a certain variable always results in the same type of array no matter the number of steps selected. - Python: fix for string attributes: return a string, not a list of one element which is a string, to be consistent with string global values and with other APIs. --- .../api_python/adios2-doc-read-filereader.py | 8 ++- ...-doc-read.py => adios2-doc-read-stream.py} | 4 +- .../source/api_python/python_example.rst | 24 ++++--- python/adios2/file_reader.py | 4 -- python/adios2/stream.py | 62 ++++++++++++++----- testing/adios2/python/TestFileReader.py | 58 ++++++++++++----- 6 files changed, 113 insertions(+), 47 deletions(-) rename docs/user_guide/source/api_python/{adios2-doc-read.py => adios2-doc-read-stream.py} (89%) diff --git a/docs/user_guide/source/api_python/adios2-doc-read-filereader.py b/docs/user_guide/source/api_python/adios2-doc-read-filereader.py index 3211e82642..73fa255005 100644 --- a/docs/user_guide/source/api_python/adios2-doc-read-filereader.py +++ b/docs/user_guide/source/api_python/adios2-doc-read-filereader.py @@ -9,14 +9,18 @@ for key, value in info.items(): print("\t" + key + ": " + value, end=" ") print() + print() nproc = s.read("nproc") - print(f"nproc is {nproc} of type {type(nproc)}") + print(f"nproc is {nproc} of type {type(nproc)} with ndim {nproc.ndim}") # read variables return a numpy array with corresponding selection steps = int(vars["physical_time"]["AvailableStepsCount"]) physical_time = s.read("physical_time", step_selection=[0, steps]) - print(f"physical_time is {physical_time} of type {type(physical_time)}") + print( + f"physical_time is {physical_time} of type {type(physical_time)} with " + f"ndim {physical_time.ndim} shape = {physical_time.shape}" + ) steps = int(vars["temperature"]["AvailableStepsCount"]) temperature = s.read("temperature", step_selection=[0, steps]) diff --git a/docs/user_guide/source/api_python/adios2-doc-read.py b/docs/user_guide/source/api_python/adios2-doc-read-stream.py similarity index 89% rename from docs/user_guide/source/api_python/adios2-doc-read.py rename to docs/user_guide/source/api_python/adios2-doc-read-stream.py index 5f7d1ee2cf..0d24844ed6 100644 --- a/docs/user_guide/source/api_python/adios2-doc-read.py +++ b/docs/user_guide/source/api_python/adios2-doc-read-stream.py @@ -16,7 +16,7 @@ if s.current_step() == 0: nproc = s.read("nproc") - print(f"nproc is {nproc} of type {type(nproc)}") + print(f"nproc is {nproc} of type {type(nproc)} with ndim {nproc.ndim}") # read variables return a numpy array with corresponding selection physical_time = s.read("physical_time") @@ -26,6 +26,6 @@ print(f"temperature array size is {temperature.size} of shape {temperature.shape}") print(f"temperature unit is {temp_unit} of type {type(temp_unit)}") pressure = s.read("pressure") - press_unit = s.read_attribute("pressure/unit") + press_unit = s.read_attribute("unit", "pressure") print(f"pressure unit is {press_unit} of type {type(press_unit)}") print() diff --git a/docs/user_guide/source/api_python/python_example.rst b/docs/user_guide/source/api_python/python_example.rst index d2458a12c2..58ac0b9375 100644 --- a/docs/user_guide/source/api_python/python_example.rst +++ b/docs/user_guide/source/api_python/python_example.rst @@ -44,6 +44,7 @@ Python Write example count = [nx] temperature = np.zeros(nx, dtype=np.double) + pressure = np.ones(nx, dtype=np.double) delta_time = 0.01 physical_time = 0.0 nsteps = 5 @@ -125,8 +126,8 @@ Python Read "step-by-step" example nproc is 4 of type physical_time is 0.0 of type temperature array size is 40 of shape (40,) - temperature unit is ['K'] of type - pressure unit is ['Pa'] of type + temperature unit is K of type + pressure unit is Pa of type Current step is 1 variable_name: physical_time AvailableStepsCount: 1 Max: 0.01 Min: 0.01 Shape: SingleValue: true Type: double @@ -134,8 +135,8 @@ Python Read "step-by-step" example variable_name: temperature AvailableStepsCount: 1 Max: 0 Min: 0 Shape: 40 SingleValue: false Type: double physical_time is 0.01 of type temperature array size is 40 of shape (40,) - temperature unit is ['K'] of type - pressure unit is ['Pa'] of type + temperature unit is K of type + pressure unit is Pa of type ... @@ -156,14 +157,18 @@ Python Read Random Access example for key, value in info.items(): print("\t" + key + ": " + value, end=" ") print() + print() nproc = s.read("nproc") - print(f"nproc is {nproc} of type {type(nproc)}") + print(f"nproc is {nproc} of type {type(nproc)} with ndim {nproc.ndim}") # read variables return a numpy array with corresponding selection steps = int(vars['physical_time']['AvailableStepsCount']) physical_time = s.read("physical_time", step_selection=[0, steps]) - print(f"physical_time is {physical_time} of type {type(physical_time)}") + print( + f"physical_time is {physical_time} of type {type(physical_time)} with " + f"ndim {physical_time.ndim} shape = {physical_time.shape}" + ) steps = int(vars['temperature']['AvailableStepsCount']) temperature = s.read("temperature", step_selection=[0, steps]) @@ -183,8 +188,9 @@ Python Read Random Access example variable_name: physical_time AvailableStepsCount: 5 Max: 0.04 Min: 0 Shape: SingleValue: true Type: double variable_name: pressure AvailableStepsCount: 5 Max: 1 Min: 1 Shape: 40 SingleValue: false Type: double variable_name: temperature AvailableStepsCount: 5 Max: 0 Min: 0 Shape: 40 SingleValue: false Type: double - nproc is 4 of type - physical_time is [0. 0.01 0.02 0.03 0.04] of type + + nproc is 4 of type with ndim 0 + physical_time is [0. 0.01 0.02 0.03 0.04] of type with ndim 1 shape = (5,) temperature array size is 200 of shape (200,) - temperature unit is ['K'] of type + temperature unit is K of type diff --git a/python/adios2/file_reader.py b/python/adios2/file_reader.py index 74f0f640fe..bc0eb44452 100644 --- a/python/adios2/file_reader.py +++ b/python/adios2/file_reader.py @@ -25,7 +25,3 @@ def _(self, io: IO, path, comm=None): super().__init__(io, path, "rra", comm) # pylint: enable=E1121 - - def variables(self): - """Returns the list of variables contained in the opened file""" - return [self._io.inquire_variable(var) for var in self.available_variables()] diff --git a/python/adios2/stream.py b/python/adios2/stream.py index 826316d9cd..0285507cc9 100644 --- a/python/adios2/stream.py +++ b/python/adios2/stream.py @@ -325,11 +325,9 @@ def _(self, name, content, shape=[], start=[], count=[], operations=None): self.write(variable, content) - @singledispatchmethod - def read(self, variable: Variable): + def _read_var(self, variable: Variable): """ - Random access read allowed to select steps, - only valid with Stream Engines + Internal common function to read. Settings must be done to Variable before the call. Parameters variable @@ -342,6 +340,7 @@ def read(self, variable: Variable): """ dtype = type_adios_to_numpy(variable.type()) count = variable.count() + if count != []: # array # steps = variable.get_steps_from_step_selection() @@ -365,7 +364,8 @@ def read(self, variable: Variable): else: # scalar size_all_steps = variable.selection_size() - if size_all_steps > 1: + # if size_all_steps > 1: + if self._mode == bindings.Mode.ReadRandomAccess and variable.steps() > 1: output_shape = [size_all_steps] else: output_shape = [] @@ -374,15 +374,17 @@ def read(self, variable: Variable): self._engine.get(variable, output) return output - @read.register(str) - def _(self, name: str, start=[], count=[], block_id=None, step_selection=None): + @singledispatchmethod + def read(self, variable: Variable, start=[], count=[], block_id=None, step_selection=None): """ - Random access read allowed to select steps, - only valid with Stream Engines + Read a variable. + Random access read allowed to select steps. Parameters - name - variable to be read + variable + adios2.Variable object to be read + Use variable.set_selection(), set_block_selection(), set_step_selection() + to prepare a read start variable offset dimensions @@ -400,10 +402,6 @@ def _(self, name: str, start=[], count=[], block_id=None, step_selection=None): array resulting array from selection """ - variable = self._io.inquire_variable(name) - if not variable: - raise ValueError() - if step_selection is not None and not self._mode == bindings.Mode.ReadRandomAccess: raise RuntimeError("step_selection parameter requires 'rra' mode") @@ -419,7 +417,39 @@ def _(self, name: str, start=[], count=[], block_id=None, step_selection=None): if start != [] and count != []: variable.set_selection([start, count]) - return self.read(variable) + return self._read_var(variable) + + @read.register(str) + def _(self, name: str, start=[], count=[], block_id=None, step_selection=None): + """ + Read a variable. + Random access read allowed to select steps. + + Parameters + name + variable to be read + + start + variable offset dimensions + + count + variable local dimensions from offset + + block_id + (int) Required for reading local variables, local array, and local + value. + + step_selection + (list): On the form of [start, count]. + Returns + array + resulting array from selection + """ + variable = self._io.inquire_variable(name) + if not variable: + raise ValueError() + + return self.read(variable, start, count, block_id, step_selection) def write_attribute(self, name, content, variable_name="", separator="/"): """ diff --git a/testing/adios2/python/TestFileReader.py b/testing/adios2/python/TestFileReader.py index cda2dc6533..fbd712e681 100644 --- a/testing/adios2/python/TestFileReader.py +++ b/testing/adios2/python/TestFileReader.py @@ -1,45 +1,75 @@ -from adios2 import FileReader, Stream +from adios2 import FileReader, Stream, LocalValueDim from random import randint import unittest -class TestStream(unittest.TestCase): +class TestFileReader(unittest.TestCase): def test_basic(self): with Stream("pythonfiletest.bp", "w") as s: for _ in s.steps(10): if s.current_step() == 0: + # String s.write("Outlook", "Good") + # Global Value single step + s.write("LocationID", 42) + # Global Array s.write( - "temp", + "Temp", content=[randint(15, 35), randint(15, 35), randint(15, 35)], shape=[3], start=[0], count=[3], ) + # Local Value + s.write("Wind", [5], shape=[LocalValueDim]) + # Local Array + s.write("Coords", [38, -46], [], [], [2]) + # Global Value every step + s.write("Hour", 8 + s.current_step()) with FileReader("pythonfiletest.bp") as f: self.assertEqual(len(f.all_blocks_info("temp")), 10) - for var in f.variables(): + outlook_var = f.inquire_variable("Outlook") + outlook = f.read(outlook_var) + self.assertEqual(outlook, "Good") + + for name in f.available_variables(): + var = f.inquire_variable(name) if not var.type() == "string": - self.assertEqual(var.steps(), 10) + if var.steps() > 1: + self.assertEqual(var.steps(), 10) var.set_step_selection([0, var.steps()]) - - output = f.read(var.name()) - print(f"var:{var.name()} output:{output}") + output = f.read(var) + print(f"var:{var.name()} output:{output}") with FileReader("pythonfiletest.bp") as f: - output = f.read("temp", step_selection=[0, 10]) + output = f.read("LocationID") + self.assertEqual(output.ndim, 0) + self.assertEqual(output.size, 1) + self.assertEqual(output, 42) + + output = f.read("Hour", step_selection=[0, 10]) + self.assertEqual(output.ndim, 1) + self.assertEqual(output.size, 10) + self.assertTrue((output == [8, 9, 10, 11, 12, 13, 14, 15, 16, 17]).all()) + + output = f.read("Hour", step_selection=[5, 1]) + self.assertEqual(output.ndim, 1) + self.assertEqual(output.size, 1) + self.assertEqual(output, [13]) + + output = f.read("Temp", step_selection=[0, 10]) self.assertEqual(len(output), 30) - print(f"var:temp output:{output}") + print(f"var:Temp output:{output}") - output = f.read("temp", step_selection=[0, 5]) + output = f.read("Temp", step_selection=[0, 5]) self.assertEqual(len(output), 15) - print(f"var:temp output:{output}") + print(f"var:Temp output:{output}") - output = f.read("temp", start=[0], count=[2], step_selection=[0, 10]) + output = f.read("Temp", start=[0], count=[2], step_selection=[0, 10]) self.assertEqual(len(output), 20) - print(f"var:temp output:{output}") + print(f"var:Temp output:{output}") if __name__ == "__main__": From 9486fd753cc16fdad667f4c32de553ab8685fae9 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Fri, 8 Mar 2024 12:08:46 -0500 Subject: [PATCH 5/5] Add test for single string attribute vs string array attribute with a single element --- .../python/TestBPWriteTypesHighLevelAPI.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/testing/adios2/python/TestBPWriteTypesHighLevelAPI.py b/testing/adios2/python/TestBPWriteTypesHighLevelAPI.py index dbd9cf6635..f77aac8599 100644 --- a/testing/adios2/python/TestBPWriteTypesHighLevelAPI.py +++ b/testing/adios2/python/TestBPWriteTypesHighLevelAPI.py @@ -84,7 +84,8 @@ s.write_attribute("attr_int_list", data.int_list) s.write_attribute("attr_float_list", data.float_list) s.write_attribute("attr_complex_list", data.complex_list) - s.write_attribute("attrStrArray", ["string1", "string2", "string3"]) + s.write_attribute("attrStrArray1", ["string1"]) + s.write_attribute("attrStrArray3", ["string1", "string2", "string3"]) # array attributes with numpy arrays s.write_attribute("attrI8Array", data.i8) @@ -285,8 +286,10 @@ print(f"a_complex_list = {a_complex_list} of type {type(a_complex_list)}") # Array attribute - inTag = fr_step.read_attribute("attrStrArray") - print(f"attrStrArray = {inTag}") + inStr1 = fr_step.read_attribute("attrStrArray1") + print(f"attrStrArray1 = {inStr1}") + inStr3 = fr_step.read_attribute("attrStrArray3") + print(f"attrStrArray3 = {inStr3}") inI8 = fr_step.read_attribute("attrI8Array") inI16 = fr_step.read_attribute("attrI16Array") print(f"attrI16Array = {inI16}") @@ -299,8 +302,11 @@ inR32 = fr_step.read_attribute("attrR32Array") inR64 = fr_step.read_attribute("attrR64Array") - if inTag != ["string1", "string2", "string3"]: - raise ValueError("attrStrArray read failed") + if inStr1 != ["string1"]: + raise ValueError("attrStrArray1 read failed") + + if inStr3 != ["string1", "string2", "string3"]: + raise ValueError("attrStrArray3 read failed") if not (inI8 == data.i8).all(): raise ValueError("attrI8 array read failed")