Skip to content

Commit

Permalink
Merge pull request #4076 from pnorbert/python-scalar-attributes
Browse files Browse the repository at this point in the history
Python: add the same treatment to attributes as to variables before: …
  • Loading branch information
pnorbert authored Mar 8, 2024
2 parents 269f1f7 + 9486fd7 commit c317c35
Show file tree
Hide file tree
Showing 15 changed files with 224 additions and 91 deletions.
6 changes: 6 additions & 0 deletions bindings/Python/py11Attribute.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,12 @@ std::string Attribute::Type() const
return ToString(m_Attribute->m_Type);
}

bool Attribute::SingleValue() const
{
helper::CheckForNullptr(m_Attribute, "in call to Attribute::SingleValue");
return m_Attribute->m_IsSingleValue;
}

std::vector<std::string> Attribute::DataString()
{
helper::CheckForNullptr(m_Attribute, "in call to Attribute::DataStrings");
Expand Down
2 changes: 2 additions & 0 deletions bindings/Python/py11Attribute.h
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,8 @@ class Attribute

std::string Type() const;

bool SingleValue() const;

pybind11::array Data();

std::vector<std::string> DataString();
Expand Down
11 changes: 9 additions & 2 deletions bindings/Python/py11IO.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -180,8 +180,15 @@ Attribute IO::DefineAttribute(const std::string &name, const pybind11::array &ar
else if (pybind11::isinstance<pybind11::array_t<T, pybind11::array::c_style>>(array)) \
{ \
const T *data = reinterpret_cast<const T *>(array.data()); \
const size_t size = static_cast<size_t>(array.size()); \
attribute = &m_IO->DefineAttribute<T>(name, data, size, variableName, separator); \
if (array.ndim()) \
{ \
const size_t size = static_cast<size_t>(array.size()); \
attribute = &m_IO->DefineAttribute<T>(name, data, size, variableName, separator); \
} \
else \
{ \
attribute = &m_IO->DefineAttribute<T>(name, data[0], variableName, separator); \
} \
}
ADIOS2_FOREACH_NUMPY_ATTRIBUTE_TYPE_1ARG(declare_type)
#undef declare_type
Expand Down
3 changes: 2 additions & 1 deletion bindings/Python/py11glue.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -395,7 +395,8 @@ PYBIND11_MODULE(ADIOS2_PYTHON_MODULE_NAME, m)
.def("Name", &adios2::py11::Attribute::Name)
.def("Type", &adios2::py11::Attribute::Type)
.def("DataString", &adios2::py11::Attribute::DataString)
.def("Data", &adios2::py11::Attribute::Data);
.def("Data", &adios2::py11::Attribute::Data)
.def("SingleValue", &adios2::py11::Attribute::SingleValue);

pybind11::class_<adios2::py11::Engine>(m, "Engine")
// Python 2
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,18 @@
for key, value in info.items():
print("\t" + key + ": " + value, end=" ")
print()
print()

nproc = s.read("nproc")
print(f"nproc is {nproc} of type {type(nproc)}")
print(f"nproc is {nproc} of type {type(nproc)} with ndim {nproc.ndim}")

# read variables return a numpy array with corresponding selection
steps = int(vars["physical_time"]["AvailableStepsCount"])
physical_time = s.read("physical_time", step_selection=[0, steps])
print(f"physical_time is {physical_time} of type {type(physical_time)}")
print(
f"physical_time is {physical_time} of type {type(physical_time)} with "
f"ndim {physical_time.ndim} shape = {physical_time.shape}"
)

steps = int(vars["temperature"]["AvailableStepsCount"])
temperature = s.read("temperature", step_selection=[0, steps])
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@

if s.current_step() == 0:
nproc = s.read("nproc")
print(f"nproc is {nproc} of type {type(nproc)}")
print(f"nproc is {nproc} of type {type(nproc)} with ndim {nproc.ndim}")

# read variables return a numpy array with corresponding selection
physical_time = s.read("physical_time")
Expand All @@ -26,6 +26,6 @@
print(f"temperature array size is {temperature.size} of shape {temperature.shape}")
print(f"temperature unit is {temp_unit} of type {type(temp_unit)}")
pressure = s.read("pressure")
press_unit = s.read_attribute("pressure/unit")
press_unit = s.read_attribute("unit", "pressure")
print(f"pressure unit is {press_unit} of type {type(press_unit)}")
print()
24 changes: 15 additions & 9 deletions docs/user_guide/source/api_python/python_example.rst
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ Python Write example
count = [nx]
temperature = np.zeros(nx, dtype=np.double)
pressure = np.ones(nx, dtype=np.double)
delta_time = 0.01
physical_time = 0.0
nsteps = 5
Expand Down Expand Up @@ -125,17 +126,17 @@ Python Read "step-by-step" example
nproc is 4 of type <class 'numpy.ndarray'>
physical_time is 0.0 of type <class 'numpy.ndarray'>
temperature array size is 40 of shape (40,)
temperature unit is ['K'] of type <class 'list'>
pressure unit is ['Pa'] of type <class 'list'>
temperature unit is K of type <class 'str'>
pressure unit is Pa of type <class 'str'>
Current step is 1
variable_name: physical_time AvailableStepsCount: 1 Max: 0.01 Min: 0.01 Shape: SingleValue: true Type: double
variable_name: pressure AvailableStepsCount: 1 Max: 1 Min: 1 Shape: 40 SingleValue: false Type: double
variable_name: temperature AvailableStepsCount: 1 Max: 0 Min: 0 Shape: 40 SingleValue: false Type: double
physical_time is 0.01 of type <class 'numpy.ndarray'>
temperature array size is 40 of shape (40,)
temperature unit is ['K'] of type <class 'list'>
pressure unit is ['Pa'] of type <class 'list'>
temperature unit is K of type <class 'str'>
pressure unit is Pa of type <class 'str'>
...
Expand All @@ -156,14 +157,18 @@ Python Read Random Access example
for key, value in info.items():
print("\t" + key + ": " + value, end=" ")
print()
print()
nproc = s.read("nproc")
print(f"nproc is {nproc} of type {type(nproc)}")
print(f"nproc is {nproc} of type {type(nproc)} with ndim {nproc.ndim}")
# read variables return a numpy array with corresponding selection
steps = int(vars['physical_time']['AvailableStepsCount'])
physical_time = s.read("physical_time", step_selection=[0, steps])
print(f"physical_time is {physical_time} of type {type(physical_time)}")
print(
f"physical_time is {physical_time} of type {type(physical_time)} with "
f"ndim {physical_time.ndim} shape = {physical_time.shape}"
)
steps = int(vars['temperature']['AvailableStepsCount'])
temperature = s.read("temperature", step_selection=[0, steps])
Expand All @@ -183,8 +188,9 @@ Python Read Random Access example
variable_name: physical_time AvailableStepsCount: 5 Max: 0.04 Min: 0 Shape: SingleValue: true Type: double
variable_name: pressure AvailableStepsCount: 5 Max: 1 Min: 1 Shape: 40 SingleValue: false Type: double
variable_name: temperature AvailableStepsCount: 5 Max: 0 Min: 0 Shape: 40 SingleValue: false Type: double
nproc is 4 of type <class 'numpy.ndarray'>
physical_time is [0. 0.01 0.02 0.03 0.04] of type <class 'numpy.ndarray'>
nproc is 4 of type <class 'numpy.ndarray'> with ndim 0
physical_time is [0. 0.01 0.02 0.03 0.04] of type <class 'numpy.ndarray'> with ndim 1 shape = (5,)
temperature array size is 200 of shape (200,)
temperature unit is ['K'] of type <class 'list'>
temperature unit is K of type <class 'str'>
31 changes: 22 additions & 9 deletions examples/hello/bpReader/bpReaderHeatMap2D.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,27 +35,40 @@
for j in range(0, Ny):
value = iGlobal * shape[1] + j
temperatures[i * Nx + j] = value
# print(str(i) + "," + str(j) + " " + str(value))

with Stream("HeatMap2D_py.bp", "w", comm) as obpStream:
obpStream.write("temperature2D", temperatures, shape, start, count)
if not rank:
obpStream.write("N", [size, Nx, Ny]) # will be an array in output
obpStream.write("Nx", numpy.array(Nx)) # will be a scalar in output
obpStream.write("Ny", Ny) # will be a scalar in output
obpStream.write_attribute("nproc", size) # will be a single value attribute in output
obpStream.write_attribute("dimensions", [size * Nx, Ny], "temperature2D")

if not rank:
with FileReader("HeatMap2D_py.bp", MPI.COMM_SELF) as ibpFile:
# scalar variables are read as a numpy array with 0 dimension
in_nx = ibpFile.read("Nx")
in_ny = ibpFile.read("Ny")
print(f"Incoming nx, ny = {in_nx}, {in_ny}")

# single value attribute is read as a numpy array with 0 dimension
in_nproc = ibpFile.read_attribute("nproc")
print(f"Incoming nproc = {in_nproc}")
# array attribute is read as a numpy array or string list
in_dims = ibpFile.read_attribute("temperature2D/dimensions")
print(f"Incoming diumensions = {in_dims}")

# On option is to inquire a variable to know its type, shape
# directly, not as strings, and then we can use the variable
# object to set selection and/or set steps to read
var_inTemperature = ibpFile.inquire_variable("temperature2D")
if var_inTemperature is not None:
var_inTemperature.set_selection([[2, 2], [4, 4]])
inTemperatures = ibpFile.read(var_inTemperature)

in_nx = ibpFile.read("Nx") # scalar is read as a numpy array with 1 element
in_ny = ibpFile.read("Ny") # scalar is read as a numpy array with 1 element
print(f"Incoming nx, ny = {in_nx}, {in_ny}")

print("Incoming temperature map")
for i in range(0, inTemperatures.shape[1]):
print(str(inTemperatures[i]))
print(
f"Incoming temperature map with selection "
f"start = {var_inTemperature.start()}, count = {var_inTemperature.count()}"
)
for i in range(0, inTemperatures.shape[1]):
print(str(inTemperatures[i]))
13 changes: 13 additions & 0 deletions python/adios2/attribute.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,13 +42,24 @@ def type(self):
"""
return self.impl.Type()

def single_value(self):
"""
True if the attribute is a single value, False if it is an array
Returns:
True or False.
"""
return self.impl.SingleValue()

def data(self):
"""
Content of the Attribute
Returns:
Content of the Attribute as a non string.
"""
if self.single_value():
return self.impl.Data()[0]
return self.impl.Data()

def data_string(self):
Expand All @@ -58,4 +69,6 @@ def data_string(self):
Returns:
Content of the Attribute as a str.
"""
if self.single_value():
return self.impl.DataString()[0]
return self.impl.DataString()
4 changes: 0 additions & 4 deletions python/adios2/file_reader.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,3 @@ def _(self, io: IO, path, comm=None):
super().__init__(io, path, "rra", comm)

# pylint: enable=E1121

def variables(self):
"""Returns the list of variables contained in the opened file"""
return [self._io.inquire_variable(var) for var in self.available_variables()]
62 changes: 46 additions & 16 deletions python/adios2/stream.py
Original file line number Diff line number Diff line change
Expand Up @@ -325,11 +325,9 @@ def _(self, name, content, shape=[], start=[], count=[], operations=None):

self.write(variable, content)

@singledispatchmethod
def read(self, variable: Variable):
def _read_var(self, variable: Variable):
"""
Random access read allowed to select steps,
only valid with Stream Engines
Internal common function to read. Settings must be done to Variable before the call.
Parameters
variable
Expand All @@ -342,6 +340,7 @@ def read(self, variable: Variable):
"""
dtype = type_adios_to_numpy(variable.type())
count = variable.count()

if count != []:
# array
# steps = variable.get_steps_from_step_selection()
Expand All @@ -365,7 +364,8 @@ def read(self, variable: Variable):
else:
# scalar
size_all_steps = variable.selection_size()
if size_all_steps > 1:
# if size_all_steps > 1:
if self._mode == bindings.Mode.ReadRandomAccess and variable.steps() > 1:
output_shape = [size_all_steps]
else:
output_shape = []
Expand All @@ -374,15 +374,17 @@ def read(self, variable: Variable):
self._engine.get(variable, output)
return output

@read.register(str)
def _(self, name: str, start=[], count=[], block_id=None, step_selection=None):
@singledispatchmethod
def read(self, variable: Variable, start=[], count=[], block_id=None, step_selection=None):
"""
Random access read allowed to select steps,
only valid with Stream Engines
Read a variable.
Random access read allowed to select steps.
Parameters
name
variable to be read
variable
adios2.Variable object to be read
Use variable.set_selection(), set_block_selection(), set_step_selection()
to prepare a read
start
variable offset dimensions
Expand All @@ -400,10 +402,6 @@ def _(self, name: str, start=[], count=[], block_id=None, step_selection=None):
array
resulting array from selection
"""
variable = self._io.inquire_variable(name)
if not variable:
raise ValueError()

if step_selection is not None and not self._mode == bindings.Mode.ReadRandomAccess:
raise RuntimeError("step_selection parameter requires 'rra' mode")

Expand All @@ -419,7 +417,39 @@ def _(self, name: str, start=[], count=[], block_id=None, step_selection=None):
if start != [] and count != []:
variable.set_selection([start, count])

return self.read(variable)
return self._read_var(variable)

@read.register(str)
def _(self, name: str, start=[], count=[], block_id=None, step_selection=None):
"""
Read a variable.
Random access read allowed to select steps.
Parameters
name
variable to be read
start
variable offset dimensions
count
variable local dimensions from offset
block_id
(int) Required for reading local variables, local array, and local
value.
step_selection
(list): On the form of [start, count].
Returns
array
resulting array from selection
"""
variable = self._io.inquire_variable(name)
if not variable:
raise ValueError()

return self.read(variable, start, count, block_id, step_selection)

def write_attribute(self, name, content, variable_name="", separator="/"):
"""
Expand Down
4 changes: 2 additions & 2 deletions testing/adios2/python/TestAttribute.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,14 +12,14 @@ def test_create_write(self):
with adios.declare_io("BPWriter") as io:
ts = io.define_attribute("timestamp", "20231122")
self.assertEqual(ts.name(), "timestamp")
self.assertEqual(ts.data_string(), ["20231122"])
self.assertEqual(ts.data_string(), "20231122")

def test_create_reader(self):
adios = Adios()
with adios.declare_io("BPWriter") as io:
ts = io.define_attribute("timestamp", "20231122")
self.assertEqual(ts.name(), "timestamp")
self.assertEqual(ts.data_string(), ["20231122"])
self.assertEqual(ts.data_string(), "20231122")

def test_create_write_ndarray(self):
adios = Adios()
Expand Down
Loading

0 comments on commit c317c35

Please sign in to comment.