Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Python: add the same treatment to attributes as to variables before: … #4076

Merged
merged 5 commits into from
Mar 8, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions bindings/Python/py11Attribute.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,12 @@ std::string Attribute::Type() const
return ToString(m_Attribute->m_Type);
}

bool Attribute::SingleValue() const
{
helper::CheckForNullptr(m_Attribute, "in call to Attribute::SingleValue");
return m_Attribute->m_IsSingleValue;
}

std::vector<std::string> Attribute::DataString()
{
helper::CheckForNullptr(m_Attribute, "in call to Attribute::DataStrings");
Expand Down
2 changes: 2 additions & 0 deletions bindings/Python/py11Attribute.h
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,8 @@ class Attribute

std::string Type() const;

bool SingleValue() const;

pybind11::array Data();

std::vector<std::string> DataString();
Expand Down
11 changes: 9 additions & 2 deletions bindings/Python/py11IO.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -180,8 +180,15 @@ Attribute IO::DefineAttribute(const std::string &name, const pybind11::array &ar
else if (pybind11::isinstance<pybind11::array_t<T, pybind11::array::c_style>>(array)) \
{ \
const T *data = reinterpret_cast<const T *>(array.data()); \
const size_t size = static_cast<size_t>(array.size()); \
attribute = &m_IO->DefineAttribute<T>(name, data, size, variableName, separator); \
if (array.ndim()) \
{ \
const size_t size = static_cast<size_t>(array.size()); \
attribute = &m_IO->DefineAttribute<T>(name, data, size, variableName, separator); \
} \
else \
{ \
attribute = &m_IO->DefineAttribute<T>(name, data[0], variableName, separator); \
} \
}
ADIOS2_FOREACH_NUMPY_ATTRIBUTE_TYPE_1ARG(declare_type)
#undef declare_type
Expand Down
3 changes: 2 additions & 1 deletion bindings/Python/py11glue.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -395,7 +395,8 @@ PYBIND11_MODULE(ADIOS2_PYTHON_MODULE_NAME, m)
.def("Name", &adios2::py11::Attribute::Name)
.def("Type", &adios2::py11::Attribute::Type)
.def("DataString", &adios2::py11::Attribute::DataString)
.def("Data", &adios2::py11::Attribute::Data);
.def("Data", &adios2::py11::Attribute::Data)
.def("SingleValue", &adios2::py11::Attribute::SingleValue);

pybind11::class_<adios2::py11::Engine>(m, "Engine")
// Python 2
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,18 @@
for key, value in info.items():
print("\t" + key + ": " + value, end=" ")
print()
print()

nproc = s.read("nproc")
print(f"nproc is {nproc} of type {type(nproc)}")
print(f"nproc is {nproc} of type {type(nproc)} with ndim {nproc.ndim}")

# read variables return a numpy array with corresponding selection
steps = int(vars["physical_time"]["AvailableStepsCount"])
physical_time = s.read("physical_time", step_selection=[0, steps])
print(f"physical_time is {physical_time} of type {type(physical_time)}")
print(
f"physical_time is {physical_time} of type {type(physical_time)} with "
f"ndim {physical_time.ndim} shape = {physical_time.shape}"
)

steps = int(vars["temperature"]["AvailableStepsCount"])
temperature = s.read("temperature", step_selection=[0, steps])
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@

if s.current_step() == 0:
nproc = s.read("nproc")
print(f"nproc is {nproc} of type {type(nproc)}")
print(f"nproc is {nproc} of type {type(nproc)} with ndim {nproc.ndim}")

# read variables return a numpy array with corresponding selection
physical_time = s.read("physical_time")
Expand All @@ -26,6 +26,6 @@
print(f"temperature array size is {temperature.size} of shape {temperature.shape}")
print(f"temperature unit is {temp_unit} of type {type(temp_unit)}")
pressure = s.read("pressure")
press_unit = s.read_attribute("pressure/unit")
press_unit = s.read_attribute("unit", "pressure")
print(f"pressure unit is {press_unit} of type {type(press_unit)}")
print()
24 changes: 15 additions & 9 deletions docs/user_guide/source/api_python/python_example.rst
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ Python Write example
count = [nx]
temperature = np.zeros(nx, dtype=np.double)
pressure = np.ones(nx, dtype=np.double)
delta_time = 0.01
physical_time = 0.0
nsteps = 5
Expand Down Expand Up @@ -125,17 +126,17 @@ Python Read "step-by-step" example
nproc is 4 of type <class 'numpy.ndarray'>
physical_time is 0.0 of type <class 'numpy.ndarray'>
temperature array size is 40 of shape (40,)
temperature unit is ['K'] of type <class 'list'>
pressure unit is ['Pa'] of type <class 'list'>
temperature unit is K of type <class 'str'>
pressure unit is Pa of type <class 'str'>
Current step is 1
variable_name: physical_time AvailableStepsCount: 1 Max: 0.01 Min: 0.01 Shape: SingleValue: true Type: double
variable_name: pressure AvailableStepsCount: 1 Max: 1 Min: 1 Shape: 40 SingleValue: false Type: double
variable_name: temperature AvailableStepsCount: 1 Max: 0 Min: 0 Shape: 40 SingleValue: false Type: double
physical_time is 0.01 of type <class 'numpy.ndarray'>
temperature array size is 40 of shape (40,)
temperature unit is ['K'] of type <class 'list'>
pressure unit is ['Pa'] of type <class 'list'>
temperature unit is K of type <class 'str'>
pressure unit is Pa of type <class 'str'>
...
Expand All @@ -156,14 +157,18 @@ Python Read Random Access example
for key, value in info.items():
print("\t" + key + ": " + value, end=" ")
print()
print()
nproc = s.read("nproc")
print(f"nproc is {nproc} of type {type(nproc)}")
print(f"nproc is {nproc} of type {type(nproc)} with ndim {nproc.ndim}")
# read variables return a numpy array with corresponding selection
steps = int(vars['physical_time']['AvailableStepsCount'])
physical_time = s.read("physical_time", step_selection=[0, steps])
print(f"physical_time is {physical_time} of type {type(physical_time)}")
print(
f"physical_time is {physical_time} of type {type(physical_time)} with "
f"ndim {physical_time.ndim} shape = {physical_time.shape}"
)
steps = int(vars['temperature']['AvailableStepsCount'])
temperature = s.read("temperature", step_selection=[0, steps])
Expand All @@ -183,8 +188,9 @@ Python Read Random Access example
variable_name: physical_time AvailableStepsCount: 5 Max: 0.04 Min: 0 Shape: SingleValue: true Type: double
variable_name: pressure AvailableStepsCount: 5 Max: 1 Min: 1 Shape: 40 SingleValue: false Type: double
variable_name: temperature AvailableStepsCount: 5 Max: 0 Min: 0 Shape: 40 SingleValue: false Type: double
nproc is 4 of type <class 'numpy.ndarray'>
physical_time is [0. 0.01 0.02 0.03 0.04] of type <class 'numpy.ndarray'>
nproc is 4 of type <class 'numpy.ndarray'> with ndim 0
physical_time is [0. 0.01 0.02 0.03 0.04] of type <class 'numpy.ndarray'> with ndim 1 shape = (5,)
temperature array size is 200 of shape (200,)
temperature unit is ['K'] of type <class 'list'>
temperature unit is K of type <class 'str'>
31 changes: 22 additions & 9 deletions examples/hello/bpReader/bpReaderHeatMap2D.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,27 +35,40 @@
for j in range(0, Ny):
value = iGlobal * shape[1] + j
temperatures[i * Nx + j] = value
# print(str(i) + "," + str(j) + " " + str(value))

with Stream("HeatMap2D_py.bp", "w", comm) as obpStream:
obpStream.write("temperature2D", temperatures, shape, start, count)
if not rank:
obpStream.write("N", [size, Nx, Ny]) # will be an array in output
obpStream.write("Nx", numpy.array(Nx)) # will be a scalar in output
obpStream.write("Ny", Ny) # will be a scalar in output
obpStream.write_attribute("nproc", size) # will be a single value attribute in output
obpStream.write_attribute("dimensions", [size * Nx, Ny], "temperature2D")

if not rank:
with FileReader("HeatMap2D_py.bp", MPI.COMM_SELF) as ibpFile:
# scalar variables are read as a numpy array with 0 dimension
in_nx = ibpFile.read("Nx")
in_ny = ibpFile.read("Ny")
print(f"Incoming nx, ny = {in_nx}, {in_ny}")

# single value attribute is read as a numpy array with 0 dimension
in_nproc = ibpFile.read_attribute("nproc")
print(f"Incoming nproc = {in_nproc}")
# array attribute is read as a numpy array or string list
in_dims = ibpFile.read_attribute("temperature2D/dimensions")
print(f"Incoming diumensions = {in_dims}")

# On option is to inquire a variable to know its type, shape
# directly, not as strings, and then we can use the variable
# object to set selection and/or set steps to read
var_inTemperature = ibpFile.inquire_variable("temperature2D")
if var_inTemperature is not None:
var_inTemperature.set_selection([[2, 2], [4, 4]])
inTemperatures = ibpFile.read(var_inTemperature)

in_nx = ibpFile.read("Nx") # scalar is read as a numpy array with 1 element
in_ny = ibpFile.read("Ny") # scalar is read as a numpy array with 1 element
print(f"Incoming nx, ny = {in_nx}, {in_ny}")

print("Incoming temperature map")
for i in range(0, inTemperatures.shape[1]):
print(str(inTemperatures[i]))
print(
f"Incoming temperature map with selection "
f"start = {var_inTemperature.start()}, count = {var_inTemperature.count()}"
)
for i in range(0, inTemperatures.shape[1]):
print(str(inTemperatures[i]))
13 changes: 13 additions & 0 deletions python/adios2/attribute.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,13 +42,24 @@ def type(self):
"""
return self.impl.Type()

def single_value(self):
"""
True if the attribute is a single value, False if it is an array

Returns:
True or False.
"""
return self.impl.SingleValue()

def data(self):
"""
Content of the Attribute

Returns:
Content of the Attribute as a non string.
"""
if self.single_value():
return self.impl.Data()[0]
return self.impl.Data()

def data_string(self):
Expand All @@ -58,4 +69,6 @@ def data_string(self):
Returns:
Content of the Attribute as a str.
"""
if self.single_value():
return self.impl.DataString()[0]
return self.impl.DataString()
4 changes: 0 additions & 4 deletions python/adios2/file_reader.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,3 @@ def _(self, io: IO, path, comm=None):
super().__init__(io, path, "rra", comm)

# pylint: enable=E1121

def variables(self):
"""Returns the list of variables contained in the opened file"""
return [self._io.inquire_variable(var) for var in self.available_variables()]
62 changes: 46 additions & 16 deletions python/adios2/stream.py
Original file line number Diff line number Diff line change
Expand Up @@ -325,11 +325,9 @@ def _(self, name, content, shape=[], start=[], count=[], operations=None):

self.write(variable, content)

@singledispatchmethod
def read(self, variable: Variable):
def _read_var(self, variable: Variable):
"""
Random access read allowed to select steps,
only valid with Stream Engines
Internal common function to read. Settings must be done to Variable before the call.
Parameters
variable
Expand All @@ -342,6 +340,7 @@ def read(self, variable: Variable):
"""
dtype = type_adios_to_numpy(variable.type())
count = variable.count()

if count != []:
# array
# steps = variable.get_steps_from_step_selection()
Expand All @@ -365,7 +364,8 @@ def read(self, variable: Variable):
else:
# scalar
size_all_steps = variable.selection_size()
if size_all_steps > 1:
# if size_all_steps > 1:
if self._mode == bindings.Mode.ReadRandomAccess and variable.steps() > 1:
output_shape = [size_all_steps]
else:
output_shape = []
Expand All @@ -374,15 +374,17 @@ def read(self, variable: Variable):
self._engine.get(variable, output)
return output

@read.register(str)
def _(self, name: str, start=[], count=[], block_id=None, step_selection=None):
@singledispatchmethod
def read(self, variable: Variable, start=[], count=[], block_id=None, step_selection=None):
"""
Random access read allowed to select steps,
only valid with Stream Engines
Read a variable.
Random access read allowed to select steps.
Parameters
name
variable to be read
variable
adios2.Variable object to be read
Use variable.set_selection(), set_block_selection(), set_step_selection()
to prepare a read
start
variable offset dimensions
Expand All @@ -400,10 +402,6 @@ def _(self, name: str, start=[], count=[], block_id=None, step_selection=None):
array
resulting array from selection
"""
variable = self._io.inquire_variable(name)
if not variable:
raise ValueError()

if step_selection is not None and not self._mode == bindings.Mode.ReadRandomAccess:
raise RuntimeError("step_selection parameter requires 'rra' mode")

Expand All @@ -419,7 +417,39 @@ def _(self, name: str, start=[], count=[], block_id=None, step_selection=None):
if start != [] and count != []:
variable.set_selection([start, count])

return self.read(variable)
return self._read_var(variable)

@read.register(str)
def _(self, name: str, start=[], count=[], block_id=None, step_selection=None):
"""
Read a variable.
Random access read allowed to select steps.
Parameters
name
variable to be read
start
variable offset dimensions
count
variable local dimensions from offset
block_id
(int) Required for reading local variables, local array, and local
value.
step_selection
(list): On the form of [start, count].
Returns
array
resulting array from selection
"""
variable = self._io.inquire_variable(name)
if not variable:
raise ValueError()

return self.read(variable, start, count, block_id, step_selection)

def write_attribute(self, name, content, variable_name="", separator="/"):
"""
Expand Down
4 changes: 2 additions & 2 deletions testing/adios2/python/TestAttribute.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,14 +12,14 @@ def test_create_write(self):
with adios.declare_io("BPWriter") as io:
ts = io.define_attribute("timestamp", "20231122")
self.assertEqual(ts.name(), "timestamp")
self.assertEqual(ts.data_string(), ["20231122"])
self.assertEqual(ts.data_string(), "20231122")

def test_create_reader(self):
adios = Adios()
with adios.declare_io("BPWriter") as io:
ts = io.define_attribute("timestamp", "20231122")
self.assertEqual(ts.name(), "timestamp")
self.assertEqual(ts.data_string(), ["20231122"])
self.assertEqual(ts.data_string(), "20231122")

def test_create_write_ndarray(self):
adios = Adios()
Expand Down
Loading
Loading