Skip to content

Commit

Permalink
Make new jaggedArray backwards compatible (#1778)
Browse files Browse the repository at this point in the history
Make the new jagged data unpacking routine compatible with the old database format (prior to #1726). Add a unit test to check compatibility.
  • Loading branch information
mgjarrett authored Jul 12, 2024
1 parent aa758f3 commit b3986c1
Show file tree
Hide file tree
Showing 2 changed files with 47 additions and 3 deletions.
8 changes: 5 additions & 3 deletions armi/bookkeeping/db/jaggedArray.py
Original file line number Diff line number Diff line change
Expand Up @@ -173,17 +173,19 @@ def unpack(self):
List of numpy arrays with varying dimensions (i.e., jagged arrays)
"""
unpackedJaggedData: List[Optional[np.ndarray]] = []
numElements = len(self.offsets) + len(self.nones)
shapeIndices = [i for i, x in enumerate(self.shapes) if sum(x) != 0]
numElements = len(shapeIndices) + len(self.nones)
j = 0 # non-None element counter
for i in range(numElements):
if i in self.nones:
unpackedJaggedData.append(None)
else:
k = shapeIndices[j]
unpackedJaggedData.append(
np.ndarray(
self.shapes[j],
self.shapes[k],
dtype=self.dtype,
buffer=self.flattenedArray[self.offsets[j] :],
buffer=self.flattenedArray[self.offsets[k] :],
)
)
j += 1
Expand Down
42 changes: 42 additions & 0 deletions armi/bookkeeping/db/tests/test_jaggedArray.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,48 @@ def test_flatten(self):
flatArray = JaggedArray.flatten(testdata)
self.assertEqual(flatArray, [1, 2, 3, 4, 5, None, 6, 7, 8, 9])

def test_backwardsCompatible(self):
"""
Test that the new JaggedArray can unpack the old database jagged data format.
The "old" database format contains shapes and offsets for locations that have None.
The "new" database format only contains shapes and offsets for non-None values.
The "new" unpacking routine is able to read either format.
"""
paramName = "test_old"
data = [[1, 2], None, [3, 4, 5], None, None, [6, 7, 8, 9]]
flattenedArray = numpy.array([1, 2, 3, 4, 5, 6, 7, 8, 9])
shapes = [(2,), (0,), (3,), (0,), (0,), (4,)]
offsets = [0, 2, 2, 5, 5, 5, 5]
nones = [1, 3, 4]
h5file = "test_oldFormat.h5"
with h5py.File(h5file, "w") as hf:
dset = hf.create_dataset(
data=flattenedArray,
name=paramName,
)
dset.attrs["jagged"] = True
dset.attrs["offsets"] = offsets
dset.attrs["shapes"] = shapes
dset.attrs["noneLocations"] = nones

with h5py.File(h5file, "r") as hf:
dataset = hf[paramName]
values = dataset[()]
offsets = dataset.attrs["offsets"]
shapes = dataset.attrs["shapes"]
nones = dataset.attrs["noneLocations"]

roundTrip = JaggedArray.fromH5(
values,
offsets,
shapes,
nones,
dtype=flattenedArray.dtype,
paramName=paramName,
)
self._compareArrays(data, roundTrip)

def _compareRoundTrip(self, data, paramName):
"""Make sure that data is unchanged by packing/unpacking."""
jaggedArray = JaggedArray(data, paramName)
Expand Down

0 comments on commit b3986c1

Please sign in to comment.