From 01e2aca57296ba5ab53c2f739dbe04353e322efe Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Tue, 1 Oct 2024 08:56:51 +0200 Subject: [PATCH 1/2] Remove h5io from dependencies --- .ci_support/environment-docs.yml | 3 +-- .ci_support/environment-mpich.yml | 1 - .ci_support/environment-old.yml | 1 - .ci_support/environment-openmpi.yml | 1 - .ci_support/environment-win.yml | 1 - binder/environment.yml | 1 - executorlib/cache/hdf.py | 26 +++++++------------------- pyproject.toml | 1 - 8 files changed, 8 insertions(+), 27 deletions(-) diff --git a/.ci_support/environment-docs.yml b/.ci_support/environment-docs.yml index b6a4d246..8f92efb8 100644 --- a/.ci_support/environment-docs.yml +++ b/.ci_support/environment-docs.yml @@ -12,5 +12,4 @@ dependencies: - pyzmq =26.2.0 - flux-core - jupyter-book =1.0.0 -- h5py =3.11.0 -- h5io =0.2.4 \ No newline at end of file +- h5py =3.11.0 \ No newline at end of file diff --git a/.ci_support/environment-mpich.yml b/.ci_support/environment-mpich.yml index 8f67c272..7b779f40 100644 --- a/.ci_support/environment-mpich.yml +++ b/.ci_support/environment-mpich.yml @@ -9,7 +9,6 @@ dependencies: - mpi4py =4.0.0 - pyzmq =26.2.0 - h5py =3.11.0 -- h5io =0.2.4 - matplotlib =3.9.2 - networkx =3.3 - pygraphviz =1.13 diff --git a/.ci_support/environment-old.yml b/.ci_support/environment-old.yml index f2bb44f2..e936c64b 100644 --- a/.ci_support/environment-old.yml +++ b/.ci_support/environment-old.yml @@ -9,7 +9,6 @@ dependencies: - mpi4py =3.1.4 - pyzmq =25.0.0 - h5py =3.6.0 -- h5io =0.2.1 - matplotlib =3.5.3 - networkx =2.8.8 - ipython =7.33.0 diff --git a/.ci_support/environment-openmpi.yml b/.ci_support/environment-openmpi.yml index f712a8e4..9c56a991 100644 --- a/.ci_support/environment-openmpi.yml +++ b/.ci_support/environment-openmpi.yml @@ -9,7 +9,6 @@ dependencies: - mpi4py =4.0.0 - pyzmq =26.2.0 - h5py =3.11.0 -- h5io =0.2.4 - matplotlib =3.9.2 - networkx =3.3 - pygraphviz =1.13 diff --git a/.ci_support/environment-win.yml b/.ci_support/environment-win.yml index 8c4300f0..d0350241 100644 --- a/.ci_support/environment-win.yml +++ b/.ci_support/environment-win.yml @@ -9,7 +9,6 @@ dependencies: - mpi4py =4.0.0 - pyzmq =26.2.0 - h5py =3.11.0 -- h5io =0.2.4 - matplotlib =3.9.2 - networkx =3.3 - pygraphviz =1.13 diff --git a/binder/environment.yml b/binder/environment.yml index 34fd4f2b..879d65f3 100644 --- a/binder/environment.yml +++ b/binder/environment.yml @@ -11,4 +11,3 @@ dependencies: - flux-pmix =0.5.0 - versioneer =0.28 - h5py =3.11.0 -- h5io =0.2.2 diff --git a/executorlib/cache/hdf.py b/executorlib/cache/hdf.py index 2deda68f..7f0af98f 100644 --- a/executorlib/cache/hdf.py +++ b/executorlib/cache/hdf.py @@ -1,7 +1,6 @@ from typing import Tuple import cloudpickle -import h5io import h5py import numpy as np @@ -23,14 +22,11 @@ def dump(file_name: str, data_dict: dict) -> None: with h5py.File(file_name, "a") as fname: for data_key, data_value in data_dict.items(): if data_key in group_dict.keys(): - h5io.write_hdf5( - fname=fname, - data=np.void(cloudpickle.dumps(data_value)), - overwrite="update", - title=group_dict[data_key], + fname.create_dataset( + name="/" + group_dict[data_key], + data=np.void(cloudpickle.dumps(data_value)) ) - def load(file_name: str) -> dict: """ Load data dictionary from HDF5 file @@ -44,21 +40,15 @@ def load(file_name: str) -> dict: with h5py.File(file_name, "r") as hdf: data_dict = {} if "function" in hdf: - data_dict["fn"] = cloudpickle.loads( - h5io.read_hdf5(fname=hdf, title="function", slash="ignore") - ) + data_dict["fn"] = cloudpickle.loads(np.void(hdf["/function"])) else: raise TypeError("Function not found in HDF5 file.") if "input_args" in hdf: - data_dict["args"] = cloudpickle.loads( - h5io.read_hdf5(fname=hdf, title="input_args", slash="ignore") - ) + data_dict["args"] = cloudpickle.loads(np.void(hdf["/input_args"])) else: data_dict["args"] = () if "input_kwargs" in hdf: - data_dict["kwargs"] = cloudpickle.loads( - h5io.read_hdf5(fname=hdf, title="input_kwargs", slash="ignore") - ) + data_dict["kwargs"] = cloudpickle.loads(np.void(hdf["/input_kwargs"])) else: data_dict["kwargs"] = {} return data_dict @@ -76,8 +66,6 @@ def get_output(file_name: str) -> Tuple[bool, object]: """ with h5py.File(file_name, "r") as hdf: if "output" in hdf: - return True, cloudpickle.loads( - h5io.read_hdf5(fname=hdf, title="output", slash="ignore") - ) + return True, cloudpickle.loads(np.void(hdf["/output"])) else: return False, None diff --git a/pyproject.toml b/pyproject.toml index 8fd66bfc..afbc7b92 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,7 +39,6 @@ conda = ["conda_subprocess==0.0.4"] mpi = ["mpi4py==4.0.0"] hdf = [ "h5py==3.11.0", - "h5io==0.2.4", ] graph = [ "pygraphviz==1.13", From eb6a4a36518fa68b032510a9b3f82cf856c98cbb Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 1 Oct 2024 07:19:58 +0000 Subject: [PATCH 2/2] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- executorlib/cache/hdf.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/executorlib/cache/hdf.py b/executorlib/cache/hdf.py index 7f0af98f..9e8c8798 100644 --- a/executorlib/cache/hdf.py +++ b/executorlib/cache/hdf.py @@ -24,9 +24,10 @@ def dump(file_name: str, data_dict: dict) -> None: if data_key in group_dict.keys(): fname.create_dataset( name="/" + group_dict[data_key], - data=np.void(cloudpickle.dumps(data_value)) + data=np.void(cloudpickle.dumps(data_value)), ) + def load(file_name: str) -> dict: """ Load data dictionary from HDF5 file