From 278be682b2aece3ab335149a7d5d25eb03eeddb4 Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Fri, 29 Mar 2024 11:33:30 +0000 Subject: [PATCH 1/6] Enable control of matminer `ignore_errors` arg from MODFeaturizer Linting --- modnet/featurizers/featurizers.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/modnet/featurizers/featurizers.py b/modnet/featurizers/featurizers.py index 8de3d5a..ee103fa 100644 --- a/modnet/featurizers/featurizers.py +++ b/modnet/featurizers/featurizers.py @@ -141,7 +141,10 @@ def _fit_apply_featurizers( _featurizers.set_n_jobs(self._n_jobs) return _featurizers.featurize_dataframe( - df, column, multiindex=True, ignore_errors=True + df, + column, + multiindex=True, + ignore_errors=getattr(self, "ignore_errors", True), ) elif mode == "single": @@ -164,7 +167,10 @@ def _fit_apply_featurizers( ) start = time.monotonic_ns() df = featurizer.featurize_dataframe( - df, column, multiindex=True, ignore_errors=True + df, + column, + multiindex=True, + ignore_errors=getattr(self, "ignore_errors", True), ) LOG.info( f"Applied featurizer {featurizer.__class__.__name__} to column {column!r} in {(time.monotonic_ns() - start) * 1e-9} seconds" @@ -244,7 +250,11 @@ def featurize_composition(self, df: pd.DataFrame) -> pd.DataFrame: else: df = CompositionToOxidComposition( max_sites=-1 if getattr(self, "continuous_only", False) else None - ).featurize_dataframe(df, col_id=col_comp, ignore_errors=True) + ).featurize_dataframe( + df, + col_id=col_comp, + ignore_errors=getattr(self, "ignore_errors", True), + ) df = self._fit_apply_featurizers( df, self.oxid_composition_featurizers, @@ -311,7 +321,10 @@ def featurize_site( fingerprint, stats=self.site_stats ) df = site_stats_fingerprint.featurize_dataframe( - df, "Input data|structure", multiindex=False, ignore_errors=True + df, + "Input data|structure", + multiindex=False, + ignore_errors=getattr(self, "ignore_errors", True), ) if aliases: From 10b8f59edf9ec180b0f679ac4e4a5e768d35a87f Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Fri, 29 Mar 2024 11:33:54 +0000 Subject: [PATCH 2/6] Dynamically patch old structural data to work with latest pymatgen when depickling --- modnet/tests/conftest.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/modnet/tests/conftest.py b/modnet/tests/conftest.py index d5061c9..8a433c3 100644 --- a/modnet/tests/conftest.py +++ b/modnet/tests/conftest.py @@ -2,6 +2,7 @@ from pathlib import Path from modnet.utils import get_hash_of_file +from pymatgen.core import Structure _TEST_DATA_HASHES = { @@ -41,7 +42,22 @@ def _load_moddata(filename): # what it was when created assert get_hash_of_file(data_file) == _TEST_DATA_HASHES[filename] - return MODData.load(data_file) + moddata = MODData.load(data_file) + # For forwards compatibility with pymatgen, we have to patch our old test data to have the following attributes + # to allow for depickling + # This is hopefully only a temporary solution, and in future, we should serialize pymatgen objects + # with Monty's `from_dict`/`to_dict` to avoid having to hack this private interface + for ind, s in enumerate(moddata.structures): + if isinstance(moddata.structures[ind], Structure): + # assume all previous data was periodic + moddata.structures[ind].lattice._pbc = [True, True, True] + for jnd, site in enumerate(s.sites): + # assume all of our previous data had ordered sites + moddata.structures[ind].sites[jnd].label = str(next(iter(site.species))) + # required for the global structure.is_ordered to work + moddata.structures[ind].sites[jnd].species._n_atoms = 1.0 + + return moddata @pytest.fixture(scope="function") From 75914bb914bf2f7a880207d9ef09a5f1cc077120 Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Fri, 29 Mar 2024 11:34:35 +0000 Subject: [PATCH 3/6] Refactor tests to allow new featurizer columns to exist as long as old ones are present --- modnet/tests/test_preprocessing.py | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/modnet/tests/test_preprocessing.py b/modnet/tests/test_preprocessing.py index e27fa47..51c294c 100644 --- a/modnet/tests/test_preprocessing.py +++ b/modnet/tests/test_preprocessing.py @@ -12,8 +12,14 @@ def check_column_values(new: MODData, reference: MODData, tolerance=0.03): Allows for some columns to be checked more loosely (see inline comment below). """ + new_cols = set(new.df_featurized.columns) + old_cols = set(reference.df_featurized.columns) + + # Check that the new df only adds new columns and is not missing anything + assert not (old_cols - new_cols) + error_cols = set() - for col in new.df_featurized.columns: + for col in old_cols: if not ( np.absolute( ( @@ -349,14 +355,6 @@ def test_small_moddata_featurization(small_moddata_2023, featurizer_mode): featurizer.featurizer_mode = featurizer_mode new = MODData(structures, targets, target_names=names, featurizer=featurizer) new.featurize(fast=False, n_jobs=1) - - new_cols = sorted(new.df_featurized.columns.tolist()) - old_cols = sorted(old.df_featurized.columns.tolist()) - - for i in range(len(old_cols)): - assert new_cols[i] == old_cols[i] - - np.testing.assert_array_equal(old_cols, new_cols) check_column_values(new, old, tolerance=0.03) @@ -376,13 +374,6 @@ def test_small_moddata_composition_featurization( new = MODData(materials=compositions, featurizer=featurizer) new.featurize(fast=False, n_jobs=1) - new_cols = sorted(new.df_featurized.columns.tolist()) - ref_cols = sorted(reference.df_featurized.columns.tolist()) - - for i in range(len(ref_cols)): - # print(new_cols[i], ref_cols[i]) - assert new_cols[i] == ref_cols[i] - # assert relative error below 3 percent check_column_values(new, reference, tolerance=0.03) From 0221dd123a6dd98bf2e365779868d6b185e99ef0 Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Fri, 29 Mar 2024 13:04:30 +0000 Subject: [PATCH 4/6] Now try to update pymatgen again --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ff4c4e3..81c0afa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,5 +3,5 @@ pandas==1.5.2 scikit-learn==1.3.2 matminer==0.9.2 numpy>=1.25 -pymatgen==2023.11.12 +pymatgen==2024.3.1 scikit-learn==1.3.2 From d829766de75841f8bf4d7ce49a25154f11abdce7 Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Fri, 29 Mar 2024 14:42:35 +0000 Subject: [PATCH 5/6] Also fix composition container loading --- modnet/tests/conftest.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/modnet/tests/conftest.py b/modnet/tests/conftest.py index 8a433c3..44e8aa1 100644 --- a/modnet/tests/conftest.py +++ b/modnet/tests/conftest.py @@ -1,5 +1,6 @@ import pytest from pathlib import Path +from modnet.preprocessing import CompositionContainer from modnet.utils import get_hash_of_file from pymatgen.core import Structure @@ -48,7 +49,7 @@ def _load_moddata(filename): # This is hopefully only a temporary solution, and in future, we should serialize pymatgen objects # with Monty's `from_dict`/`to_dict` to avoid having to hack this private interface for ind, s in enumerate(moddata.structures): - if isinstance(moddata.structures[ind], Structure): + if isinstance(s, Structure): # assume all previous data was periodic moddata.structures[ind].lattice._pbc = [True, True, True] for jnd, site in enumerate(s.sites): @@ -56,6 +57,8 @@ def _load_moddata(filename): moddata.structures[ind].sites[jnd].label = str(next(iter(site.species))) # required for the global structure.is_ordered to work moddata.structures[ind].sites[jnd].species._n_atoms = 1.0 + elif isinstance(s, CompositionContainer): + moddata.structures[ind].composition._n_atoms = s.composition._natoms return moddata From 6d8d6ffe052ed2884fdccdc656005e57b75793d1 Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Tue, 2 Apr 2024 15:44:16 +0100 Subject: [PATCH 6/6] Add tensorflow upper bound from other PR --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e650934..28781da 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,7 @@ packages=setuptools.find_packages(), install_requires=[ "pandas~=1.5", - "tensorflow~=2.10", + "tensorflow~=2.10,<2.12", "pymatgen>=2023", "matminer~=0.9", "numpy>=1.24",