diff --git a/matbench_discovery/plots.py b/matbench_discovery/plots.py index 0dd47d9e..cc94fe76 100644 --- a/matbench_discovery/plots.py +++ b/matbench_discovery/plots.py @@ -63,6 +63,7 @@ def unit(text: str) -> str: formula="Formula", ) model_labels = dict( + alignn="ALIGNN", bowsr_megnet="BOWSR + MEGNet", chgnet="CHGNet", chgnet_megnet="CHGNet + MEGNet", diff --git a/matbench_discovery/preds.py b/matbench_discovery/preds.py index eb33d0f2..2dc80f1f 100644 --- a/matbench_discovery/preds.py +++ b/matbench_discovery/preds.py @@ -60,6 +60,9 @@ class PredFiles(Files): # wrenformer 10-member ensemble wrenformer = "wrenformer/2022-11-15-wrenformer-IS2RE-preds.csv" + alignn = "alignn/2023-06-02-alignn-wbm-IS2RE.csv.gz" + alignn_pretrained = "alignn/2023-06-03-mp-e-form-alignn-wbm-IS2RE.csv.gz" + # model_labels remaps model keys to pretty plot labels (see Files) PRED_FILES = PredFiles(root=f"{ROOT}/models", key_map=model_labels) diff --git a/matbench_discovery/structure.py b/matbench_discovery/structure.py index 89fe9f4f..4c267235 100644 --- a/matbench_discovery/structure.py +++ b/matbench_discovery/structure.py @@ -13,6 +13,9 @@ def perturb_structure(struct: Structure, gamma: float = 1.5) -> Structure: """Perturb the atomic coordinates of a pymatgen structure. Used for CGCNN+P training set augmentation. + Not identical but very similar to the perturbation method used in + https://nature.com/articles/s41524-022-00891-8#Fig5. + Args: struct (Structure): pymatgen structure to be perturbed gamma (float, optional): Weibull distribution parameter. Defaults to 1.5. diff --git a/models/alignn/2023-06-02-alignn-wbm-IS2RE.csv.gz b/models/alignn/2023-06-02-alignn-wbm-IS2RE.csv.gz new file mode 100644 index 00000000..a3fa74e0 Binary files /dev/null and b/models/alignn/2023-06-02-alignn-wbm-IS2RE.csv.gz differ diff --git a/models/alignn/2023-06-03-mp-e-form-alignn-wbm-IS2RE.csv.gz b/models/alignn/2023-06-03-mp-e-form-alignn-wbm-IS2RE.csv.gz new file mode 100644 index 00000000..e084a446 Binary files /dev/null and b/models/alignn/2023-06-03-mp-e-form-alignn-wbm-IS2RE.csv.gz differ diff --git a/models/alignn/alignn-2023.01.10.patch b/models/alignn/alignn-2023.01.10.patch new file mode 100644 index 00000000..507e0060 --- /dev/null +++ b/models/alignn/alignn-2023.01.10.patch @@ -0,0 +1,179 @@ +diff --git a/alignn/data.py b/alignn/data.py +index 175b915..42ce26e 100644 +--- a/alignn/data.py ++++ b/alignn/data.py +@@ -171,8 +171,9 @@ def get_id_train_val_test( + # full train/val test split + # ids = ids[::-1] + id_train = ids[:n_train] +- id_val = ids[-(n_val + n_test) : -n_test] # noqa:E203 +- id_test = ids[-n_test:] ++ id_val = ids[-(n_val + n_test) : -n_test] if n_test > 0 else ids[-(n_val + n_test) :] # noqa:E203 ++ id_test = ids[n_test:] if n_test > 0 else [] ++ + return id_train, id_val, id_test + + +@@ -508,7 +509,7 @@ def get_train_val_loaders( + classification=classification_threshold is not None, + output_dir=output_dir, + tmp_name="test_data", +- ) ++ ) if len(dataset_test) > 0 else None + + collate_fn = train_data.collate + # print("line_graph,line_dih_graph", line_graph, line_dih_graph) +@@ -528,7 +529,7 @@ def get_train_val_loaders( + + val_loader = DataLoader( + val_data, +- batch_size=batch_size, ++ batch_size=1, + shuffle=False, + collate_fn=collate_fn, + drop_last=True, +@@ -549,9 +550,13 @@ def get_train_val_loaders( + torch.save(train_loader, train_sample) + torch.save(val_loader, val_sample) + torch.save(test_loader, test_sample) ++ + print("n_train:", len(train_loader.dataset)) +- print("n_val:", len(val_loader.dataset)) +- print("n_test:", len(test_loader.dataset)) ++ if val_loader.dataset is not None: ++ print("n_val:", len(val_loader.dataset)) ++ if test_loader.dataset is not None: ++ print("n_test:", len(test_loader.dataset)) ++ + return ( + train_loader, + val_loader, +diff --git a/alignn/train.py b/alignn/train.py +index 4fa072e..0b6330a 100644 +--- a/alignn/train.py ++++ b/alignn/train.py +@@ -69,7 +69,7 @@ torch.set_default_dtype(torch.float32) + + device = "cpu" + if torch.cuda.is_available(): +- device = torch.device("cuda") ++ device = torch.device("cuda:0") + + + def activated_output_transform(output): +@@ -817,6 +817,8 @@ def train_dgl( + train_eos = EpochOutputStore() + train_eos.attach(train_evaluator) + ++ best_mae = np.inf ++ + # collect evaluation performance + @trainer.on(Events.EPOCH_COMPLETED) + def log_results(engine): +@@ -839,6 +841,20 @@ def train_dgl( + history["train"][metric].append(tm) + history["validation"][metric].append(vm) + ++## Adapted so that the best model is saved ++## ---------------------------------------------------------------------------- ++ nonlocal best_mae ++ ++ if metric == 'mae' and vm < best_mae: ++ ++ best_mae = vm ++ ++ out_path = os.path.join(config.output_dir, 'best-model.pth') ++ print(f'Saving model with MAE={vm} to file "{out_path}"') ++ torch.save(net.state_dict(), out_path) ++ ++## ---------------------------------------------------------------------------- ++ + # for metric in metrics.keys(): + # history["train"][metric].append(tmetrics[metric]) + # history["validation"][metric].append(vmetrics[metric]) +@@ -978,39 +994,40 @@ def train_dgl( + and not classification + and config.model.output_features == 1 + ): +- net.eval() +- f = open( +- os.path.join(config.output_dir, "prediction_results_test_set.csv"), +- "w", +- ) +- f.write("id,target,prediction\n") +- targets = [] +- predictions = [] +- with torch.no_grad(): +- ids = test_loader.dataset.ids # [test_loader.dataset.indices] +- for dat, id in zip(test_loader, ids): +- g, lg, target = dat +- out_data = net([g.to(device), lg.to(device)]) +- out_data = out_data.cpu().numpy().tolist() +- if config.standard_scalar_and_pca: +- sc = pk.load( +- open(os.path.join(tmp_output_dir, "sc.pkl"), "rb") +- ) +- out_data = sc.transform(np.array(out_data).reshape(-1, 1))[ +- 0 +- ][0] +- target = target.cpu().numpy().flatten().tolist() +- if len(target) == 1: +- target = target[0] +- f.write("%s, %6f, %6f\n" % (id, target, out_data)) +- targets.append(target) +- predictions.append(out_data) +- f.close() ++ if test_loader is not None: ++ net.eval() ++ f = open( ++ os.path.join(config.output_dir, "prediction_results_test_set.csv"), ++ "w", ++ ) ++ f.write("id,target,prediction\n") ++ targets = [] ++ predictions = [] ++ with torch.no_grad(): ++ ids = test_loader.dataset.ids # [test_loader.dataset.indices] ++ for dat, id in zip(test_loader, ids): ++ g, lg, target = dat ++ out_data = net([g.to(device), lg.to(device)]) ++ out_data = out_data.cpu().numpy().tolist() ++ if config.standard_scalar_and_pca: ++ sc = pk.load( ++ open(os.path.join(tmp_output_dir, "sc.pkl"), "rb") ++ ) ++ out_data = sc.transform(np.array(out_data).reshape(-1, 1))[ ++ 0 ++ ][0] ++ target = target.cpu().numpy().flatten().tolist() ++ if len(target) == 1: ++ target = target[0] ++ f.write("%s, %6f, %6f\n" % (id, target, out_data)) ++ targets.append(target) ++ predictions.append(out_data) ++ f.close() + +- print( +- "Test MAE:", +- mean_absolute_error(np.array(targets), np.array(predictions)), +- ) ++ print( ++ "Test MAE:", ++ mean_absolute_error(np.array(targets), np.array(predictions)), ++ ) + if config.store_outputs and not classification: + x = [] + y = [] +diff --git a/alignn/train_folder.py b/alignn/train_folder.py +index b532d4e..c9eb4a5 100644 +--- a/alignn/train_folder.py ++++ b/alignn/train_folder.py +@@ -180,7 +180,7 @@ def train_for_folder( + train_val_test_loaders=[ + train_loader, + val_loader, +- test_loader, ++ None, + prepare_batch, + ], + ) diff --git a/models/alignn/alignn-config.json b/models/alignn/alignn-config.json new file mode 100644 index 00000000..2ae7b43f --- /dev/null +++ b/models/alignn/alignn-config.json @@ -0,0 +1,54 @@ +{ + "version": "112bbedebdaecf59fb18e11c929080fb2f358246", + "dataset": "user_data", + "target": "target", + "atom_features": "cgcnn", + "neighbor_strategy": "k-nearest", + "id_tag": "jid", + "random_seed": 123, + "classification_threshold": null, + "n_val": null, + "n_test": null, + "n_train": null, + "train_ratio": 0.9, + "val_ratio": 0.1, + "test_ratio": 0.0, + "target_multiplication_factor": null, + "epochs": 1000, + "n_early_stopping": 100, + "batch_size": 128, + "weight_decay": 1e-5, + "learning_rate": 0.001, + "filename": "sample", + "warmup_steps": 2000, + "criterion": "l1", + "optimizer": "adamw", + "scheduler": "onecycle", + "pin_memory": true, + "save_dataloader": false, + "write_checkpoint": true, + "write_predictions": true, + "store_outputs": true, + "progress": true, + "log_tensorboard": false, + "standard_scalar_and_pca": false, + "use_canonize": true, + "num_workers": 4, + "cutoff": 8.0, + "max_neighbors": 12, + "keep_data_order": false, + "model": { + "name": "alignn", + "alignn_layers": 4, + "gcn_layers": 4, + "atom_input_features": 92, + "edge_input_features": 80, + "triplet_input_features": 40, + "embedding_features": 64, + "hidden_features": 256, + "output_features": 1, + "link": "identity", + "zero_inflated": false, + "classification": false + } +} diff --git a/models/alignn/make_train_data.py b/models/alignn/make_train_data.py new file mode 100644 index 00000000..b52507e1 --- /dev/null +++ b/models/alignn/make_train_data.py @@ -0,0 +1,60 @@ +# %% Imports +import os + +import pandas as pd +from pymatgen.core import Structure +from sklearn.model_selection import train_test_split +from tqdm import tqdm, trange + +from matbench_discovery.data import DATA_FILES +from matbench_discovery.structure import perturb_structure + +__author__ = "Philipp Benner" +__date__ = "2023-06-02" + + +# %% +target_col = "formation_energy_per_atom" +input_col = "structure" +id_col = "material_id" +n_perturb = 0 + + +# %% load structures +df_cse = pd.read_json(DATA_FILES.mp_computed_structure_entries).set_index(id_col) +df_cse[input_col] = [ + Structure.from_dict(cse[input_col]) for cse in tqdm(df_cse.entry, disable=None) +] + +# load energies +df = pd.read_csv(DATA_FILES.mp_energies).set_index(id_col) +df[input_col] = df_cse[input_col] +assert target_col in df + + +# %% augment with randomly perturbed structures +df_aug = df.copy() +structs = df_aug.pop(input_col) +for idx in trange(n_perturb, desc="Generating perturbed structures"): + df_aug[input_col] = [perturb_structure(x) for x in structs] + df = pd.concat([df, df_aug.set_index(f"{x}-aug={idx+1}" for x in df_aug.index)]) + +del df_aug + + +# %% export data +X_train, X_test, y_train, y_test = train_test_split( + df[input_col], df[target_col], test_size=0.05, random_state=42 +) + +for samples, targets, label in ((X_train, y_train, "train"), (X_test, y_test, "test")): + out_dir = f"{label}-data" + os.makedirs(out_dir, exist_ok=True) + + targets.to_csv(f"{out_dir}/targets.csv") + + struct: Structure + for mat_id, struct in tqdm( + samples.items(), desc="Saving structures", total=len(samples) + ): + struct.to(f"{out_dir}/{mat_id}.poscar", fmt="POSCAR") diff --git a/models/alignn/metadata.yml b/models/alignn/metadata.yml new file mode 100644 index 00000000..f7ed783a --- /dev/null +++ b/models/alignn/metadata.yml @@ -0,0 +1,31 @@ +model_name: ALIGNN +model_version: 2023.01.10 +matbench_discovery_version: 1.0 +date_added: "2023-06-02" +date_published: "2021-02-22" +authors: + - name: Kamal Choudhary + affiliation: National Institute of Standards and Technology + email: kamal.choudhary@nist.gov + orcid: https://orcid.org/0000-0001-9737-8074 + - name: Brian DeCost + affiliation: National Institute of Standards and Technology + orcid: https://orcid.org/0000-0002-3459-5888 + email: zhongpc@berkeley.edu + - name: Philipp Benner + affiliation: Bundesanstalt für Materialforschung und -prüfung BAM + orcid: https://orcid.org/0000-0002-0912-8137 + github: https://github.com/pbenner +repo: https://github.com/usnistgov/alignn +url: https://jarvis.nist.gov/jalignn +doi: https://nature.com/articles/s41524-021-00650-1 +preprint: https://arxiv.org/abs/2209.05554 +requirements: + ase: 3.22.0 + dgl-cu111: 0.6.1 + numpy: 1.24.3 + pandas: 2.0.1 + scikit-learn: 1.2.2 + torch: 1.9.0+cu111 +trained_for_benchmark: true +# hyperparams: see align-config.json diff --git a/models/alignn/readme.md b/models/alignn/readme.md new file mode 100644 index 00000000..dd809767 --- /dev/null +++ b/models/alignn/readme.md @@ -0,0 +1,21 @@ +## ALIGNN formation energy predictions on WBM test set + +ALIGNN is trained using L1 loss and 1000 epochs. The model that performs best on the validation set is saved and used for predictions (requires minor adaptation of the ALIGNN source). The modifications to the ALIGNN source code are provided as patch `alignn-2023.01.10.patch`, which was applied to ALIGNN version `2023.01.10`. In addition, all Python requirements are given as `requirements.txt`. + +To reproduce the `alignn` package state used for this submission, run + +```bash +pip install alignn==2023.01.10 +alignn_dir=$(python -c "import alignn; print(alignn.__path__[0])") +cd $alignn_dir +git apply /path/to/alignn-2023.01.10.patch +``` + +Replace `/path/to/` with the actual path to the patch file. + +The directory contains the following files, which must be executed in the given order to reproduce the results: + +1. `train_data.py`: Export Matbench Discovery training data to ALIGNN compatible format. This script outputs training data in the directory `data_train`. In addition, a small test data set is set apart and stored in the directory `data_test` +1. `train_alignn.py`: Train an ALIGNN model on previously exported data. The resulting model is stored in the directory `data-train-result` +1. `test_data.py`: Export WBM test data in ALIGNN-compatible format. The data is stored in the directory `data-test-wbm` +1. `test_alignn.py`: Test a trained ALIGNN model on the WBM data. Predictions are stored in the file `test_alignn_result.json` diff --git a/models/alignn/test_alignn.py b/models/alignn/test_alignn.py new file mode 100644 index 00000000..44f50770 --- /dev/null +++ b/models/alignn/test_alignn.py @@ -0,0 +1,137 @@ +# %% +from __future__ import annotations + +import json +import os +from importlib.metadata import version + +import pandas as pd +import torch +import wandb +from alignn.config import TrainingConfig +from alignn.models.alignn import ALIGNN +from alignn.pretrained import all_models, get_figshare_model +from jarvis.core.graphs import Graph +from pymatgen.core import Structure +from pymatgen.io.jarvis import JarvisAtomsAdaptor +from sklearn.metrics import r2_score +from tqdm import tqdm + +from matbench_discovery import DEBUG, today +from matbench_discovery.data import DATA_FILES, df_wbm +from matbench_discovery.plots import wandb_scatter +from matbench_discovery.slurm import slurm_submit + +__author__ = "Janosh Riebesell, Philipp Benner" +__date__ = "2023-06-03" + +module_dir = os.path.dirname(__file__) + + +# %% +model_name = "mp_e_form_alignn" # pre-trained by NIST +# model_name = f"{module_dir}/data-train-result/best-model.pth" +task_type = "IS2RE" +target_col = "e_form_per_atom_mp2020_corrected" +input_col = "initial_structure" +id_col = "material_id" +device = "cuda" if torch.cuda.is_available() else "cpu" +job_name = f"{model_name}-wbm-{task_type}{'-debug' if DEBUG else ''}" +out_dir = os.getenv("SBATCH_OUTPUT", f"{module_dir}/{today}-{job_name}") + + +if model_name in all_models: # load pre-trained model + model = get_figshare_model(model_name) + pred_col = "e_form_per_atom_alignn_pretrained" +elif os.path.isfile(model_name): + pred_col = "e_form_per_atom_alignn" + with open(f"{module_dir}/alignn-config.json") as file: + config = TrainingConfig(**json.load(file)) + + model = ALIGNN(config.model) + # load trained ALIGNN model + state_dict = torch.load(model_name, map_location=device) + model.load_state_dict(state_dict) + model = model.to(device) +else: + raise ValueError( + f"{model_name=} not found, train a model or use pre-trained {list(all_models)}" + ) + +slurm_vars = slurm_submit( + job_name=job_name, + partition="ampere", + account="LEE-SL3-GPU", + time="12:0:0", + out_dir=out_dir, + slurm_flags="--nodes 1 --gpus-per-node 1", + pre_cmd=". /etc/profile.d/modules.sh; module load rhel8/default-amp;" + "module load cuda/11.8", +) + + +# %% Load data +data_path = { + "IS2RE": DATA_FILES.wbm_initial_structures, + "RS2RE": DATA_FILES.wbm_computed_structure_entries, +}[task_type] +input_col = {"IS2RE": "initial_structure", "RS2RE": "relaxed_structure"}[task_type] + +df_in = pd.read_json(data_path).set_index(id_col) + +df_in[target_col] = df_wbm[target_col] +if task_type == "RS2RE": + df_in[input_col] = [x["structure"] for x in df_in.computed_structure_entry] +assert input_col in df_in, f"{input_col=} not in {list(df_in)}" + +df_in[input_col] = [ + JarvisAtomsAdaptor.get_atoms(Structure.from_dict(x)) + for x in tqdm(df_in[input_col], leave=False, desc="Converting to JARVIS atoms") +] + + +# %% +run_params = dict( + data_path=data_path, + **{f"{dep}_version": version(dep) for dep in ("megnet", "numpy")}, + model_name=model_name, + task_type=task_type, + target_col=target_col, + df=dict(shape=str(df_in.shape), columns=", ".join(df_in)), + slurm_vars=slurm_vars, +) + +wandb.init(project="matbench-discovery", name=job_name, config=run_params) + + +# %% Predict +model.eval() +e_form_preds: dict[str, float] = {} +with torch.no_grad(): # get predictions + for material_id, atoms in tqdm( + df_in[input_col].items(), + total=len(df_in), + desc=f"Predicting {target_col=} {task_type}", + ): + atom_graph, line_graph = Graph.atom_dgl_multigraph(atoms) + e_form = model([atom_graph.to(device), line_graph.to(device)]).item() + + e_form_preds[material_id] = e_form + +df_wbm[pred_col] = e_form_preds + +df_wbm[pred_col] -= df_wbm.e_correction_per_atom_mp_legacy +df_wbm[pred_col] += df_wbm.e_correction_per_atom_mp2020 + +df_wbm[pred_col].round(4).to_csv(f"{module_dir}/{today}-{model_name}-wbm-IS2RE.csv.gz") + + +# %% +table = wandb.Table(dataframe=df_wbm[[target_col, pred_col]].reset_index()) + +MAE = (df_wbm[target_col] - df_wbm[pred_col]).abs().mean() +R2 = r2_score(df_wbm[target_col], df_wbm[pred_col]) +title = f"{model_name} {task_type} {MAE=:.4} {R2=:.4}" +print(title) + +wandb_scatter(table, fields=dict(x=target_col, y=pred_col), title=title) diff --git a/models/alignn/train_alignn.py b/models/alignn/train_alignn.py new file mode 100644 index 00000000..e6243cd2 --- /dev/null +++ b/models/alignn/train_alignn.py @@ -0,0 +1,19 @@ +"""Train a ALIGNN on target_col of data_path.""" + + +# %% +from alignn.train_folder import train_for_folder + +__author__ = "Philipp Benner" +__date__ = "2023-06-02" + + +# %% +train_for_folder( + root_dir="data_train", + config_name="alignn-config.json", + keep_data_order=False, + output_dir="data-train-result", + epochs=1000, + file_format="poscar", +) diff --git a/models/cgcnn/test_cgcnn.py b/models/cgcnn/test_cgcnn.py index 1f2c7f30..55b52cf8 100644 --- a/models/cgcnn/test_cgcnn.py +++ b/models/cgcnn/test_cgcnn.py @@ -60,7 +60,7 @@ df[input_col] = [x["structure"] for x in df.computed_structure_entry] assert input_col in df, f"{input_col=} not in {list(df)}" -df[input_col] = [Structure.from_dict(x) for x in tqdm(df[input_col], disable=None)] +df[input_col] = [Structure.from_dict(dct) for dct in tqdm(df[input_col], disable=None)] filters = { # "display_name": {"$regex": "^train-cgcnn-robust-augment=3-"}, @@ -117,7 +117,7 @@ ) slurm_job_id = os.getenv("SLURM_JOB_ID", "debug") -df.round(4).to_csv(f"{out_dir}/{job_name}-preds-{slurm_job_id}.csv") +df.round(4).to_csv(f"{out_dir}/{job_name}-preds-{slurm_job_id}.csv.gz") pred_col = f"{e_form_col}_pred_ens" assert pred_col in df, f"{pred_col=} not in {list(df)}" table = wandb.Table(dataframe=df[[e_form_col, pred_col]].reset_index()) diff --git a/models/cgcnn/train_cgcnn.py b/models/cgcnn/train_cgcnn.py index dad4c803..b3c34f0f 100644 --- a/models/cgcnn/train_cgcnn.py +++ b/models/cgcnn/train_cgcnn.py @@ -29,9 +29,10 @@ target_col = "formation_energy_per_atom" input_col = "structure" id_col = "material_id" -perturb = 0 # 0 for no perturbation, n>1 means train on n perturbations of each crystal +# 0 for no perturbation, n>1 means train on n perturbations of each crystal # in the training set all assigned the same original target energy -job_name = f"train-cgcnn-robust-{perturb=}{'-debug' if DEBUG else ''}" +n_perturb = 0 +job_name = f"train-cgcnn-robust-{n_perturb=}{'-debug' if DEBUG else ''}" print(f"{job_name=}") robust = "robust" in job_name.lower() ensemble_size = 10 @@ -61,19 +62,23 @@ # %% data_path = DATA_FILES.mp_energies print(f"{data_path=}") -df = pd.read_json(data_path).set_index(id_col) -df[input_col] = [Structure.from_dict(s) for s in tqdm(df[input_col], disable=None)] -assert target_col in df +df_in = pd.read_json(data_path).set_index(id_col) +df_in[input_col] = [ + Structure.from_dict(s) for s in tqdm(df_in[input_col], disable=None) +] +assert target_col in df_in -df_aug = df.copy() +df_aug = df_in.copy() structs = df_aug.pop(input_col) -for idx in trange(perturb, desc="Generating perturbed structures"): +for idx in trange(n_perturb, desc="Generating perturbed structures"): df_aug[input_col] = [perturb_structure(x) for x in structs] - df = pd.concat([df, df_aug.set_index(f"{x}-aug={idx+1}" for x in df_aug.index)]) + df_in = pd.concat( + [df_in, df_aug.set_index(f"{x}-aug={idx+1}" for x in df_aug.index)] + ) del df_aug -train_df, test_df = df_train_test_split(df, test_size=0.05) +train_df, test_df = df_train_test_split(df_in, test_size=0.05) print(f"{train_df.shape=}") train_data = CrystalGraphData(train_df, task_dict={target_col: task_type}) @@ -88,7 +93,7 @@ ) # 1 for regression, n_classes for classification -n_targets = [1 if task_type == "regression" else df[target_col].max() + 1] +n_targets = [1 if task_type == "regression" else df_in[target_col].max() + 1] model_params = dict( n_targets=n_targets, @@ -106,7 +111,7 @@ train_df=dict(shape=str(train_data.df.shape), columns=", ".join(train_df)), test_df=dict(shape=str(test_data.df.shape), columns=", ".join(test_df)), slurm_vars=slurm_vars, - perturb=perturb, + n_perturb=n_perturb, input_col=input_col, ) diff --git a/models/chgnet/join_chgnet_results.py b/models/chgnet/join_chgnet_results.py index 691a1242..ea26fc41 100644 --- a/models/chgnet/join_chgnet_results.py +++ b/models/chgnet/join_chgnet_results.py @@ -66,7 +66,7 @@ # %% out_path = f"{module_dir}/{today}-chgnet-wbm-{task_type}" df_chgnet = df_chgnet.round(4) -df_chgnet.select_dtypes("number").to_csv(f"{out_path}.csv") +df_chgnet.select_dtypes("number").to_csv(f"{out_path}.csv.gz") df_chgnet.reset_index().to_json(f"{out_path}.json.gz", default_handler=as_dict_handler) # in_path = f"{module_dir}/2023-03-04-chgnet-wbm-IS2RE.json.gz" diff --git a/models/m3gnet/join_m3gnet_results.py b/models/m3gnet/join_m3gnet_results.py index 8396934e..1ef88e39 100644 --- a/models/m3gnet/join_m3gnet_results.py +++ b/models/m3gnet/join_m3gnet_results.py @@ -93,7 +93,7 @@ # %% out_path = f"{module_dir}/{today}-m3gnet-{model_type}-wbm-{task_type}" df_m3gnet = df_m3gnet.round(4) -df_m3gnet.select_dtypes("number").to_csv(f"{out_path}.csv") +df_m3gnet.select_dtypes("number").to_csv(f"{out_path}.csv.gz") df_m3gnet.reset_index().to_json(f"{out_path}.json.gz", default_handler=as_dict_handler) diff --git a/models/megnet/test_megnet.py b/models/megnet/test_megnet.py index f143e687..34e28b01 100644 --- a/models/megnet/test_megnet.py +++ b/models/megnet/test_megnet.py @@ -51,7 +51,7 @@ # %% slurm_array_task_id = int(os.getenv("SLURM_ARRAY_TASK_ID", "0")) -out_path = f"{out_dir}/megnet-e-form-preds.csv" +out_path = f"{out_dir}/megnet-e-form-preds.csv.gz" if os.path.isfile(out_path): raise SystemExit(f"{out_path = } already exists, exciting early") @@ -121,8 +121,7 @@ + df_wbm.e_correction_per_atom_mp2020 ).to_frame(name=pred_col) -df_megnet.round(4).to_csv("2022-11-18-megnet-wbm-IS2RE/megnet-e-form-preds.csv") - +df_megnet.round(4).to_csv(out_path) # df_megnet = pd.read_csv(f"{ROOT}/models/{PRED_FILES.megnet}").set_index("material_id") diff --git a/models/voronoi/train_test_voronoi_rf.py b/models/voronoi/train_test_voronoi_rf.py index 60f7635f..20a878c9 100644 --- a/models/voronoi/train_test_voronoi_rf.py +++ b/models/voronoi/train_test_voronoi_rf.py @@ -28,7 +28,7 @@ print(f"{task_type=}") out_dir = f"{module_dir}/{today}-train-test" -out_path = f"{out_dir}/e-form-preds-{task_type}.csv" +out_path = f"{out_dir}/e-form-preds-{task_type}.csv.gz" if os.path.isfile(out_path): raise SystemExit(f"{out_path = } already exists, exciting early") diff --git a/models/voronoi/voronoi_featurize_dataset.py b/models/voronoi/voronoi_featurize_dataset.py index 839ea7a0..964e9d25 100644 --- a/models/voronoi/voronoi_featurize_dataset.py +++ b/models/voronoi/voronoi_featurize_dataset.py @@ -70,7 +70,9 @@ elif data_name == "wbm" and input_col == "initial_structure": struct_dicts = df_in.initial_structure -df_in[input_col] = [Structure.from_dict(x) for x in tqdm(struct_dicts, disable=None)] +df_in[input_col] = [ + Structure.from_dict(dct) for dct in tqdm(struct_dicts, disable=None) +] # %% diff --git a/models/wrenformer/test_wrenformer.py b/models/wrenformer/test_wrenformer.py index 25180974..1399ea51 100644 --- a/models/wrenformer/test_wrenformer.py +++ b/models/wrenformer/test_wrenformer.py @@ -104,7 +104,7 @@ df = df.round(4) slurm_job_id = os.getenv("SLURM_JOB_ID", "debug") -df.to_csv(f"{out_dir}/{job_name}-preds-{slurm_job_id}.csv") +df.to_csv(f"{out_dir}/{job_name}-preds-{slurm_job_id}.csv.gz") # %% diff --git a/pyproject.toml b/pyproject.toml index 8b8e8225..e00c4851 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,7 +55,9 @@ Package = "https://pypi.org/project/matbench-discovery" test = ["pytest", "pytest-cov"] # how to specify git deps: https://stackoverflow.com/a/73572379 running-models = [ + "alignn", "chgnet", + "jarvis-tools", # torch needs to install before aviary "torch", diff --git a/scripts/calc_wandb_model_runtimes.py b/scripts/calc_wandb_model_runtimes.py index 9889b995..aeba003b 100644 --- a/scripts/calc_wandb_model_runtimes.py +++ b/scripts/calc_wandb_model_runtimes.py @@ -18,7 +18,7 @@ from pymatviz.utils import save_fig from tqdm import tqdm -from matbench_discovery import FIGS, PDF_FIGS, WANDB_PATH +from matbench_discovery import FIGS, MODELS, PDF_FIGS, WANDB_PATH from matbench_discovery.preds import df_metrics, df_preds __author__ = "Janosh Riebesell" @@ -122,7 +122,7 @@ df_stats.attrs["All Models Run Time"] = df_stats[time_col].sum() print(f"{df_stats[time_col].sum()=:.0f} hours") -# df_stats.round(2).to_json(f"{MODELS}/model-stats.json", orient="index") +df_stats.round(2).to_json(f"{MODELS}/model-stats.json", orient="index") df_time = ( df_stats.sort_index() .filter(like=time_col) diff --git a/scripts/make_metrics_tables.py b/scripts/make_metrics_tables.py index 0d07a910..49f0b121 100644 --- a/scripts/make_metrics_tables.py +++ b/scripts/make_metrics_tables.py @@ -46,7 +46,8 @@ # %% -ontology = { +ontology = { # (training type, test type, model type) + "ALIGNN": ("RS2RE", "IS2RE", "GNN"), "CHGNet": ("S2EFSM", "IS2RE-SR", "UIP-GNN"), "M3GNet": ("S2EFS", "IS2RE-SR", "UIP-GNN"), "MEGNet": ("RS2RE", "IS2E", "GNN"), @@ -74,8 +75,8 @@ for label, df, extra_hide_metrics in ( # hide redundant metrics (TPR = Recall, FPR = 1 - TNR, FNR = 1 - TPR) - ("", df_metrics, []), ("-first-10k", df_metrics_10k, ["TPR", "TNR"]), + ("", df_metrics, []), ): df_table = pd.concat([df, df_ont]).rename(index={"R2": R2_col}) df_table.index.name = "Model" diff --git a/scripts/readme.md b/scripts/readme.md index 0e571a4b..e95c6947 100644 --- a/scripts/readme.md +++ b/scripts/readme.md @@ -1,8 +1 @@ -# About `scripts/` - -This folder contains - -- plotting scripts -- [`ctk_structure_viewer.py`](ctk_structure_viewer.py) a [Crystal Toolkit app](https://github.com/materialsproject/crystaltoolkit) to 3d-render crystals when hovering corresponding points in a `plotly` figure -- [`update_wandb_runs.py`](update_wandb_runs.py) use the [WandB](https://wandb.ai/janosh/matbench-discovery) API to update run metadata -- [`make_api_docs.py`](make_api_docs.py): [auto-generate markdown API docs](https://github.com/ml-tooling/lazydocs) for the `matbench-discovery` PyPI package +See the module doc strings for the purpose of each script. diff --git a/scripts/scatter_e_above_hull_models.py b/scripts/scatter_e_above_hull_models.py index c2c6105e..f9927dd8 100644 --- a/scripts/scatter_e_above_hull_models.py +++ b/scripts/scatter_e_above_hull_models.py @@ -52,7 +52,7 @@ legend_order = list(df_metrics.T.MAE.sort_values().index) -# %% determine each point's classification to color them by +# determine each point's classification to color them by true_pos, false_neg, false_pos, true_neg = classify_stable( df_bin[each_true_col], df_bin[each_pred_col] ) @@ -220,7 +220,7 @@ **axis_titles, ) # fig.layout.height = 1000 -fig.layout.width = 1100 +# fig.layout.width = 1100 fig.layout.margin.update(l=40, r=10, t=10, b=50) fig.update_xaxes(matches=None) fig.update_yaxes(matches=None) diff --git a/site/src/app.css b/site/src/app.css index 6fb74511..f5f2b95a 100644 --- a/site/src/app.css +++ b/site/src/app.css @@ -68,7 +68,6 @@ button { border-radius: 3pt; background: teal; padding: 2pt 4pt; - font-size: 12pt; } a { color: var(--blue); diff --git a/site/src/figs/metrics-table-first-10k.svelte b/site/src/figs/metrics-table-first-10k.svelte index b96c8cb3..b5937210 100644 --- a/site/src/figs/metrics-table-first-10k.svelte +++ b/site/src/figs/metrics-table-first-10k.svelte @@ -51,70 +51,81 @@ GNN - CGCNN+P - 0.74 - 3.51 - 0.58 - 0.58 - 0.21 - 0.29 - 0.02 + ALIGNN + 0.75 + 3.57 + 0.59 + 0.59 + 0.19 + 0.27 + 0.12 GNN - Wrenformer + CGCNN+P 0.74 3.51 0.58 0.58 - 0.19 - 0.25 - 0.21 - Transformer + 0.21 + 0.29 + 0.02 + GNN - BOWSR + MEGNet - 0.69 - 3.14 - 0.52 - 0.52 - 0.27 - 0.33 - -1.10 - BO+GNN + Wrenformer + 0.74 + 3.51 + 0.58 + 0.58 + 0.19 + 0.25 + 0.21 + Transformer - MEGNet - 0.65 - 2.86 - 0.48 - 0.48 - 0.33 - 0.36 - -0.82 - GNN + BOWSR + MEGNet + 0.69 + 3.14 + 0.52 + 0.52 + 0.27 + 0.33 + -1.10 + BO+GNN - Voronoi RF - 0.58 - 2.47 - 0.41 - 0.41 - 0.36 - 0.43 - -0.83 - Fingerprint+RF + MEGNet + 0.65 + 2.86 + 0.48 + 0.48 + 0.33 + 0.36 + -0.82 + GNN - Dummy - 0.19 - 1.00 - 0.17 - 0.68 - 0.12 - 0.18 - 0.00 - scikit-learn + Voronoi RF + 0.58 + 2.47 + 0.41 + 0.41 + 0.36 + 0.43 + -0.83 + Fingerprint+RF + + + Dummy + 0.19 + 1.00 + 0.17 + 0.68 + 0.12 + 0.18 + 0.00 + scikit-learn @@ -199,135 +210,154 @@ background-color: #4ac16d; color: #000000; } - #T__row3_col0, - #T__row4_col0 { - background-color: #65cb5e; + #T__row3_col0 { + background-color: #6ccd5a; color: #000000; } #T__row3_col1, + #T__row3_col2, + #T__row9_col3 { + background-color: #2eb37c; + color: #f1f1f1; + } + #T__row3_col3 { + background-color: #26828e; + color: #f1f1f1; + } + #T__row3_col4, + #T__row9_col6 { + background-color: #1fa287; + color: #f1f1f1; + } + #T__row3_col5 { + background-color: #21908d; + color: #f1f1f1; + } + #T__row3_col6 { + background-color: #2cb17e; + color: #f1f1f1; + } + #T__row4_col0, + #T__row5_col0 { + background-color: #65cb5e; + color: #000000; + } #T__row4_col1, - #T__row6_col0 { + #T__row5_col1, + #T__row7_col0 { background-color: #2ab07f; color: #f1f1f1; } - #T__row3_col2, - #T__row4_col2 { + #T__row4_col2, + #T__row5_col2 { background-color: #29af7f; color: #f1f1f1; } - #T__row3_col3, - #T__row4_col3 { + #T__row4_col3, + #T__row5_col3 { background-color: #287c8e; color: #f1f1f1; } - #T__row3_col4 { + #T__row4_col4 { background-color: #20928c; color: #f1f1f1; } - #T__row3_col5 { + #T__row4_col5 { background-color: #277f8e; color: #f1f1f1; } - #T__row3_col6 { + #T__row4_col6 { background-color: #20a486; color: #f1f1f1; } - #T__row4_col4 { + #T__row5_col4 { background-color: #21a685; color: #f1f1f1; } - #T__row4_col5, - #T__row5_col1 { + #T__row5_col5, + #T__row6_col1 { background-color: #1f9a8a; color: #f1f1f1; } - #T__row4_col6 { + #T__row5_col6 { background-color: #3bbb75; color: #f1f1f1; } - #T__row5_col0 { + #T__row6_col0 { background-color: #40bd72; color: #f1f1f1; } - #T__row5_col2 { + #T__row6_col2 { background-color: #1f998a; color: #f1f1f1; } - #T__row5_col3 { + #T__row6_col3 { background-color: #39568c; color: #f1f1f1; } - #T__row5_col4 { + #T__row6_col4 { background-color: #31678e; color: #f1f1f1; } - #T__row5_col5 { + #T__row6_col5 { background-color: #33638d; color: #f1f1f1; } - #T__row5_col6, - #T__row7_col3, - #T__row7_col4, - #T__row7_col5, - #T__row8_col0, - #T__row8_col1, - #T__row8_col2 { + #T__row6_col6, + #T__row8_col3, + #T__row8_col4, + #T__row8_col5, + #T__row9_col0, + #T__row9_col1, + #T__row9_col2 { background-color: #440154; color: #f1f1f1; } - #T__row6_col1, - #T__row6_col2 { + #T__row7_col1, + #T__row7_col2 { background-color: #23888e; color: #f1f1f1; } - #T__row6_col3 { + #T__row7_col3 { background-color: #453581; color: #f1f1f1; } - #T__row6_col4 { + #T__row7_col4 { background-color: #482576; color: #f1f1f1; } - #T__row6_col5 { + #T__row7_col5 { background-color: #3e4a89; color: #f1f1f1; } - #T__row6_col6 { + #T__row7_col6 { background-color: #46337f; color: #f1f1f1; } - #T__row7_col0 { + #T__row8_col0 { background-color: #1e9b8a; color: #f1f1f1; } - #T__row7_col1 { + #T__row8_col1 { background-color: #2c718e; color: #f1f1f1; } - #T__row7_col2 { + #T__row8_col2 { background-color: #2d718e; color: #f1f1f1; } - #T__row7_col6 { + #T__row8_col6 { background-color: #46327e; color: #f1f1f1; } - #T__row8_col3 { - background-color: #2eb37c; - color: #f1f1f1; - } - #T__row8_col4 { + #T__row9_col4 { background-color: #90d743; color: #000000; } - #T__row8_col5 { + #T__row9_col5 { background-color: #67cc5c; color: #000000; } - #T__row8_col6 { - background-color: #1fa287; - color: #f1f1f1; - } #T_ :is(td, th):nth-last-child(4) { border-left: 1px dotted white; } diff --git a/site/src/figs/metrics-table.svelte b/site/src/figs/metrics-table.svelte index 54898c14..3e7b8195 100644 --- a/site/src/figs/metrics-table.svelte +++ b/site/src/figs/metrics-table.svelte @@ -9,13 +9,13 @@ F1 DAF Precision - Accuracy - TPR - TNR - MAE - RMSE - R2 - Model Class + Accuracy + TPR + TNR + MAE + RMSE + R2 + Model Class @@ -24,117 +24,130 @@ 0.59 3.06 0.52 - 0.84 - 0.67 - 0.87 - 0.07 - 0.11 - 0.61 - UIP-GNN + 0.84 + 0.67 + 0.87 + 0.07 + 0.11 + 0.61 + UIP-GNN M3GNet 0.58 2.66 0.45 - 0.80 - 0.79 - 0.80 - 0.07 - 0.12 - 0.59 - UIP-GNN + 0.80 + 0.79 + 0.80 + 0.07 + 0.12 + 0.59 + UIP-GNN - MEGNet - 0.52 - 2.70 - 0.46 - 0.81 - 0.59 - 0.86 - 0.13 - 0.20 - -0.27 - GNN + ALIGNN + 0.57 + 2.87 + 0.49 + 0.82 + 0.66 + 0.86 + 0.09 + 0.15 + 0.27 + GNN - CGCNN + MEGNet 0.52 - 2.62 - 0.45 - 0.81 - 0.60 - 0.85 - 0.14 - 0.23 - -0.61 - GNN + 2.70 + 0.46 + 0.81 + 0.59 + 0.86 + 0.13 + 0.20 + -0.27 + GNN - CGCNN+P - 0.51 - 2.38 - 0.41 - 0.78 - 0.69 - 0.79 - 0.11 - 0.18 - 0.02 - GNN + CGCNN + 0.52 + 2.62 + 0.45 + 0.81 + 0.60 + 0.85 + 0.14 + 0.23 + -0.61 + GNN - Wrenformer - 0.48 - 2.13 - 0.36 - 0.74 - 0.71 - 0.74 - 0.10 - 0.18 - -0.04 - Transformer + CGCNN+P + 0.51 + 2.38 + 0.41 + 0.78 + 0.69 + 0.79 + 0.11 + 0.18 + 0.02 + GNN - BOWSR + MEGNet - 0.44 - 1.90 - 0.32 - 0.68 + Wrenformer + 0.48 + 2.13 + 0.36 + 0.74 + 0.71 0.74 - 0.67 - 0.11 - 0.16 - 0.15 - BO+GNN + 0.10 + 0.18 + -0.04 + Transformer - Voronoi RF - 0.34 - 1.51 - 0.26 - 0.66 - 0.52 - 0.69 - 0.14 - 0.21 - -0.32 - Fingerprint+RF + BOWSR + MEGNet + 0.44 + 1.90 + 0.32 + 0.68 + 0.74 + 0.67 + 0.11 + 0.16 + 0.15 + BO+GNN - Dummy - 0.19 - 1.00 - 0.17 - 0.68 - 0.23 - 0.77 - 0.12 - 0.18 - 0.00 - scikit-learn + Voronoi RF + 0.34 + 1.51 + 0.26 + 0.66 + 0.52 + 0.69 + 0.14 + 0.21 + -0.32 + Fingerprint+RF + + + Dummy + 0.19 + 1.00 + 0.17 + 0.68 + 0.23 + 0.77 + 0.12 + 0.18 + 0.00 + scikit-learn @@ -162,17 +175,17 @@ #T__row0_col0, #T__row0_col1, #T__row0_col2, - #T__row0_col4, + #T__row0_col3, + #T__row0_col5, + #T__row0_col6, #T__row0_col7, - #T__row0_col13, - #T__row0_col14, - #T__row0_col15, - #T__row1_col5 { + #T__row0_col8, + #T__row1_col4 { background-color: #fde725; color: #000000; } - #T__row0_col5, - #T__row3_col1 { + #T__row0_col4, + #T__row4_col1 { background-color: #75d054; color: #000000; } @@ -181,7 +194,7 @@ color: #000000; } #T__row1_col1, - #T__row4_col0 { + #T__row5_col0 { background-color: #81d34d; color: #000000; } @@ -189,231 +202,264 @@ background-color: #7fd34e; color: #000000; } - #T__row1_col4 { + #T__row1_col3 { background-color: #73d056; color: #000000; } - #T__row1_col7 { + #T__row1_col5 { background-color: #32b67a; color: #f1f1f1; } - #T__row1_col13 { + #T__row1_col6 { background-color: #f6e620; color: #000000; } - #T__row1_col14 { + #T__row1_col7 { background-color: #efe51c; color: #000000; } - #T__row1_col15 { + #T__row1_col8 { background-color: #f8e621; color: #000000; } #T__row2_col0 { - background-color: #89d548; + background-color: #dae319; color: #000000; } #T__row2_col1 { - background-color: #8bd646; + background-color: #c5e021; color: #000000; } #T__row2_col2 { - background-color: #90d743; + background-color: #cae11f; color: #000000; } - #T__row2_col4, - #T__row5_col5 { - background-color: #a0da39; + #T__row2_col3 { + background-color: #d2e21b; color: #000000; } - #T__row2_col5 { - background-color: #2ab07f; + #T__row2_col4 { + background-color: #69cd5b; + color: #000000; + } + #T__row2_col5, + #T__row3_col5 { + background-color: #d8e219; + color: #000000; + } + #T__row2_col6 { + background-color: #46c06f; color: #f1f1f1; } #T__row2_col7 { - background-color: #d8e219; + background-color: #2db27d; + color: #f1f1f1; + } + #T__row2_col8 { + background-color: #50c46a; + color: #000000; + } + #T__row3_col0 { + background-color: #89d548; + color: #000000; + } + #T__row3_col1 { + background-color: #8bd646; + color: #000000; + } + #T__row3_col2 { + background-color: #90d743; + color: #000000; + } + #T__row3_col3, + #T__row6_col4 { + background-color: #a0da39; color: #000000; } - #T__row2_col13 { + #T__row3_col4 { + background-color: #2ab07f; + color: #f1f1f1; + } + #T__row3_col6 { background-color: #414287; color: #f1f1f1; } - #T__row2_col14 { + #T__row3_col7 { background-color: #3e4989; color: #f1f1f1; } - #T__row2_col15 { + #T__row3_col8 { background-color: #375b8d; color: #f1f1f1; } - #T__row3_col0, - #T__row4_col5 { + #T__row4_col0, + #T__row5_col4 { background-color: #84d44b; color: #000000; } - #T__row3_col2 { + #T__row4_col2 { background-color: #7ad151; color: #000000; } - #T__row3_col4 { + #T__row4_col3 { background-color: #86d549; color: #000000; } - #T__row3_col5 { + #T__row4_col4 { background-color: #34b679; color: #f1f1f1; } - #T__row3_col7 { + #T__row4_col5 { background-color: #b0dd2f; color: #000000; } - #T__row3_col13 { + #T__row4_col6 { background-color: #482475; color: #f1f1f1; } - #T__row3_col14, - #T__row3_col15, - #T__row6_col7, - #T__row7_col4, - #T__row7_col13, - #T__row8_col0, - #T__row8_col1, - #T__row8_col2, - #T__row8_col5 { + #T__row4_col7, + #T__row4_col8, + #T__row7_col5, + #T__row8_col3, + #T__row8_col6, + #T__row9_col0, + #T__row9_col1, + #T__row9_col2, + #T__row9_col4 { background-color: #440154; color: #f1f1f1; } - #T__row4_col1 { + #T__row5_col1 { background-color: #38b977; color: #f1f1f1; } - #T__row4_col2 { + #T__row5_col2 { background-color: #3bbb75; color: #f1f1f1; } - #T__row4_col4 { + #T__row5_col3 { background-color: #2fb47c; color: #f1f1f1; } - #T__row4_col7 { + #T__row5_col5 { background-color: #25ac82; color: #f1f1f1; } - #T__row4_col13 { + #T__row5_col6 { background-color: #24878e; color: #f1f1f1; } - #T__row4_col14 { + #T__row5_col7 { background-color: #277f8e; color: #f1f1f1; } - #T__row4_col15, - #T__row5_col13 { + #T__row5_col8, + #T__row6_col6 { background-color: #1f958b; color: #f1f1f1; } - #T__row5_col0 { + #T__row6_col0 { background-color: #54c568; color: #000000; } - #T__row5_col1 { + #T__row6_col1 { background-color: #1e9c89; color: #f1f1f1; } - #T__row5_col2 { + #T__row6_col2 { background-color: #1f9e89; color: #f1f1f1; } - #T__row5_col4 { + #T__row6_col3 { background-color: #26818e; color: #f1f1f1; } - #T__row5_col7 { + #T__row6_col5 { background-color: #2d718e; color: #f1f1f1; } - #T__row5_col14 { + #T__row6_col7 { background-color: #2b748e; color: #f1f1f1; } - #T__row5_col15 { + #T__row6_col8 { background-color: #23898e; color: #f1f1f1; } - #T__row6_col0 { + #T__row7_col0 { background-color: #29af7f; color: #f1f1f1; } - #T__row6_col1 { + #T__row7_col1 { background-color: #26828e; color: #f1f1f1; } - #T__row6_col2, - #T__row8_col14 { + #T__row7_col2, + #T__row9_col7 { background-color: #287c8e; color: #f1f1f1; } - #T__row6_col4 { + #T__row7_col3 { background-color: #482979; color: #f1f1f1; } - #T__row6_col5 { + #T__row7_col4 { background-color: #c8e020; color: #000000; } - #T__row6_col13 { + #T__row7_col6 { background-color: #2a768e; color: #f1f1f1; } - #T__row6_col14 { + #T__row7_col7 { background-color: #1fa088; color: #f1f1f1; } - #T__row6_col15 { + #T__row7_col8 { background-color: #28ae80; color: #f1f1f1; } - #T__row7_col0 { + #T__row8_col0 { background-color: #2c738e; color: #f1f1f1; } - #T__row7_col1 { + #T__row8_col1 { background-color: #3b518b; color: #f1f1f1; } - #T__row7_col2 { + #T__row8_col2 { background-color: #3a548c; color: #f1f1f1; } - #T__row7_col5 { + #T__row8_col4 { background-color: #20938c; color: #f1f1f1; } - #T__row7_col7 { + #T__row8_col5 { background-color: #482878; color: #f1f1f1; } - #T__row7_col14 { + #T__row8_col7 { background-color: #424086; color: #f1f1f1; } - #T__row7_col15 { + #T__row8_col8 { background-color: #3c508b; color: #f1f1f1; } - #T__row8_col4 { + #T__row9_col3 { background-color: #482576; color: #f1f1f1; } - #T__row8_col7 { + #T__row9_col5 { background-color: #21908d; color: #f1f1f1; } - #T__row8_col13 { + #T__row9_col6 { background-color: #34618d; color: #f1f1f1; } - #T__row8_col15 { + #T__row9_col8 { background-color: #21918c; color: #f1f1f1; } diff --git a/site/src/routes/models/+page.svelte b/site/src/routes/models/+page.svelte index 7b8514e1..b65b8bb2 100644 --- a/site/src/routes/models/+page.svelte +++ b/site/src/routes/models/+page.svelte @@ -46,7 +46,7 @@ } -
+

Models

@@ -93,10 +93,8 @@ {/each} -
-
-

Per-Element Model Error Heatmaps

+

Per-Element Model Error Heatmaps

diff --git a/site/src/routes/models/model-stats.json b/site/src/routes/models/model-stats.json index a488a20b..ca143ce5 100644 --- a/site/src/routes/models/model-stats.json +++ b/site/src/routes/models/model-stats.json @@ -1 +1 @@ -{"Voronoi RF":{"F1":0.34,"R2":-0.32,"DAF":1.51,"Precision":0.26,"Recall":0.52,"TPR":0.52,"FPR":0.31,"TNR":0.69,"FNR":0.48,"Accuracy":0.66,"MAE":0.14,"RMSE":0.21,"Run Time (h)":204.7,"GPU":0.0,"CPU":76.0,"Slurm Jobs":68.0,"missing_preds":17,"missing_percent":"0.01%"},"BOWSR + MEGNet":{"F1":0.44,"R2":0.15,"DAF":1.9,"Precision":0.32,"Recall":0.74,"TPR":0.74,"FPR":0.33,"TNR":0.67,"FNR":0.26,"Accuracy":0.68,"MAE":0.11,"RMSE":0.16,"Run Time (h)":2776.57,"GPU":0.0,"CPU":32.0,"Slurm Jobs":500.0,"missing_preds":6184,"missing_percent":"2.41%"},"Wrenformer":{"F1":0.48,"R2":-0.04,"DAF":2.13,"Precision":0.36,"Recall":0.71,"TPR":0.71,"FPR":0.26,"TNR":0.74,"FNR":0.29,"Accuracy":0.74,"MAE":0.1,"RMSE":0.18,"Run Time (h)":57.89,"GPU":1.0,"CPU":128.0,"Slurm Jobs":10.0,"missing_preds":0,"missing_percent":"0.00%"},"CGCNN+P":{"F1":0.51,"R2":0.02,"DAF":2.38,"Precision":0.41,"Recall":0.69,"TPR":0.69,"FPR":0.21,"TNR":0.79,"FNR":0.31,"Accuracy":0.78,"MAE":0.11,"RMSE":0.18,"Run Time (h)":null,"GPU":null,"CPU":null,"Slurm Jobs":null,"missing_preds":2,"missing_percent":"0.00%"},"CGCNN":{"F1":0.52,"R2":-0.61,"DAF":2.62,"Precision":0.45,"Recall":0.6,"TPR":0.6,"FPR":0.15,"TNR":0.85,"FNR":0.4,"Accuracy":0.8,"MAE":0.14,"RMSE":0.23,"Run Time (h)":11.35,"GPU":1.0,"CPU":128.0,"Slurm Jobs":10.0,"missing_preds":2,"missing_percent":"0.00%"},"MEGNet":{"F1":0.52,"R2":-0.27,"DAF":2.7,"Precision":0.46,"Recall":0.59,"TPR":0.59,"FPR":0.14,"TNR":0.86,"FNR":0.41,"Accuracy":0.81,"MAE":0.13,"RMSE":0.2,"Run Time (h)":3.44,"GPU":1.0,"CPU":128.0,"Slurm Jobs":1.0,"missing_preds":0,"missing_percent":"0.00%"},"CHGNet + MEGNet":{"F1":0.55,"R2":0.34,"DAF":2.56,"Precision":0.43,"Recall":0.75,"TPR":0.75,"FPR":0.21,"TNR":0.79,"FNR":0.25,"Accuracy":0.79,"MAE":0.09,"RMSE":0.15,"Run Time (h)":null,"GPU":null,"CPU":null,"Slurm Jobs":null,"missing_preds":5225,"missing_percent":"2.03%"},"M3GNet + MEGNet":{"F1":0.55,"R2":0.35,"DAF":2.54,"Precision":0.43,"Recall":0.77,"TPR":0.77,"FPR":0.21,"TNR":0.79,"FNR":0.23,"Accuracy":0.79,"MAE":0.09,"RMSE":0.14,"Run Time (h)":87.09,"GPU":0.0,"CPU":76.0,"Slurm Jobs":99.0,"missing_preds":2576,"missing_percent":"1.00%"},"M3GNet":{"F1":0.58,"R2":0.6,"DAF":2.66,"Precision":0.45,"Recall":0.79,"TPR":0.79,"FPR":0.2,"TNR":0.8,"FNR":0.21,"Accuracy":0.8,"MAE":0.07,"RMSE":0.12,"Run Time (h)":83.65,"GPU":0.0,"CPU":76.0,"Slurm Jobs":99.0,"missing_preds":2569,"missing_percent":"1.00%"},"CHGNet":{"F1":0.59,"R2":0.61,"DAF":3.06,"Precision":0.52,"Recall":0.67,"TPR":0.67,"FPR":0.13,"TNR":0.87,"FNR":0.33,"Accuracy":0.84,"MAE":0.07,"RMSE":0.11,"Run Time (h)":142.48,"GPU":1.0,"CPU":128.0,"Slurm Jobs":102.0,"missing_preds":2655,"missing_percent":"1.03%"}} +{"CHGNet":{"F1":0.59,"DAF":3.06,"Precision":0.52,"Recall":0.67,"Accuracy":0.84,"TPR":0.67,"FPR":0.13,"TNR":0.87,"FNR":0.33,"TP":29561.0,"FP":27118.0,"TN":183325.0,"FN":14304.0,"MAE":0.07,"RMSE":0.11,"R2":0.61,"Train Run Time (h)":null,"Train GPU":null,"Train CPU":null,"Train Slurm Jobs":null,"Test Run Time (h)":142.48,"Test GPU":1.0,"Test CPU":128.0,"Test Slurm Jobs":100.0,"Run Time (h)":142.48,"missing_preds":2655,"missing_percent":"1.03%"},"M3GNet":{"F1":0.58,"DAF":2.66,"Precision":0.45,"Recall":0.79,"Accuracy":0.8,"TPR":0.79,"FPR":0.2,"TNR":0.8,"FNR":0.21,"TP":34556.0,"FP":41743.0,"TN":168956.0,"FN":9139.0,"MAE":0.07,"RMSE":0.12,"R2":0.6,"Train Run Time (h)":null,"Train GPU":null,"Train CPU":null,"Train Slurm Jobs":null,"Test Run Time (h)":83.65,"Test GPU":0.0,"Test CPU":76.0,"Test Slurm Jobs":99.0,"Run Time (h)":83.65,"missing_preds":2569,"missing_percent":"1.00%"},"ALIGNN":{"F1":0.57,"DAF":2.87,"Precision":0.49,"Recall":0.66,"Accuracy":0.82,"TPR":0.66,"FPR":0.14,"TNR":0.86,"FNR":0.34,"TP":29246.0,"FP":30061.0,"TN":182810.0,"FN":14846.0,"MAE":0.09,"RMSE":0.15,"R2":0.27,"Train Run Time (h)":null,"Train GPU":null,"Train CPU":null,"Train Slurm Jobs":null,"Test Run Time (h)":null,"Test GPU":null,"Test CPU":null,"Test Slurm Jobs":null,"Run Time (h)":0.0,"missing_preds":0,"missing_percent":"0.00%"},"MEGNet":{"F1":0.52,"DAF":2.7,"Precision":0.46,"Recall":0.59,"Accuracy":0.81,"TPR":0.59,"FPR":0.14,"TNR":0.86,"FNR":0.41,"TP":25892.0,"FP":29983.0,"TN":182888.0,"FN":18200.0,"MAE":0.13,"RMSE":0.2,"R2":-0.27,"Train Run Time (h)":null,"Train GPU":null,"Train CPU":null,"Train Slurm Jobs":null,"Test Run Time (h)":3.44,"Test GPU":1.0,"Test CPU":128.0,"Test Slurm Jobs":1.0,"Run Time (h)":3.44,"missing_preds":0,"missing_percent":"0.00%"},"CGCNN":{"F1":0.52,"DAF":2.62,"Precision":0.45,"Recall":0.6,"Accuracy":0.8,"TPR":0.6,"FPR":0.15,"TNR":0.85,"FNR":0.4,"TP":26552.0,"FP":32463.0,"TN":180406.0,"FN":17540.0,"MAE":0.14,"RMSE":0.23,"R2":-0.61,"Train Run Time (h)":11.35,"Train GPU":1.0,"Train CPU":128.0,"Train Slurm Jobs":10.0,"Test Run Time (h)":0.17,"Test GPU":1.0,"Test CPU":128.0,"Test Slurm Jobs":1.0,"Run Time (h)":11.52,"missing_preds":2,"missing_percent":"0.00%"},"CGCNN+P":{"F1":0.51,"DAF":2.38,"Precision":0.41,"Recall":0.69,"Accuracy":0.78,"TPR":0.69,"FPR":0.21,"TNR":0.79,"FNR":0.31,"TP":30321.0,"FP":43760.0,"TN":169109.0,"FN":13771.0,"MAE":0.11,"RMSE":0.18,"R2":0.02,"Train Run Time (h)":null,"Train GPU":null,"Train CPU":null,"Train Slurm Jobs":null,"Test Run Time (h)":null,"Test GPU":null,"Test CPU":null,"Test Slurm Jobs":null,"Run Time (h)":0.0,"missing_preds":2,"missing_percent":"0.00%"},"Wrenformer":{"F1":0.48,"DAF":2.13,"Precision":0.36,"Recall":0.71,"Accuracy":0.74,"TPR":0.71,"FPR":0.26,"TNR":0.74,"FNR":0.29,"TP":31381.0,"FP":54514.0,"TN":158357.0,"FN":12711.0,"MAE":0.1,"RMSE":0.18,"R2":-0.04,"Train Run Time (h)":57.89,"Train GPU":1.0,"Train CPU":128.0,"Train Slurm Jobs":10.0,"Test Run Time (h)":0.45,"Test GPU":1.0,"Test CPU":128.0,"Test Slurm Jobs":1.0,"Run Time (h)":58.34,"missing_preds":0,"missing_percent":"0.00%"},"BOWSR + MEGNet":{"F1":0.44,"DAF":1.9,"Precision":0.32,"Recall":0.74,"Accuracy":0.68,"TPR":0.74,"FPR":0.33,"TNR":0.67,"FNR":0.26,"TP":31607.0,"FP":68808.0,"TN":139481.0,"FN":10883.0,"MAE":0.11,"RMSE":0.16,"R2":0.15,"Train Run Time (h)":null,"Train GPU":null,"Train CPU":null,"Train Slurm Jobs":null,"Test Run Time (h)":2705.45,"Test GPU":0.0,"Test CPU":32.0,"Test Slurm Jobs":476.0,"Run Time (h)":2705.45,"missing_preds":6184,"missing_percent":"2.41%"},"Voronoi RF":{"F1":0.34,"DAF":1.51,"Precision":0.26,"Recall":0.52,"Accuracy":0.66,"TPR":0.52,"FPR":0.31,"TNR":0.69,"FNR":0.48,"TP":22885.0,"FP":65514.0,"TN":147341.0,"FN":21206.0,"MAE":0.14,"RMSE":0.21,"R2":-0.32,"Train Run Time (h)":173.78,"Train GPU":0.0,"Train CPU":76.0,"Train Slurm Jobs":48.0,"Test Run Time (h)":30.92,"Test GPU":0.0,"Test CPU":76.0,"Test Slurm Jobs":20.0,"Run Time (h)":204.7,"missing_preds":17,"missing_percent":"0.01%"}} diff --git a/tests/test_preds.py b/tests/test_preds.py index 85629e28..ee1baae8 100644 --- a/tests/test_preds.py +++ b/tests/test_preds.py @@ -66,7 +66,10 @@ def test_load_df_wbm_with_preds_raises() -> None: def test_pred_files() -> None: assert len(PRED_FILES) >= 6 - assert all(path.endswith((".csv", ".json")) for path in PRED_FILES.values()) + assert all( + path.endswith((".csv", ".csv.gz", ".json", ".json.gz")) + for path in PRED_FILES.values() + ) for model, path in PRED_FILES.items(): msg = f"Missing preds file for {model=}, expected at {path=}" assert os.path.isfile(path), msg