From d21a10984ccae69e72d6c675f4543bf1ada2c2bc Mon Sep 17 00:00:00 2001 From: CompRhys Date: Tue, 6 Feb 2024 21:22:20 -0800 Subject: [PATCH 01/12] fix: pass preds rather than err to v-plots --- matbench_discovery/plots.py | 19 +++--- .../hist_classified_stable_vs_hull_dist.py | 5 +- .../rolling_mae_vs_hull_dist_models.py | 2 +- .../rolling_mae_vs_hull_dist_wbm_batches.py | 65 ++++++++++--------- scripts/rolling_mae_vs_hull_dist.py | 4 +- 5 files changed, 47 insertions(+), 48 deletions(-) diff --git a/matbench_discovery/plots.py b/matbench_discovery/plots.py index fc88fb3f..7fce2bb0 100644 --- a/matbench_discovery/plots.py +++ b/matbench_discovery/plots.py @@ -265,7 +265,7 @@ def hist_classified_stable_vs_hull_dist( def rolling_mae_vs_hull_dist( e_above_hull_true: pd.Series, - e_above_hull_errors: pd.DataFrame | dict[str, pd.Series], + e_above_hull_preds: pd.DataFrame | dict[str, pd.Series], df_rolling_err: pd.DataFrame | None = None, df_err_std: pd.DataFrame | None = None, window: float = 0.02, @@ -291,9 +291,8 @@ def rolling_mae_vs_hull_dist( Args: e_above_hull_true (pd.Series): Distance to convex hull according to DFT ground truth (in eV / atom). - e_above_hull_errors (pd.DataFrame | dict[str, pd.Series]): Error in - model-predicted distance to convex hull, i.e. actual hull distance minus - predicted hull distance (in eV / atom). + e_above_hull_preds (pd.DataFrame | dict[str, pd.Series]): Predicted distance to + convex hull by models, one column per model (in eV / atom). df_rolling_err (pd.DataFrame, optional): Cached rolling MAE(s) as returned by previous call to this function. Defaults to None. df_err_std (pd.DataFrame, optional): Cached standard error in the mean of the @@ -332,7 +331,7 @@ def rolling_mae_vs_hull_dist( standard error in the mean. """ bins = np.arange(*x_lim, bin_width) - models = list(e_above_hull_errors) + models = list(e_above_hull_preds) if df_rolling_err is None or df_err_std is None: df_rolling_err = pd.DataFrame(columns=models, index=bins) @@ -342,20 +341,20 @@ def rolling_mae_vs_hull_dist( prog_bar := tqdm(models, desc="Calculating rolling MAE", disable=not pbar) ): prog_bar.set_postfix_str(model) - for idx, bin_center in enumerate(bins): + for bin_center in bins: low = bin_center - window high = bin_center + window mask = (e_above_hull_true <= high) & (e_above_hull_true > low) - bin_mae = e_above_hull_errors[model].loc[mask].abs().mean() - df_rolling_err[model].iloc[idx] = bin_mae + bin_mae = (e_above_hull_preds[model]-e_above_hull_true).loc[mask].abs().mean() + df_rolling_err.loc[bin_center, model] = bin_mae # drop NaNs to avoid error, scipy doesn't ignore NaNs each_std = scipy.stats.sem( - e_above_hull_errors[model].loc[mask].dropna().abs() + (e_above_hull_preds[model]-e_above_hull_true).loc[mask].dropna().abs() ) - df_err_std[model].iloc[idx] = each_std + df_err_std.loc[bin_center, model] = each_std else: print("Using pre-calculated rolling MAE") diff --git a/scripts/hist_classified_stable_vs_hull_dist.py b/scripts/hist_classified_stable_vs_hull_dist.py index b435a3fe..d32e1b82 100644 --- a/scripts/hist_classified_stable_vs_hull_dist.py +++ b/scripts/hist_classified_stable_vs_hull_dist.py @@ -1,7 +1,6 @@ """Histogram of the energy difference (either according to DFT ground truth [default] or model predicted energy) to the convex hull for materials in the WBM data set. The histogram stacks true/false positives/negatives with different colors. - See fig. S1 in https://science.org/doi/10.1126/sciadv.abn4117. """ @@ -22,7 +21,7 @@ # %% # model_name = "Wrenformer" -model_name = "CGCNN" +model_name = "CHGNet" # model_name = "CGCNN+P" which_energy: Final = "pred" df_each_pred[Key.each_true] = df_wbm[Key.each_true] @@ -41,7 +40,7 @@ if backend == "plotly": # fig.layout.title.update(text=model_name, x=0.5) fig.layout.margin.update(l=0, r=0, b=0, t=30) - # fig.update_yaxes(range=[0, 12000]) + fig.update_yaxes(range=[0, 12000]) fig.show() diff --git a/scripts/model_figs/rolling_mae_vs_hull_dist_models.py b/scripts/model_figs/rolling_mae_vs_hull_dist_models.py index a150591c..ff93ae4b 100644 --- a/scripts/model_figs/rolling_mae_vs_hull_dist_models.py +++ b/scripts/model_figs/rolling_mae_vs_hull_dist_models.py @@ -22,7 +22,7 @@ fig, df_err, df_std = rolling_mae_vs_hull_dist( e_above_hull_true=df_preds[Key.each_true], - e_above_hull_errors=df_each_pred[models], + e_above_hull_preds=df_each_pred[models], backend=backend, with_sem=False, df_rolling_err=df_err, diff --git a/scripts/model_figs/rolling_mae_vs_hull_dist_wbm_batches.py b/scripts/model_figs/rolling_mae_vs_hull_dist_wbm_batches.py index d165cc53..403cf900 100644 --- a/scripts/model_figs/rolling_mae_vs_hull_dist_wbm_batches.py +++ b/scripts/model_figs/rolling_mae_vs_hull_dist_wbm_batches.py @@ -19,6 +19,35 @@ df_err, df_std = None, None # variables to cache rolling MAE and std models = globals().get("models", all_models) +# %% plotly version +for model in models: + df_pivot = df_each_pred.pivot(columns=batch_col, values=model) + + fig, df_err, df_std = rolling_mae_vs_hull_dist( + e_above_hull_true=df_preds[Key.each_true], + e_above_hull_preds=df_pivot, + # df_rolling_err=df_err, + # df_err_std=df_std, + backend="plotly", + show_dummy_mae=False, + with_sem=False, + ) + fig.layout.legend.update( + title=f"{model}", x=0.02, y=0.02, bgcolor="rgba(0,0,0,0)" + ) + fig.layout.margin.update(l=10, r=10, b=10, t=10) + fig.layout.update(hovermode="x unified", hoverlabel_bgcolor="black") + fig.update_traces( + hovertemplate="y=%{y:.3f} eV", + selector=lambda trace: trace.name.startswith("Batch"), + ) + fig.show() + + model_snake_case = model.lower().replace(" + ", "-").replace(" ", "-") + img_path = f"rolling-mae-vs-hull-dist-wbm-batches-{model_snake_case}" + save_fig(fig, f"{SITE_FIGS}/{img_path}.svelte") + save_fig(fig, f"{PDF_FIGS}/{img_path}.pdf", width=500, height=330) + # %% matplotlib version fig, ax = plt.subplots(1, figsize=(10, 9)) @@ -29,13 +58,15 @@ for idx, marker in enumerate(markers, 1): # select all rows from WBM step=idx df_step = df_preds[df_preds.index.str.startswith(f"wbm-{idx}-")] + df_each_step = df_each_pred[df_each_pred.index.str.startswith(f"wbm-{idx}-")] title = f"Batch {idx} ({len(df_step.filter(like='e_').dropna()):,})" assert 1e4 < len(df_step) < 1e5, print(f"{len(df_step) = :,}") + assert (df_step.index == df_each_step.index).all() ax, df_err, df_std = rolling_mae_vs_hull_dist( e_above_hull_true=df_step[Key.each_true], - e_above_hull_errors={title: df_step[Key.e_form] - df_step[model]}, + e_above_hull_preds={title: df_each_step[model]}, label=title, marker=marker, markevery=20, @@ -52,34 +83,4 @@ ax.set(title=f"{today} {model}") for line in ax.lines: line._linewidth *= 3 # noqa: SLF001 - line.set_markersize(10) - - -# %% plotly version -for model in models: - df_pivot = df_each_pred.pivot(columns=batch_col, values=model) - - fig, df_err, df_std = rolling_mae_vs_hull_dist( - e_above_hull_true=df_preds[Key.each_true], - e_above_hull_errors=df_pivot, - # df_rolling_err=df_err, - # df_err_std=df_std, - backend="plotly", - show_dummy_mae=False, - with_sem=False, - ) - fig.layout.legend.update( - title=f"{model}", x=0.02, y=0.02, bgcolor="rgba(0,0,0,0)" - ) - fig.layout.margin.update(l=10, r=10, b=10, t=10) - fig.layout.update(hovermode="x unified", hoverlabel_bgcolor="black") - fig.update_traces( - hovertemplate="y=%{y:.3f} eV", - selector=lambda trace: trace.name.startswith("Batch"), - ) - fig.show() - - model_snake_case = model.lower().replace(" + ", "-").replace(" ", "-") - img_path = f"rolling-mae-vs-hull-dist-wbm-batches-{model_snake_case}" - save_fig(fig, f"{SITE_FIGS}/{img_path}.svelte") - save_fig(fig, f"{PDF_FIGS}/{img_path}.pdf", width=500, height=330) + line.set_markersize(10) \ No newline at end of file diff --git a/scripts/rolling_mae_vs_hull_dist.py b/scripts/rolling_mae_vs_hull_dist.py index 0be71713..7696bfdc 100644 --- a/scripts/rolling_mae_vs_hull_dist.py +++ b/scripts/rolling_mae_vs_hull_dist.py @@ -6,7 +6,7 @@ from matbench_discovery import PDF_FIGS, Key, today from matbench_discovery.plots import rolling_mae_vs_hull_dist -from matbench_discovery.preds import df_metrics, df_preds +from matbench_discovery.preds import df_each_pred, df_metrics, df_preds __author__ = "Rhys Goodall, Janosh Riebesell" __date__ = "2022-06-18" @@ -19,7 +19,7 @@ ax, df_err, df_std = rolling_mae_vs_hull_dist( e_above_hull_true=df_preds[Key.each_true], - e_above_hull_errors={model: df_preds[Key.e_form] - df_preds[model]}, + e_above_hull_preds={model: df_each_pred[model]}, backend=(backend := "plotly"), ) From 1b8cb3d3c4eb6de9a27188091be0f9b76bae7eec Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 7 Feb 2024 05:24:02 +0000 Subject: [PATCH 02/12] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- matbench_discovery/plots.py | 12 ++++++++++-- .../rolling_mae_vs_hull_dist_wbm_batches.py | 3 ++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/matbench_discovery/plots.py b/matbench_discovery/plots.py index 7fce2bb0..1c3fede5 100644 --- a/matbench_discovery/plots.py +++ b/matbench_discovery/plots.py @@ -347,12 +347,20 @@ def rolling_mae_vs_hull_dist( mask = (e_above_hull_true <= high) & (e_above_hull_true > low) - bin_mae = (e_above_hull_preds[model]-e_above_hull_true).loc[mask].abs().mean() + bin_mae = ( + (e_above_hull_preds[model] - e_above_hull_true) + .loc[mask] + .abs() + .mean() + ) df_rolling_err.loc[bin_center, model] = bin_mae # drop NaNs to avoid error, scipy doesn't ignore NaNs each_std = scipy.stats.sem( - (e_above_hull_preds[model]-e_above_hull_true).loc[mask].dropna().abs() + (e_above_hull_preds[model] - e_above_hull_true) + .loc[mask] + .dropna() + .abs() ) df_err_std.loc[bin_center, model] = each_std else: diff --git a/scripts/model_figs/rolling_mae_vs_hull_dist_wbm_batches.py b/scripts/model_figs/rolling_mae_vs_hull_dist_wbm_batches.py index 403cf900..795df859 100644 --- a/scripts/model_figs/rolling_mae_vs_hull_dist_wbm_batches.py +++ b/scripts/model_figs/rolling_mae_vs_hull_dist_wbm_batches.py @@ -19,6 +19,7 @@ df_err, df_std = None, None # variables to cache rolling MAE and std models = globals().get("models", all_models) + # %% plotly version for model in models: df_pivot = df_each_pred.pivot(columns=batch_col, values=model) @@ -83,4 +84,4 @@ ax.set(title=f"{today} {model}") for line in ax.lines: line._linewidth *= 3 # noqa: SLF001 - line.set_markersize(10) \ No newline at end of file + line.set_markersize(10) From e1c0b332c7e1bd1a928bf8eef603f205a2c1b779 Mon Sep 17 00:00:00 2001 From: CompRhys Date: Tue, 6 Feb 2024 21:32:58 -0800 Subject: [PATCH 03/12] fix: fast pivot by dropping nan --- matbench_discovery/plots.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/matbench_discovery/plots.py b/matbench_discovery/plots.py index 7fce2bb0..1209e390 100644 --- a/matbench_discovery/plots.py +++ b/matbench_discovery/plots.py @@ -341,18 +341,21 @@ def rolling_mae_vs_hull_dist( prog_bar := tqdm(models, desc="Calculating rolling MAE", disable=not pbar) ): prog_bar.set_postfix_str(model) + e_above_hull_pred = e_above_hull_preds[model].dropna() + e_above_hull_ref = e_above_hull_true.loc[e_above_hull_pred.index] + for bin_center in bins: low = bin_center - window high = bin_center + window - mask = (e_above_hull_true <= high) & (e_above_hull_true > low) + mask = (e_above_hull_ref <= high) & (e_above_hull_ref > low) - bin_mae = (e_above_hull_preds[model]-e_above_hull_true).loc[mask].abs().mean() + bin_mae = (e_above_hull_pred-e_above_hull_ref).loc[mask].abs().mean() df_rolling_err.loc[bin_center, model] = bin_mae # drop NaNs to avoid error, scipy doesn't ignore NaNs each_std = scipy.stats.sem( - (e_above_hull_preds[model]-e_above_hull_true).loc[mask].dropna().abs() + (e_above_hull_pred-e_above_hull_ref).loc[mask].dropna().abs() ) df_err_std.loc[bin_center, model] = each_std else: From f3533ec1b186e685bd9af20f387447d506515662 Mon Sep 17 00:00:00 2001 From: CompRhys Date: Tue, 6 Feb 2024 22:19:43 -0800 Subject: [PATCH 04/12] patch: rough solution to recreating plots for unique proto set --- pyproject.toml | 1 + scripts/model_figs/analyze_model_disagreement.py | 8 +++++++- scripts/model_figs/cumulative_metrics.py | 6 ++++++ .../hist_classified_stable_vs_hull_dist_models.py | 8 +++++++- scripts/model_figs/make_hull_dist_box_plot.py | 10 ++++++++-- scripts/model_figs/parity_energy_models.py | 8 +++++++- scripts/model_figs/per_element_errors.py | 13 ++++++++++++- scripts/model_figs/roc_prc_curves_models.py | 5 +++++ .../model_figs/rolling_mae_vs_hull_dist_models.py | 9 ++++++++- .../rolling_mae_vs_hull_dist_wbm_batches.py | 5 +++++ scripts/rolling_mae_vs_hull_dist.py | 9 ++++++++- 11 files changed, 74 insertions(+), 8 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c06a0768..6a15952c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,6 +37,7 @@ dependencies = [ "pymatviz[export-figs,df-pdf-export]", "scikit-learn", "scipy", + "seaborn", "tqdm", "wandb", ] diff --git a/scripts/model_figs/analyze_model_disagreement.py b/scripts/model_figs/analyze_model_disagreement.py index 4ad2bd4a..cfd9f812 100644 --- a/scripts/model_figs/analyze_model_disagreement.py +++ b/scripts/model_figs/analyze_model_disagreement.py @@ -7,7 +7,6 @@ import sys import pandas as pd -from crystal_toolkit.helpers.utils import hook_up_fig_with_struct_viewer from pymatviz.io import save_fig from pymatviz.utils import add_identity_line @@ -18,6 +17,11 @@ __author__ = "Janosh Riebesell" __date__ = "2023-02-15" +use_unique_proto = True + +if use_unique_proto: + df_preds = df_preds.query(Key.uniq_proto) + # %% scatter plot of largest model errors vs. DFT hull distance # while some points lie on a horizontal line of constant error, more follow the identity @@ -87,6 +91,8 @@ # %% struct viewer +from crystal_toolkit.helpers.utils import hook_up_fig_with_struct_viewer + # only run this in Jupyter Notebook is_jupyter = "ipykernel" in sys.modules if is_jupyter: diff --git a/scripts/model_figs/cumulative_metrics.py b/scripts/model_figs/cumulative_metrics.py index 890ff631..3b70a59f 100644 --- a/scripts/model_figs/cumulative_metrics.py +++ b/scripts/model_figs/cumulative_metrics.py @@ -19,6 +19,12 @@ __author__ = "Janosh Riebesell, Rhys Goodall" __date__ = "2022-12-04" +use_unique_proto = True + +if use_unique_proto: + df_preds = df_preds.query(Key.uniq_proto) + df_each_pred = df_each_pred.loc[df_preds.index] + # %% metrics: tuple[str, ...] = globals().get("metrics", ("Precision", "Recall")) diff --git a/scripts/model_figs/hist_classified_stable_vs_hull_dist_models.py b/scripts/model_figs/hist_classified_stable_vs_hull_dist_models.py index 8e2d8445..9752c434 100644 --- a/scripts/model_figs/hist_classified_stable_vs_hull_dist_models.py +++ b/scripts/model_figs/hist_classified_stable_vs_hull_dist_models.py @@ -12,12 +12,18 @@ from matbench_discovery import PDF_FIGS, SITE_FIGS, Key, today from matbench_discovery.plots import hist_classified_stable_vs_hull_dist, plt -from matbench_discovery.preds import df_metrics, df_preds +from matbench_discovery.preds import df_metrics, df_preds, df_metrics_uniq_protos __author__ = "Janosh Riebesell" __date__ = "2022-12-01" +use_unique_proto = True + +if use_unique_proto: + df_preds = df_preds.query(Key.uniq_proto) + df_metrics = df_metrics_uniq_protos + # %% hover_cols = ( df_preds.index.name, diff --git a/scripts/model_figs/make_hull_dist_box_plot.py b/scripts/model_figs/make_hull_dist_box_plot.py index be662e95..5a47a1f7 100644 --- a/scripts/model_figs/make_hull_dist_box_plot.py +++ b/scripts/model_figs/make_hull_dist_box_plot.py @@ -4,13 +4,19 @@ import seaborn as sns from pymatviz.io import save_fig -from matbench_discovery import PDF_FIGS, SITE_FIGS, Quantity -from matbench_discovery.preds import df_each_err, models +from matbench_discovery import PDF_FIGS, SITE_FIGS, Quantity, Key +from matbench_discovery.preds import df_each_err, models, df_preds __author__ = "Janosh Riebesell" __date__ = "2023-05-25" +use_unique_proto = True + +if use_unique_proto: + df_preds = df_preds.query(Key.uniq_proto) + df_each_err = df_each_err.loc[df_preds.index] + # %% ax = df_each_err[models].plot.box( showfliers=False, diff --git a/scripts/model_figs/parity_energy_models.py b/scripts/model_figs/parity_energy_models.py index b9321584..b1dda871 100644 --- a/scripts/model_figs/parity_energy_models.py +++ b/scripts/model_figs/parity_energy_models.py @@ -15,7 +15,7 @@ from matbench_discovery import PDF_FIGS, SITE_FIGS, Key from matbench_discovery.plots import clf_colors -from matbench_discovery.preds import df_metrics, df_preds +from matbench_discovery.preds import df_metrics, df_preds, df_metrics_uniq_protos __author__ = "Janosh Riebesell" __date__ = "2022-11-28" @@ -31,6 +31,12 @@ e_true_col = Key.e_form e_pred_col = Key.e_form_pred +use_unique_proto = True + +if use_unique_proto: + df_preds = df_preds.query(Key.uniq_proto) + df_metrics = df_metrics_uniq_protos + # %% facet_col = "Model" diff --git a/scripts/model_figs/per_element_errors.py b/scripts/model_figs/per_element_errors.py index 1d4fdabf..f5efd30d 100644 --- a/scripts/model_figs/per_element_errors.py +++ b/scripts/model_figs/per_element_errors.py @@ -15,11 +15,22 @@ from matbench_discovery import PDF_FIGS, ROOT, SITE_FIGS, SITE_LIB, Key from matbench_discovery.data import df_wbm -from matbench_discovery.preds import df_each_err, df_metrics, df_preds +from matbench_discovery.preds import df_each_err, df_metrics, df_preds, df_metrics_uniq_protos __author__ = "Janosh Riebesell" __date__ = "2023-02-15" + +# %% +use_unique_proto = True + +if use_unique_proto: + df_preds = df_preds.query(Key.uniq_proto) + df_each_err = df_each_err.loc[df_preds.index] + df_metrics = df_metrics_uniq_protos + df_wbm = df_wbm.loc[df_preds.index] + + for df in (df_each_err, df_preds): df[Key.model_mean_err] = df_each_err.abs().mean(axis=1) diff --git a/scripts/model_figs/roc_prc_curves_models.py b/scripts/model_figs/roc_prc_curves_models.py index 78371910..6dbaddad 100644 --- a/scripts/model_figs/roc_prc_curves_models.py +++ b/scripts/model_figs/roc_prc_curves_models.py @@ -23,6 +23,11 @@ n_cols = 3 n_rows = math.ceil(len(models) / n_cols) +use_unique_proto = True + +if use_unique_proto: + df_preds = df_preds.query(Key.uniq_proto) + df_each_pred = df_each_pred.loc[df_preds.index] # %% df_roc = pd.DataFrame() diff --git a/scripts/model_figs/rolling_mae_vs_hull_dist_models.py b/scripts/model_figs/rolling_mae_vs_hull_dist_models.py index ff93ae4b..5c434f4d 100644 --- a/scripts/model_figs/rolling_mae_vs_hull_dist_models.py +++ b/scripts/model_figs/rolling_mae_vs_hull_dist_models.py @@ -9,7 +9,7 @@ from matbench_discovery import PDF_FIGS, SITE_FIGS, Key from matbench_discovery.plots import rolling_mae_vs_hull_dist -from matbench_discovery.preds import df_each_pred, df_metrics, df_preds, models +from matbench_discovery.preds import df_each_pred, df_metrics, df_preds, models, df_metrics_uniq_protos __author__ = "Rhys Goodall, Janosh Riebesell" __date__ = "2022-06-18" @@ -19,6 +19,13 @@ # %% backend: Final = "plotly" +use_unique_proto = True + +if use_unique_proto: + df_preds = df_preds.query(Key.uniq_proto) + df_each_pred = df_each_pred.loc[df_preds.index] + df_metrics = df_metrics_uniq_protos + fig, df_err, df_std = rolling_mae_vs_hull_dist( e_above_hull_true=df_preds[Key.each_true], diff --git a/scripts/model_figs/rolling_mae_vs_hull_dist_wbm_batches.py b/scripts/model_figs/rolling_mae_vs_hull_dist_wbm_batches.py index 795df859..9b252dbc 100644 --- a/scripts/model_figs/rolling_mae_vs_hull_dist_wbm_batches.py +++ b/scripts/model_figs/rolling_mae_vs_hull_dist_wbm_batches.py @@ -19,6 +19,11 @@ df_err, df_std = None, None # variables to cache rolling MAE and std models = globals().get("models", all_models) +use_unique_proto = True + +if use_unique_proto: + df_preds = df_preds.query(Key.uniq_proto) + df_each_pred = df_each_pred.loc[df_preds.index] # %% plotly version for model in models: diff --git a/scripts/rolling_mae_vs_hull_dist.py b/scripts/rolling_mae_vs_hull_dist.py index 7696bfdc..2256a759 100644 --- a/scripts/rolling_mae_vs_hull_dist.py +++ b/scripts/rolling_mae_vs_hull_dist.py @@ -6,13 +6,20 @@ from matbench_discovery import PDF_FIGS, Key, today from matbench_discovery.plots import rolling_mae_vs_hull_dist -from matbench_discovery.preds import df_each_pred, df_metrics, df_preds +from matbench_discovery.preds import df_each_pred, df_metrics, df_preds, df_metrics_uniq_protos __author__ = "Rhys Goodall, Janosh Riebesell" __date__ = "2022-06-18" # %% +use_unique_proto = True + +if use_unique_proto: + df_preds = df_preds.query(Key.uniq_proto) + df_each_pred = df_each_pred.loc[df_preds.index] + df_metrics = df_metrics_uniq_protos + model = "Wrenformer" model = "MEGNet" model = "CHGNet" From c2c2921c9b5cd6f06f0e765546dad17a1c0202a9 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 7 Feb 2024 06:22:48 +0000 Subject: [PATCH 05/12] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../hist_classified_stable_vs_hull_dist_models.py | 3 ++- scripts/model_figs/make_hull_dist_box_plot.py | 5 +++-- scripts/model_figs/parity_energy_models.py | 2 +- scripts/model_figs/per_element_errors.py | 7 ++++++- scripts/model_figs/roc_prc_curves_models.py | 1 + scripts/model_figs/rolling_mae_vs_hull_dist_models.py | 8 +++++++- .../model_figs/rolling_mae_vs_hull_dist_wbm_batches.py | 1 + scripts/rolling_mae_vs_hull_dist.py | 7 ++++++- 8 files changed, 27 insertions(+), 7 deletions(-) diff --git a/scripts/model_figs/hist_classified_stable_vs_hull_dist_models.py b/scripts/model_figs/hist_classified_stable_vs_hull_dist_models.py index 9752c434..d917cfc9 100644 --- a/scripts/model_figs/hist_classified_stable_vs_hull_dist_models.py +++ b/scripts/model_figs/hist_classified_stable_vs_hull_dist_models.py @@ -12,7 +12,7 @@ from matbench_discovery import PDF_FIGS, SITE_FIGS, Key, today from matbench_discovery.plots import hist_classified_stable_vs_hull_dist, plt -from matbench_discovery.preds import df_metrics, df_preds, df_metrics_uniq_protos +from matbench_discovery.preds import df_metrics, df_metrics_uniq_protos, df_preds __author__ = "Janosh Riebesell" __date__ = "2022-12-01" @@ -24,6 +24,7 @@ df_preds = df_preds.query(Key.uniq_proto) df_metrics = df_metrics_uniq_protos + # %% hover_cols = ( df_preds.index.name, diff --git a/scripts/model_figs/make_hull_dist_box_plot.py b/scripts/model_figs/make_hull_dist_box_plot.py index 5a47a1f7..7bf84278 100644 --- a/scripts/model_figs/make_hull_dist_box_plot.py +++ b/scripts/model_figs/make_hull_dist_box_plot.py @@ -4,8 +4,8 @@ import seaborn as sns from pymatviz.io import save_fig -from matbench_discovery import PDF_FIGS, SITE_FIGS, Quantity, Key -from matbench_discovery.preds import df_each_err, models, df_preds +from matbench_discovery import PDF_FIGS, SITE_FIGS, Key, Quantity +from matbench_discovery.preds import df_each_err, df_preds, models __author__ = "Janosh Riebesell" __date__ = "2023-05-25" @@ -17,6 +17,7 @@ df_preds = df_preds.query(Key.uniq_proto) df_each_err = df_each_err.loc[df_preds.index] + # %% ax = df_each_err[models].plot.box( showfliers=False, diff --git a/scripts/model_figs/parity_energy_models.py b/scripts/model_figs/parity_energy_models.py index b1dda871..6414d8fd 100644 --- a/scripts/model_figs/parity_energy_models.py +++ b/scripts/model_figs/parity_energy_models.py @@ -15,7 +15,7 @@ from matbench_discovery import PDF_FIGS, SITE_FIGS, Key from matbench_discovery.plots import clf_colors -from matbench_discovery.preds import df_metrics, df_preds, df_metrics_uniq_protos +from matbench_discovery.preds import df_metrics, df_metrics_uniq_protos, df_preds __author__ = "Janosh Riebesell" __date__ = "2022-11-28" diff --git a/scripts/model_figs/per_element_errors.py b/scripts/model_figs/per_element_errors.py index f5efd30d..a2ee5757 100644 --- a/scripts/model_figs/per_element_errors.py +++ b/scripts/model_figs/per_element_errors.py @@ -15,7 +15,12 @@ from matbench_discovery import PDF_FIGS, ROOT, SITE_FIGS, SITE_LIB, Key from matbench_discovery.data import df_wbm -from matbench_discovery.preds import df_each_err, df_metrics, df_preds, df_metrics_uniq_protos +from matbench_discovery.preds import ( + df_each_err, + df_metrics, + df_metrics_uniq_protos, + df_preds, +) __author__ = "Janosh Riebesell" __date__ = "2023-02-15" diff --git a/scripts/model_figs/roc_prc_curves_models.py b/scripts/model_figs/roc_prc_curves_models.py index 6dbaddad..cbc59775 100644 --- a/scripts/model_figs/roc_prc_curves_models.py +++ b/scripts/model_figs/roc_prc_curves_models.py @@ -29,6 +29,7 @@ df_preds = df_preds.query(Key.uniq_proto) df_each_pred = df_each_pred.loc[df_preds.index] + # %% df_roc = pd.DataFrame() diff --git a/scripts/model_figs/rolling_mae_vs_hull_dist_models.py b/scripts/model_figs/rolling_mae_vs_hull_dist_models.py index 5c434f4d..3934e09a 100644 --- a/scripts/model_figs/rolling_mae_vs_hull_dist_models.py +++ b/scripts/model_figs/rolling_mae_vs_hull_dist_models.py @@ -9,7 +9,13 @@ from matbench_discovery import PDF_FIGS, SITE_FIGS, Key from matbench_discovery.plots import rolling_mae_vs_hull_dist -from matbench_discovery.preds import df_each_pred, df_metrics, df_preds, models, df_metrics_uniq_protos +from matbench_discovery.preds import ( + df_each_pred, + df_metrics, + df_metrics_uniq_protos, + df_preds, + models, +) __author__ = "Rhys Goodall, Janosh Riebesell" __date__ = "2022-06-18" diff --git a/scripts/model_figs/rolling_mae_vs_hull_dist_wbm_batches.py b/scripts/model_figs/rolling_mae_vs_hull_dist_wbm_batches.py index 9b252dbc..d4b0f0a0 100644 --- a/scripts/model_figs/rolling_mae_vs_hull_dist_wbm_batches.py +++ b/scripts/model_figs/rolling_mae_vs_hull_dist_wbm_batches.py @@ -25,6 +25,7 @@ df_preds = df_preds.query(Key.uniq_proto) df_each_pred = df_each_pred.loc[df_preds.index] + # %% plotly version for model in models: df_pivot = df_each_pred.pivot(columns=batch_col, values=model) diff --git a/scripts/rolling_mae_vs_hull_dist.py b/scripts/rolling_mae_vs_hull_dist.py index 2256a759..4a0627de 100644 --- a/scripts/rolling_mae_vs_hull_dist.py +++ b/scripts/rolling_mae_vs_hull_dist.py @@ -6,7 +6,12 @@ from matbench_discovery import PDF_FIGS, Key, today from matbench_discovery.plots import rolling_mae_vs_hull_dist -from matbench_discovery.preds import df_each_pred, df_metrics, df_preds, df_metrics_uniq_protos +from matbench_discovery.preds import ( + df_each_pred, + df_metrics, + df_metrics_uniq_protos, + df_preds, +) __author__ = "Rhys Goodall, Janosh Riebesell" __date__ = "2022-06-18" From a8712fde7e7440c0c3c7787a25793f8f0fb29109 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 7 Feb 2024 09:48:32 +0000 Subject: [PATCH 06/12] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/model_figs/rolling_mae_vs_hull_dist_wbm_batches.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/model_figs/rolling_mae_vs_hull_dist_wbm_batches.py b/scripts/model_figs/rolling_mae_vs_hull_dist_wbm_batches.py index de1029e5..4cdafc9b 100644 --- a/scripts/model_figs/rolling_mae_vs_hull_dist_wbm_batches.py +++ b/scripts/model_figs/rolling_mae_vs_hull_dist_wbm_batches.py @@ -21,6 +21,7 @@ use_unique_proto = True + # %% plotly version for model in models: df_pivot = df_each_pred.pivot(columns=batch_col, values=model) From bc4550908cf21cd66b1c5385c29fd9a39e34a566 Mon Sep 17 00:00:00 2001 From: Janosh Riebesell Date: Wed, 7 Feb 2024 10:58:15 +0100 Subject: [PATCH 07/12] fix ruff E402 --- scripts/model_figs/analyze_model_disagreement.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/scripts/model_figs/analyze_model_disagreement.py b/scripts/model_figs/analyze_model_disagreement.py index cfd9f812..21ca1a04 100644 --- a/scripts/model_figs/analyze_model_disagreement.py +++ b/scripts/model_figs/analyze_model_disagreement.py @@ -90,12 +90,11 @@ df_cse = pd.read_json(DATA_FILES.wbm_cses_plus_init_structs).set_index(Key.mat_id) -# %% struct viewer -from crystal_toolkit.helpers.utils import hook_up_fig_with_struct_viewer - -# only run this in Jupyter Notebook +# %% CTK structure viewer is_jupyter = "ipykernel" in sys.modules -if is_jupyter: +if is_jupyter: # only run this in Jupyter Notebook + from crystal_toolkit.helpers.utils import hook_up_fig_with_struct_viewer + app = hook_up_fig_with_struct_viewer( fig, df_cse, From 7a437373d876e088528c7a0e2b74af0315d59468 Mon Sep 17 00:00:00 2001 From: Janosh Riebesell Date: Wed, 7 Feb 2024 11:22:11 +0100 Subject: [PATCH 08/12] fix empty train_size_col in make_metrics_tables.py --- matbench_discovery/preds.py | 2 +- ...t_classified_stable_vs_hull_dist_models.py | 12 ++-- scripts/model_figs/make_metrics_tables.py | 68 +++++++++---------- .../rolling_mae_vs_hull_dist_models.py | 4 +- 4 files changed, 39 insertions(+), 47 deletions(-) diff --git a/matbench_discovery/preds.py b/matbench_discovery/preds.py index 5e4c2d6e..486ba12e 100644 --- a/matbench_discovery/preds.py +++ b/matbench_discovery/preds.py @@ -61,7 +61,7 @@ class PredFiles(Files): # alignn_pretrained = "alignn/2023-06-03-mp-e-form-alignn-wbm-IS2RE.csv.gz" # alignn_ff = "alignn_ff/2023-07-11-alignn-ff-wbm-IS2RE.csv.gz" - gnome = "gnome/2023-11-01-gnome-preds-50076332.csv.gz" + # gnome = "gnome/2023-11-01-gnome-preds-50076332.csv.gz" # key_map maps model keys to pretty labels diff --git a/scripts/model_figs/hist_classified_stable_vs_hull_dist_models.py b/scripts/model_figs/hist_classified_stable_vs_hull_dist_models.py index d917cfc9..5bc23f59 100644 --- a/scripts/model_figs/hist_classified_stable_vs_hull_dist_models.py +++ b/scripts/model_figs/hist_classified_stable_vs_hull_dist_models.py @@ -26,15 +26,11 @@ # %% -hover_cols = ( - df_preds.index.name, - Key.e_form, - Key.each_true, - Key.formula, -) +hover_cols = (df_preds.index.name, Key.e_form, Key.each_true, Key.formula) facet_col = "Model" -# sort facet plots by model's F1 scores (optionally only show top n=6) -models = list(df_metrics.T.F1.sort_values().index)[::-1] +# sort models by F1 scores so that facet plots are ordered by model performance +# (optionally only show top n=6) +models = list(df_preds.filter([*df_metrics.sort_values("F1", axis=1)]))[::-1] df_melt = df_preds.melt( id_vars=hover_cols, diff --git a/scripts/model_figs/make_metrics_tables.py b/scripts/model_figs/make_metrics_tables.py index df6ec1c2..00cfc2eb 100644 --- a/scripts/model_figs/make_metrics_tables.py +++ b/scripts/model_figs/make_metrics_tables.py @@ -35,33 +35,33 @@ "M3GNet→MEGNet": "M3GNet", "CHGNet→MEGNet": "CHGNet", } +df_met = df_metrics_uniq_protos +df_met.loc[Key.train_size.label] = "" + for model in df_metrics: model_name = name_map.get(model, model) if not (model_data := MODEL_METADATA.get(model_name)): continue n_structs = model_data["training_set"]["n_structures"] - n_structs_str = si_fmt(n_structs) + train_size_str = si_fmt(n_structs) if n_materials := model_data["training_set"].get("n_materials"): - n_structs_str += f" ({si_fmt(n_materials)})" - - for df_m in (df_metrics, df_metrics_10k, df_metrics_uniq_protos): - if Key.train_size not in df_m.index: - df_m.loc[Key.train_size.label] = "" - df_m.loc[Key.train_size.label, model] = n_structs_str - model_params = model_data.get(Key.model_params) - df_m.loc[Key.model_params.label, model] = ( - si_fmt(model_params) if isinstance(model_params, int) else model_params - ) - for key in ( - Key.openness, - Key.model_type, - Key.train_task, - Key.test_task, - Key.targets, - ): - default = {Key.openness: Open.OSOD}.get(key, pd.NA) - df_m.loc[key.label, model] = model_data.get(key, default) + train_size_str += f" ({si_fmt(n_materials)})" + + df_met.loc[Key.train_size.label, model] = train_size_str + model_params = model_data.get(Key.model_params) + df_met.loc[Key.model_params.label, model] = ( + si_fmt(model_params) if isinstance(model_params, int) else model_params + ) + for key in ( + Key.openness, + Key.model_type, + Key.train_task, + Key.test_task, + Key.targets, + ): + default = {Key.openness: Open.OSOD}.get(key, pd.NA) + df_met.loc[key.label, model] = model_data.get(key, default) # %% add dummy classifier results to df_metrics(_10k, _uniq_protos) @@ -112,18 +112,18 @@ Key.model_type.label, Key.targets.label, ] -show_cols = [ - *f"F1,DAF,Precision,Accuracy,TPR,TNR,MAE,RMSE,{R2_col}".split(","), - *meta_cols, -] +show_cols = [*f"F1,DAF,Prec,Acc,TPR,TNR,MAE,RMSE,{R2_col}".split(","), *meta_cols] for label, df in ( ("", df_metrics), ("-uniq-protos", df_metrics_uniq_protos), ("-first-10k", df_metrics_10k), ): - df_table = df.rename(index={"R2": R2_col}) - df_table.index.name = "Model" + # abbreviate long column names + df = df.rename(index={"R2": R2_col, "Precision": "Prec", "Accuracy": "Acc"}) + df.index.name = "Model" + # only keep columns we want to show + df_table = df.T.filter(show_cols) if make_uip_megnet_comparison: df_table = df_table.filter(regex="MEGNet|CHGNet|M3GNet") # |Dummy @@ -133,29 +133,25 @@ "hint: for make_uip_megnet_comparison, uncomment the lines " "chgnet_megnet and m3gnet_megnet in PredFiles" ) - df_filtered = df_table.T[show_cols] # only keep columns we want to show - - # abbreviate long column names - df_filtered = df_filtered.rename(columns={"Precision": "Prec", "Accuracy": "Acc"}) if "-first-10k" in label: # hide redundant metrics for first 10k preds (all TPR = 1, TNR = 0) - df_filtered = df_filtered.drop(["TPR", "TNR"], axis="columns") + df_table = df_table.drop(["TPR", "TNR"], axis="columns") if label != "-uniq-protos": # only show training size and model type once - df_filtered = df_filtered.drop(meta_cols, axis="columns") + df_table = df_table.drop(meta_cols, axis="columns", errors="ignore") styler = ( - df_filtered.style.format( + df_table.style.format( # render integers without decimal places dict.fromkeys("TP FN FP TN".split(), "{:,.0f}"), precision=2, # render floats with 2 decimals na_rep="", # render NaNs as empty string ) .background_gradient( - cmap="viridis", subset=list(higher_is_better & {*df_filtered}) + cmap="viridis", subset=list(higher_is_better & {*df_table}) ) .background_gradient( # reverse color map if lower=better - cmap="viridis_r", subset=list(lower_is_better & {*df_filtered}) + cmap="viridis_r", subset=list(lower_is_better & {*df_table}) ) ) # add up/down arrows to indicate which metrics are better when higher/lower @@ -163,7 +159,7 @@ lower_is_better, " ↓" ) styler.relabel_index( - [f"{col}{arrow_suffix.get(col, '')}" for col in df_filtered], + [f"{col}{arrow_suffix.get(col, '')}" for col in df_table], axis="columns", ) diff --git a/scripts/model_figs/rolling_mae_vs_hull_dist_models.py b/scripts/model_figs/rolling_mae_vs_hull_dist_models.py index a0fc7187..0533670d 100644 --- a/scripts/model_figs/rolling_mae_vs_hull_dist_models.py +++ b/scripts/model_figs/rolling_mae_vs_hull_dist_models.py @@ -7,7 +7,7 @@ import numpy as np from pymatviz.io import save_fig -from matbench_discovery import PDF_FIGS, SITE_FIGS, Key, Model +from matbench_discovery import PDF_FIGS, SITE_FIGS, Key from matbench_discovery.plots import rolling_mae_vs_hull_dist from matbench_discovery.preds import ( df_each_pred, @@ -35,7 +35,7 @@ fig, df_err, df_std = rolling_mae_vs_hull_dist( e_above_hull_true=df_preds[Key.each_true], - e_above_hull_preds=df_each_pred[models].drop(columns=Model.gnome), + e_above_hull_preds=df_each_pred[models], backend=backend, with_sem=False, df_rolling_err=df_err, From 3cacd9fd07b37b9a12365993bbf459c7578ebea4 Mon Sep 17 00:00:00 2001 From: Janosh Riebesell Date: Wed, 7 Feb 2024 12:19:09 +0100 Subject: [PATCH 09/12] add/update run times to model-stats*.json --- .pre-commit-config.yaml | 2 +- matbench_discovery/plots.py | 2 +- scripts/model_figs/compile_model_stats.py | 4 +- scripts/rolling_mae_vs_hull_dist.py | 2 +- site/src/lib/model-stats-10k.json | 322 +++++++++++++++++++++- site/src/lib/model-stats-uniq-protos.json | 322 +++++++++++++++++++++- site/src/lib/model-stats.json | 322 +++++++++++++++++++++- site/src/routes/+page.svelte | 4 +- 8 files changed, 970 insertions(+), 10 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e026455a..97af8d4b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -53,7 +53,7 @@ repos: - prettier - prettier-plugin-svelte - svelte - exclude: ^(site/src/figs/.+\.svelte|data/wbm/20.+\..+|site/src/.+\.(yaml|json)|changelog.md)$ + exclude: ^(site/src/figs/.+\.svelte|data/wbm/20.+\..+|site/src/routes.+\.(yaml|json)|changelog.md)$ - repo: https://github.com/pre-commit/mirrors-eslint rev: v9.0.0-alpha.2 diff --git a/matbench_discovery/plots.py b/matbench_discovery/plots.py index a3dfdc5a..08461e88 100644 --- a/matbench_discovery/plots.py +++ b/matbench_discovery/plots.py @@ -542,7 +542,7 @@ def rolling_mae_vs_hull_dist( xanchor="right", xref="x", ) - fig.add_shape(type="rect", x0=x0, y0=y0, x1=x0 - window, y1=y0 + window / 5) + fig.add_shape(type="rect", x0=x0, y0=y0, x1=x0 - window, y1=y0 + 0.006) from matbench_discovery.preds import model_styles diff --git a/scripts/model_figs/compile_model_stats.py b/scripts/model_figs/compile_model_stats.py index cb56badd..b2805009 100644 --- a/scripts/model_figs/compile_model_stats.py +++ b/scripts/model_figs/compile_model_stats.py @@ -26,6 +26,7 @@ df_preds, model_styles, ) +from matbench_discovery.preds import models as all_models __author__ = "Janosh Riebesell" __date__ = "2022-11-28" @@ -128,8 +129,7 @@ df_tmp[time_col] = df_tmp.filter(like=time_col).sum(axis="columns") # write model metrics to json for website use - in_both = [*set(df_metrics) & set(df_preds)] - df_tmp["missing_preds"] = df_preds[in_both].isna().sum() + df_tmp["missing_preds"] = df_preds[all_models].isna().sum() df_tmp["missing_percent"] = [ f"{x / len(df_preds):.2%}" for x in df_tmp.missing_preds ] diff --git a/scripts/rolling_mae_vs_hull_dist.py b/scripts/rolling_mae_vs_hull_dist.py index 43ba0f53..4ba7c289 100644 --- a/scripts/rolling_mae_vs_hull_dist.py +++ b/scripts/rolling_mae_vs_hull_dist.py @@ -36,6 +36,6 @@ # %% -img_name = "rolling-mae-vs-hull-dist" +img_name = f"rolling-mae-vs-hull-dist-{model}" save_fig(fig, f"{PDF_FIGS}/{img_name}.pdf") save_fig(fig, f"{SITE_FIGS}/{img_name}.svelte") diff --git a/site/src/lib/model-stats-10k.json b/site/src/lib/model-stats-10k.json index b4dedc6a..361fd26e 100644 --- a/site/src/lib/model-stats-10k.json +++ b/site/src/lib/model-stats-10k.json @@ -1 +1,321 @@ -{"GNoME":{"F1":0.97,"DAF":5.62,"Precision":0.94,"Recall":1.0,"Accuracy":0.94,"TPR":1.0,"FPR":1.0,"TNR":0.0,"FNR":0.0,"TP":9371.0,"FP":629.0,"TN":0.0,"FN":0.0,"MAE":0.04,"RMSE":0.09,"R2":0.86,"Train Run Time (h)":null,"Train GPU":null,"Train CPU":null,"Train Slurm Jobs":null,"Test Run Time (h)":null,"Test GPU":null,"Test CPU":null,"Test Slurm Jobs":null,"Run Time (h)":0.0,"missing_preds":1742,"missing_percent":"0.68%"},"CHGNet":{"F1":0.94,"DAF":5.29,"Precision":0.88,"Recall":1.0,"Accuracy":0.88,"TPR":1.0,"FPR":1.0,"TNR":0.0,"FNR":0.0,"TP":8817.0,"FP":1183.0,"TN":0.0,"FN":0.0,"MAE":0.06,"RMSE":0.11,"R2":0.81,"Train Run Time (h)":null,"Train GPU":null,"Train CPU":null,"Train Slurm Jobs":null,"Test Run Time (h)":142.48,"Test GPU":1.0,"Test CPU":128.0,"Test Slurm Jobs":100.0,"Run Time (h)":142.48,"missing_preds":0,"missing_percent":"0.00%"},"MACE":{"F1":0.9,"DAF":4.88,"Precision":0.81,"Recall":1.0,"Accuracy":0.81,"TPR":1.0,"FPR":1.0,"TNR":0.0,"FNR":0.0,"TP":8125.0,"FP":1875.0,"TN":0.0,"FN":0.0,"MAE":0.09,"RMSE":0.23,"R2":0.22,"Train Run Time (h)":null,"Train GPU":null,"Train CPU":null,"Train Slurm Jobs":null,"Test Run Time (h)":null,"Test GPU":null,"Test CPU":null,"Test Slurm Jobs":null,"Run Time (h)":0.0,"missing_preds":7735,"missing_percent":"3.01%"},"M3GNet":{"F1":0.89,"DAF":4.79,"Precision":0.8,"Recall":1.0,"Accuracy":0.8,"TPR":1.0,"FPR":1.0,"TNR":0.0,"FNR":0.0,"TP":7980.0,"FP":2020.0,"TN":0.0,"FN":0.0,"MAE":0.1,"RMSE":0.16,"R2":0.62,"Train Run Time (h)":null,"Train GPU":null,"Train CPU":null,"Train Slurm Jobs":null,"Test Run Time (h)":null,"Test GPU":null,"Test CPU":null,"Test Slurm Jobs":null,"Run Time (h)":0.0,"missing_preds":353,"missing_percent":"0.14%"},"CGCNN":{"F1":0.76,"DAF":3.66,"Precision":0.61,"Recall":1.0,"Accuracy":0.61,"TPR":1.0,"FPR":1.0,"TNR":0.0,"FNR":0.0,"TP":6102.0,"FP":3898.0,"TN":0.0,"FN":0.0,"MAE":0.17,"RMSE":0.24,"R2":0.27,"Train Run Time (h)":11.35,"Train GPU":1.0,"Train CPU":128.0,"Train Slurm Jobs":10.0,"Test Run Time (h)":0.17,"Test GPU":1.0,"Test CPU":128.0,"Test Slurm Jobs":1.0,"Run Time (h)":11.52,"missing_preds":2,"missing_percent":"0.00%"},"ALIGNN":{"F1":0.75,"DAF":3.58,"Precision":0.6,"Recall":1.0,"Accuracy":0.6,"TPR":1.0,"FPR":1.0,"TNR":0.0,"FNR":0.0,"TP":5965.0,"FP":4035.0,"TN":0.0,"FN":0.0,"MAE":0.19,"RMSE":0.26,"R2":0.14,"Train Run Time (h)":null,"Train GPU":null,"Train CPU":null,"Train Slurm Jobs":null,"Test Run Time (h)":null,"Test GPU":null,"Test CPU":null,"Test Slurm Jobs":null,"Run Time (h)":0.0,"missing_preds":0,"missing_percent":"0.00%"},"CGCNN+P":{"F1":0.74,"DAF":3.53,"Precision":0.59,"Recall":1.0,"Accuracy":0.59,"TPR":1.0,"FPR":1.0,"TNR":0.0,"FNR":0.0,"TP":5887.0,"FP":4113.0,"TN":0.0,"FN":0.0,"MAE":0.21,"RMSE":0.29,"R2":0.03,"Train Run Time (h)":null,"Train GPU":null,"Train CPU":null,"Train Slurm Jobs":null,"Test Run Time (h)":null,"Test GPU":null,"Test CPU":null,"Test Slurm Jobs":null,"Run Time (h)":0.0,"missing_preds":2,"missing_percent":"0.00%"},"Wrenformer":{"F1":0.74,"DAF":3.52,"Precision":0.59,"Recall":1.0,"Accuracy":0.59,"TPR":1.0,"FPR":1.0,"TNR":0.0,"FNR":0.0,"TP":5873.0,"FP":4127.0,"TN":0.0,"FN":0.0,"MAE":0.19,"RMSE":0.25,"R2":0.22,"Train Run Time (h)":57.89,"Train GPU":1.0,"Train CPU":128.0,"Train Slurm Jobs":10.0,"Test Run Time (h)":null,"Test GPU":null,"Test CPU":null,"Test Slurm Jobs":null,"Run Time (h)":57.89,"missing_preds":0,"missing_percent":"0.00%"},"BOWSR":{"F1":0.69,"DAF":3.14,"Precision":0.52,"Recall":1.0,"Accuracy":0.52,"TPR":1.0,"FPR":1.0,"TNR":0.0,"FNR":0.0,"TP":5227.0,"FP":4773.0,"TN":0.0,"FN":0.0,"MAE":0.26,"RMSE":0.33,"R2":-1.1,"Train Run Time (h)":null,"Train GPU":null,"Train CPU":null,"Train Slurm Jobs":null,"Test Run Time (h)":2705.45,"Test GPU":0.0,"Test CPU":32.0,"Test Slurm Jobs":476.0,"Run Time (h)":2705.45,"missing_preds":6184,"missing_percent":"2.41%"},"MEGNet":{"F1":0.64,"DAF":2.84,"Precision":0.47,"Recall":1.0,"Accuracy":0.47,"TPR":1.0,"FPR":1.0,"TNR":0.0,"FNR":0.0,"TP":4727.0,"FP":5273.0,"TN":0.0,"FN":0.0,"MAE":0.33,"RMSE":0.36,"R2":-0.82,"Train Run Time (h)":null,"Train GPU":null,"Train CPU":null,"Train Slurm Jobs":null,"Test Run Time (h)":null,"Test GPU":null,"Test CPU":null,"Test Slurm Jobs":null,"Run Time (h)":0.0,"missing_preds":0,"missing_percent":"0.00%"},"Voronoi RF":{"F1":0.58,"DAF":2.45,"Precision":0.41,"Recall":1.0,"Accuracy":0.41,"TPR":1.0,"FPR":1.0,"TNR":0.0,"FNR":0.0,"TP":4082.0,"FP":5918.0,"TN":0.0,"FN":0.0,"MAE":0.36,"RMSE":0.43,"R2":-0.81,"Train Run Time (h)":173.78,"Train GPU":0.0,"Train CPU":76.0,"Train Slurm Jobs":48.0,"Test Run Time (h)":null,"Test GPU":null,"Test CPU":null,"Test Slurm Jobs":null,"Run Time (h)":173.78,"missing_preds":17,"missing_percent":"0.01%"}} +{ + "GNoME": { + "F1": 0.97, + "DAF": 5.62, + "Precision": 0.94, + "Recall": 1.0, + "Accuracy": 0.94, + "TPR": 1.0, + "FPR": 1.0, + "TNR": 0.0, + "FNR": 0.0, + "TP": 9371.0, + "FP": 629.0, + "TN": 0.0, + "FN": 0.0, + "MAE": 0.04, + "RMSE": 0.09, + "R2": 0.86, + "Train Run Time (h)": null, + "Train GPU": null, + "Train CPU": null, + "Train Slurm Jobs": null, + "Test Run Time (h)": null, + "Test GPU": null, + "Test CPU": null, + "Test Slurm Jobs": null, + "Run Time (h)": 0.0, + "missing_preds": 1742, + "missing_percent": "0.68%" + }, + "CHGNet": { + "F1": 0.94, + "DAF": 5.29, + "Precision": 0.88, + "Recall": 1.0, + "Accuracy": 0.88, + "TPR": 1.0, + "FPR": 1.0, + "TNR": 0.0, + "FNR": 0.0, + "TP": 8817.0, + "FP": 1183.0, + "TN": 0.0, + "FN": 0.0, + "MAE": 0.06, + "RMSE": 0.11, + "R2": 0.81, + "Train Run Time (h)": null, + "Train GPU": null, + "Train CPU": null, + "Train Slurm Jobs": null, + "Test Run Time (h)": 142.48, + "Test GPU": 1.0, + "Test CPU": 128.0, + "Test Slurm Jobs": 100.0, + "Run Time (h)": 142.48, + "missing_preds": 0, + "missing_percent": "0.00%" + }, + "MACE": { + "F1": 0.9, + "DAF": 4.88, + "Precision": 0.81, + "Recall": 1.0, + "Accuracy": 0.81, + "TPR": 1.0, + "FPR": 1.0, + "TNR": 0.0, + "FNR": 0.0, + "TP": 8125.0, + "FP": 1875.0, + "TN": 0.0, + "FN": 0.0, + "MAE": 0.09, + "RMSE": 0.23, + "R2": 0.22, + "Train Run Time (h)": null, + "Train GPU": null, + "Train CPU": null, + "Train Slurm Jobs": null, + "Test Run Time (h)": 111.9, + "Test GPU": 4.0, + "Test CPU": 64.0, + "Test Slurm Jobs": 100.0, + "Run Time (h)": 111.9, + "missing_preds": 7735, + "missing_percent": "3.01%" + }, + "M3GNet": { + "F1": 0.89, + "DAF": 4.79, + "Precision": 0.8, + "Recall": 1.0, + "Accuracy": 0.8, + "TPR": 1.0, + "FPR": 1.0, + "TNR": 0.0, + "FNR": 0.0, + "TP": 7980.0, + "FP": 2020.0, + "TN": 0.0, + "FN": 0.0, + "MAE": 0.1, + "RMSE": 0.16, + "R2": 0.62, + "Train Run Time (h)": null, + "Train GPU": null, + "Train CPU": null, + "Train Slurm Jobs": null, + "Test Run Time (h)": 83.65, + "Test GPU": 0.0, + "Test CPU": 76.0, + "Test Slurm Jobs": 99.0, + "Run Time (h)": 83.65, + "missing_preds": 353, + "missing_percent": "0.14%" + }, + "CGCNN": { + "F1": 0.76, + "DAF": 3.66, + "Precision": 0.61, + "Recall": 1.0, + "Accuracy": 0.61, + "TPR": 1.0, + "FPR": 1.0, + "TNR": 0.0, + "FNR": 0.0, + "TP": 6102.0, + "FP": 3898.0, + "TN": 0.0, + "FN": 0.0, + "MAE": 0.17, + "RMSE": 0.24, + "R2": 0.27, + "Train Run Time (h)": 11.35, + "Train GPU": 1.0, + "Train CPU": 128.0, + "Train Slurm Jobs": 10.0, + "Test Run Time (h)": 0.17, + "Test GPU": 1.0, + "Test CPU": 128.0, + "Test Slurm Jobs": 1.0, + "Run Time (h)": 11.52, + "missing_preds": 2, + "missing_percent": "0.00%" + }, + "ALIGNN": { + "F1": 0.75, + "DAF": 3.58, + "Precision": 0.6, + "Recall": 1.0, + "Accuracy": 0.6, + "TPR": 1.0, + "FPR": 1.0, + "TNR": 0.0, + "FNR": 0.0, + "TP": 5965.0, + "FP": 4035.0, + "TN": 0.0, + "FN": 0.0, + "MAE": 0.19, + "RMSE": 0.26, + "R2": 0.14, + "Train Run Time (h)": null, + "Train GPU": null, + "Train CPU": null, + "Train Slurm Jobs": null, + "Test Run Time (h)": null, + "Test GPU": null, + "Test CPU": null, + "Test Slurm Jobs": null, + "Run Time (h)": 0.0, + "missing_preds": 0, + "missing_percent": "0.00%" + }, + "CGCNN+P": { + "F1": 0.74, + "DAF": 3.53, + "Precision": 0.59, + "Recall": 1.0, + "Accuracy": 0.59, + "TPR": 1.0, + "FPR": 1.0, + "TNR": 0.0, + "FNR": 0.0, + "TP": 5887.0, + "FP": 4113.0, + "TN": 0.0, + "FN": 0.0, + "MAE": 0.21, + "RMSE": 0.29, + "R2": 0.03, + "Train Run Time (h)": null, + "Train GPU": null, + "Train CPU": null, + "Train Slurm Jobs": null, + "Test Run Time (h)": null, + "Test GPU": null, + "Test CPU": null, + "Test Slurm Jobs": null, + "Run Time (h)": 0.0, + "missing_preds": 2, + "missing_percent": "0.00%" + }, + "Wrenformer": { + "F1": 0.74, + "DAF": 3.52, + "Precision": 0.59, + "Recall": 1.0, + "Accuracy": 0.59, + "TPR": 1.0, + "FPR": 1.0, + "TNR": 0.0, + "FNR": 0.0, + "TP": 5873.0, + "FP": 4127.0, + "TN": 0.0, + "FN": 0.0, + "MAE": 0.19, + "RMSE": 0.25, + "R2": 0.22, + "Train Run Time (h)": 57.89, + "Train GPU": 1.0, + "Train CPU": 128.0, + "Train Slurm Jobs": 10.0, + "Test Run Time (h)": 0.45, + "Test GPU": 1.0, + "Test CPU": 128.0, + "Test Slurm Jobs": 1.0, + "Run Time (h)": 58.34, + "missing_preds": 0, + "missing_percent": "0.00%" + }, + "BOWSR": { + "F1": 0.69, + "DAF": 3.14, + "Precision": 0.52, + "Recall": 1.0, + "Accuracy": 0.52, + "TPR": 1.0, + "FPR": 1.0, + "TNR": 0.0, + "FNR": 0.0, + "TP": 5227.0, + "FP": 4773.0, + "TN": 0.0, + "FN": 0.0, + "MAE": 0.26, + "RMSE": 0.33, + "R2": -1.1, + "Train Run Time (h)": null, + "Train GPU": null, + "Train CPU": null, + "Train Slurm Jobs": null, + "Test Run Time (h)": 2705.45, + "Test GPU": 0.0, + "Test CPU": 32.0, + "Test Slurm Jobs": 476.0, + "Run Time (h)": 2705.45, + "missing_preds": 6184, + "missing_percent": "2.41%" + }, + "MEGNet": { + "F1": 0.64, + "DAF": 2.84, + "Precision": 0.47, + "Recall": 1.0, + "Accuracy": 0.47, + "TPR": 1.0, + "FPR": 1.0, + "TNR": 0.0, + "FNR": 0.0, + "TP": 4727.0, + "FP": 5273.0, + "TN": 0.0, + "FN": 0.0, + "MAE": 0.33, + "RMSE": 0.36, + "R2": -0.82, + "Train Run Time (h)": null, + "Train GPU": null, + "Train CPU": null, + "Train Slurm Jobs": null, + "Test Run Time (h)": 3.44, + "Test GPU": 1.0, + "Test CPU": 128.0, + "Test Slurm Jobs": 1.0, + "Run Time (h)": 3.44, + "missing_preds": 0, + "missing_percent": "0.00%" + }, + "Voronoi RF": { + "F1": 0.58, + "DAF": 2.45, + "Precision": 0.41, + "Recall": 1.0, + "Accuracy": 0.41, + "TPR": 1.0, + "FPR": 1.0, + "TNR": 0.0, + "FNR": 0.0, + "TP": 4082.0, + "FP": 5918.0, + "TN": 0.0, + "FN": 0.0, + "MAE": 0.36, + "RMSE": 0.43, + "R2": -0.81, + "Train Run Time (h)": 173.78, + "Train GPU": 0.0, + "Train CPU": 76.0, + "Train Slurm Jobs": 48.0, + "Test Run Time (h)": 30.92, + "Test GPU": 0.0, + "Test CPU": 76.0, + "Test Slurm Jobs": 20.0, + "Run Time (h)": 204.7, + "missing_preds": 17, + "missing_percent": "0.01%" + } +} diff --git a/site/src/lib/model-stats-uniq-protos.json b/site/src/lib/model-stats-uniq-protos.json index 55c21490..83acb697 100644 --- a/site/src/lib/model-stats-uniq-protos.json +++ b/site/src/lib/model-stats-uniq-protos.json @@ -1 +1,321 @@ -{"GNoME":{"F1":0.83,"DAF":5.49,"Precision":0.84,"Recall":0.82,"Accuracy":0.95,"TPR":0.82,"FPR":0.03,"TNR":0.97,"FNR":0.18,"TP":27178.0,"FP":5009.0,"TN":175827.0,"FN":5957.0,"MAE":0.04,"RMSE":0.08,"R2":0.78,"Train Run Time (h)":null,"Train GPU":null,"Train CPU":null,"Train Slurm Jobs":null,"Test Run Time (h)":null,"Test GPU":null,"Test CPU":null,"Test Slurm Jobs":null,"Run Time (h)":0.0,"missing_preds":1742,"missing_percent":"0.68%"},"MACE":{"F1":0.67,"DAF":3.84,"Precision":0.58,"Recall":0.8,"Accuracy":0.88,"TPR":0.8,"FPR":0.11,"TNR":0.89,"FNR":0.2,"TP":25734.0,"FP":18924.0,"TN":157508.0,"FN":6560.0,"MAE":0.06,"RMSE":0.11,"R2":0.66,"Train Run Time (h)":null,"Train GPU":null,"Train CPU":null,"Train Slurm Jobs":null,"Test Run Time (h)":null,"Test GPU":null,"Test CPU":null,"Test Slurm Jobs":null,"Run Time (h)":0.0,"missing_preds":7735,"missing_percent":"3.01%"},"CHGNet":{"F1":0.61,"DAF":3.32,"Precision":0.51,"Recall":0.76,"Accuracy":0.85,"TPR":0.76,"FPR":0.13,"TNR":0.87,"FNR":0.24,"TP":25313.0,"FP":23955.0,"TN":158159.0,"FN":8061.0,"MAE":0.06,"RMSE":0.1,"R2":0.69,"Train Run Time (h)":null,"Train GPU":null,"Train CPU":null,"Train Slurm Jobs":null,"Test Run Time (h)":142.48,"Test GPU":1.0,"Test CPU":128.0,"Test Slurm Jobs":100.0,"Run Time (h)":142.48,"missing_preds":0,"missing_percent":"0.00%"},"M3GNet":{"F1":0.57,"DAF":2.84,"Precision":0.44,"Recall":0.8,"Accuracy":0.81,"TPR":0.8,"FPR":0.19,"TNR":0.81,"FNR":0.2,"TP":26797.0,"FP":34034.0,"TN":147781.0,"FN":6577.0,"MAE":0.08,"RMSE":0.12,"R2":0.58,"Train Run Time (h)":null,"Train GPU":null,"Train CPU":null,"Train Slurm Jobs":null,"Test Run Time (h)":null,"Test GPU":null,"Test CPU":null,"Test Slurm Jobs":null,"Run Time (h)":0.0,"missing_preds":353,"missing_percent":"0.14%"},"ALIGNN":{"F1":0.57,"DAF":3.16,"Precision":0.49,"Recall":0.67,"Accuracy":0.84,"TPR":0.67,"FPR":0.13,"TNR":0.87,"FNR":0.33,"TP":22436.0,"FP":23346.0,"TN":158768.0,"FN":10938.0,"MAE":0.09,"RMSE":0.15,"R2":0.3,"Train Run Time (h)":null,"Train GPU":null,"Train CPU":null,"Train Slurm Jobs":null,"Test Run Time (h)":null,"Test GPU":null,"Test CPU":null,"Test Slurm Jobs":null,"Run Time (h)":0.0,"missing_preds":0,"missing_percent":"0.00%"},"MEGNet":{"F1":0.51,"DAF":2.92,"Precision":0.45,"Recall":0.58,"Accuracy":0.83,"TPR":0.58,"FPR":0.13,"TNR":0.87,"FNR":0.42,"TP":19537.0,"FP":23651.0,"TN":158463.0,"FN":13837.0,"MAE":0.13,"RMSE":0.21,"R2":-0.25,"Train Run Time (h)":null,"Train GPU":null,"Train CPU":null,"Train Slurm Jobs":null,"Test Run Time (h)":null,"Test GPU":null,"Test CPU":null,"Test Slurm Jobs":null,"Run Time (h)":0.0,"missing_preds":0,"missing_percent":"0.00%"},"CGCNN":{"F1":0.51,"DAF":2.82,"Precision":0.44,"Recall":0.6,"Accuracy":0.82,"TPR":0.6,"FPR":0.14,"TNR":0.86,"FNR":0.4,"TP":20191.0,"FP":26074.0,"TN":156038.0,"FN":13183.0,"MAE":0.14,"RMSE":0.23,"R2":-0.6,"Train Run Time (h)":11.35,"Train GPU":1.0,"Train CPU":128.0,"Train Slurm Jobs":10.0,"Test Run Time (h)":0.17,"Test GPU":1.0,"Test CPU":128.0,"Test Slurm Jobs":1.0,"Run Time (h)":11.52,"missing_preds":2,"missing_percent":"0.00%"},"CGCNN+P":{"F1":0.5,"DAF":2.53,"Precision":0.39,"Recall":0.69,"Accuracy":0.79,"TPR":0.69,"FPR":0.2,"TNR":0.8,"FNR":0.31,"TP":23117.0,"FP":35893.0,"TN":146219.0,"FN":10257.0,"MAE":0.11,"RMSE":0.18,"R2":0.02,"Train Run Time (h)":null,"Train GPU":null,"Train CPU":null,"Train Slurm Jobs":null,"Test Run Time (h)":null,"Test GPU":null,"Test CPU":null,"Test Slurm Jobs":null,"Run Time (h)":0.0,"missing_preds":2,"missing_percent":"0.00%"},"Wrenformer":{"F1":0.47,"DAF":2.23,"Precision":0.34,"Recall":0.72,"Accuracy":0.74,"TPR":0.72,"FPR":0.25,"TNR":0.75,"FNR":0.28,"TP":23990.0,"FP":45586.0,"TN":136528.0,"FN":9384.0,"MAE":0.11,"RMSE":0.19,"R2":-0.04,"Train Run Time (h)":57.89,"Train GPU":1.0,"Train CPU":128.0,"Train Slurm Jobs":10.0,"Test Run Time (h)":null,"Test GPU":null,"Test CPU":null,"Test Slurm Jobs":null,"Run Time (h)":57.89,"missing_preds":0,"missing_percent":"0.00%"},"BOWSR":{"F1":0.43,"DAF":2.0,"Precision":0.3,"Recall":0.74,"Accuracy":0.7,"TPR":0.74,"FPR":0.31,"TNR":0.69,"FNR":0.26,"TP":23963.0,"FP":55843.0,"TN":122841.0,"FN":8357.0,"MAE":0.12,"RMSE":0.17,"R2":0.15,"Train Run Time (h)":null,"Train GPU":null,"Train CPU":null,"Train Slurm Jobs":null,"Test Run Time (h)":2705.45,"Test GPU":0.0,"Test CPU":32.0,"Test Slurm Jobs":476.0,"Run Time (h)":2705.45,"missing_preds":6184,"missing_percent":"2.41%"},"Voronoi RF":{"F1":0.33,"DAF":1.56,"Precision":0.24,"Recall":0.54,"Accuracy":0.67,"TPR":0.54,"FPR":0.31,"TNR":0.69,"FNR":0.46,"TP":17854.0,"FP":56122.0,"TN":125990.0,"FN":15520.0,"MAE":0.15,"RMSE":0.21,"R2":-0.33,"Train Run Time (h)":173.78,"Train GPU":0.0,"Train CPU":76.0,"Train Slurm Jobs":48.0,"Test Run Time (h)":null,"Test GPU":null,"Test CPU":null,"Test Slurm Jobs":null,"Run Time (h)":173.78,"missing_preds":17,"missing_percent":"0.01%"}} +{ + "GNoME": { + "F1": 0.83, + "DAF": 5.49, + "Precision": 0.84, + "Recall": 0.82, + "Accuracy": 0.95, + "TPR": 0.82, + "FPR": 0.03, + "TNR": 0.97, + "FNR": 0.18, + "TP": 27178.0, + "FP": 5009.0, + "TN": 175827.0, + "FN": 5957.0, + "MAE": 0.04, + "RMSE": 0.08, + "R2": 0.78, + "Train Run Time (h)": null, + "Train GPU": null, + "Train CPU": null, + "Train Slurm Jobs": null, + "Test Run Time (h)": null, + "Test GPU": null, + "Test CPU": null, + "Test Slurm Jobs": null, + "Run Time (h)": 0.0, + "missing_preds": 1742, + "missing_percent": "0.68%" + }, + "MACE": { + "F1": 0.67, + "DAF": 3.84, + "Precision": 0.58, + "Recall": 0.8, + "Accuracy": 0.88, + "TPR": 0.8, + "FPR": 0.11, + "TNR": 0.89, + "FNR": 0.2, + "TP": 25734.0, + "FP": 18924.0, + "TN": 157508.0, + "FN": 6560.0, + "MAE": 0.06, + "RMSE": 0.11, + "R2": 0.66, + "Train Run Time (h)": null, + "Train GPU": null, + "Train CPU": null, + "Train Slurm Jobs": null, + "Test Run Time (h)": 111.9, + "Test GPU": 4.0, + "Test CPU": 64.0, + "Test Slurm Jobs": 100.0, + "Run Time (h)": 111.9, + "missing_preds": 7735, + "missing_percent": "3.01%" + }, + "CHGNet": { + "F1": 0.61, + "DAF": 3.32, + "Precision": 0.51, + "Recall": 0.76, + "Accuracy": 0.85, + "TPR": 0.76, + "FPR": 0.13, + "TNR": 0.87, + "FNR": 0.24, + "TP": 25313.0, + "FP": 23955.0, + "TN": 158159.0, + "FN": 8061.0, + "MAE": 0.06, + "RMSE": 0.1, + "R2": 0.69, + "Train Run Time (h)": null, + "Train GPU": null, + "Train CPU": null, + "Train Slurm Jobs": null, + "Test Run Time (h)": 142.48, + "Test GPU": 1.0, + "Test CPU": 128.0, + "Test Slurm Jobs": 100.0, + "Run Time (h)": 142.48, + "missing_preds": 0, + "missing_percent": "0.00%" + }, + "M3GNet": { + "F1": 0.57, + "DAF": 2.84, + "Precision": 0.44, + "Recall": 0.8, + "Accuracy": 0.81, + "TPR": 0.8, + "FPR": 0.19, + "TNR": 0.81, + "FNR": 0.2, + "TP": 26797.0, + "FP": 34034.0, + "TN": 147781.0, + "FN": 6577.0, + "MAE": 0.08, + "RMSE": 0.12, + "R2": 0.58, + "Train Run Time (h)": null, + "Train GPU": null, + "Train CPU": null, + "Train Slurm Jobs": null, + "Test Run Time (h)": 83.65, + "Test GPU": 0.0, + "Test CPU": 76.0, + "Test Slurm Jobs": 99.0, + "Run Time (h)": 83.65, + "missing_preds": 353, + "missing_percent": "0.14%" + }, + "ALIGNN": { + "F1": 0.57, + "DAF": 3.16, + "Precision": 0.49, + "Recall": 0.67, + "Accuracy": 0.84, + "TPR": 0.67, + "FPR": 0.13, + "TNR": 0.87, + "FNR": 0.33, + "TP": 22436.0, + "FP": 23346.0, + "TN": 158768.0, + "FN": 10938.0, + "MAE": 0.09, + "RMSE": 0.15, + "R2": 0.3, + "Train Run Time (h)": null, + "Train GPU": null, + "Train CPU": null, + "Train Slurm Jobs": null, + "Test Run Time (h)": null, + "Test GPU": null, + "Test CPU": null, + "Test Slurm Jobs": null, + "Run Time (h)": 0.0, + "missing_preds": 0, + "missing_percent": "0.00%" + }, + "MEGNet": { + "F1": 0.51, + "DAF": 2.92, + "Precision": 0.45, + "Recall": 0.58, + "Accuracy": 0.83, + "TPR": 0.58, + "FPR": 0.13, + "TNR": 0.87, + "FNR": 0.42, + "TP": 19537.0, + "FP": 23651.0, + "TN": 158463.0, + "FN": 13837.0, + "MAE": 0.13, + "RMSE": 0.21, + "R2": -0.25, + "Train Run Time (h)": null, + "Train GPU": null, + "Train CPU": null, + "Train Slurm Jobs": null, + "Test Run Time (h)": 3.44, + "Test GPU": 1.0, + "Test CPU": 128.0, + "Test Slurm Jobs": 1.0, + "Run Time (h)": 3.44, + "missing_preds": 0, + "missing_percent": "0.00%" + }, + "CGCNN": { + "F1": 0.51, + "DAF": 2.82, + "Precision": 0.44, + "Recall": 0.6, + "Accuracy": 0.82, + "TPR": 0.6, + "FPR": 0.14, + "TNR": 0.86, + "FNR": 0.4, + "TP": 20191.0, + "FP": 26074.0, + "TN": 156038.0, + "FN": 13183.0, + "MAE": 0.14, + "RMSE": 0.23, + "R2": -0.6, + "Train Run Time (h)": 11.35, + "Train GPU": 1.0, + "Train CPU": 128.0, + "Train Slurm Jobs": 10.0, + "Test Run Time (h)": 0.17, + "Test GPU": 1.0, + "Test CPU": 128.0, + "Test Slurm Jobs": 1.0, + "Run Time (h)": 11.52, + "missing_preds": 2, + "missing_percent": "0.00%" + }, + "CGCNN+P": { + "F1": 0.5, + "DAF": 2.53, + "Precision": 0.39, + "Recall": 0.69, + "Accuracy": 0.79, + "TPR": 0.69, + "FPR": 0.2, + "TNR": 0.8, + "FNR": 0.31, + "TP": 23117.0, + "FP": 35893.0, + "TN": 146219.0, + "FN": 10257.0, + "MAE": 0.11, + "RMSE": 0.18, + "R2": 0.02, + "Train Run Time (h)": null, + "Train GPU": null, + "Train CPU": null, + "Train Slurm Jobs": null, + "Test Run Time (h)": null, + "Test GPU": null, + "Test CPU": null, + "Test Slurm Jobs": null, + "Run Time (h)": 0.0, + "missing_preds": 2, + "missing_percent": "0.00%" + }, + "Wrenformer": { + "F1": 0.47, + "DAF": 2.23, + "Precision": 0.34, + "Recall": 0.72, + "Accuracy": 0.74, + "TPR": 0.72, + "FPR": 0.25, + "TNR": 0.75, + "FNR": 0.28, + "TP": 23990.0, + "FP": 45586.0, + "TN": 136528.0, + "FN": 9384.0, + "MAE": 0.11, + "RMSE": 0.19, + "R2": -0.04, + "Train Run Time (h)": 57.89, + "Train GPU": 1.0, + "Train CPU": 128.0, + "Train Slurm Jobs": 10.0, + "Test Run Time (h)": 0.45, + "Test GPU": 1.0, + "Test CPU": 128.0, + "Test Slurm Jobs": 1.0, + "Run Time (h)": 58.34, + "missing_preds": 0, + "missing_percent": "0.00%" + }, + "BOWSR": { + "F1": 0.43, + "DAF": 2.0, + "Precision": 0.3, + "Recall": 0.74, + "Accuracy": 0.7, + "TPR": 0.74, + "FPR": 0.31, + "TNR": 0.69, + "FNR": 0.26, + "TP": 23963.0, + "FP": 55843.0, + "TN": 122841.0, + "FN": 8357.0, + "MAE": 0.12, + "RMSE": 0.17, + "R2": 0.15, + "Train Run Time (h)": null, + "Train GPU": null, + "Train CPU": null, + "Train Slurm Jobs": null, + "Test Run Time (h)": 2705.45, + "Test GPU": 0.0, + "Test CPU": 32.0, + "Test Slurm Jobs": 476.0, + "Run Time (h)": 2705.45, + "missing_preds": 6184, + "missing_percent": "2.41%" + }, + "Voronoi RF": { + "F1": 0.33, + "DAF": 1.56, + "Precision": 0.24, + "Recall": 0.54, + "Accuracy": 0.67, + "TPR": 0.54, + "FPR": 0.31, + "TNR": 0.69, + "FNR": 0.46, + "TP": 17854.0, + "FP": 56122.0, + "TN": 125990.0, + "FN": 15520.0, + "MAE": 0.15, + "RMSE": 0.21, + "R2": -0.33, + "Train Run Time (h)": 173.78, + "Train GPU": 0.0, + "Train CPU": 76.0, + "Train Slurm Jobs": 48.0, + "Test Run Time (h)": 30.92, + "Test GPU": 0.0, + "Test CPU": 76.0, + "Test Slurm Jobs": 20.0, + "Run Time (h)": 204.7, + "missing_preds": 17, + "missing_percent": "0.01%" + } +} diff --git a/site/src/lib/model-stats.json b/site/src/lib/model-stats.json index 1ebe4200..684c757f 100644 --- a/site/src/lib/model-stats.json +++ b/site/src/lib/model-stats.json @@ -1 +1,321 @@ -{"GNoME":{"F1":0.81,"DAF":4.84,"Precision":0.82,"Recall":0.8,"Accuracy":0.94,"TPR":0.8,"FPR":0.04,"TNR":0.96,"FNR":0.2,"TP":35082.0,"FP":7422.0,"TN":203981.0,"FN":8736.0,"MAE":0.03,"RMSE":0.08,"R2":0.78,"Train Run Time (h)":null,"Train GPU":null,"Train CPU":null,"Train Slurm Jobs":null,"Test Run Time (h)":null,"Test GPU":null,"Test CPU":null,"Test Slurm Jobs":null,"Run Time (h)":0.0,"missing_preds":1742,"missing_percent":"0.68%"},"MACE":{"F1":0.67,"DAF":3.5,"Precision":0.58,"Recall":0.78,"Accuracy":0.87,"TPR":0.78,"FPR":0.12,"TNR":0.88,"FNR":0.22,"TP":33428.0,"FP":23943.0,"TN":182497.0,"FN":9360.0,"MAE":0.06,"RMSE":0.1,"R2":0.66,"Train Run Time (h)":null,"Train GPU":null,"Train CPU":null,"Train Slurm Jobs":null,"Test Run Time (h)":null,"Test GPU":null,"Test CPU":null,"Test Slurm Jobs":null,"Run Time (h)":0.0,"missing_preds":7735,"missing_percent":"3.01%"},"CHGNet":{"F1":0.61,"DAF":3.04,"Precision":0.52,"Recall":0.74,"Accuracy":0.84,"TPR":0.74,"FPR":0.14,"TNR":0.86,"FNR":0.26,"TP":32642.0,"FP":29980.0,"TN":182891.0,"FN":11450.0,"MAE":0.06,"RMSE":0.1,"R2":0.68,"Train Run Time (h)":null,"Train GPU":null,"Train CPU":null,"Train Slurm Jobs":null,"Test Run Time (h)":142.48,"Test GPU":1.0,"Test CPU":128.0,"Test Slurm Jobs":100.0,"Run Time (h)":142.48,"missing_preds":0,"missing_percent":"0.00%"},"M3GNet":{"F1":0.58,"DAF":2.65,"Precision":0.45,"Recall":0.79,"Accuracy":0.8,"TPR":0.79,"FPR":0.2,"TNR":0.8,"FNR":0.21,"TP":34731.0,"FP":41738.0,"TN":170780.0,"FN":9361.0,"MAE":0.07,"RMSE":0.12,"R2":0.58,"Train Run Time (h)":null,"Train GPU":null,"Train CPU":null,"Train Slurm Jobs":null,"Test Run Time (h)":null,"Test GPU":null,"Test CPU":null,"Test Slurm Jobs":null,"Run Time (h)":0.0,"missing_preds":353,"missing_percent":"0.14%"},"ALIGNN":{"F1":0.56,"DAF":2.92,"Precision":0.5,"Recall":0.65,"Accuracy":0.83,"TPR":0.65,"FPR":0.13,"TNR":0.87,"FNR":0.35,"TP":28598.0,"FP":28465.0,"TN":184406.0,"FN":15494.0,"MAE":0.09,"RMSE":0.15,"R2":0.27,"Train Run Time (h)":null,"Train GPU":null,"Train CPU":null,"Train Slurm Jobs":null,"Test Run Time (h)":null,"Test GPU":null,"Test CPU":null,"Test Slurm Jobs":null,"Run Time (h)":0.0,"missing_preds":0,"missing_percent":"0.00%"},"MEGNet":{"F1":0.51,"DAF":2.7,"Precision":0.46,"Recall":0.57,"Accuracy":0.81,"TPR":0.57,"FPR":0.14,"TNR":0.86,"FNR":0.43,"TP":25311.0,"FP":29343.0,"TN":183528.0,"FN":18781.0,"MAE":0.13,"RMSE":0.2,"R2":-0.28,"Train Run Time (h)":null,"Train GPU":null,"Train CPU":null,"Train Slurm Jobs":null,"Test Run Time (h)":null,"Test GPU":null,"Test CPU":null,"Test Slurm Jobs":null,"Run Time (h)":0.0,"missing_preds":0,"missing_percent":"0.00%"},"CGCNN":{"F1":0.51,"DAF":2.63,"Precision":0.45,"Recall":0.59,"Accuracy":0.81,"TPR":0.59,"FPR":0.15,"TNR":0.85,"FNR":0.41,"TP":25895.0,"FP":31475.0,"TN":181394.0,"FN":18197.0,"MAE":0.14,"RMSE":0.23,"R2":-0.62,"Train Run Time (h)":11.35,"Train GPU":1.0,"Train CPU":128.0,"Train Slurm Jobs":10.0,"Test Run Time (h)":0.17,"Test GPU":1.0,"Test CPU":128.0,"Test Slurm Jobs":1.0,"Run Time (h)":11.52,"missing_preds":2,"missing_percent":"0.00%"},"CGCNN+P":{"F1":0.51,"DAF":2.4,"Precision":0.41,"Recall":0.67,"Accuracy":0.78,"TPR":0.67,"FPR":0.2,"TNR":0.8,"FNR":0.33,"TP":29557.0,"FP":42282.0,"TN":170587.0,"FN":14535.0,"MAE":0.11,"RMSE":0.18,"R2":0.03,"Train Run Time (h)":null,"Train GPU":null,"Train CPU":null,"Train Slurm Jobs":null,"Test Run Time (h)":null,"Test GPU":null,"Test CPU":null,"Test Slurm Jobs":null,"Run Time (h)":0.0,"missing_preds":2,"missing_percent":"0.00%"},"Wrenformer":{"F1":0.48,"DAF":2.13,"Precision":0.36,"Recall":0.69,"Accuracy":0.74,"TPR":0.69,"FPR":0.25,"TNR":0.75,"FNR":0.31,"TP":30570.0,"FP":53088.0,"TN":159783.0,"FN":13522.0,"MAE":0.1,"RMSE":0.18,"R2":-0.04,"Train Run Time (h)":57.89,"Train GPU":1.0,"Train CPU":128.0,"Train Slurm Jobs":10.0,"Test Run Time (h)":null,"Test GPU":null,"Test CPU":null,"Test Slurm Jobs":null,"Run Time (h)":57.89,"missing_preds":0,"missing_percent":"0.00%"},"BOWSR":{"F1":0.44,"DAF":1.91,"Precision":0.32,"Recall":0.74,"Accuracy":0.68,"TPR":0.74,"FPR":0.33,"TNR":0.67,"FNR":0.26,"TP":31347.0,"FP":68140.0,"TN":140149.0,"FN":11143.0,"MAE":0.12,"RMSE":0.16,"R2":0.14,"Train Run Time (h)":null,"Train GPU":null,"Train CPU":null,"Train Slurm Jobs":null,"Test Run Time (h)":2705.45,"Test GPU":0.0,"Test CPU":32.0,"Test Slurm Jobs":476.0,"Run Time (h)":2705.45,"missing_preds":6184,"missing_percent":"2.41%"},"Voronoi RF":{"F1":0.34,"DAF":1.51,"Precision":0.26,"Recall":0.51,"Accuracy":0.66,"TPR":0.51,"FPR":0.3,"TNR":0.7,"FNR":0.49,"TP":22517.0,"FP":64431.0,"TN":148424.0,"FN":21574.0,"MAE":0.14,"RMSE":0.21,"R2":-0.31,"Train Run Time (h)":173.78,"Train GPU":0.0,"Train CPU":76.0,"Train Slurm Jobs":48.0,"Test Run Time (h)":null,"Test GPU":null,"Test CPU":null,"Test Slurm Jobs":null,"Run Time (h)":173.78,"missing_preds":17,"missing_percent":"0.01%"}} +{ + "GNoME": { + "F1": 0.81, + "DAF": 4.84, + "Precision": 0.82, + "Recall": 0.8, + "Accuracy": 0.94, + "TPR": 0.8, + "FPR": 0.04, + "TNR": 0.96, + "FNR": 0.2, + "TP": 35082.0, + "FP": 7422.0, + "TN": 203981.0, + "FN": 8736.0, + "MAE": 0.03, + "RMSE": 0.08, + "R2": 0.78, + "Train Run Time (h)": null, + "Train GPU": null, + "Train CPU": null, + "Train Slurm Jobs": null, + "Test Run Time (h)": null, + "Test GPU": null, + "Test CPU": null, + "Test Slurm Jobs": null, + "Run Time (h)": 0.0, + "missing_preds": 1742, + "missing_percent": "0.68%" + }, + "MACE": { + "F1": 0.67, + "DAF": 3.5, + "Precision": 0.58, + "Recall": 0.78, + "Accuracy": 0.87, + "TPR": 0.78, + "FPR": 0.12, + "TNR": 0.88, + "FNR": 0.22, + "TP": 33428.0, + "FP": 23943.0, + "TN": 182497.0, + "FN": 9360.0, + "MAE": 0.06, + "RMSE": 0.1, + "R2": 0.66, + "Train Run Time (h)": null, + "Train GPU": null, + "Train CPU": null, + "Train Slurm Jobs": null, + "Test Run Time (h)": 111.9, + "Test GPU": 4.0, + "Test CPU": 64.0, + "Test Slurm Jobs": 100.0, + "Run Time (h)": 111.9, + "missing_preds": 7735, + "missing_percent": "3.01%" + }, + "CHGNet": { + "F1": 0.61, + "DAF": 3.04, + "Precision": 0.52, + "Recall": 0.74, + "Accuracy": 0.84, + "TPR": 0.74, + "FPR": 0.14, + "TNR": 0.86, + "FNR": 0.26, + "TP": 32642.0, + "FP": 29980.0, + "TN": 182891.0, + "FN": 11450.0, + "MAE": 0.06, + "RMSE": 0.1, + "R2": 0.68, + "Train Run Time (h)": null, + "Train GPU": null, + "Train CPU": null, + "Train Slurm Jobs": null, + "Test Run Time (h)": 142.48, + "Test GPU": 1.0, + "Test CPU": 128.0, + "Test Slurm Jobs": 100.0, + "Run Time (h)": 142.48, + "missing_preds": 0, + "missing_percent": "0.00%" + }, + "M3GNet": { + "F1": 0.58, + "DAF": 2.65, + "Precision": 0.45, + "Recall": 0.79, + "Accuracy": 0.8, + "TPR": 0.79, + "FPR": 0.2, + "TNR": 0.8, + "FNR": 0.21, + "TP": 34731.0, + "FP": 41738.0, + "TN": 170780.0, + "FN": 9361.0, + "MAE": 0.07, + "RMSE": 0.12, + "R2": 0.58, + "Train Run Time (h)": null, + "Train GPU": null, + "Train CPU": null, + "Train Slurm Jobs": null, + "Test Run Time (h)": 83.65, + "Test GPU": 0.0, + "Test CPU": 76.0, + "Test Slurm Jobs": 99.0, + "Run Time (h)": 83.65, + "missing_preds": 353, + "missing_percent": "0.14%" + }, + "ALIGNN": { + "F1": 0.56, + "DAF": 2.92, + "Precision": 0.5, + "Recall": 0.65, + "Accuracy": 0.83, + "TPR": 0.65, + "FPR": 0.13, + "TNR": 0.87, + "FNR": 0.35, + "TP": 28598.0, + "FP": 28465.0, + "TN": 184406.0, + "FN": 15494.0, + "MAE": 0.09, + "RMSE": 0.15, + "R2": 0.27, + "Train Run Time (h)": null, + "Train GPU": null, + "Train CPU": null, + "Train Slurm Jobs": null, + "Test Run Time (h)": null, + "Test GPU": null, + "Test CPU": null, + "Test Slurm Jobs": null, + "Run Time (h)": 0.0, + "missing_preds": 0, + "missing_percent": "0.00%" + }, + "MEGNet": { + "F1": 0.51, + "DAF": 2.7, + "Precision": 0.46, + "Recall": 0.57, + "Accuracy": 0.81, + "TPR": 0.57, + "FPR": 0.14, + "TNR": 0.86, + "FNR": 0.43, + "TP": 25311.0, + "FP": 29343.0, + "TN": 183528.0, + "FN": 18781.0, + "MAE": 0.13, + "RMSE": 0.2, + "R2": -0.28, + "Train Run Time (h)": null, + "Train GPU": null, + "Train CPU": null, + "Train Slurm Jobs": null, + "Test Run Time (h)": 3.44, + "Test GPU": 1.0, + "Test CPU": 128.0, + "Test Slurm Jobs": 1.0, + "Run Time (h)": 3.44, + "missing_preds": 0, + "missing_percent": "0.00%" + }, + "CGCNN": { + "F1": 0.51, + "DAF": 2.63, + "Precision": 0.45, + "Recall": 0.59, + "Accuracy": 0.81, + "TPR": 0.59, + "FPR": 0.15, + "TNR": 0.85, + "FNR": 0.41, + "TP": 25895.0, + "FP": 31475.0, + "TN": 181394.0, + "FN": 18197.0, + "MAE": 0.14, + "RMSE": 0.23, + "R2": -0.62, + "Train Run Time (h)": 11.35, + "Train GPU": 1.0, + "Train CPU": 128.0, + "Train Slurm Jobs": 10.0, + "Test Run Time (h)": 0.17, + "Test GPU": 1.0, + "Test CPU": 128.0, + "Test Slurm Jobs": 1.0, + "Run Time (h)": 11.52, + "missing_preds": 2, + "missing_percent": "0.00%" + }, + "CGCNN+P": { + "F1": 0.51, + "DAF": 2.4, + "Precision": 0.41, + "Recall": 0.67, + "Accuracy": 0.78, + "TPR": 0.67, + "FPR": 0.2, + "TNR": 0.8, + "FNR": 0.33, + "TP": 29557.0, + "FP": 42282.0, + "TN": 170587.0, + "FN": 14535.0, + "MAE": 0.11, + "RMSE": 0.18, + "R2": 0.03, + "Train Run Time (h)": null, + "Train GPU": null, + "Train CPU": null, + "Train Slurm Jobs": null, + "Test Run Time (h)": null, + "Test GPU": null, + "Test CPU": null, + "Test Slurm Jobs": null, + "Run Time (h)": 0.0, + "missing_preds": 2, + "missing_percent": "0.00%" + }, + "Wrenformer": { + "F1": 0.48, + "DAF": 2.13, + "Precision": 0.36, + "Recall": 0.69, + "Accuracy": 0.74, + "TPR": 0.69, + "FPR": 0.25, + "TNR": 0.75, + "FNR": 0.31, + "TP": 30570.0, + "FP": 53088.0, + "TN": 159783.0, + "FN": 13522.0, + "MAE": 0.1, + "RMSE": 0.18, + "R2": -0.04, + "Train Run Time (h)": 57.89, + "Train GPU": 1.0, + "Train CPU": 128.0, + "Train Slurm Jobs": 10.0, + "Test Run Time (h)": 0.45, + "Test GPU": 1.0, + "Test CPU": 128.0, + "Test Slurm Jobs": 1.0, + "Run Time (h)": 58.34, + "missing_preds": 0, + "missing_percent": "0.00%" + }, + "BOWSR": { + "F1": 0.44, + "DAF": 1.91, + "Precision": 0.32, + "Recall": 0.74, + "Accuracy": 0.68, + "TPR": 0.74, + "FPR": 0.33, + "TNR": 0.67, + "FNR": 0.26, + "TP": 31347.0, + "FP": 68140.0, + "TN": 140149.0, + "FN": 11143.0, + "MAE": 0.12, + "RMSE": 0.16, + "R2": 0.14, + "Train Run Time (h)": null, + "Train GPU": null, + "Train CPU": null, + "Train Slurm Jobs": null, + "Test Run Time (h)": 2705.45, + "Test GPU": 0.0, + "Test CPU": 32.0, + "Test Slurm Jobs": 476.0, + "Run Time (h)": 2705.45, + "missing_preds": 6184, + "missing_percent": "2.41%" + }, + "Voronoi RF": { + "F1": 0.34, + "DAF": 1.51, + "Precision": 0.26, + "Recall": 0.51, + "Accuracy": 0.66, + "TPR": 0.51, + "FPR": 0.3, + "TNR": 0.7, + "FNR": 0.49, + "TP": 22517.0, + "FP": 64431.0, + "TN": 148424.0, + "FN": 21574.0, + "MAE": 0.14, + "RMSE": 0.21, + "R2": -0.31, + "Train Run Time (h)": 173.78, + "Train GPU": 0.0, + "Train CPU": 76.0, + "Train Slurm Jobs": 48.0, + "Test Run Time (h)": 30.92, + "Test GPU": 0.0, + "Test CPU": 76.0, + "Test Slurm Jobs": 20.0, + "Run Time (h)": 204.7, + "missing_preds": 17, + "missing_percent": "0.01%" + } +} diff --git a/site/src/routes/+page.svelte b/site/src/routes/+page.svelte index 2165805e..9ddb92c6 100644 --- a/site/src/routes/+page.svelte +++ b/site/src/routes/+page.svelte @@ -36,8 +36,8 @@
{#if best_model} - {@const { model_name, F1, R2, DAF, repo, doi } = best_model} - We find {model_name} (paper) to achieve the + {@const { model_name, F1, R2, DAF, repo, paper } = best_model} + We find {model_name} (paper) to achieve the highest F1 score of {F1}, R2 of {R2} and a discovery acceleration factor (DAF) of {DAF} (i.e. a ~{Number(DAF).toFixed(1)}x higher rate of stable structures compared to From 0eea3af007e0d62648919b5fa91a896d9758aa92 Mon Sep 17 00:00:00 2001 From: Janosh Riebesell Date: Wed, 7 Feb 2024 13:08:43 +0100 Subject: [PATCH 10/12] add TestSubset Enum for extensible test set slices in future analysis --- matbench_discovery/enums.py | 29 ++++++++++++------- .../model_figs/analyze_model_disagreement.py | 7 +++-- scripts/model_figs/cumulative_metrics.py | 8 +++-- ...t_classified_stable_vs_hull_dist_models.py | 7 +++-- scripts/model_figs/make_hull_dist_box_plot.py | 7 +++-- scripts/model_figs/parity_energy_models.py | 8 +++-- scripts/model_figs/per_element_errors.py | 7 +++-- scripts/model_figs/roc_prc_curves_models.py | 8 +++-- .../rolling_mae_vs_hull_dist_models.py | 8 +++-- .../rolling_mae_vs_hull_dist_wbm_batches.py | 12 ++++++-- 10 files changed, 64 insertions(+), 37 deletions(-) diff --git a/matbench_discovery/enums.py b/matbench_discovery/enums.py index 957d3ffd..03f21702 100644 --- a/matbench_discovery/enums.py +++ b/matbench_discovery/enums.py @@ -135,7 +135,16 @@ class Open(LabelEnum): CSCD = "CSCD", "closed source, closed data" -ev_per_atom = styled_html_tag( +@unique +class TestSubset(LabelEnum): + """Test set subsets.""" + + uniq_protos = "uniq_protos", "Unique Structure Prototypes" + ten_k_most_stable = "10k_most_stable", "10k Most Stable" + full = "full", "Full Test Set" + + +eV_per_atom = styled_html_tag( # noqa: N816 "(eV/atom)", tag="span", style="font-size: 0.8em; font-weight: lighter;" ) @@ -149,16 +158,16 @@ class Quantity(LabelEnum): spg_num = "Space group" n_wyckoff = "Number of Wyckoff positions" n_sites = "Number of atoms" - energy_per_atom = f"Energy {ev_per_atom}" - e_form = f"DFT Eform {ev_per_atom}" - e_above_hull = f"Ehull dist {ev_per_atom}" - e_above_hull_mp2020_corrected_ppd_mp = f"DFT Ehull dist {ev_per_atom}" - e_above_hull_pred = f"Predicted Ehull dist {ev_per_atom}" - e_above_hull_mp = f"Eabove MP hull {ev_per_atom}" - e_above_hull_error = f"Error in Ehull dist {ev_per_atom}" + energy_per_atom = f"Energy {eV_per_atom}" + e_form = f"DFT Eform {eV_per_atom}" + e_above_hull = f"Ehull dist {eV_per_atom}" + e_above_hull_mp2020_corrected_ppd_mp = f"DFT Ehull dist {eV_per_atom}" + e_above_hull_pred = f"Predicted Ehull dist {eV_per_atom}" + e_above_hull_mp = f"Eabove MP hull {eV_per_atom}" + e_above_hull_error = f"Error in Ehull dist {eV_per_atom}" vol_diff = "Volume difference (A^3)" - e_form_per_atom_mp2020_corrected = f"DFT Eform {ev_per_atom}" - e_form_per_atom_pred = f"Predicted Eform {ev_per_atom}" + e_form_per_atom_mp2020_corrected = f"DFT Eform {eV_per_atom}" + e_form_per_atom_pred = f"Predicted Eform {eV_per_atom}" material_id = "Material ID" band_gap = "Band gap (eV)" formula = "Formula" diff --git a/scripts/model_figs/analyze_model_disagreement.py b/scripts/model_figs/analyze_model_disagreement.py index 21ca1a04..18f7d6f4 100644 --- a/scripts/model_figs/analyze_model_disagreement.py +++ b/scripts/model_figs/analyze_model_disagreement.py @@ -10,16 +10,17 @@ from pymatviz.io import save_fig from pymatviz.utils import add_identity_line -from matbench_discovery import PDF_FIGS, SITE_FIGS, Key +from matbench_discovery import PDF_FIGS, SITE_FIGS from matbench_discovery.data import DATA_FILES +from matbench_discovery.enums import Key, TestSubset from matbench_discovery.preds import df_preds __author__ = "Janosh Riebesell" __date__ = "2023-02-15" -use_unique_proto = True +test_subset = globals().get("test_subset", TestSubset.full) -if use_unique_proto: +if test_subset == TestSubset.uniq_protos: df_preds = df_preds.query(Key.uniq_proto) diff --git a/scripts/model_figs/cumulative_metrics.py b/scripts/model_figs/cumulative_metrics.py index 3b70a59f..d31e33ed 100644 --- a/scripts/model_figs/cumulative_metrics.py +++ b/scripts/model_figs/cumulative_metrics.py @@ -12,16 +12,18 @@ import pandas as pd from pymatviz.io import save_fig -from matbench_discovery import PDF_FIGS, SITE_FIGS, Key +from matbench_discovery import PDF_FIGS, SITE_FIGS +from matbench_discovery.enums import Key, TestSubset from matbench_discovery.plots import cumulative_metrics from matbench_discovery.preds import df_each_pred, df_preds, model_styles, models __author__ = "Janosh Riebesell, Rhys Goodall" __date__ = "2022-12-04" -use_unique_proto = True -if use_unique_proto: +test_subset = globals().get("test_subset", TestSubset.full) + +if test_subset == TestSubset.uniq_protos: df_preds = df_preds.query(Key.uniq_proto) df_each_pred = df_each_pred.loc[df_preds.index] diff --git a/scripts/model_figs/hist_classified_stable_vs_hull_dist_models.py b/scripts/model_figs/hist_classified_stable_vs_hull_dist_models.py index 5bc23f59..00a988c9 100644 --- a/scripts/model_figs/hist_classified_stable_vs_hull_dist_models.py +++ b/scripts/model_figs/hist_classified_stable_vs_hull_dist_models.py @@ -10,7 +10,8 @@ from pymatviz.io import save_fig -from matbench_discovery import PDF_FIGS, SITE_FIGS, Key, today +from matbench_discovery import PDF_FIGS, SITE_FIGS, today +from matbench_discovery.enums import Key, TestSubset from matbench_discovery.plots import hist_classified_stable_vs_hull_dist, plt from matbench_discovery.preds import df_metrics, df_metrics_uniq_protos, df_preds @@ -18,9 +19,9 @@ __date__ = "2022-12-01" -use_unique_proto = True +test_subset = globals().get("test_subset", TestSubset.full) -if use_unique_proto: +if test_subset == TestSubset.uniq_protos: df_preds = df_preds.query(Key.uniq_proto) df_metrics = df_metrics_uniq_protos diff --git a/scripts/model_figs/make_hull_dist_box_plot.py b/scripts/model_figs/make_hull_dist_box_plot.py index 7bf84278..435afe67 100644 --- a/scripts/model_figs/make_hull_dist_box_plot.py +++ b/scripts/model_figs/make_hull_dist_box_plot.py @@ -4,16 +4,17 @@ import seaborn as sns from pymatviz.io import save_fig -from matbench_discovery import PDF_FIGS, SITE_FIGS, Key, Quantity +from matbench_discovery import PDF_FIGS, SITE_FIGS +from matbench_discovery.enums import Key, Quantity, TestSubset from matbench_discovery.preds import df_each_err, df_preds, models __author__ = "Janosh Riebesell" __date__ = "2023-05-25" -use_unique_proto = True +test_subset = globals().get("test_subset", TestSubset.full) -if use_unique_proto: +if test_subset == TestSubset.uniq_protos: df_preds = df_preds.query(Key.uniq_proto) df_each_err = df_each_err.loc[df_preds.index] diff --git a/scripts/model_figs/parity_energy_models.py b/scripts/model_figs/parity_energy_models.py index 6414d8fd..cbb70555 100644 --- a/scripts/model_figs/parity_energy_models.py +++ b/scripts/model_figs/parity_energy_models.py @@ -13,7 +13,8 @@ from pymatviz.io import save_fig from pymatviz.utils import add_identity_line, bin_df_cols -from matbench_discovery import PDF_FIGS, SITE_FIGS, Key +from matbench_discovery import PDF_FIGS, SITE_FIGS +from matbench_discovery.enums import Key, TestSubset from matbench_discovery.plots import clf_colors from matbench_discovery.preds import df_metrics, df_metrics_uniq_protos, df_preds @@ -31,9 +32,10 @@ e_true_col = Key.e_form e_pred_col = Key.e_form_pred -use_unique_proto = True -if use_unique_proto: +test_subset = globals().get("test_subset", TestSubset.full) + +if test_subset == TestSubset.uniq_protos: df_preds = df_preds.query(Key.uniq_proto) df_metrics = df_metrics_uniq_protos diff --git a/scripts/model_figs/per_element_errors.py b/scripts/model_figs/per_element_errors.py index b3654aa1..973dbda3 100644 --- a/scripts/model_figs/per_element_errors.py +++ b/scripts/model_figs/per_element_errors.py @@ -13,8 +13,9 @@ from pymatviz.utils import bin_df_cols, df_ptable from tqdm import tqdm -from matbench_discovery import PDF_FIGS, ROOT, SITE_FIGS, SITE_LIB, Key, Model +from matbench_discovery import PDF_FIGS, ROOT, SITE_FIGS, SITE_LIB from matbench_discovery.data import df_wbm +from matbench_discovery.enums import Key, Model, TestSubset from matbench_discovery.preds import ( df_each_err, df_metrics, @@ -27,9 +28,9 @@ # %% -use_unique_proto = True +test_subset = globals().get("test_subset", TestSubset.full) -if use_unique_proto: +if test_subset == TestSubset.uniq_protos: df_preds = df_preds.query(Key.uniq_proto) df_each_err = df_each_err.loc[df_preds.index] df_metrics = df_metrics_uniq_protos diff --git a/scripts/model_figs/roc_prc_curves_models.py b/scripts/model_figs/roc_prc_curves_models.py index cbc59775..2d5cf3b6 100644 --- a/scripts/model_figs/roc_prc_curves_models.py +++ b/scripts/model_figs/roc_prc_curves_models.py @@ -9,8 +9,9 @@ from sklearn.metrics import auc, precision_recall_curve, roc_curve from tqdm import tqdm -from matbench_discovery import PDF_FIGS, SITE_FIGS, STABILITY_THRESHOLD, Key +from matbench_discovery import PDF_FIGS, SITE_FIGS, STABILITY_THRESHOLD from matbench_discovery import plots as plots +from matbench_discovery.enums import Key, TestSubset from matbench_discovery.preds import df_each_pred, df_preds, model_styles, models __author__ = "Janosh Riebesell" @@ -23,9 +24,10 @@ n_cols = 3 n_rows = math.ceil(len(models) / n_cols) -use_unique_proto = True -if use_unique_proto: +test_subset = globals().get("test_subset", TestSubset.full) + +if test_subset == TestSubset.uniq_protos: df_preds = df_preds.query(Key.uniq_proto) df_each_pred = df_each_pred.loc[df_preds.index] diff --git a/scripts/model_figs/rolling_mae_vs_hull_dist_models.py b/scripts/model_figs/rolling_mae_vs_hull_dist_models.py index 0533670d..5be94693 100644 --- a/scripts/model_figs/rolling_mae_vs_hull_dist_models.py +++ b/scripts/model_figs/rolling_mae_vs_hull_dist_models.py @@ -7,7 +7,8 @@ import numpy as np from pymatviz.io import save_fig -from matbench_discovery import PDF_FIGS, SITE_FIGS, Key +from matbench_discovery import PDF_FIGS, SITE_FIGS +from matbench_discovery.enums import Key, TestSubset from matbench_discovery.plots import rolling_mae_vs_hull_dist from matbench_discovery.preds import ( df_each_pred, @@ -25,9 +26,10 @@ # %% backend: Final = "plotly" -use_unique_proto = True -if use_unique_proto: +test_subset = globals().get("test_subset", TestSubset.full) + +if test_subset == TestSubset.uniq_protos: df_preds = df_preds.query(Key.uniq_proto) df_each_pred = df_each_pred.loc[df_preds.index] df_metrics = df_metrics_uniq_protos diff --git a/scripts/model_figs/rolling_mae_vs_hull_dist_wbm_batches.py b/scripts/model_figs/rolling_mae_vs_hull_dist_wbm_batches.py index 4cdafc9b..fc3646cb 100644 --- a/scripts/model_figs/rolling_mae_vs_hull_dist_wbm_batches.py +++ b/scripts/model_figs/rolling_mae_vs_hull_dist_wbm_batches.py @@ -6,7 +6,8 @@ # %% from pymatviz.io import save_fig -from matbench_discovery import PDF_FIGS, SITE_FIGS, Key, Model, today +from matbench_discovery import PDF_FIGS, SITE_FIGS, today +from matbench_discovery.enums import Key, Model, TestSubset from matbench_discovery.plots import plt, rolling_mae_vs_hull_dist from matbench_discovery.preds import df_each_pred, df_preds from matbench_discovery.preds import models as all_models @@ -19,12 +20,17 @@ df_err, df_std = None, None # variables to cache rolling MAE and std models = globals().get("models", all_models) -use_unique_proto = True + +test_subset = globals().get("test_subset", TestSubset.full) + +if test_subset == TestSubset.uniq_protos: + df_preds = df_preds.query(Key.uniq_proto) + df_each_pred = df_each_pred.loc[df_preds.index] # %% plotly version for model in models: - df_pivot = df_each_pred.pivot(columns=batch_col, values=model) + df_pivot = df_each_pred.pivot(columns=batch_col, values=model) # noqa: PD010 fig, df_err, df_std = rolling_mae_vs_hull_dist( e_above_hull_true=df_preds[Key.each_true], From 69065aaa1a79b0dfb2608c0fe4450309399e63f2 Mon Sep 17 00:00:00 2001 From: Janosh Riebesell Date: Wed, 7 Feb 2024 13:10:55 +0100 Subject: [PATCH 11/12] update all site figs --- ...ment-counts-mp+wbm-normalized=False.svelte | 2 +- site/src/figs/box-hull-dist-errors.svelte | 2 +- site/src/figs/cumulative-mae.svelte | 2 +- .../figs/cumulative-precision-recall.svelte | 2 +- site/src/figs/metrics-table-first-10k.svelte | 374 +++++------ .../src/figs/metrics-table-uniq-protos.svelte | 592 +++++++++--------- site/src/figs/metrics-table.svelte | 512 +++++++-------- site/src/figs/prc-models-4x3.svelte | 2 +- site/src/figs/roc-models-all-in-one.svelte | 2 +- .../rolling-mae-vs-hull-dist-models.svelte | 2 +- ...mae-vs-hull-dist-wbm-batches-alignn.svelte | 2 +- ...-mae-vs-hull-dist-wbm-batches-bowsr.svelte | 2 +- ...ae-vs-hull-dist-wbm-batches-cgcnn+p.svelte | 2 +- ...-mae-vs-hull-dist-wbm-batches-cgcnn.svelte | 2 +- ...mae-vs-hull-dist-wbm-batches-chgnet.svelte | 2 +- ...mae-vs-hull-dist-wbm-batches-m3gnet.svelte | 2 +- ...g-mae-vs-hull-dist-wbm-batches-mace.svelte | 2 +- ...mae-vs-hull-dist-wbm-batches-megnet.svelte | 2 +- ...vs-hull-dist-wbm-batches-voronoi-rf.svelte | 2 +- ...vs-hull-dist-wbm-batches-wrenformer.svelte | 2 +- ...ls-mean-vs-true-hull-dist-actinides.svelte | 2 +- ...s-models-mean-vs-true-hull-dist-all.svelte | 2 +- ...dels-mean-vs-true-hull-dist-borides.svelte | 2 +- ...els-mean-vs-true-hull-dist-carbides.svelte | 2 +- ...ean-vs-true-hull-dist-chalcogenides.svelte | 2 +- ...dels-mean-vs-true-hull-dist-halides.svelte | 2 +- ...els-mean-vs-true-hull-dist-hydrides.svelte | 2 +- ...-mean-vs-true-hull-dist-lanthanides.svelte | 2 +- ...els-mean-vs-true-hull-dist-nitrides.svelte | 2 +- ...odels-mean-vs-true-hull-dist-oxides.svelte | 2 +- ...-mean-vs-true-hull-dist-oxynitrides.svelte | 2 +- ...ls-mean-vs-true-hull-dist-pnictides.svelte | 2 +- ...els-mean-vs-true-hull-dist-sulfides.svelte | 2 +- 33 files changed, 722 insertions(+), 816 deletions(-) diff --git a/site/src/figs/bar-element-counts-mp+wbm-normalized=False.svelte b/site/src/figs/bar-element-counts-mp+wbm-normalized=False.svelte index 55617797..df711add 100644 --- a/site/src/figs/bar-element-counts-mp+wbm-normalized=False.svelte +++ b/site/src/figs/bar-element-counts-mp+wbm-normalized=False.svelte @@ -1 +1 @@ -
+
diff --git a/site/src/figs/box-hull-dist-errors.svelte b/site/src/figs/box-hull-dist-errors.svelte index cd9dc758..963e1e66 100644 --- a/site/src/figs/box-hull-dist-errors.svelte +++ b/site/src/figs/box-hull-dist-errors.svelte @@ -1 +1 @@ -
+
diff --git a/site/src/figs/cumulative-mae.svelte b/site/src/figs/cumulative-mae.svelte index dd4c042d..8ab455cf 100644 --- a/site/src/figs/cumulative-mae.svelte +++ b/site/src/figs/cumulative-mae.svelte @@ -1 +1 @@ -
+
diff --git a/site/src/figs/cumulative-precision-recall.svelte b/site/src/figs/cumulative-precision-recall.svelte index dbc6c26c..dbda9400 100644 --- a/site/src/figs/cumulative-precision-recall.svelte +++ b/site/src/figs/cumulative-precision-recall.svelte @@ -1 +1 @@ -
+
diff --git a/site/src/figs/metrics-table-first-10k.svelte b/site/src/figs/metrics-table-first-10k.svelte index 1b20466e..d546addb 100644 --- a/site/src/figs/metrics-table-first-10k.svelte +++ b/site/src/figs/metrics-table-first-10k.svelte @@ -27,229 +27,201 @@ color: #000000; } #T__row1_col0 { - background-color: #e5e419; + background-color: #dde318; color: #000000; } -#T__row1_col1, #T__row1_col2 { - background-color: #d0e11c; +#T__row1_col1 { + background-color: #c0df25; color: #000000; } -#T__row1_col3, #T__row3_col0 { - background-color: #bade28; +#T__row1_col2, #T__row1_col4, #T__row2_col6 { + background-color: #bddf26; color: #000000; } -#T__row1_col4 { - background-color: #cae11f; +#T__row1_col3 { + background-color: #9dd93b; color: #000000; } #T__row1_col5 { - background-color: #dde318; - color: #000000; + background-color: #28ae80; + color: #f1f1f1; } -#T__row1_col6 { - background-color: #efe51c; - color: #000000; +#T__row1_col6, #T__row6_col6 { + background-color: #40bd72; + color: #f1f1f1; } #T__row2_col0 { - background-color: #c2df23; + background-color: #d5e21a; color: #000000; } -#T__row2_col1, #T__row2_col2 { - background-color: #93d741; +#T__row2_col1 { + background-color: #b2dd2d; + color: #000000; +} +#T__row2_col2 { + background-color: #b0dd2f; color: #000000; } #T__row2_col3 { - background-color: #65cb5e; + background-color: #89d548; color: #000000; } #T__row2_col4 { - background-color: #8bd646; + background-color: #a8db34; color: #000000; } #T__row2_col5 { - background-color: #22a785; - color: #f1f1f1; + background-color: #93d741; + color: #000000; } -#T__row2_col6 { - background-color: #3aba76; - color: #f1f1f1; +#T__row3_col0 { + background-color: #63cb5f; + color: #000000; } #T__row3_col1, #T__row3_col2 { - background-color: #86d549; - color: #000000; + background-color: #26ad81; + color: #f1f1f1; } -#T__row3_col3 { - background-color: #56c667; - color: #000000; +#T__row3_col3, #T__row8_col1, #T__row8_col2 { + background-color: #277f8e; + color: #f1f1f1; } #T__row3_col4 { - background-color: #7ad151; - color: #000000; + background-color: #29af7f; + color: #f1f1f1; } #T__row3_col5 { - background-color: #77d153; - color: #000000; + background-color: #22a785; + color: #f1f1f1; } #T__row3_col6 { - background-color: #addc30; + background-color: #4ec36b; color: #000000; } #T__row4_col0 { - background-color: #52c569; + background-color: #5ac864; color: #000000; } -#T__row4_col1, #T__row4_col2, #T__row6_col6 { - background-color: #1fa287; +#T__row4_col1, #T__row4_col2 { + background-color: #22a884; color: #f1f1f1; } #T__row4_col3 { - background-color: #2c738e; + background-color: #2a778e; color: #f1f1f1; } -#T__row4_col4, #T__row9_col0 { - background-color: #20a386; +#T__row4_col4 { + background-color: #1fa188; color: #f1f1f1; } #T__row4_col5 { - background-color: #1fa088; + background-color: #20928c; color: #f1f1f1; } #T__row4_col6 { - background-color: #44bf70; + background-color: #2fb47c; color: #f1f1f1; } -#T__row5_col0 { - background-color: #4ac16d; +#T__row5_col0, #T__row6_col0 { + background-color: #56c667; color: #000000; } -#T__row5_col1, #T__row5_col2 { - background-color: #1f9e89; +#T__row5_col1, #T__row5_col2, #T__row5_col6 { + background-color: #21a685; color: #f1f1f1; } #T__row5_col3 { - background-color: #2e6d8e; + background-color: #2c738e; color: #f1f1f1; } -#T__row5_col4, #T__row7_col5 { - background-color: #1f958b; +#T__row5_col4, #T__row7_col1, #T__row7_col2 { + background-color: #21908d; color: #f1f1f1; } #T__row5_col5 { - background-color: #228c8d; - color: #f1f1f1; -} -#T__row5_col6 { - background-color: #2ab07f; - color: #f1f1f1; -} -#T__row6_col0, #T__row7_col0 { - background-color: #46c06f; + background-color: #27808e; color: #f1f1f1; } #T__row6_col1, #T__row6_col2 { - background-color: #1e9c89; + background-color: #21a585; color: #f1f1f1; } #T__row6_col3 { - background-color: #306a8e; + background-color: #2c728e; color: #f1f1f1; } #T__row6_col4 { - background-color: #24868e; + background-color: #20a386; color: #f1f1f1; } #T__row6_col5 { - background-color: #297a8e; + background-color: #1e9d89; color: #f1f1f1; } -#T__row7_col1, #T__row7_col2 { - background-color: #1e9b8a; +#T__row7_col0 { + background-color: #34b679; color: #f1f1f1; } #T__row7_col3 { - background-color: #30698e; + background-color: #3c508b; color: #f1f1f1; } #T__row7_col4 { - background-color: #1f978b; + background-color: #33638d; color: #f1f1f1; } -#T__row7_col6 { - background-color: #38b977; +#T__row7_col5 { + background-color: #33628d; color: #f1f1f1; } -#T__row8_col0 { - background-color: #2cb17e; +#T__row7_col6, #T__row9_col3, #T__row9_col4, #T__row9_col5, #T__row10_col0, #T__row10_col1, #T__row10_col2 { + background-color: #440154; color: #f1f1f1; } -#T__row8_col1, #T__row8_col2 { - background-color: #24878e; +#T__row8_col0 { + background-color: #23a983; color: #f1f1f1; } #T__row8_col3 { - background-color: #3f4889; + background-color: #46307e; color: #f1f1f1; } #T__row8_col4 { - background-color: #365c8d; + background-color: #482173; color: #f1f1f1; } #T__row8_col5 { - background-color: #355e8d; - color: #f1f1f1; -} -#T__row8_col6, #T__row10_col3, #T__row10_col4, #T__row10_col5, #T__row11_col0, #T__row11_col1, #T__row11_col2 { - background-color: #440154; - color: #f1f1f1; -} -#T__row9_col1, #T__row9_col2 { - background-color: #2a778e; + background-color: #3e4989; color: #f1f1f1; } -#T__row9_col3 { - background-color: #472c7a; +#T__row8_col6 { + background-color: #46337f; color: #f1f1f1; } -#T__row9_col4 { - background-color: #481f70; +#T__row9_col0 { + background-color: #1f948c; color: #f1f1f1; } -#T__row9_col5 { - background-color: #404688; +#T__row9_col1, #T__row9_col2 { + background-color: #30698e; color: #f1f1f1; } #T__row9_col6 { - background-color: #46327e; - color: #f1f1f1; -} -#T__row10_col0 { - background-color: #21908d; - color: #f1f1f1; -} -#T__row10_col1, #T__row10_col2 { - background-color: #33638d; - color: #f1f1f1; -} -#T__row10_col6 { - background-color: #463480; + background-color: #453581; color: #f1f1f1; } -#T__row11_col3 { - background-color: #20938c; +#T__row10_col3, #T__row10_col6 { + background-color: #1fa287; color: #f1f1f1; } -#T__row11_col4 { - background-color: #58c765; +#T__row10_col4 { + background-color: #7fd34e; color: #000000; } -#T__row11_col5 { - background-color: #54c568; +#T__row10_col5 { + background-color: #6ccd5a; color: #000000; } -#T__row11_col6 { - background-color: #1f9f88; - color: #f1f1f1; -} #T_ :is(td, th):nth-child(6) { border-left: 1px solid white; } table { scrollbar-width: none; /* Firefox */ @@ -266,7 +238,7 @@ id="T_"> - Model +   F1 ↑ DAF ↑ Prec ↑ @@ -278,124 +250,114 @@ - GNoME - 0.97 - 5.62 - 0.94 - 0.94 - 0.04 - 0.09 - 0.86 + CHGNet + 0.94 + 5.29 + 0.88 + 0.88 + 0.06 + 0.11 + 0.81 - CHGNet - 0.94 - 5.29 - 0.88 - 0.88 - 0.06 - 0.11 - 0.81 + MACE + 0.90 + 4.88 + 0.81 + 0.81 + 0.09 + 0.23 + 0.22 - MACE - 0.90 - 4.88 - 0.81 - 0.81 - 0.09 - 0.23 - 0.22 + M3GNet + 0.89 + 4.79 + 0.80 + 0.80 + 0.10 + 0.16 + 0.62 - M3GNet - 0.89 - 4.79 - 0.80 - 0.80 - 0.10 - 0.16 - 0.62 + CGCNN + 0.76 + 3.66 + 0.61 + 0.61 + 0.17 + 0.24 + 0.27 - CGCNN - 0.76 - 3.66 - 0.61 - 0.61 - 0.17 - 0.24 - 0.27 + ALIGNN + 0.75 + 3.58 + 0.60 + 0.60 + 0.19 + 0.27 + 0.14 - ALIGNN - 0.75 - 3.58 - 0.60 - 0.60 - 0.19 - 0.27 - 0.14 + CGCNN+P + 0.74 + 3.53 + 0.59 + 0.59 + 0.21 + 0.29 + 0.03 - CGCNN+P + Wrenformer 0.74 - 3.53 + 3.52 0.59 0.59 - 0.21 - 0.29 - 0.03 - - - Wrenformer - 0.74 - 3.52 - 0.59 - 0.59 - 0.19 - 0.25 - 0.22 + 0.19 + 0.25 + 0.22 - BOWSR - 0.69 - 3.14 - 0.52 - 0.52 - 0.27 - 0.33 - -1.10 + BOWSR + 0.69 + 3.14 + 0.52 + 0.52 + 0.27 + 0.33 + -1.10 - MEGNet - 0.64 - 2.84 - 0.47 - 0.47 - 0.33 - 0.36 - -0.82 + MEGNet + 0.64 + 2.84 + 0.47 + 0.47 + 0.33 + 0.36 + -0.82 - Voronoi RF - 0.58 - 2.45 - 0.41 - 0.41 - 0.36 - 0.43 - -0.81 + Voronoi RF + 0.58 + 2.45 + 0.41 + 0.41 + 0.36 + 0.43 + -0.81 - Dummy - 0.19 - 1.00 - 0.17 - 0.68 - 0.12 - 0.18 - 0.00 + Dummy + 0.19 + 1.00 + 0.17 + 0.68 + 0.12 + 0.18 + 0.00 diff --git a/site/src/figs/metrics-table-uniq-protos.svelte b/site/src/figs/metrics-table-uniq-protos.svelte index 7abf72e6..0f63a2ba 100644 --- a/site/src/figs/metrics-table-uniq-protos.svelte +++ b/site/src/figs/metrics-table-uniq-protos.svelte @@ -22,304 +22,292 @@ font-weight: normal; padding: 3pt; } -#T__row0_col0, #T__row0_col1, #T__row0_col2, #T__row0_col3, #T__row0_col4, #T__row0_col5, #T__row0_col6, #T__row0_col7, #T__row0_col8 { +#T__row0_col0, #T__row0_col1, #T__row0_col2, #T__row0_col3, #T__row0_col5, #T__row0_col6, #T__row1_col7, #T__row1_col8, #T__row2_col4 { background-color: #fde725; color: #000000; } -#T__row1_col0, #T__row1_col3, #T__row4_col4 { - background-color: #5cc863; +#T__row0_col4 { + background-color: #f8e621; color: #000000; } -#T__row1_col1, #T__row2_col5, #T__row6_col4 { - background-color: #2ab07f; - color: #f1f1f1; +#T__row0_col7 { + background-color: #ece51b; + color: #000000; } -#T__row1_col2 { - background-color: #25ab82; - color: #f1f1f1; +#T__row0_col8 { + background-color: #f1e51d; + color: #000000; } -#T__row1_col4 { - background-color: #e5e419; +#T__row1_col0, #T__row2_col7 { + background-color: #b2dd2d; color: #000000; } -#T__row1_col5 { - background-color: #50c46a; +#T__row1_col1 { + background-color: #84d44b; color: #000000; } -#T__row1_col6 { - background-color: #7fd34e; +#T__row1_col2, #T__row7_col4 { + background-color: #9dd93b; color: #000000; } -#T__row1_col7 { - background-color: #9bd93c; +#T__row1_col3 { + background-color: #aadc32; color: #000000; } -#T__row1_col8 { - background-color: #c5e021; +#T__row1_col4, #T__row2_col8 { + background-color: #cae11f; color: #000000; } -#T__row2_col0 { - background-color: #34b679; - color: #f1f1f1; +#T__row1_col5 { + background-color: #addc30; + color: #000000; +} +#T__row1_col6 { + background-color: #d5e21a; + color: #000000; +} +#T__row2_col0, #T__row3_col2 { + background-color: #77d153; + color: #000000; } #T__row2_col1 { - background-color: #1f948c; + background-color: #2eb37c; color: #f1f1f1; } -#T__row2_col2 { - background-color: #1f958b; +#T__row2_col2, #T__row2_col3 { + background-color: #3bbb75; color: #f1f1f1; } -#T__row2_col3, #T__row4_col5 { - background-color: #2fb47c; +#T__row2_col5 { + background-color: #25ab82; color: #f1f1f1; } -#T__row2_col4 { - background-color: #bade28; - color: #000000; -} #T__row2_col6 { - background-color: #5ec962; - color: #000000; -} -#T__row2_col7 { - background-color: #addc30; + background-color: #7cd250; color: #000000; } -#T__row2_col8 { - background-color: #d2e21b; +#T__row3_col0 { + background-color: #75d054; color: #000000; } -#T__row3_col0, #T__row4_col0 { - background-color: #21a685; - color: #f1f1f1; -} #T__row3_col1 { - background-color: #297b8e; - color: #f1f1f1; -} -#T__row3_col2 { - background-color: #287c8e; - color: #f1f1f1; + background-color: #63cb5f; + color: #000000; } #T__row3_col3 { - background-color: #20928c; - color: #f1f1f1; + background-color: #89d548; + color: #000000; } #T__row3_col4 { - background-color: #ece51b; + background-color: #69cd5b; color: #000000; } #T__row3_col5 { - background-color: #26828e; - color: #f1f1f1; + background-color: #bade28; + color: #000000; } -#T__row3_col6, #T__row4_col8 { - background-color: #2eb37c; +#T__row3_col6 { + background-color: #23a983; color: #f1f1f1; } #T__row3_col7 { - background-color: #6ccd5a; - color: #000000; + background-color: #24aa83; + color: #f1f1f1; } #T__row3_col8 { - background-color: #a0da39; - color: #000000; + background-color: #42be71; + color: #f1f1f1; +} +#T__row4_col0, #T__row5_col2 { + background-color: #37b878; + color: #f1f1f1; } #T__row4_col1 { - background-color: #228c8d; + background-color: #38b977; color: #f1f1f1; } -#T__row4_col2, #T__row4_col6, #T__row7_col0 { - background-color: #228d8d; +#T__row4_col2 { + background-color: #46c06f; color: #f1f1f1; } #T__row4_col3 { - background-color: #25ac82; + background-color: #5ec962; + color: #000000; +} +#T__row4_col4 { + background-color: #26ad81; color: #f1f1f1; } -#T__row4_col7, #T__row6_col3 { - background-color: #1f988b; +#T__row4_col5 { + background-color: #b5de2b; + color: #000000; +} +#T__row4_col6, #T__row9_col1 { + background-color: #414287; color: #f1f1f1; } -#T__row5_col0 { - background-color: #21918c; +#T__row4_col7, #T__row9_col2 { + background-color: #404688; color: #f1f1f1; } -#T__row5_col1 { - background-color: #277f8e; +#T__row4_col8 { + background-color: #38598c; color: #f1f1f1; } -#T__row5_col2 { - background-color: #27808e; +#T__row5_col0 { + background-color: #35b779; color: #f1f1f1; } -#T__row5_col3 { - background-color: #1f9f88; +#T__row5_col1 { + background-color: #2cb17e; color: #f1f1f1; } +#T__row5_col3 { + background-color: #4ac16d; + color: #000000; +} #T__row5_col4 { - background-color: #22a884; + background-color: #31b57b; color: #f1f1f1; } #T__row5_col5 { - background-color: #2db27d; - color: #f1f1f1; + background-color: #8bd646; + color: #000000; } #T__row5_col6 { - background-color: #453781; - color: #f1f1f1; -} -#T__row5_col7 { - background-color: #433e85; + background-color: #482878; color: #f1f1f1; } -#T__row5_col8 { - background-color: #3a538b; +#T__row5_col7, #T__row5_col8, #T__row8_col5, #T__row9_col3, #T__row9_col6, #T__row10_col0, #T__row10_col1, #T__row10_col2, #T__row10_col4 { + background-color: #440154; color: #f1f1f1; } #T__row6_col0 { - background-color: #21908d; + background-color: #2fb47c; color: #f1f1f1; } #T__row6_col1 { - background-color: #29798e; - color: #f1f1f1; -} -#T__row6_col2, #T__row7_col5, #T__row8_col8 { - background-color: #297a8e; + background-color: #1f998a; color: #f1f1f1; } -#T__row6_col5 { - background-color: #22a785; +#T__row6_col2, #T__row6_col5 { + background-color: #1fa088; color: #f1f1f1; } -#T__row6_col6 { - background-color: #482071; +#T__row6_col3 { + background-color: #1f9f88; color: #f1f1f1; } -#T__row6_col7, #T__row6_col8, #T__row9_col5, #T__row10_col3, #T__row10_col6, #T__row11_col0, #T__row11_col1, #T__row11_col2, #T__row11_col4 { - background-color: #440154; - color: #f1f1f1; +#T__row6_col4 { + background-color: #7fd34e; + color: #000000; } -#T__row7_col1 { - background-color: #306a8e; +#T__row6_col6 { + background-color: #2b748e; color: #f1f1f1; } -#T__row7_col2, #T__row7_col7 { - background-color: #2f6b8e; +#T__row6_col7 { + background-color: #2a768e; color: #f1f1f1; } -#T__row7_col3 { - background-color: #287d8e; +#T__row6_col8 { + background-color: #228c8d; color: #f1f1f1; } -#T__row7_col4 { - background-color: #70cf57; - color: #000000; -} -#T__row7_col6, #T__row8_col7 { - background-color: #33628d; +#T__row7_col0 { + background-color: #20a386; color: #f1f1f1; } -#T__row7_col8, #T__row9_col7 { - background-color: #25838e; +#T__row7_col1 { + background-color: #27808e; color: #f1f1f1; } -#T__row8_col0, #T__row11_col8 { - background-color: #26818e; +#T__row7_col2 { + background-color: #25858e; color: #f1f1f1; } -#T__row8_col1 { - background-color: #38588c; +#T__row7_col3 { + background-color: #2d708e; color: #f1f1f1; } -#T__row8_col2, #T__row8_col3 { - background-color: #38598c; +#T__row7_col5, #T__row9_col0 { + background-color: #34618d; color: #f1f1f1; } -#T__row8_col4 { - background-color: #8ed645; - color: #000000; -} -#T__row8_col5 { - background-color: #3e4989; +#T__row7_col6 { + background-color: #287c8e; color: #f1f1f1; } -#T__row8_col6 { - background-color: #30698e; +#T__row7_col7, #T__row8_col1 { + background-color: #2e6d8e; color: #f1f1f1; } -#T__row9_col0 { - background-color: #2c718e; +#T__row7_col8 { + background-color: #26828e; color: #f1f1f1; } -#T__row9_col1 { - background-color: #3e4a89; +#T__row8_col0 { + background-color: #21918c; color: #f1f1f1; } -#T__row9_col2 { - background-color: #3f4788; +#T__row8_col2 { + background-color: #2f6b8e; color: #f1f1f1; } -#T__row9_col3 { - background-color: #482475; +#T__row8_col3 { + background-color: #472f7d; color: #f1f1f1; } -#T__row9_col4 { - background-color: #a5db36; +#T__row8_col4 { + background-color: #b8de29; color: #000000; } -#T__row9_col6 { - background-color: #39558c; - color: #f1f1f1; -} -#T__row9_col8 { - background-color: #1e9b8a; +#T__row8_col6 { + background-color: #31678e; color: #f1f1f1; } -#T__row10_col0 { - background-color: #3e4c8a; +#T__row8_col7 { + background-color: #20928c; color: #f1f1f1; } -#T__row10_col1 { - background-color: #472c7a; +#T__row8_col8 { + background-color: #20a486; color: #f1f1f1; } -#T__row10_col2 { - background-color: #472d7b; +#T__row9_col4 { + background-color: #1f978b; color: #f1f1f1; } -#T__row10_col4 { - background-color: #20938c; +#T__row9_col5 { + background-color: #460a5d; color: #f1f1f1; } -#T__row10_col5 { - background-color: #46075a; +#T__row9_col7 { + background-color: #453882; color: #f1f1f1; } -#T__row10_col7 { - background-color: #46327e; +#T__row9_col8 { + background-color: #3f4788; color: #f1f1f1; } -#T__row10_col8 { - background-color: #414287; +#T__row10_col3 { + background-color: #482173; color: #f1f1f1; } -#T__row11_col3 { - background-color: #481a6c; +#T__row10_col5 { + background-color: #2a788e; color: #f1f1f1; } -#T__row11_col5 { - background-color: #365c8d; +#T__row10_col6 { + background-color: #39568c; color: #f1f1f1; } -#T__row11_col6 { - background-color: #3f4889; +#T__row10_col7 { + background-color: #2c728e; color: #f1f1f1; } -#T__row11_col7 { - background-color: #31678e; +#T__row10_col8 { + background-color: #23888e; color: #f1f1f1; } #T_ :is(td, th):nth-child(8) { border-left: 1px solid white; } @@ -338,7 +326,7 @@ id="T_"> - Model +   F1 ↑ DAF ↑ Prec ↑ @@ -356,196 +344,180 @@ - GNoME - 0.83 - 5.49 - 0.84 - 0.95 - 0.82 - 0.97 - 0.04 - 0.09 - 0.79 - 96.0M (8.0M) - 16.2M + MACE + 0.67 + 3.85 + 0.58 + 0.88 + 0.80 + 0.89 + 0.06 + 0.11 + 0.66 + 1.6M (145.9K) + 4.7M UIP EFS - MACE - 0.67 - 3.85 - 0.58 - 0.88 - 0.80 - 0.89 + CHGNet + 0.61 + 3.32 + 0.51 + 0.85 + 0.76 + 0.87 0.06 - 0.11 - 0.66 + 0.10 + 0.69 1.6M (145.9K) - 4.7M + 412.5K UIP - EFS + EFSM - CHGNet - 0.61 - 3.32 - 0.51 - 0.85 - 0.76 - 0.87 - 0.06 - 0.10 - 0.69 - 1.6M (145.9K) - 412.5K + M3GNet + 0.57 + 2.84 + 0.44 + 0.81 + 0.80 + 0.81 + 0.07 + 0.12 + 0.58 + 188.3K (62.8K) + 227.5K UIP - EFSM + EFS - M3GNet + ALIGNN 0.57 - 2.84 - 0.44 - 0.81 - 0.80 - 0.81 - 0.07 - 0.12 - 0.58 - 188.3K (62.8K) - 227.5K - UIP - EFS + 3.16 + 0.49 + 0.84 + 0.67 + 0.87 + 0.09 + 0.15 + 0.30 + 154.7K + 4.0M + GNN + E - ALIGNN - 0.57 - 3.16 - 0.49 - 0.84 - 0.67 + MEGNet + 0.51 + 2.92 + 0.45 + 0.83 + 0.58 0.87 - 0.09 - 0.15 - 0.30 - 154.7K - 4.0M + 0.13 + 0.21 + -0.25 + 133.4K + 167.8K GNN E - MEGNet + CGCNN 0.51 - 2.92 - 0.45 - 0.83 - 0.58 - 0.87 - 0.13 - 0.21 - -0.25 - 133.4K - 167.8K + 2.82 + 0.44 + 0.82 + 0.60 + 0.86 + 0.14 + 0.23 + -0.60 + 154.7K + 128.4K GNN E - CGCNN - 0.51 - 2.82 - 0.44 - 0.82 - 0.60 - 0.86 - 0.14 - 0.23 - -0.60 + CGCNN+P + 0.50 + 2.53 + 0.39 + 0.79 + 0.69 + 0.80 + 0.11 + 0.18 + 0.02 154.7K 128.4K GNN E - CGCNN+P - 0.50 - 2.53 - 0.39 - 0.79 - 0.69 - 0.80 + Wrenformer + 0.47 + 2.23 + 0.34 + 0.74 + 0.72 + 0.75 0.11 - 0.18 - 0.02 + 0.19 + -0.04 154.7K - 128.4K - GNN + 5.2M + Transformer E - Wrenformer - 0.47 - 2.23 - 0.34 - 0.74 - 0.72 - 0.75 - 0.11 - 0.19 - -0.04 - 154.7K - 5.2M - Transformer + BOWSR + 0.43 + 2.00 + 0.30 + 0.70 + 0.74 + 0.69 + 0.12 + 0.17 + 0.15 + 133.4K + 167.8K + BO-GNN E - BOWSR - 0.43 - 2.00 - 0.30 - 0.70 - 0.74 + Voronoi RF + 0.33 + 1.56 + 0.24 + 0.67 + 0.54 0.69 - 0.12 - 0.17 - 0.15 - 133.4K - 167.8K - BO-GNN + 0.15 + 0.21 + -0.33 + 154.7K + 0.0 + Fingerprint E - Voronoi RF - 0.33 - 1.56 - 0.24 - 0.67 - 0.54 - 0.69 - 0.15 - 0.21 - -0.33 - 154.7K - 0.0 - Fingerprint - E - - - Dummy - 0.18 - 1.00 - 0.15 - 0.69 - 0.23 - 0.77 - 0.12 - 0.18 - 0.00 - - - - + Dummy + 0.18 + 1.00 + 0.15 + 0.69 + 0.23 + 0.77 + 0.12 + 0.18 + 0.00 + + + + diff --git a/site/src/figs/metrics-table.svelte b/site/src/figs/metrics-table.svelte index 39e72905..3a18728e 100644 --- a/site/src/figs/metrics-table.svelte +++ b/site/src/figs/metrics-table.svelte @@ -22,316 +22,300 @@ font-weight: normal; padding: 3pt; } -#T__row0_col0, #T__row0_col1, #T__row0_col2, #T__row0_col3, #T__row0_col4, #T__row0_col5, #T__row0_col6, #T__row0_col7, #T__row0_col8 { +#T__row0_col0, #T__row0_col1, #T__row0_col2, #T__row0_col3, #T__row0_col5, #T__row0_col6, #T__row1_col7, #T__row1_col8, #T__row2_col4 { background-color: #fde725; color: #000000; } -#T__row1_col0 { - background-color: #65cb5e; +#T__row0_col4 { + background-color: #f6e620; color: #000000; } -#T__row1_col1 { - background-color: #2fb47c; - color: #f1f1f1; +#T__row0_col7 { + background-color: #f1e51d; + color: #000000; } -#T__row1_col2 { - background-color: #29af7f; - color: #f1f1f1; +#T__row0_col8 { + background-color: #f4e61e; + color: #000000; } -#T__row1_col3 { - background-color: #58c765; +#T__row1_col0, #T__row2_col7 { + background-color: #b2dd2d; color: #000000; } -#T__row1_col4 { - background-color: #eae51a; +#T__row1_col1, #T__row3_col3 { + background-color: #84d44b; color: #000000; } -#T__row1_col5 { - background-color: #4ec36b; +#T__row1_col2, #T__row5_col5 { + background-color: #9bd93c; color: #000000; } -#T__row1_col6 { - background-color: #7fd34e; +#T__row1_col3 { + background-color: #a5db36; color: #000000; } -#T__row1_col7 { - background-color: #a8db34; +#T__row1_col4, #T__row8_col4 { + background-color: #c5e021; color: #000000; } -#T__row1_col8 { - background-color: #c8e020; +#T__row1_col5 { + background-color: #b0dd2f; + color: #000000; +} +#T__row1_col6 { + background-color: #d2e21b; color: #000000; } #T__row2_col0 { - background-color: #38b977; - color: #f1f1f1; + background-color: #7fd34e; + color: #000000; } #T__row2_col1 { - background-color: #1f978b; + background-color: #32b67a; color: #f1f1f1; } #T__row2_col2 { - background-color: #1f998a; + background-color: #3fbc73; color: #f1f1f1; } -#T__row2_col3, #T__row2_col5, #T__row4_col8 { - background-color: #2cb17e; +#T__row2_col3 { + background-color: #3aba76; color: #f1f1f1; } -#T__row2_col4 { - background-color: #b8de29; - color: #000000; -} -#T__row2_col6 { - background-color: #5cc863; - color: #000000; +#T__row2_col5 { + background-color: #26ad81; + color: #f1f1f1; } -#T__row2_col7, #T__row9_col4 { - background-color: #b5de2b; +#T__row2_col6, #T__row3_col2 { + background-color: #7cd250; color: #000000; } #T__row2_col8 { - background-color: #d2e21b; + background-color: #cde11d; color: #000000; } #T__row3_col0 { - background-color: #25ac82; - color: #f1f1f1; + background-color: #70cf57; + color: #000000; } #T__row3_col1 { - background-color: #277f8e; - color: #f1f1f1; -} -#T__row3_col2 { - background-color: #26818e; - color: #f1f1f1; -} -#T__row3_col3, #T__row4_col1 { - background-color: #21918c; - color: #f1f1f1; + background-color: #67cc5c; + color: #000000; } #T__row3_col4 { - background-color: #f1e51d; + background-color: #5ec962; color: #000000; } #T__row3_col5 { - background-color: #25838e; - color: #f1f1f1; + background-color: #c8e020; + color: #000000; } -#T__row3_col6, #T__row5_col5 { - background-color: #2eb37c; +#T__row3_col6 { + background-color: #1fa287; color: #f1f1f1; } -#T__row3_col7 { - background-color: #73d056; - color: #000000; +#T__row3_col7, #T__row6_col2 { + background-color: #21a585; + color: #f1f1f1; } #T__row3_col8 { - background-color: #a2da37; - color: #000000; + background-color: #3dbc74; + color: #f1f1f1; } -#T__row4_col0, #T__row5_col4 { - background-color: #22a884; +#T__row4_col0 { + background-color: #38b977; color: #f1f1f1; } -#T__row4_col2, #T__row6_col0, #T__row7_col0 { - background-color: #20928c; +#T__row4_col1, #T__row5_col2 { + background-color: #3bbb75; color: #f1f1f1; } +#T__row4_col2 { + background-color: #4ac16d; + color: #000000; +} #T__row4_col3 { - background-color: #23a983; - color: #f1f1f1; + background-color: #56c667; + color: #000000; } #T__row4_col4 { - background-color: #54c568; - color: #000000; + background-color: #25ac82; + color: #f1f1f1; } #T__row4_col5 { - background-color: #34b679; - color: #f1f1f1; + background-color: #bade28; + color: #000000; } -#T__row4_col6, #T__row7_col8 { - background-color: #24878e; +#T__row4_col6 { + background-color: #453882; color: #f1f1f1; } #T__row4_col7 { - background-color: #1f968b; + background-color: #414487; color: #f1f1f1; } -#T__row5_col0 { - background-color: #20938c; +#T__row4_col8 { + background-color: #39558c; color: #f1f1f1; } -#T__row5_col1, #T__row7_col5, #T__row11_col8 { - background-color: #26828e; +#T__row5_col0, #T__row6_col0 { + background-color: #35b779; color: #f1f1f1; } -#T__row5_col2 { - background-color: #25848e; +#T__row5_col1 { + background-color: #31b57b; color: #f1f1f1; } -#T__row5_col3, #T__row9_col8 { - background-color: #1e9b8a; +#T__row5_col3 { + background-color: #46c06f; color: #f1f1f1; } -#T__row5_col6, #T__row10_col1 { - background-color: #472e7c; +#T__row5_col4 { + background-color: #2cb17e; color: #f1f1f1; } -#T__row5_col7 { - background-color: #433d84; +#T__row5_col6 { + background-color: #481d6f; color: #f1f1f1; } -#T__row5_col8 { - background-color: #3c508b; +#T__row5_col7, #T__row5_col8, #T__row8_col5, #T__row9_col3, #T__row9_col6, #T__row10_col0, #T__row10_col1, #T__row10_col2, #T__row10_col4 { + background-color: #440154; color: #f1f1f1; } #T__row6_col1 { - background-color: #277e8e; - color: #f1f1f1; -} -#T__row6_col2 { - background-color: #27808e; + background-color: #1f9f88; color: #f1f1f1; } #T__row6_col3 { - background-color: #1f958b; + background-color: #1fa188; color: #f1f1f1; } #T__row6_col4 { - background-color: #27ad81; - color: #f1f1f1; + background-color: #73d056; + color: #000000; } #T__row6_col5 { - background-color: #25ab82; + background-color: #24aa83; color: #f1f1f1; } #T__row6_col6 { - background-color: #48186a; + background-color: #2a768e; color: #f1f1f1; } -#T__row6_col7, #T__row6_col8, #T__row9_col5, #T__row10_col3, #T__row10_col6, #T__row11_col0, #T__row11_col1, #T__row11_col2, #T__row11_col4 { - background-color: #440154; +#T__row6_col7 { + background-color: #29798e; + color: #f1f1f1; +} +#T__row6_col8 { + background-color: #218f8d; + color: #f1f1f1; +} +#T__row7_col0 { + background-color: #23a983; color: #f1f1f1; } #T__row7_col1 { - background-color: #2d708e; + background-color: #25848e; color: #f1f1f1; } -#T__row7_col2 { - background-color: #2d718e; +#T__row7_col2, #T__row10_col8 { + background-color: #238a8d; color: #f1f1f1; } #T__row7_col3 { - background-color: #287d8e; + background-color: #2c728e; color: #f1f1f1; } #T__row7_col4 { - background-color: #69cd5b; + background-color: #8ed645; color: #000000; } +#T__row7_col5 { + background-color: #2d718e; + color: #f1f1f1; +} #T__row7_col6 { - background-color: #33638d; + background-color: #277e8e; color: #f1f1f1; } #T__row7_col7 { background-color: #2e6e8e; color: #f1f1f1; } +#T__row7_col8 { + background-color: #25838e; + color: #f1f1f1; +} #T__row8_col0 { - background-color: #24868e; + background-color: #1f968b; color: #f1f1f1; } #T__row8_col1 { - background-color: #355e8d; + background-color: #2e6f8e; color: #f1f1f1; } #T__row8_col2 { - background-color: #355f8d; + background-color: #2e6d8e; color: #f1f1f1; } #T__row8_col3 { - background-color: #375a8c; - color: #f1f1f1; -} -#T__row8_col4 { - background-color: #81d34d; - color: #000000; -} -#T__row8_col5 { - background-color: #39568c; + background-color: #482374; color: #f1f1f1; } -#T__row8_col6, #T__row11_col7 { - background-color: #306a8e; +#T__row8_col6 { + background-color: #33628d; color: #f1f1f1; } #T__row8_col7 { - background-color: #32648e; + background-color: #20928c; color: #f1f1f1; } #T__row8_col8 { - background-color: #287c8e; + background-color: #20a486; color: #f1f1f1; } #T__row9_col0 { - background-color: #2a788e; + background-color: #32648e; color: #f1f1f1; } #T__row9_col1 { - background-color: #3d4e8a; + background-color: #404588; color: #f1f1f1; } #T__row9_col2 { - background-color: #3e4a89; + background-color: #3e4989; color: #f1f1f1; } -#T__row9_col3 { - background-color: #481a6c; +#T__row9_col4 { + background-color: #21918c; color: #f1f1f1; } -#T__row9_col6 { - background-color: #3b518b; +#T__row9_col5 { + background-color: #482979; color: #f1f1f1; } #T__row9_col7 { - background-color: #25858e; - color: #f1f1f1; -} -#T__row10_col0 { - background-color: #3c4f8a; + background-color: #433d84; color: #f1f1f1; } -#T__row10_col2 { - background-color: #46307e; +#T__row9_col8 { + background-color: #3d4e8a; color: #f1f1f1; } -#T__row10_col4 { - background-color: #218e8d; +#T__row10_col3 { + background-color: #481c6e; color: #f1f1f1; } #T__row10_col5 { - background-color: #481f70; - color: #f1f1f1; -} -#T__row10_col7 { - background-color: #453781; - color: #f1f1f1; -} -#T__row10_col8 { - background-color: #3f4889; - color: #f1f1f1; -} -#T__row11_col3 { - background-color: #481668; + background-color: #24868e; color: #f1f1f1; } -#T__row11_col5 { - background-color: #31688e; +#T__row10_col6 { + background-color: #3b528b; color: #f1f1f1; } -#T__row11_col6 { - background-color: #414487; +#T__row10_col7 { + background-color: #2b748e; color: #f1f1f1; } #T_ :is(td, th):nth-child(8) { border-left: 1px solid white; } @@ -350,7 +334,7 @@ id="T_"> - Model +   F1 ↑ DAF ↑ Prec ↑ @@ -364,148 +348,136 @@ - GNoME - 0.81 - 4.84 - 0.82 - 0.94 - 0.80 - 0.96 - 0.03 - 0.09 - 0.78 + MACE + 0.67 + 3.50 + 0.58 + 0.87 + 0.78 + 0.88 + 0.06 + 0.10 + 0.66 - MACE - 0.67 - 3.50 - 0.58 - 0.87 - 0.78 - 0.88 + CHGNet + 0.61 + 3.04 + 0.52 + 0.84 + 0.74 + 0.86 0.06 0.10 - 0.66 + 0.69 - CHGNet - 0.61 - 3.04 - 0.52 - 0.84 - 0.74 - 0.86 - 0.06 - 0.10 - 0.69 + M3GNet + 0.58 + 2.65 + 0.45 + 0.80 + 0.79 + 0.80 + 0.07 + 0.12 + 0.58 - M3GNet - 0.58 - 2.65 - 0.45 - 0.80 - 0.79 - 0.80 - 0.07 - 0.12 - 0.58 + ALIGNN + 0.56 + 2.92 + 0.50 + 0.83 + 0.65 + 0.87 + 0.09 + 0.15 + 0.27 - ALIGNN - 0.56 - 2.92 - 0.50 - 0.83 - 0.65 - 0.87 - 0.09 - 0.15 - 0.27 + MEGNet + 0.51 + 2.70 + 0.46 + 0.81 + 0.57 + 0.86 + 0.13 + 0.20 + -0.28 - MEGNet + CGCNN 0.51 - 2.70 - 0.46 + 2.63 + 0.45 0.81 - 0.57 - 0.86 - 0.13 - 0.20 - -0.28 + 0.59 + 0.85 + 0.14 + 0.23 + -0.62 - CGCNN + CGCNN+P 0.51 - 2.63 - 0.45 - 0.81 - 0.59 - 0.85 - 0.14 - 0.23 - -0.62 + 2.40 + 0.41 + 0.78 + 0.67 + 0.80 + 0.11 + 0.18 + 0.03 - CGCNN+P - 0.51 - 2.40 - 0.41 - 0.78 - 0.67 - 0.80 - 0.11 + Wrenformer + 0.48 + 2.13 + 0.36 + 0.74 + 0.69 + 0.75 + 0.10 0.18 - 0.03 - - - Wrenformer - 0.48 - 2.13 - 0.36 - 0.74 - 0.69 - 0.75 - 0.10 - 0.18 - -0.04 + -0.04 - BOWSR - 0.44 - 1.91 - 0.32 - 0.68 - 0.74 - 0.67 - 0.12 - 0.16 - 0.14 + BOWSR + 0.44 + 1.91 + 0.32 + 0.68 + 0.74 + 0.67 + 0.12 + 0.16 + 0.14 - Voronoi RF - 0.34 - 1.51 - 0.26 - 0.67 - 0.51 - 0.70 - 0.14 - 0.21 - -0.31 + Voronoi RF + 0.34 + 1.51 + 0.26 + 0.67 + 0.51 + 0.70 + 0.14 + 0.21 + -0.31 - Dummy - 0.19 - 1.00 - 0.17 - 0.68 - 0.23 - 0.77 - 0.12 - 0.18 - 0.00 + Dummy + 0.19 + 1.00 + 0.17 + 0.68 + 0.23 + 0.77 + 0.12 + 0.18 + 0.00 diff --git a/site/src/figs/prc-models-4x3.svelte b/site/src/figs/prc-models-4x3.svelte index 8068401d..b394130d 100644 --- a/site/src/figs/prc-models-4x3.svelte +++ b/site/src/figs/prc-models-4x3.svelte @@ -1 +1 @@ -
+
diff --git a/site/src/figs/roc-models-all-in-one.svelte b/site/src/figs/roc-models-all-in-one.svelte index 22f79a49..55e0f3da 100644 --- a/site/src/figs/roc-models-all-in-one.svelte +++ b/site/src/figs/roc-models-all-in-one.svelte @@ -1 +1 @@ -
+
diff --git a/site/src/figs/rolling-mae-vs-hull-dist-models.svelte b/site/src/figs/rolling-mae-vs-hull-dist-models.svelte index 22d02552..a0639bdb 100644 --- a/site/src/figs/rolling-mae-vs-hull-dist-models.svelte +++ b/site/src/figs/rolling-mae-vs-hull-dist-models.svelte @@ -1 +1 @@ -
+
diff --git a/site/src/figs/rolling-mae-vs-hull-dist-wbm-batches-alignn.svelte b/site/src/figs/rolling-mae-vs-hull-dist-wbm-batches-alignn.svelte index 7dcea211..56860cc5 100644 --- a/site/src/figs/rolling-mae-vs-hull-dist-wbm-batches-alignn.svelte +++ b/site/src/figs/rolling-mae-vs-hull-dist-wbm-batches-alignn.svelte @@ -1 +1 @@ -
+
diff --git a/site/src/figs/rolling-mae-vs-hull-dist-wbm-batches-bowsr.svelte b/site/src/figs/rolling-mae-vs-hull-dist-wbm-batches-bowsr.svelte index b403b1f2..155aab97 100644 --- a/site/src/figs/rolling-mae-vs-hull-dist-wbm-batches-bowsr.svelte +++ b/site/src/figs/rolling-mae-vs-hull-dist-wbm-batches-bowsr.svelte @@ -1 +1 @@ -
+
diff --git a/site/src/figs/rolling-mae-vs-hull-dist-wbm-batches-cgcnn+p.svelte b/site/src/figs/rolling-mae-vs-hull-dist-wbm-batches-cgcnn+p.svelte index 3d5f84e7..c0c07191 100644 --- a/site/src/figs/rolling-mae-vs-hull-dist-wbm-batches-cgcnn+p.svelte +++ b/site/src/figs/rolling-mae-vs-hull-dist-wbm-batches-cgcnn+p.svelte @@ -1 +1 @@ -
+
diff --git a/site/src/figs/rolling-mae-vs-hull-dist-wbm-batches-cgcnn.svelte b/site/src/figs/rolling-mae-vs-hull-dist-wbm-batches-cgcnn.svelte index 2fd47530..a9e0ea0d 100644 --- a/site/src/figs/rolling-mae-vs-hull-dist-wbm-batches-cgcnn.svelte +++ b/site/src/figs/rolling-mae-vs-hull-dist-wbm-batches-cgcnn.svelte @@ -1 +1 @@ -
+
diff --git a/site/src/figs/rolling-mae-vs-hull-dist-wbm-batches-chgnet.svelte b/site/src/figs/rolling-mae-vs-hull-dist-wbm-batches-chgnet.svelte index f0ec63cd..576cea7e 100644 --- a/site/src/figs/rolling-mae-vs-hull-dist-wbm-batches-chgnet.svelte +++ b/site/src/figs/rolling-mae-vs-hull-dist-wbm-batches-chgnet.svelte @@ -1 +1 @@ -
+
diff --git a/site/src/figs/rolling-mae-vs-hull-dist-wbm-batches-m3gnet.svelte b/site/src/figs/rolling-mae-vs-hull-dist-wbm-batches-m3gnet.svelte index 20f4da9d..65af10c7 100644 --- a/site/src/figs/rolling-mae-vs-hull-dist-wbm-batches-m3gnet.svelte +++ b/site/src/figs/rolling-mae-vs-hull-dist-wbm-batches-m3gnet.svelte @@ -1 +1 @@ -
+
diff --git a/site/src/figs/rolling-mae-vs-hull-dist-wbm-batches-mace.svelte b/site/src/figs/rolling-mae-vs-hull-dist-wbm-batches-mace.svelte index 366a5bf3..b31762e9 100644 --- a/site/src/figs/rolling-mae-vs-hull-dist-wbm-batches-mace.svelte +++ b/site/src/figs/rolling-mae-vs-hull-dist-wbm-batches-mace.svelte @@ -1 +1 @@ -
+
diff --git a/site/src/figs/rolling-mae-vs-hull-dist-wbm-batches-megnet.svelte b/site/src/figs/rolling-mae-vs-hull-dist-wbm-batches-megnet.svelte index d21787d1..b6823e0e 100644 --- a/site/src/figs/rolling-mae-vs-hull-dist-wbm-batches-megnet.svelte +++ b/site/src/figs/rolling-mae-vs-hull-dist-wbm-batches-megnet.svelte @@ -1 +1 @@ -
+
diff --git a/site/src/figs/rolling-mae-vs-hull-dist-wbm-batches-voronoi-rf.svelte b/site/src/figs/rolling-mae-vs-hull-dist-wbm-batches-voronoi-rf.svelte index 07a1485a..6c462f9b 100644 --- a/site/src/figs/rolling-mae-vs-hull-dist-wbm-batches-voronoi-rf.svelte +++ b/site/src/figs/rolling-mae-vs-hull-dist-wbm-batches-voronoi-rf.svelte @@ -1 +1 @@ -
+
diff --git a/site/src/figs/rolling-mae-vs-hull-dist-wbm-batches-wrenformer.svelte b/site/src/figs/rolling-mae-vs-hull-dist-wbm-batches-wrenformer.svelte index d678ddb8..267e2f2b 100644 --- a/site/src/figs/rolling-mae-vs-hull-dist-wbm-batches-wrenformer.svelte +++ b/site/src/figs/rolling-mae-vs-hull-dist-wbm-batches-wrenformer.svelte @@ -1 +1 @@ -
+
diff --git a/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-actinides.svelte b/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-actinides.svelte index 454802d4..8056f730 100644 --- a/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-actinides.svelte +++ b/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-actinides.svelte @@ -1 +1 @@ -
+
diff --git a/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-all.svelte b/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-all.svelte index 84711ce7..0fb9b303 100644 --- a/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-all.svelte +++ b/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-all.svelte @@ -1 +1 @@ -
+
diff --git a/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-borides.svelte b/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-borides.svelte index 4549af30..352a6f4f 100644 --- a/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-borides.svelte +++ b/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-borides.svelte @@ -1 +1 @@ -
+
diff --git a/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-carbides.svelte b/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-carbides.svelte index 99012878..ee846fbf 100644 --- a/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-carbides.svelte +++ b/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-carbides.svelte @@ -1 +1 @@ -
+
diff --git a/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-chalcogenides.svelte b/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-chalcogenides.svelte index ed0e0576..ba1c4b1b 100644 --- a/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-chalcogenides.svelte +++ b/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-chalcogenides.svelte @@ -1 +1 @@ -
+
diff --git a/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-halides.svelte b/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-halides.svelte index 0a35e62d..7df64260 100644 --- a/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-halides.svelte +++ b/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-halides.svelte @@ -1 +1 @@ -
+
diff --git a/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-hydrides.svelte b/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-hydrides.svelte index b5fb87e6..274e5cd3 100644 --- a/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-hydrides.svelte +++ b/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-hydrides.svelte @@ -1 +1 @@ -
+
diff --git a/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-lanthanides.svelte b/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-lanthanides.svelte index 42834c8d..91135dd2 100644 --- a/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-lanthanides.svelte +++ b/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-lanthanides.svelte @@ -1 +1 @@ -
+
diff --git a/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-nitrides.svelte b/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-nitrides.svelte index 0785f59c..32c3c215 100644 --- a/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-nitrides.svelte +++ b/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-nitrides.svelte @@ -1 +1 @@ -
+
diff --git a/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-oxides.svelte b/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-oxides.svelte index 37a109ce..47183502 100644 --- a/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-oxides.svelte +++ b/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-oxides.svelte @@ -1 +1 @@ -
+
diff --git a/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-oxynitrides.svelte b/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-oxynitrides.svelte index b91c060b..11301d41 100644 --- a/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-oxynitrides.svelte +++ b/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-oxynitrides.svelte @@ -1 +1 @@ -
+
diff --git a/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-pnictides.svelte b/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-pnictides.svelte index 53082862..244a6976 100644 --- a/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-pnictides.svelte +++ b/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-pnictides.svelte @@ -1 +1 @@ -
+
diff --git a/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-sulfides.svelte b/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-sulfides.svelte index 94cab431..4486af94 100644 --- a/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-sulfides.svelte +++ b/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-sulfides.svelte @@ -1 +1 @@ -
+
From 1b9e75b0eba11c04dbf13a1b72d36c068fd111b9 Mon Sep 17 00:00:00 2001 From: Janosh Riebesell Date: Wed, 7 Feb 2024 13:18:27 +0100 Subject: [PATCH 12/12] move per-element-each-errors.json to site/src/figs --- .pre-commit-config.yaml | 2 +- scripts/model_figs/per_element_errors.py | 4 ++-- site/src/{lib => figs}/per-element-each-errors.json | 0 site/src/routes/models/element-errors-ptable-heatmap.svelte | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) rename site/src/{lib => figs}/per-element-each-errors.json (100%) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 97af8d4b..270692eb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -53,7 +53,7 @@ repos: - prettier - prettier-plugin-svelte - svelte - exclude: ^(site/src/figs/.+\.svelte|data/wbm/20.+\..+|site/src/routes.+\.(yaml|json)|changelog.md)$ + exclude: ^(site/src/figs/.+\.svelte|data/wbm/20.+\..+|site/src/(routes|figs).+\.(yaml|json)|changelog.md)$ - repo: https://github.com/pre-commit/mirrors-eslint rev: v9.0.0-alpha.2 diff --git a/scripts/model_figs/per_element_errors.py b/scripts/model_figs/per_element_errors.py index 973dbda3..3b264948 100644 --- a/scripts/model_figs/per_element_errors.py +++ b/scripts/model_figs/per_element_errors.py @@ -13,7 +13,7 @@ from pymatviz.utils import bin_df_cols, df_ptable from tqdm import tqdm -from matbench_discovery import PDF_FIGS, ROOT, SITE_FIGS, SITE_LIB +from matbench_discovery import PDF_FIGS, ROOT, SITE_FIGS from matbench_discovery.data import df_wbm from matbench_discovery.enums import Key, Model, TestSubset from matbench_discovery.preds import ( @@ -152,7 +152,7 @@ } assert {*df_elem_err} >= expected_cols assert (df_elem_err.isna().sum() < 35).all() -df_elem_err.round(4).to_json(f"{SITE_LIB}/per-element-each-errors.json") +df_elem_err.round(4).to_json(f"{SITE_FIGS}/per-element-each-errors.json") # %% scatter plot error by element against prevalence in training set diff --git a/site/src/lib/per-element-each-errors.json b/site/src/figs/per-element-each-errors.json similarity index 100% rename from site/src/lib/per-element-each-errors.json rename to site/src/figs/per-element-each-errors.json diff --git a/site/src/routes/models/element-errors-ptable-heatmap.svelte b/site/src/routes/models/element-errors-ptable-heatmap.svelte index df085bf0..5596e489 100644 --- a/site/src/routes/models/element-errors-ptable-heatmap.svelte +++ b/site/src/routes/models/element-errors-ptable-heatmap.svelte @@ -1,6 +1,6 @@