From 3365bcd06a53541db5904b0c5f9050c10cd84b43 Mon Sep 17 00:00:00 2001 From: Sreeja Gaddamidi <37956427+sreeja-g@users.noreply.github.com> Date: Fri, 16 Sep 2022 17:53:18 -0400 Subject: [PATCH] average predictive value difference metric implementation #376 #376 implementation average predictive value difference metric --- aif360/metrics/classification_metric.py | 14 +++++++ aif360/sklearn/metrics/metrics.py | 56 ++++++++++++++++++++++++- tests/sklearn/test_metrics.py | 8 +++- 3 files changed, 75 insertions(+), 3 deletions(-) diff --git a/aif360/metrics/classification_metric.py b/aif360/metrics/classification_metric.py index 6cb9799b..22376826 100644 --- a/aif360/metrics/classification_metric.py +++ b/aif360/metrics/classification_metric.py @@ -571,6 +571,20 @@ def average_abs_odds_difference(self): return 0.5 * (np.abs(self.difference(self.false_positive_rate)) + np.abs(self.difference(self.true_positive_rate))) + def average_predictive_value_difference(self): + r"""Average of difference in PPV and FOR for unprivileged and privileged + groups: + + .. math:: + + \tfrac{1}{2}\left[(PPV_{D = \text{unprivileged}} - PPV_{D = \text{privileged}}) + + (FOR_{D = \text{unprivileged}} - FOR_{D = \text{privileged}}))\right] + + A value of 0 indicates equality of chance of success. + """ + return 0.5 * (self.difference(self.positive_predictive_value) + + self.difference(self.false_omission_rate)) + def error_rate_difference(self): r"""Difference in error rates for unprivileged and privileged groups, :math:`ERR_{D = \text{unprivileged}} - ERR_{D = \text{privileged}}`. diff --git a/aif360/sklearn/metrics/metrics.py b/aif360/sklearn/metrics/metrics.py index eb582e92..22abe9c5 100644 --- a/aif360/sklearn/metrics/metrics.py +++ b/aif360/sklearn/metrics/metrics.py @@ -3,7 +3,7 @@ import numpy as np import pandas as pd from scipy.special import rel_entr -from sklearn.metrics import make_scorer as _make_scorer, recall_score +from sklearn.metrics import make_scorer as _make_scorer, recall_score, precision_score from sklearn.metrics import multilabel_confusion_matrix from sklearn.metrics._classification import _prf_divide, _check_zero_division from sklearn.neighbors import NearestNeighbors @@ -25,7 +25,7 @@ 'smoothed_selection_rate', 'generalized_fpr', 'generalized_fnr', # group fairness 'statistical_parity_difference', 'disparate_impact_ratio', - 'equal_opportunity_difference', 'average_odds_difference', + 'equal_opportunity_difference', 'average_odds_difference', 'average_predictive_value_difference', 'average_odds_error', 'class_imbalance', 'kl_divergence', 'conditional_demographic_disparity', 'smoothed_edf', 'df_bias_amplification', 'mdss_bias_scan', 'mdss_bias_score', @@ -344,6 +344,27 @@ def specificity_score(y_true, y_pred, *, pos_label=1, sample_weight=None, return _prf_divide(tn, negs, 'specificity', 'negative', None, ('specificity',), zero_division).item() +def false_omission_score(y_true, y_pred, *, pos_label=1, sample_weight=None, + zero_division='warn'): + """Compute the false omission rate. + + Args: + y_true (array-like): Ground truth (correct) target values. + y_pred (array-like): Estimated targets as returned by a classifier. + pos_label (scalar, optional): The label of the positive class. + sample_weight (array-like, optional): Sample weights. + zero_division ('warn', 0 or 1): Sets the value to return when there is a + zero division. If set to “warn”, this acts as 0, but warnings are + also raised. + """ + _check_zero_division(zero_division) + MCM = multilabel_confusion_matrix(y_true, y_pred, labels=[pos_label], + sample_weight=sample_weight) + tn, fn = MCM[:, 0, 0], MCM[:, 1, 0] + negs = tn + fn + return _prf_divide(fn, negs, 'negative_predictive', 'negative', None, + ('negative_predictive',), zero_division).item() + def base_rate(y_true, y_pred=None, *, pos_label=1, sample_weight=None): r"""Compute the base rate, :math:`Pr(Y = \text{pos_label}) = \frac{P}{P+N}`. @@ -649,6 +670,37 @@ def average_odds_error(y_true, y_pred, *, prot_attr=None, priv_group=None, sample_weight=sample_weight) return (abs(tpr_diff) + abs(fpr_diff)) / 2 +def average_predictive_value_difference(y_true, y_pred, *, prot_attr=None, priv_group=1, + pos_label=1, sample_weight=None): + r"""Returns the average of the difference in positive predictive value and false omission rate for the unprivileged and privileged groups: + + .. math:: + + \dfrac{(PPV_{D = \text{unprivileged}} - PPV_{D = \text{privileged}}) + + (FOR_{D = \text{unprivileged}} - FOR_{D = \text{privileged}})}{2} + + A value of 0 indicates equality of chance of success. + + Args: + y_true (pandas.Series): Ground truth (correct) target values. + y_pred (array-like): Estimated targets as returned by a classifier. + prot_attr (array-like, keyword-only): Protected attribute(s). If + ``None``, all protected attributes in y_true are used. + priv_group (scalar, optional): The label of the privileged group. + pos_label (scalar, optional): The label of the positive class. + sample_weight (array-like, optional): Sample weights. + + Returns: + float: Average predictive value difference. + """ + for_diff = difference(false_omission_score, y_true, y_pred, + prot_attr=prot_attr, priv_group=priv_group, + pos_label=pos_label, sample_weight=sample_weight) + ppv_diff = difference(precision_score, y_true, y_pred, prot_attr=prot_attr, + priv_group=priv_group, pos_label=pos_label, + sample_weight=sample_weight) + return (ppv_diff + for_diff) / 2 + def class_imbalance(y_true, y_pred=None, *, prot_attr=None, priv_group=1, sample_weight=None): r"""Compute the class imbalance, :math:`\frac{N_u - N_p}{N_u + N_p}`. diff --git a/tests/sklearn/test_metrics.py b/tests/sklearn/test_metrics.py index 5888da28..0dc6ae09 100644 --- a/tests/sklearn/test_metrics.py +++ b/tests/sklearn/test_metrics.py @@ -13,7 +13,7 @@ consistency_score, specificity_score, selection_rate, base_rate, smoothed_base_rate, generalized_fpr, generalized_fnr, disparate_impact_ratio, statistical_parity_difference, - equal_opportunity_difference, average_odds_difference, + equal_opportunity_difference, average_odds_difference, average_predictive_value_difference, average_odds_error, smoothed_edf, df_bias_amplification, generalized_entropy_error, between_group_generalized_entropy_error, class_imbalance, kl_divergence, conditional_demographic_disparity, @@ -100,6 +100,12 @@ def test_average_odds_difference(): sample_weight=sample_weight) assert np.isclose(aod, cm.average_odds_difference()) +def test_average_predictive_value_difference(): + """Tests that the old and new average_predictive_value_difference matches exactly.""" + aod = average_predictive_value_difference(y, y_pred, prot_attr='sex', + sample_weight=sample_weight) + assert np.isclose(aod, cm.average_predictive_value_difference()) + def test_average_odds_error(): """Tests that the old and new average_odds_error matches exactly.""" aoe = average_odds_error(y, y_pred, prot_attr='sex',