From b0f13f445accb9248599d5a4c2b68a5af70946f9 Mon Sep 17 00:00:00 2001 From: Greg Caporaso Date: Tue, 7 Jun 2016 11:16:26 -0700 Subject: [PATCH] MAINT/API: percentile abundances now returned in separate dataframe fixes #1293 --- CHANGELOG.md | 1 + skbio/stats/composition.py | 139 ++++++++++++++++------- skbio/stats/tests/test_composition.py | 156 +++++++++++++++++++++++--- 3 files changed, 241 insertions(+), 55 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9213ae5ea5..25fb5981d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ ### Backward-incompatible changes [experimental] * `TabularMSA.append` and `TabularMSA.extend` now require one of `minter`, `index`, or `reset_index` to be provided when incorporating new sequences into an MSA. Previous behavior was to auto-increment the index labels if `minter` and `index` weren't provided and the MSA had a default integer index, otherwise error. Use `reset_index=True` to obtain the previous behavior in a more explicit way. +* `skbio.stats.ancom` now returns two `pd.DataFrame` objects, where it previously returned one. The first contains the ANCOM test results, as before, and the second contains percentile abundances of each feature in each group. The specific percentiles that are computed and returned is controlled by the new `percentiles` parameter to `skbio.stats.ancom`. In the future, this second `pd.DataFrame` will not be returned by this function, but will be available through the [contingency table API](https://github.com/biocore/scikit-bio/issues/848). ([#1293](https://github.com/biocore/scikit-bio/issues/1293)) ### Bug fixes * Fixed row and column names to `biplot_scores` in the `OrdinationResults` object from `skbio.stats.ordination`. This fix affect the `cca` and `rda` methods. ([#1322](https://github.com/biocore/scikit-bio/issues/1322)) diff --git a/skbio/stats/composition.py b/skbio/stats/composition.py index e1bb00fff8..2ca6ea13cc 100644 --- a/skbio/stats/composition.py +++ b/skbio/stats/composition.py @@ -99,6 +99,8 @@ # The full license is in the file COPYING.txt, distributed with this software. # ---------------------------------------------------------------------------- +import collections + import numpy as np import pandas as pd import scipy.stats @@ -652,7 +654,7 @@ def ancom(table, grouping, and performing a significance test to determine if there is a significant difference in feature ratios with respect to the variable of interest. - In an experiment with only two treatments, this test tests the following + In an experiment with only two treatments, this tests the following hypothesis for feature :math:`i` .. math:: @@ -697,6 +699,10 @@ def ancom(table, grouping, classes. This function must be able to accept at least two 1D array_like arguments of floats and returns a test statistic and a p-value. By default ``scipy.stats.f_oneway`` is used. + percentiles : list of floats, optional + Percentile abundances to return for each feature in each group. By + default, will return the minimum, 25th percentile, median, 75th + percentile, and maximum abundances for each feature in each group. Returns ------- @@ -709,6 +715,10 @@ def ancom(table, grouping, `"reject"` indicates if feature is significantly different or not. + pd.DataFrame + A table of features and their percentile abundances in each group. If + ``percentiles = []``, this will be an empty data frame. + See Also -------- multiplicative_replacement @@ -752,7 +762,8 @@ def ancom(table, grouping, >>> from skbio.stats.composition import ancom >>> import pandas as pd - Now let's load in a pd.DataFrame with 6 samples and 7 unknown bacteria: + Now let's load in a DataFrame with 6 samples and 7 features (e.g., + these may be bacterial OTUs): >>> table = pd.DataFrame([[12, 11, 10, 10, 10, 10, 10], ... [9, 11, 12, 10, 10, 10, 10], @@ -760,22 +771,25 @@ def ancom(table, grouping, ... [22, 21, 9, 10, 10, 10, 10], ... [20, 22, 10, 10, 13, 10, 10], ... [23, 21, 14, 10, 10, 10, 10]], - ... index=['s1','s2','s3','s4','s5','s6'], - ... columns=['b1','b2','b3','b4','b5','b6','b7']) + ... index=['s1', 's2', 's3', 's4', 's5', 's6'], + ... columns=['b1', 'b2', 'b3', 'b4', 'b5', 'b6', + ... 'b7']) - Then create a grouping vector. In this scenario, there - are only two classes, and suppose these classes correspond to the - treatment due to a drug and a control. The first three samples - are controls and the last three samples are treatments. + Then create a grouping vector. In this example, there is a treatment group + and a placebo group. - >>> grouping = pd.Series([0, 0, 0, 1, 1, 1], - ... index=['s1','s2','s3','s4','s5','s6']) + >>> grouping = pd.Series(['treatment', 'treatment', 'treatment', + ... 'placebo', 'placebo', 'placebo'], + ... index=['s1', 's2', 's3', 's4', 's5', 's6']) - Now run ``ancom`` and see if there are any features that have any - significant differences between the treatment and the control. + Now run ``ancom`` to determine if there are any features that are + significantly different in abundance between the treatment and the placebo + groups. The first DataFrame that is returned contains the ANCOM test + results, and the second contains the percentile abundance data for each + feature in each group. - >>> results = ancom(table, grouping) - >>> results['W'] + >>> ancom_df, percentile_df = ancom(table, grouping) + >>> ancom_df['W'] b1 0 b2 4 b3 1 @@ -788,10 +802,12 @@ def ancom(table, grouping, The W-statistic is the number of features that a single feature is tested to be significantly different against. In this scenario, `b2` was detected to have significantly different abundances compared to four of the other - species. To summarize the results from the W-statistic, let's take a look - at the results from the hypothesis test: + features. To summarize the results from the W-statistic, let's take a look + at the results from the hypothesis test. The `reject` column in the table + indicates whether the null hypothesis was rejected, and that a feature + was therefore observed to be differentially abundant across the groups. - >>> results['reject'] + >>> ancom_df['reject'] b1 False b2 True b3 False @@ -801,8 +817,40 @@ def ancom(table, grouping, b7 False Name: reject, dtype: bool - From this we can conclude that only `b2` was significantly - different between the treatment and the control. + From this we can conclude that only `b2` was significantly different in + abundance between the treatment and the placebo. We still don't know, for + example, in which group `b2` was more abundant. We therefore may next be + interested in comparing the abundance of `b2` across the two groups. + We can do that using the second DataFrame that was returned. Here we + compare the median (50th percentile) abundance of `b2` in the treatment and + placebo groups: + + >>> percentile_df[50.0].loc['b2'] + Group + placebo 21.0 + treatment 11.0 + Name: b2, dtype: float64 + + We can also look at a full five-number summary for ``b2`` in the treatment + and placebo groups: + + >>> percentile_df.loc['b2'] # doctest: +NORMALIZE_WHITESPACE + Percentile Group + 0.0 placebo 21.0 + 25.0 placebo 21.0 + 50.0 placebo 21.0 + 75.0 placebo 21.5 + 100.0 placebo 22.0 + 0.0 treatment 11.0 + 25.0 treatment 11.0 + 50.0 treatment 11.0 + 75.0 treatment 11.0 + 100.0 treatment 11.0 + Name: b2, dtype: float64 + + Taken together, these data tell us that `b2` is present in significantly + higher abundance in the placebo group samples than in the treatment group + samples. """ if not isinstance(table, pd.DataFrame): @@ -838,6 +886,17 @@ def ancom(table, grouping, if (table.isnull()).any().any(): raise ValueError('Cannot handle missing values in `table`.') + for percentile in percentiles: + if not 0.0 <= percentile <= 100.0: + raise ValueError('Percentiles must be in the range [0, 100], %r ' + 'was provided.' % percentile) + + if len(percentiles) != len(set(percentiles)): + counts = collections.Counter(percentiles).items() + duplicated_values = ', '.join([str(k) for k, v in counts if v > 1]) + raise ValueError('Percentile values must be unique. The following ' + 'value(s) were duplicated: %s.' % duplicated_values) + input_grouping = grouping.copy() groups, _grouping = np.unique(grouping, return_inverse=True) grouping = pd.Series(_grouping, index=grouping.index) @@ -867,22 +926,6 @@ def ancom(table, grouping, raise ValueError('`table` index and `grouping` ' 'index must be consistent.') - # Compute DataFrame of mean/std abundances for all features on a - # per category basis. - cat_values = input_grouping.values - cs = np.unique(cat_values) - cat_dists = {k: mat[cat_values == k] for k in cs} - cat_percentiles = [] - for percentile in percentiles: - data = {k: np.percentile(v, percentile, axis=0) - for k, v in cat_dists.items()} - data = pd.DataFrame.from_dict(data) - data.index = mat.columns - data.columns = ['%s: %r percentile' % (e, percentile) - for e in data.columns] - cat_percentiles.append(data) - cat_percentiles = pd.concat(cat_percentiles, axis=1) - n_feat = mat.shape[1] _logratio_mat = _log_compare(mat.values, cats.values, significance_test) @@ -913,10 +956,28 @@ def ancom(table, grouping, else: nu = cutoff[4] reject = (W >= nu*n_feat) - labs = mat.columns - ancom_df = pd.DataFrame({'W': pd.Series(W, index=labs), - 'reject': pd.Series(reject, index=labs)}) - return pd.concat([ancom_df, cat_percentiles], axis=1) + + feat_ids = mat.columns + ancom_df = pd.DataFrame({'W': pd.Series(W, index=feat_ids), + 'reject': pd.Series(reject, index=feat_ids)}) + + if len(percentiles) == 0: + return ancom_df, pd.DataFrame() + else: + category_values = input_grouping.values + data = [] + columns = [] + for category in np.unique(category_values): + feat_dists = mat[category_values == category] + for percentile in percentiles: + columns.append([percentile, category]) + data.append(np.percentile(feat_dists, percentile, axis=0)) + columns = pd.MultiIndex.from_tuples(columns, + names=['Percentile', 'Group']) + percentile_df = pd.DataFrame( + np.asarray(data).T, columns=columns, index=feat_ids) + return ancom_df, percentile_df + def _holm_bonferroni(p): """ Performs Holm-Bonferroni correction for pvalues diff --git a/skbio/stats/tests/test_composition.py b/skbio/stats/tests/test_composition.py index 554fb18acc..773396421a 100644 --- a/skbio/stats/tests/test_composition.py +++ b/skbio/stats/tests/test_composition.py @@ -550,7 +550,131 @@ def test_ancom_basic_counts(self): 'reject': np.array([True, True, False, False, False, False, False], dtype=bool)}) - assert_data_frame_almost_equal(result, exp) + assert_data_frame_almost_equal(result[0], exp) + + def test_ancom_percentiles(self): + table = pd.DataFrame([[12, 11], + [9, 11], + [1, 11], + [22, 100], + [20, 53], + [23, 1]], + index=['s1', 's2', 's3', 's4', 's5', 's6'], + columns=['b1', 'b2']) + grouping = pd.Series(['a', 'a', 'a', 'b', 'b', 'b'], + index=['s1', 's2', 's3', 's4', 's5', 's6']) + result = ancom(table, grouping)[1] + self.assertEqual(result.shape, (2, 10)) + self.assertAlmostEqual(result[(0.0, 'a')]['b1'], 1.0) + self.assertAlmostEqual(result[(0.0, 'b')]['b1'], 20.0) + self.assertAlmostEqual(result[(25.0, 'a')]['b1'], 5.0) + self.assertAlmostEqual(result[(25.0, 'b')]['b1'], 21.0) + self.assertAlmostEqual(result[(50.0, 'a')]['b1'], 9.0) + self.assertAlmostEqual(result[(50.0, 'b')]['b1'], 22.0) + self.assertAlmostEqual(result[(75.0, 'a')]['b1'], 10.5) + self.assertAlmostEqual(result[(75.0, 'b')]['b1'], 22.5) + self.assertAlmostEqual(result[(100.0, 'a')]['b1'], 12.0) + self.assertAlmostEqual(result[(100.0, 'b')]['b1'], 23.0) + + self.assertAlmostEqual(result[(0.0, 'a')]['b2'], 11.0) + self.assertAlmostEqual(result[(0.0, 'b')]['b2'], 1.0) + self.assertAlmostEqual(result[(25.0, 'a')]['b2'], 11.0) + self.assertAlmostEqual(result[(25.0, 'b')]['b2'], 27.0) + self.assertAlmostEqual(result[(50.0, 'a')]['b2'], 11.0) + self.assertAlmostEqual(result[(50.0, 'b')]['b2'], 53.0) + self.assertAlmostEqual(result[(75.0, 'a')]['b2'], 11.0) + self.assertAlmostEqual(result[(75.0, 'b')]['b2'], 76.5) + self.assertAlmostEqual(result[(100.0, 'a')]['b2'], 11.0) + self.assertAlmostEqual(result[(100.0, 'b')]['b2'], 100.0) + + def test_ancom_percentiles_alt_categories(self): + table = pd.DataFrame([[12], + [9], + [1], + [22], + [20], + [23]], + index=['s1', 's2', 's3', 's4', 's5', 's6'], + columns=['b1']) + grouping = pd.Series(['a', 'a', 'c', 'b', 'b', 'c'], + index=['s1', 's2', 's3', 's4', 's5', 's6']) + result = ancom(table, grouping)[1] + self.assertEqual(result.shape, (1, 15)) + self.assertAlmostEqual(result[(0.0, 'a')]['b1'], 9.0) + self.assertAlmostEqual(result[(0.0, 'b')]['b1'], 20.0) + self.assertAlmostEqual(result[(0.0, 'c')]['b1'], 1.0) + self.assertAlmostEqual(result[(25.0, 'a')]['b1'], 9.75) + self.assertAlmostEqual(result[(25.0, 'b')]['b1'], 20.5) + self.assertAlmostEqual(result[(25.0, 'c')]['b1'], 6.5) + self.assertAlmostEqual(result[(50.0, 'a')]['b1'], 10.5) + self.assertAlmostEqual(result[(50.0, 'b')]['b1'], 21.0) + self.assertAlmostEqual(result[(50.0, 'c')]['b1'], 12.0) + self.assertAlmostEqual(result[(75.0, 'a')]['b1'], 11.25) + self.assertAlmostEqual(result[(75.0, 'b')]['b1'], 21.5) + self.assertAlmostEqual(result[(75.0, 'c')]['b1'], 17.5) + self.assertAlmostEqual(result[(100.0, 'a')]['b1'], 12.0) + self.assertAlmostEqual(result[(100.0, 'b')]['b1'], 22.0) + self.assertAlmostEqual(result[(100.0, 'c')]['b1'], 23.0) + + def test_ancom_alt_percentiles(self): + table = pd.DataFrame([[12], + [9], + [1], + [22], + [20], + [23]], + index=['s1', 's2', 's3', 's4', 's5', 's6'], + columns=['b1']) + grouping = pd.Series(['a', 'a', 'a', 'b', 'b', 'b'], + index=['s1', 's2', 's3', 's4', 's5', 's6']) + result = ancom(table, grouping, percentiles=[42.0, 50.0])[1] + self.assertEqual(result.shape, (1, 4)) + self.assertAlmostEqual(result[(42.0, 'a')]['b1'], 7.71999999) + self.assertAlmostEqual(result[(42.0, 'b')]['b1'], 21.68) + self.assertAlmostEqual(result[(50.0, 'a')]['b1'], 9.0) + self.assertAlmostEqual(result[(50.0, 'b')]['b1'], 22.0) + + # order of percentiles in unimportant + result = ancom(table, grouping, percentiles=[50.0, 42.0])[1] + self.assertEqual(result.shape, (1, 4)) + self.assertAlmostEqual(result[(42.0, 'a')]['b1'], 7.71999999) + self.assertAlmostEqual(result[(42.0, 'b')]['b1'], 21.68) + self.assertAlmostEqual(result[(50.0, 'a')]['b1'], 9.0) + self.assertAlmostEqual(result[(50.0, 'b')]['b1'], 22.0) + + def test_ancom_no_percentiles(self): + table = pd.DataFrame([[12], + [9], + [1], + [22], + [20], + [23]], + index=['s1', 's2', 's3', 's4', 's5', 's6'], + columns=['b1']) + grouping = pd.Series(['a', 'a', 'a', 'b', 'b', 'b'], + index=['s1', 's2', 's3', 's4', 's5', 's6']) + result = ancom(table, grouping, percentiles=[])[1] + self.assertEqual(result.shape, (0, 0)) + + def test_ancom_invalid_percentiles(self): + table = pd.DataFrame([[12], + [9], + [1], + [22], + [20], + [23]], + index=['s1', 's2', 's3', 's4', 's5', 's6'], + columns=['b1']) + grouping = pd.Series(['a', 'a', 'a', 'b', 'b', 'b'], + index=['s1', 's2', 's3', 's4', 's5', 's6']) + with self.assertRaises(ValueError): + ancom(table, grouping, percentiles=[-1.0]) + with self.assertRaises(ValueError): + ancom(table, grouping, percentiles=[100.1]) + with self.assertRaises(ValueError): + ancom(table, grouping, percentiles=[10.0, 3.0, 101.0, 100]) + with self.assertRaises(ValueError): + ancom(table, grouping, percentiles=[10.0, 10.0]) def test_ancom_basic_proportions(self): # Converts from counts to proportions @@ -569,7 +693,7 @@ def test_ancom_basic_proportions(self): 'reject': np.array([True, True, False, False, False, False, False], dtype=bool)}) - assert_data_frame_almost_equal(result, exp) + assert_data_frame_almost_equal(result[0], exp) def test_ancom_multiple_groups(self): test_table = pd.DataFrame(self.table4) @@ -585,7 +709,7 @@ def test_ancom_multiple_groups(self): 'reject': np.array([True, True, False, False, True, False, False, False, False], dtype=bool)}) - assert_data_frame_almost_equal(result, exp) + assert_data_frame_almost_equal(result[0], exp) def test_ancom_noncontiguous(self): result = ancom(self.table5, @@ -595,7 +719,7 @@ def test_ancom_noncontiguous(self): 'reject': np.array([True, False, False, False, False, True, False], dtype=bool)}) - assert_data_frame_almost_equal(result, exp) + assert_data_frame_almost_equal(result[0], exp) def test_ancom_unbalanced(self): result = ancom(self.table6, @@ -605,7 +729,7 @@ def test_ancom_unbalanced(self): 'reject': np.array([True, False, False, False, False, True, False], dtype=bool)}) - assert_data_frame_almost_equal(result, exp) + assert_data_frame_almost_equal(result[0], exp) def test_ancom_letter_categories(self): result = ancom(self.table7, @@ -615,7 +739,7 @@ def test_ancom_letter_categories(self): 'reject': np.array([True, False, False, False, False, True, False], dtype=bool)}) - assert_data_frame_almost_equal(result, exp) + assert_data_frame_almost_equal(result[0], exp) def test_ancom_multiple_comparisons(self): result = ancom(self.table1, @@ -624,7 +748,7 @@ def test_ancom_multiple_comparisons(self): significance_test=scipy.stats.mannwhitneyu) exp = pd.DataFrame({'W': np.array([0]*7), 'reject': np.array([False]*7, dtype=bool)}) - assert_data_frame_almost_equal(result, exp) + assert_data_frame_almost_equal(result[0], exp) def test_ancom_alternative_test(self): result = ancom(self.table1, @@ -635,7 +759,7 @@ def test_ancom_alternative_test(self): 'reject': np.array([True, True, False, False, False, False, False], dtype=bool)}) - assert_data_frame_almost_equal(result, exp) + assert_data_frame_almost_equal(result[0], exp) def test_ancom_normal_data(self): result = ancom(self.table2, @@ -648,7 +772,7 @@ def test_ancom_normal_data(self): True, False, False, False, False], dtype=bool)}) - assert_data_frame_almost_equal(result, exp) + assert_data_frame_almost_equal(result[0], exp) def test_ancom_basic_counts_swapped(self): result = ancom(self.table8, self.cats8) @@ -656,7 +780,7 @@ def test_ancom_basic_counts_swapped(self): 'reject': np.array([True, True, False, False, False, False, False], dtype=bool)}) - assert_data_frame_almost_equal(result, exp) + assert_data_frame_almost_equal(result[0], exp) def test_ancom_no_signal(self): result = ancom(self.table3, @@ -664,7 +788,7 @@ def test_ancom_no_signal(self): multiple_comparisons_correction=None) exp = pd.DataFrame({'W': np.array([0]*7), 'reject': np.array([False]*7, dtype=bool)}) - assert_data_frame_almost_equal(result, exp) + assert_data_frame_almost_equal(result[0], exp) def test_ancom_tau(self): exp1 = pd.DataFrame({'W': np.array([8, 7, 3, 3, 7, 3, 3, 3, 3]), @@ -691,9 +815,9 @@ def test_ancom_tau(self): result2 = ancom(self.table9, self.cats9, tau=0.02) result3 = ancom(self.table10, self.cats10, tau=0.02) - assert_data_frame_almost_equal(result1, exp1) - assert_data_frame_almost_equal(result2, exp2) - assert_data_frame_almost_equal(result3, exp3) + assert_data_frame_almost_equal(result1[0], exp1) + assert_data_frame_almost_equal(result2[0], exp2) + assert_data_frame_almost_equal(result3[0], exp3) def test_ancom_theta(self): result = ancom(self.table1, self.cats1, theta=0.3) @@ -701,7 +825,7 @@ def test_ancom_theta(self): 'reject': np.array([True, True, False, False, False, False, False], dtype=bool)}) - assert_data_frame_almost_equal(result, exp) + assert_data_frame_almost_equal(result[0], exp) def test_ancom_alpha(self): result = ancom(self.table1, self.cats1, alpha=0.5) @@ -709,7 +833,7 @@ def test_ancom_alpha(self): 'reject': np.array([True, True, False, True, True, False, False], dtype=bool)}) - assert_data_frame_almost_equal(result, exp) + assert_data_frame_almost_equal(result[0], exp) def test_ancom_fail_type(self): with self.assertRaises(TypeError):