From bb0abcfce549380d60dc5c4a1693c5c34ef627c7 Mon Sep 17 00:00:00 2001 From: Matthew Gidden Date: Tue, 2 Apr 2019 11:02:31 +0200 Subject: [PATCH 1/7] add code to read r-style columns --- pyam/utils.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/pyam/utils.py b/pyam/utils.py index 5695495f4..680d866e3 100644 --- a/pyam/utils.py +++ b/pyam/utils.py @@ -132,6 +132,26 @@ def format_data(df, **kwargs): if isinstance(df, pd.Series): df = df.to_frame() + # Check for R-style year columns, converting where necessary + def convert_r_columns(c): + try: + first = c[0] + second = c[1:] + if first != 'X': + # not in the X2015 R-style, fall down to final return statement + pass + try: + # bingo! was X2015 R-style, return the integer + return int(second) + except: + # nope, not an int, fall down to final return statement + pass + except: + # not a string/iterable/etc, fall down to final return statement + pass + return c + df.columns = map(convert_r_columns, df.columns) + # if `value` is given but not `variable`, # melt value columns and use column name as `variable` if 'value' in kwargs and 'variable' not in kwargs: From b7592327c93930f7ffc199bf7767ed8582d565b1 Mon Sep 17 00:00:00 2001 From: Matthew Gidden Date: Tue, 2 Apr 2019 11:10:16 +0200 Subject: [PATCH 2/7] add test --- tests/test_cast_to_iamc.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/test_cast_to_iamc.py b/tests/test_cast_to_iamc.py index c515cb43d..3ce685754 100644 --- a/tests/test_cast_to_iamc.py +++ b/tests/test_cast_to_iamc.py @@ -79,3 +79,14 @@ def test_cast_with_variable_and_value(meta_df): assert compare(pe_df, df).empty pd.testing.assert_frame_equal(df.data, pe_df.data.reset_index(drop=True)) + + +def test_cast_from_r_df(test_pd_df): + df = test_pd_df.copy() + # last two columns are years + df.columns = list(df.columns[:-2]) + ['X{}'.format(c) + for c in df.columns[-2:]] + obs = IamDataFrame(df) + exp = IamDataFrame(test_pd_df) + assert compare(obs, exp).empty + pd.testing.assert_frame_equal(obs.data, exp.data) From a163f3fdeeb4e2dbc5b025d9f580ff59c5379b20 Mon Sep 17 00:00:00 2001 From: Matthew Gidden Date: Tue, 2 Apr 2019 11:12:11 +0200 Subject: [PATCH 3/7] added release notes --- RELEASE_NOTES.md | 1 + 1 file changed, 1 insertion(+) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 2607e0bde..3423ff2e0 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,6 +1,7 @@ # Next Release +- [#212](https://github.com/IAMconsortium/pyam/pull/212) Now natively support reading R-style data frames with year columns like "X2015" - [#202](https://github.com/IAMconsortium/pyam/pull/202) Extend the `df.rename()` function with a `check_duplicates (default True)` validation option - [#201](https://github.com/IAMconsortium/pyam/pull/201) Added native support for legends outside of plots with `pyam.plotting.OUTSIDE_LEGEND` with a tutorial - [#199](https://github.com/IAMconsortium/pyam/pull/199) Initializing an `IamDataFrame` accepts kwargs to fill or create from the data any missing required columns From 247e30f6b989d99ed4d97ba2f7da7263dd639505 Mon Sep 17 00:00:00 2001 From: Matthew Gidden Date: Tue, 2 Apr 2019 11:13:31 +0200 Subject: [PATCH 4/7] add a bit of docs --- pyam/core.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyam/core.py b/pyam/core.py index 2bd170bda..42a9b0e07 100644 --- a/pyam/core.py +++ b/pyam/core.py @@ -57,6 +57,8 @@ class IamDataFrame(object): an instance of an TimeSeries or Scenario (requires `ixmp`), or pd.DataFrame or data file with IAMC-format data columns. A pd.DataFrame can have the required data as columns or index. + Support is provided additionally for R-style data columns for years, + like "X2015", etc. kwargs: if `value=col`, melt `col` to `value` and use `col` name as `variable`; else, mapping of columns required for an `IamDataFrame` to: @@ -64,6 +66,7 @@ class IamDataFrame(object): - multiple columns, which will be concatenated by pipe - a string to be used as value for this column """ + def __init__(self, data, **kwargs): """Initialize an instance of an IamDataFrame""" # import data from pd.DataFrame or read from source From 7ac3bb14557e03e812fd0369b01d94967a0f8ab6 Mon Sep 17 00:00:00 2001 From: Matthew Gidden Date: Tue, 2 Apr 2019 11:15:03 +0200 Subject: [PATCH 5/7] reduce impl a bit --- pyam/utils.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/pyam/utils.py b/pyam/utils.py index 680d866e3..05e37016f 100644 --- a/pyam/utils.py +++ b/pyam/utils.py @@ -137,15 +137,13 @@ def convert_r_columns(c): try: first = c[0] second = c[1:] - if first != 'X': - # not in the X2015 R-style, fall down to final return statement - pass - try: - # bingo! was X2015 R-style, return the integer - return int(second) - except: - # nope, not an int, fall down to final return statement - pass + if first == 'X': + try: + # bingo! was X2015 R-style, return the integer + return int(second) + except: + # nope, not an int, fall down to final return statement + pass except: # not a string/iterable/etc, fall down to final return statement pass From 57a18f0286535d4c72a6449938167f5cb1f714e9 Mon Sep 17 00:00:00 2001 From: Matthew Gidden Date: Fri, 5 Apr 2019 10:34:02 +0200 Subject: [PATCH 6/7] use df.map instead of map --- pyam/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyam/utils.py b/pyam/utils.py index 05e37016f..8135fb281 100644 --- a/pyam/utils.py +++ b/pyam/utils.py @@ -148,7 +148,7 @@ def convert_r_columns(c): # not a string/iterable/etc, fall down to final return statement pass return c - df.columns = map(convert_r_columns, df.columns) + df.columns = df.columns.map(convert_r_columns) # if `value` is given but not `variable`, # melt value columns and use column name as `variable` From 1585c823d93992041deaa44507f041646be5e565 Mon Sep 17 00:00:00 2001 From: Matthew Gidden Date: Fri, 5 Apr 2019 10:58:43 +0200 Subject: [PATCH 7/7] add test for other path --- tests/test_cast_to_iamc.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test_cast_to_iamc.py b/tests/test_cast_to_iamc.py index 3ce685754..c190b9981 100644 --- a/tests/test_cast_to_iamc.py +++ b/tests/test_cast_to_iamc.py @@ -90,3 +90,10 @@ def test_cast_from_r_df(test_pd_df): exp = IamDataFrame(test_pd_df) assert compare(obs, exp).empty pd.testing.assert_frame_equal(obs.data, exp.data) + + +def test_cast_from_r_df_err(test_pd_df): + df = test_pd_df.copy() + # last two columns are years + df.columns = list(df.columns[:-2]) + ['Xfoo', 'Xbar'] + pytest.raises(ValueError, IamDataFrame, df)