Extend meta_df tests to include datetime

pyam/core.py

@@ -309,6 +309,31 @@ def interpolate(self, year):
         fill_values['year'] = year
         self.data = self.data.append(fill_values, ignore_index=True)
 
+    def swap_time_for_year(self, inplace=False):
+        """Convert the `time` column to `year`.
+
+        Parameters
+        ----------
+        inplace: bool, default False
+            if True, do operation inplace and return None
+
+        Raises
+        ------
+        ValueError
+            "time" is not a column of `self.data`
+        """
+        if "time" not in self.data:
+            raise ValueError("`time` must be in `self.data` to use this method")
+
+        ret = copy.deepcopy(self) if not inplace else self
+
+        ret.data["year"] = ret.data["time"].apply(lambda x: x.year)
+        ret.data = ret.data.drop("time", axis="columns")
+        ret._LONG_IDX = [v if v != "time" else "year" for v in ret._LONG_IDX]
+
+        if not inplace:
+            return ret
+
     def as_pandas(self, with_metadata=False):
         """Return this as a pd.DataFrame
 
@@ -1358,6 +1383,10 @@ def _check_rows(rows, check, in_range=True, return_test='any'):
         msg = 'Unknown checking type: {}'
         raise ValueError(msg.format(check.keys() - valid_checks))
 
+    rows = rows.copy()
+    if 'year' not in rows:
+        rows['year'] = rows['time'].apply(lambda x: x.year)
+
     where_idx = set(rows.index[rows['year'] == check['year']]) \
         if 'year' in check else set(rows.index)
     rows = rows.loc[list(where_idx)]

tests/conftest.py

@@ -167,15 +167,15 @@
 TEST_STACKPLOT_DF['scenario'] = 'a_scen'
 
 
-TIME_AXES = [
-    [2005, 2010],
-    [datetime(2005, 6, 17), datetime(2010, 7, 21)],
-    ['2005-06-17', '2010-07-21'],
-    ['2005-06-17 00:00:00', '2010-07-21 12:00:00']
-]
-
-
-@pytest.fixture(scope="function", params=TIME_AXES)
+@pytest.fixture(
+    scope="function",
+    params=[
+        [2005, 2010],
+        [datetime(2005, 6, 17), datetime(2010, 7, 21)],
+        ['2005-06-17', '2010-07-21'],
+        ['2005-06-17 00:00:00', '2010-07-21 12:00:00']
+    ]
+)
 def test_df(request):
     tdf = TEST_DF.iloc[:2]
     tdf = tdf.rename({2005: request.param[0], 2010: request.param[1]},
@@ -195,9 +195,17 @@ def test_pd_df():
     yield TEST_DF.copy()
 
 
-@pytest.fixture(scope="function")
-def meta_df():
-    df = IamDataFrame(data=TEST_DF)
+@pytest.fixture(
+    scope="function",
+    params=[
+        [2005, 2010],
+        [datetime(2005, 6, 17), datetime(2010, 7, 21)],
+    ]
+)
+def meta_df(request):
+    mdf = TEST_DF.rename({2005: request.param[0], 2010: request.param[1]},
+                         axis="columns")
+    df = IamDataFrame(data=mdf)
     yield df
 
 

tests/test_cast_to_iamc.py

@@ -1,41 +1,48 @@
+import datetime as dt
+
+
 import pytest
 import pandas as pd
 from pyam import IamDataFrame, compare
 
 
 def test_cast_from_value_col(meta_df):
     df_with_value_cols = pd.DataFrame([
-        ['model_a', 'scen_a', 'World', 'EJ/y', 2005, 1, 0.5],
-        ['model_a', 'scen_a', 'World', 'EJ/y', 2010, 6., 3],
-        ['model_a', 'scen_b', 'World', 'EJ/y', 2005, 2, None],
-        ['model_a', 'scen_b', 'World', 'EJ/y', 2010, 7, None]
+        ['model_a', 'scen_a', 'World', 'EJ/y', dt.datetime(2005, 6, 17), 1, 0.5],
+        ['model_a', 'scen_a', 'World', 'EJ/y', dt.datetime(2010, 7, 21), 6., 3],
+        ['model_a', 'scen_b', 'World', 'EJ/y', dt.datetime(2005, 6, 17), 2, None],
+        ['model_a', 'scen_b', 'World', 'EJ/y', dt.datetime(2010, 7, 21), 7, None]
     ],
-        columns=['model', 'scenario', 'region', 'unit', 'year',
+        columns=['model', 'scenario', 'region', 'unit', 'time',
                  'Primary Energy', 'Primary Energy|Coal'],
     )
     df = IamDataFrame(df_with_value_cols,
                       value=['Primary Energy', 'Primary Energy|Coal'])
+    if "year" in meta_df.data.columns:
+        df = df.swap_time_for_year()
 
     assert compare(meta_df, df).empty
-    pd.testing.assert_frame_equal(df.data, meta_df.data)
+    pd.testing.assert_frame_equal(df.data, meta_df.data, check_like=True)
 
 
 def test_cast_from_value_col_and_args(meta_df):
     # checks for issue [#210](https://github.com/IAMconsortium/pyam/issues/210)
     df_with_value_cols = pd.DataFrame([
-        ['scen_a', 'World', 'EJ/y', 2005, 1, 0.5],
-        ['scen_a', 'World', 'EJ/y', 2010, 6., 3],
-        ['scen_b', 'World', 'EJ/y', 2005, 2, None],
-        ['scen_b', 'World', 'EJ/y', 2010, 7, None]
+        ['scen_a', 'World', 'EJ/y', dt.datetime(2005, 6, 17), 1, 0.5],
+        ['scen_a', 'World', 'EJ/y', dt.datetime(2010, 7, 21), 6., 3],
+        ['scen_b', 'World', 'EJ/y', dt.datetime(2005, 6, 17), 2, None],
+        ['scen_b', 'World', 'EJ/y', dt.datetime(2010, 7, 21), 7, None]
     ],
-        columns=['scenario', 'iso', 'unit', 'year',
+        columns=['scenario', 'iso', 'unit', 'time',
                  'Primary Energy', 'Primary Energy|Coal'],
     )
     df = IamDataFrame(df_with_value_cols, model='model_a', region='iso',
                       value=['Primary Energy', 'Primary Energy|Coal'])
+    if "year" in meta_df.data.columns:
+        df = df.swap_time_for_year()
 
     assert compare(meta_df, df).empty
-    pd.testing.assert_frame_equal(df.data, meta_df.data)
+    pd.testing.assert_frame_equal(df.data, meta_df.data, check_like=True)
 
 
 def test_cast_with_model_arg_raises():
@@ -58,17 +65,21 @@ def test_cast_with_model_arg(meta_df):
 
 
 def test_cast_by_column_concat(meta_df):
+    dts = [dt.datetime(2005, 6, 17), dt.datetime(2010, 7, 21)]
     df = pd.DataFrame([
         ['scen_a', 'World', 'Primary Energy', None, 'EJ/y', 1, 6.],
         ['scen_a', 'World', 'Primary Energy', 'Coal', 'EJ/y', 0.5, 3],
         ['scen_b', 'World', 'Primary Energy', None, 'EJ/y', 2, 7],
     ],
-        columns=['scenario', 'region', 'var_1', 'var_2', 'unit', 2005, 2010],
+        columns=['scenario', 'region', 'var_1', 'var_2', 'unit'] + dts,
     )
 
     df = IamDataFrame(df, model='model_a', variable=['var_1', 'var_2'])
+    if "year" in meta_df.data.columns:
+        df = df.swap_time_for_year()
+
     assert compare(meta_df, df).empty
-    pd.testing.assert_frame_equal(df.data, meta_df.data)
+    pd.testing.assert_frame_equal(df.data, meta_df.data, check_like=True)
 
 
 def test_cast_with_variable_and_value(meta_df):

tests/test_core.py

@@ -8,7 +8,7 @@
 from numpy import testing as npt
 
 from pyam import IamDataFrame, validate, categorize, \
-    require_variable, filter_by_meta, META_IDX, IAMC_IDX, sort_data
+    require_variable, filter_by_meta, META_IDX, IAMC_IDX, sort_data, compare
 from pyam.core import _meta_idx, concat
 
 from conftest import TEST_DATA_DIR
@@ -518,22 +518,44 @@ def test_validate_up(meta_df):
     obs = meta_df.validate({'Primary Energy': {'up': 6.5}},
                            exclude_on_fail=False)
     assert len(obs) == 1
-    assert obs['year'].values[0] == 2010
+    if 'year' in meta_df.data:
+        assert obs['year'].values[0] == 2010
+    else:
+        assert (
+            pd.to_datetime(obs['time'].values[0])
+            == pd.to_datetime(datetime.datetime(2010, 7, 21))
+        )
 
     assert list(meta_df['exclude']) == [False, False]  # assert none excluded
 
 
 def test_validate_lo(meta_df):
     obs = meta_df.validate({'Primary Energy': {'up': 8, 'lo': 2.0}})
     assert len(obs) == 1
-    assert obs['year'].values[0] == 2005
+    if 'year' in meta_df.data:
+        assert obs['year'].values[0] == 2005
+    else:
+        assert (
+            pd.to_datetime(obs['time'].values[0])
+            == pd.to_datetime(datetime.datetime(2005, 6, 17))
+        )
     assert list(obs['scenario'].values) == ['scen_a']
 
 
 def test_validate_both(meta_df):
     obs = meta_df.validate({'Primary Energy': {'up': 6.5, 'lo': 2.0}})
     assert len(obs) == 2
-    assert list(obs['year'].values) == [2005, 2010]
+    if 'year' in meta_df.data:
+        assert list(obs['year'].values) == [2005, 2010]
+    else:
+        assert (
+            pd.to_datetime(obs['time'].values)
+            == pd.to_datetime([
+                datetime.datetime(2005, 6, 17),
+                datetime.datetime(2010, 7, 21),
+            ])
+        ).all()
+
     assert list(obs['scenario'].values) == ['scen_a', 'scen_b']
 
 
@@ -556,7 +578,13 @@ def test_validate_top_level(meta_df):
     obs = validate(meta_df, criteria={'Primary Energy': {'up': 6.0}},
                    exclude_on_fail=True, variable='Primary Energy')
     assert len(obs) == 1
-    assert obs['year'].values[0] == 2010
+    if 'year' in meta_df.data:
+        assert obs['year'].values[0] == 2010
+    else:
+        assert (
+            pd.to_datetime(obs['time'].values[0])
+            == pd.to_datetime(datetime.datetime(2010, 7, 21))
+        )
     assert list(meta_df['exclude']) == [False, True]
 
 
@@ -971,10 +999,37 @@ def test_normalize(meta_df):
     exp = meta_df.data.copy().reset_index(drop=True)
     exp['value'][1::2] /= exp['value'][::2].values
     exp['value'][::2] /= exp['value'][::2].values
-    obs = meta_df.normalize(year=2005).data.reset_index(drop=True)
+    if "year" in meta_df.data:
+        obs = meta_df.normalize(year=2005).data.reset_index(drop=True)
+    else:
+        obs = meta_df.normalize(
+            time=datetime.datetime(2005, 6, 17)
+        ).data.reset_index(drop=True)
     pd.testing.assert_frame_equal(obs, exp)
 
 
 def test_normalize_not_time(meta_df):
     pytest.raises(ValueError, meta_df.normalize, variable='foo')
     pytest.raises(ValueError, meta_df.normalize, year=2015, variable='foo')
+
+
+@pytest.mark.parametrize("inplace", [True, False])
+def test_swap_time_to_year(test_df, inplace):
+    if "year" in test_df.data:
+        with pytest.raises(ValueError):
+            test_df.swap_time_for_year(inplace=inplace)
+        return
+
+    exp = test_df.data.copy()
+    exp["year"] = exp["time"].apply(lambda x: x.year)
+    exp = exp.drop("time", axis="columns")
+    exp = IamDataFrame(exp)
+
+    obs = test_df.swap_time_for_year(inplace=inplace)
+
+    if inplace:
+        assert obs is None
+        assert compare(test_df, exp).empty
+    else:
+        assert compare(obs, exp).empty
+        assert not "year" in test_df.data.columns

tests/test_feature_aggregate.py

@@ -1,3 +1,6 @@
+import datetime as dt
+
+
 import numpy as np
 import pandas as pd
 from pyam import check_aggregate, IAMC_IDX
@@ -9,13 +12,19 @@ def test_do_aggregate_append(meta_df):
     meta_df.aggregate('Primary Energy', append=True)
     obs = meta_df.filter(variable='Primary Energy').timeseries()
 
+    dts = [dt.datetime(2005, 6, 17), dt.datetime(2010, 7, 21)]
+    times = [2005, 2010] if "year" in meta_df.data else dts
     exp = pd.DataFrame([
         ['model_a', 'scen_a', 'World', 'Primary Energy', 'EJ/y', 1.5, 9.],
         ['model_a', 'scen_b', 'World', 'Primary Energy', 'EJ/y', 2, 7],
     ],
-        columns=['model', 'scenario', 'region', 'variable', 'unit', 2005, 2010]
+        columns=['model', 'scenario', 'region', 'variable', 'unit'] + times
     ).set_index(IAMC_IDX)
-    exp.columns = list(map(int, exp.columns))
+    if "year" in meta_df.data:
+        exp.columns = list(map(int, exp.columns))
+    else:
+        exp.columns = pd.to_datetime(exp.columns)
+
     pd.testing.assert_frame_equal(obs, exp)
 
 

tests/test_feature_append_rename_convert.py

@@ -1,5 +1,6 @@
 import copy
 import pytest
+import datetime as dt
 
 import numpy as np
 import pandas as pd
@@ -136,13 +137,18 @@ def test_rename_index(meta_df):
     obs = meta_df.rename(mapping, scenario={'scen_a': 'scen_c'})
 
     # test data changes
+    dts = [dt.datetime(2005, 6, 17), dt.datetime(2010, 7, 21)]
+    times = [2005, 2010] if "year" in meta_df.data else dts
     exp = pd.DataFrame([
         ['model_b', 'scen_c', 'World', 'Primary Energy', 'EJ/y', 1, 6.],
         ['model_b', 'scen_c', 'World', 'Primary Energy|Coal', 'EJ/y', 0.5, 3],
         ['model_a', 'scen_b', 'World', 'Primary Energy', 'EJ/y', 2, 7],
-    ], columns=['model', 'scenario', 'region', 'variable', 'unit', 2005, 2010]
+    ], columns=['model', 'scenario', 'region', 'variable', 'unit'] + times
     ).set_index(IAMC_IDX).sort_index()
-    exp.columns = exp.columns.map(int)
+    if "year" in meta_df.data:
+        exp.columns = list(map(int, exp.columns))
+    else:
+        exp.columns = pd.to_datetime(exp.columns)
     pd.testing.assert_frame_equal(obs.timeseries().sort_index(), exp)
 
     # test meta changes
@@ -160,15 +166,20 @@ def test_rename_append(meta_df):
     obs = meta_df.rename(mapping, append=True)
 
     # test data changes
+    dts = [dt.datetime(2005, 6, 17), dt.datetime(2010, 7, 21)]
+    times = [2005, 2010] if "year" in meta_df.data else dts
     exp = pd.DataFrame([
         ['model_a', 'scen_a', 'World', 'Primary Energy', 'EJ/y', 1, 6.],
         ['model_a', 'scen_a', 'World', 'Primary Energy|Coal', 'EJ/y', 0.5, 3],
         ['model_a', 'scen_b', 'World', 'Primary Energy', 'EJ/y', 2, 7],
         ['model_b', 'scen_c', 'World', 'Primary Energy', 'EJ/y', 1, 6.],
         ['model_b', 'scen_c', 'World', 'Primary Energy|Coal', 'EJ/y', 0.5, 3],
-    ], columns=['model', 'scenario', 'region', 'variable', 'unit', 2005, 2010]
+    ], columns=['model', 'scenario', 'region', 'variable', 'unit'] + times
     ).set_index(IAMC_IDX).sort_index()
-    exp.columns = exp.columns.map(int)
+    if "year" in meta_df.data:
+        exp.columns = list(map(int, exp.columns))
+    else:
+        exp.columns = pd.to_datetime(exp.columns)
     pd.testing.assert_frame_equal(obs.timeseries().sort_index(), exp)
 
     # test meta changes