Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Extend meta_df tests to include datetime #236

Merged
merged 18 commits into from
May 29, 2019
Merged
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 29 additions & 0 deletions pyam/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -309,6 +309,31 @@ def interpolate(self, year):
fill_values['year'] = year
self.data = self.data.append(fill_values, ignore_index=True)

def swap_time_for_year(self, inplace=False):
"""Convert the `time` column to `year`.

Parameters
----------
inplace: bool, default False
if True, do operation inplace and return None

Raises
------
ValueError
"time" is not a column of `self.data`
"""
if "time" not in self.data:
raise ValueError("`time` must be in `self.data` to use this method")
znicholls marked this conversation as resolved.
Show resolved Hide resolved

ret = copy.deepcopy(self) if not inplace else self
znicholls marked this conversation as resolved.
Show resolved Hide resolved

ret.data["year"] = ret.data["time"].apply(lambda x: x.year)
znicholls marked this conversation as resolved.
Show resolved Hide resolved
ret.data = ret.data.drop("time", axis="columns")
ret._LONG_IDX = [v if v != "time" else "year" for v in ret._LONG_IDX]

if not inplace:
return ret

def as_pandas(self, with_metadata=False):
"""Return this as a pd.DataFrame

Expand Down Expand Up @@ -1358,6 +1383,10 @@ def _check_rows(rows, check, in_range=True, return_test='any'):
msg = 'Unknown checking type: {}'
raise ValueError(msg.format(check.keys() - valid_checks))

rows = rows.copy()
znicholls marked this conversation as resolved.
Show resolved Hide resolved
if 'year' not in rows:
rows['year'] = rows['time'].apply(lambda x: x.year)

where_idx = set(rows.index[rows['year'] == check['year']]) \
if 'year' in check else set(rows.index)
rows = rows.loc[list(where_idx)]
Expand Down
32 changes: 20 additions & 12 deletions tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -167,15 +167,15 @@
TEST_STACKPLOT_DF['scenario'] = 'a_scen'


TIME_AXES = [
[2005, 2010],
[datetime(2005, 6, 17), datetime(2010, 7, 21)],
['2005-06-17', '2010-07-21'],
['2005-06-17 00:00:00', '2010-07-21 12:00:00']
]


@pytest.fixture(scope="function", params=TIME_AXES)
@pytest.fixture(
znicholls marked this conversation as resolved.
Show resolved Hide resolved
scope="function",
params=[
[2005, 2010],
[datetime(2005, 6, 17), datetime(2010, 7, 21)],
['2005-06-17', '2010-07-21'],
['2005-06-17 00:00:00', '2010-07-21 12:00:00']
]
)
def test_df(request):
tdf = TEST_DF.iloc[:2]
tdf = tdf.rename({2005: request.param[0], 2010: request.param[1]},
Expand All @@ -195,9 +195,17 @@ def test_pd_df():
yield TEST_DF.copy()


@pytest.fixture(scope="function")
def meta_df():
df = IamDataFrame(data=TEST_DF)
@pytest.fixture(
scope="function",
params=[
[2005, 2010],
[datetime(2005, 6, 17), datetime(2010, 7, 21)],
znicholls marked this conversation as resolved.
Show resolved Hide resolved
]
)
def meta_df(request):
mdf = TEST_DF.rename({2005: request.param[0], 2010: request.param[1]},
axis="columns")
df = IamDataFrame(data=mdf)
yield df


Expand Down
39 changes: 25 additions & 14 deletions tests/test_cast_to_iamc.py
Original file line number Diff line number Diff line change
@@ -1,41 +1,48 @@
import datetime as dt
znicholls marked this conversation as resolved.
Show resolved Hide resolved


import pytest
import pandas as pd
from pyam import IamDataFrame, compare


def test_cast_from_value_col(meta_df):
df_with_value_cols = pd.DataFrame([
['model_a', 'scen_a', 'World', 'EJ/y', 2005, 1, 0.5],
['model_a', 'scen_a', 'World', 'EJ/y', 2010, 6., 3],
['model_a', 'scen_b', 'World', 'EJ/y', 2005, 2, None],
['model_a', 'scen_b', 'World', 'EJ/y', 2010, 7, None]
['model_a', 'scen_a', 'World', 'EJ/y', dt.datetime(2005, 6, 17), 1, 0.5],
znicholls marked this conversation as resolved.
Show resolved Hide resolved
['model_a', 'scen_a', 'World', 'EJ/y', dt.datetime(2010, 7, 21), 6., 3],
znicholls marked this conversation as resolved.
Show resolved Hide resolved
['model_a', 'scen_b', 'World', 'EJ/y', dt.datetime(2005, 6, 17), 2, None],
znicholls marked this conversation as resolved.
Show resolved Hide resolved
['model_a', 'scen_b', 'World', 'EJ/y', dt.datetime(2010, 7, 21), 7, None]
znicholls marked this conversation as resolved.
Show resolved Hide resolved
],
columns=['model', 'scenario', 'region', 'unit', 'year',
columns=['model', 'scenario', 'region', 'unit', 'time',
'Primary Energy', 'Primary Energy|Coal'],
)
df = IamDataFrame(df_with_value_cols,
value=['Primary Energy', 'Primary Energy|Coal'])
if "year" in meta_df.data.columns:
df = df.swap_time_for_year()

assert compare(meta_df, df).empty
pd.testing.assert_frame_equal(df.data, meta_df.data)
pd.testing.assert_frame_equal(df.data, meta_df.data, check_like=True)


def test_cast_from_value_col_and_args(meta_df):
# checks for issue [#210](https://github.com/IAMconsortium/pyam/issues/210)
df_with_value_cols = pd.DataFrame([
['scen_a', 'World', 'EJ/y', 2005, 1, 0.5],
['scen_a', 'World', 'EJ/y', 2010, 6., 3],
['scen_b', 'World', 'EJ/y', 2005, 2, None],
['scen_b', 'World', 'EJ/y', 2010, 7, None]
['scen_a', 'World', 'EJ/y', dt.datetime(2005, 6, 17), 1, 0.5],
['scen_a', 'World', 'EJ/y', dt.datetime(2010, 7, 21), 6., 3],
['scen_b', 'World', 'EJ/y', dt.datetime(2005, 6, 17), 2, None],
['scen_b', 'World', 'EJ/y', dt.datetime(2010, 7, 21), 7, None]
],
columns=['scenario', 'iso', 'unit', 'year',
columns=['scenario', 'iso', 'unit', 'time',
'Primary Energy', 'Primary Energy|Coal'],
)
df = IamDataFrame(df_with_value_cols, model='model_a', region='iso',
value=['Primary Energy', 'Primary Energy|Coal'])
if "year" in meta_df.data.columns:
df = df.swap_time_for_year()

assert compare(meta_df, df).empty
pd.testing.assert_frame_equal(df.data, meta_df.data)
pd.testing.assert_frame_equal(df.data, meta_df.data, check_like=True)


def test_cast_with_model_arg_raises():
Expand All @@ -58,17 +65,21 @@ def test_cast_with_model_arg(meta_df):


def test_cast_by_column_concat(meta_df):
dts = [dt.datetime(2005, 6, 17), dt.datetime(2010, 7, 21)]
znicholls marked this conversation as resolved.
Show resolved Hide resolved
df = pd.DataFrame([
['scen_a', 'World', 'Primary Energy', None, 'EJ/y', 1, 6.],
['scen_a', 'World', 'Primary Energy', 'Coal', 'EJ/y', 0.5, 3],
['scen_b', 'World', 'Primary Energy', None, 'EJ/y', 2, 7],
],
columns=['scenario', 'region', 'var_1', 'var_2', 'unit', 2005, 2010],
columns=['scenario', 'region', 'var_1', 'var_2', 'unit'] + dts,
)

df = IamDataFrame(df, model='model_a', variable=['var_1', 'var_2'])
if "year" in meta_df.data.columns:
df = df.swap_time_for_year()

assert compare(meta_df, df).empty
pd.testing.assert_frame_equal(df.data, meta_df.data)
pd.testing.assert_frame_equal(df.data, meta_df.data, check_like=True)


def test_cast_with_variable_and_value(meta_df):
Expand Down
67 changes: 61 additions & 6 deletions tests/test_core.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
from numpy import testing as npt

from pyam import IamDataFrame, validate, categorize, \
require_variable, filter_by_meta, META_IDX, IAMC_IDX, sort_data
require_variable, filter_by_meta, META_IDX, IAMC_IDX, sort_data, compare
from pyam.core import _meta_idx, concat

from conftest import TEST_DATA_DIR
Expand Down Expand Up @@ -518,22 +518,44 @@ def test_validate_up(meta_df):
obs = meta_df.validate({'Primary Energy': {'up': 6.5}},
exclude_on_fail=False)
assert len(obs) == 1
assert obs['year'].values[0] == 2010
if 'year' in meta_df.data:
assert obs['year'].values[0] == 2010
else:
assert (
pd.to_datetime(obs['time'].values[0])
== pd.to_datetime(datetime.datetime(2010, 7, 21))
znicholls marked this conversation as resolved.
Show resolved Hide resolved
)

assert list(meta_df['exclude']) == [False, False] # assert none excluded


def test_validate_lo(meta_df):
obs = meta_df.validate({'Primary Energy': {'up': 8, 'lo': 2.0}})
assert len(obs) == 1
assert obs['year'].values[0] == 2005
if 'year' in meta_df.data:
assert obs['year'].values[0] == 2005
else:
assert (
pd.to_datetime(obs['time'].values[0])
== pd.to_datetime(datetime.datetime(2005, 6, 17))
znicholls marked this conversation as resolved.
Show resolved Hide resolved
)
assert list(obs['scenario'].values) == ['scen_a']


def test_validate_both(meta_df):
obs = meta_df.validate({'Primary Energy': {'up': 6.5, 'lo': 2.0}})
assert len(obs) == 2
assert list(obs['year'].values) == [2005, 2010]
if 'year' in meta_df.data:
assert list(obs['year'].values) == [2005, 2010]
else:
assert (
pd.to_datetime(obs['time'].values)
== pd.to_datetime([
znicholls marked this conversation as resolved.
Show resolved Hide resolved
datetime.datetime(2005, 6, 17),
datetime.datetime(2010, 7, 21),
])
).all()

assert list(obs['scenario'].values) == ['scen_a', 'scen_b']


Expand All @@ -556,7 +578,13 @@ def test_validate_top_level(meta_df):
obs = validate(meta_df, criteria={'Primary Energy': {'up': 6.0}},
exclude_on_fail=True, variable='Primary Energy')
assert len(obs) == 1
assert obs['year'].values[0] == 2010
if 'year' in meta_df.data:
assert obs['year'].values[0] == 2010
else:
assert (
pd.to_datetime(obs['time'].values[0])
== pd.to_datetime(datetime.datetime(2010, 7, 21))
znicholls marked this conversation as resolved.
Show resolved Hide resolved
)
assert list(meta_df['exclude']) == [False, True]


Expand Down Expand Up @@ -971,10 +999,37 @@ def test_normalize(meta_df):
exp = meta_df.data.copy().reset_index(drop=True)
exp['value'][1::2] /= exp['value'][::2].values
exp['value'][::2] /= exp['value'][::2].values
obs = meta_df.normalize(year=2005).data.reset_index(drop=True)
if "year" in meta_df.data:
obs = meta_df.normalize(year=2005).data.reset_index(drop=True)
else:
obs = meta_df.normalize(
time=datetime.datetime(2005, 6, 17)
).data.reset_index(drop=True)
pd.testing.assert_frame_equal(obs, exp)


def test_normalize_not_time(meta_df):
pytest.raises(ValueError, meta_df.normalize, variable='foo')
pytest.raises(ValueError, meta_df.normalize, year=2015, variable='foo')


@pytest.mark.parametrize("inplace", [True, False])
def test_swap_time_to_year(test_df, inplace):
if "year" in test_df.data:
with pytest.raises(ValueError):
test_df.swap_time_for_year(inplace=inplace)
return

exp = test_df.data.copy()
exp["year"] = exp["time"].apply(lambda x: x.year)
exp = exp.drop("time", axis="columns")
exp = IamDataFrame(exp)

obs = test_df.swap_time_for_year(inplace=inplace)

if inplace:
assert obs is None
assert compare(test_df, exp).empty
else:
assert compare(obs, exp).empty
assert not "year" in test_df.data.columns
znicholls marked this conversation as resolved.
Show resolved Hide resolved
13 changes: 11 additions & 2 deletions tests/test_feature_aggregate.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
import datetime as dt
znicholls marked this conversation as resolved.
Show resolved Hide resolved


import numpy as np
import pandas as pd
from pyam import check_aggregate, IAMC_IDX
Expand All @@ -9,13 +12,19 @@ def test_do_aggregate_append(meta_df):
meta_df.aggregate('Primary Energy', append=True)
obs = meta_df.filter(variable='Primary Energy').timeseries()

dts = [dt.datetime(2005, 6, 17), dt.datetime(2010, 7, 21)]
znicholls marked this conversation as resolved.
Show resolved Hide resolved
times = [2005, 2010] if "year" in meta_df.data else dts
exp = pd.DataFrame([
['model_a', 'scen_a', 'World', 'Primary Energy', 'EJ/y', 1.5, 9.],
['model_a', 'scen_b', 'World', 'Primary Energy', 'EJ/y', 2, 7],
],
columns=['model', 'scenario', 'region', 'variable', 'unit', 2005, 2010]
columns=['model', 'scenario', 'region', 'variable', 'unit'] + times
).set_index(IAMC_IDX)
exp.columns = list(map(int, exp.columns))
if "year" in meta_df.data:
exp.columns = list(map(int, exp.columns))
else:
exp.columns = pd.to_datetime(exp.columns)

pd.testing.assert_frame_equal(obs, exp)


Expand Down
19 changes: 15 additions & 4 deletions tests/test_feature_append_rename_convert.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import copy
import pytest
import datetime as dt

import numpy as np
import pandas as pd
Expand Down Expand Up @@ -136,13 +137,18 @@ def test_rename_index(meta_df):
obs = meta_df.rename(mapping, scenario={'scen_a': 'scen_c'})

# test data changes
dts = [dt.datetime(2005, 6, 17), dt.datetime(2010, 7, 21)]
znicholls marked this conversation as resolved.
Show resolved Hide resolved
times = [2005, 2010] if "year" in meta_df.data else dts
exp = pd.DataFrame([
['model_b', 'scen_c', 'World', 'Primary Energy', 'EJ/y', 1, 6.],
['model_b', 'scen_c', 'World', 'Primary Energy|Coal', 'EJ/y', 0.5, 3],
['model_a', 'scen_b', 'World', 'Primary Energy', 'EJ/y', 2, 7],
], columns=['model', 'scenario', 'region', 'variable', 'unit', 2005, 2010]
], columns=['model', 'scenario', 'region', 'variable', 'unit'] + times
).set_index(IAMC_IDX).sort_index()
exp.columns = exp.columns.map(int)
if "year" in meta_df.data:
exp.columns = list(map(int, exp.columns))
else:
exp.columns = pd.to_datetime(exp.columns)
pd.testing.assert_frame_equal(obs.timeseries().sort_index(), exp)

# test meta changes
Expand All @@ -160,15 +166,20 @@ def test_rename_append(meta_df):
obs = meta_df.rename(mapping, append=True)

# test data changes
dts = [dt.datetime(2005, 6, 17), dt.datetime(2010, 7, 21)]
times = [2005, 2010] if "year" in meta_df.data else dts
znicholls marked this conversation as resolved.
Show resolved Hide resolved
exp = pd.DataFrame([
['model_a', 'scen_a', 'World', 'Primary Energy', 'EJ/y', 1, 6.],
['model_a', 'scen_a', 'World', 'Primary Energy|Coal', 'EJ/y', 0.5, 3],
['model_a', 'scen_b', 'World', 'Primary Energy', 'EJ/y', 2, 7],
['model_b', 'scen_c', 'World', 'Primary Energy', 'EJ/y', 1, 6.],
['model_b', 'scen_c', 'World', 'Primary Energy|Coal', 'EJ/y', 0.5, 3],
], columns=['model', 'scenario', 'region', 'variable', 'unit', 2005, 2010]
], columns=['model', 'scenario', 'region', 'variable', 'unit'] + times
).set_index(IAMC_IDX).sort_index()
exp.columns = exp.columns.map(int)
if "year" in meta_df.data:
exp.columns = list(map(int, exp.columns))
else:
exp.columns = pd.to_datetime(exp.columns)
pd.testing.assert_frame_equal(obs.timeseries().sort_index(), exp)

# test meta changes
Expand Down
Loading