Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add ability to use array suffixes in JSON reform files #1505

Merged
merged 6 commits into from
Aug 10, 2017
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions RELEASES.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,9 @@ Release 0.9.3 on 2017-??-??
- Add validation checking of policy parameters involved in a reform
[[#1502](https://github.com/open-source-economics/Tax-Calculator/pull/1502)
by Martin Holmer]
- Add option to use policy parameter suffixes in JSON reform files
[[#1505](https://github.com/open-source-economics/Tax-Calculator/pull/1505)
by Martin Holmer]

**Bug Fixes**
- None
Expand Down
12 changes: 6 additions & 6 deletions taxcalc/calculate.py
Original file line number Diff line number Diff line change
Expand Up @@ -512,9 +512,9 @@ def _read_json_policy_reform_text(text_string, arrays_not_lists):
an extended example of a commented JSON policy reform text
that can be read by this method.

Returned dictionary rpol_dict has integer years as primary keys and
Returned dictionary prdict has integer years as primary keys and
string parameters as secondary keys. This returned dictionary is
suitable as the argument to the Policy implement_reform(rpol_dict)
suitable as the argument to the Policy implement_reform(prdict)
method ONLY if the function argument arrays_not_lists is True.
"""
# strip out //-comments without changing line numbers
Expand Down Expand Up @@ -546,10 +546,10 @@ def _read_json_policy_reform_text(text_string, arrays_not_lists):
if rkey in Calculator.REQUIRED_ASSUMP_KEYS:
msg = 'key "{}" should be in economic assumption file'
raise ValueError(msg.format(rkey))
# convert the policy dictionary in raw_dict
rpol_dict = Calculator._convert_parameter_dict(raw_dict['policy'],
arrays_not_lists)
return rpol_dict
# convert raw_dict['policy'] dictionary into prdict
tdict = Policy.translate_json_reform_suffixes(raw_dict['policy'])
prdict = Calculator._convert_parameter_dict(tdict, arrays_not_lists)
return prdict

@staticmethod
def _read_json_econ_assump_text(text_string, arrays_not_lists):
Expand Down
103 changes: 93 additions & 10 deletions taxcalc/policy.py
Original file line number Diff line number Diff line change
Expand Up @@ -204,20 +204,103 @@ def current_law_version(self):
clv.set_year(self.current_year)
return clv

JSON_REFORM_SUFFIXES = set([
'single', 'joint', 'separate', 'headhousehold', 'widow', # MARS
'0kids', '1kid', '2kids', '3+kids', # EIC
'medical', 'statelocal', 'realestate', 'casualty', # idedtype
'misc', 'interest', 'charity' # idedtype
])
JSON_REFORM_SUFFIXES = {
# MARS-indexed suffixes and list index numbers
'single': 0,
'joint': 1,
'separate': 2,
'headhousehold': 3,
'widow': 4,
# EIC-indexed suffixes and list index numbers
'0kids': 0,
'1kid': 1,
'2kids': 2,
'3+kids': 3,
# idedtype-indexed suffixes and list index numbers
'medical': 0,
'statelocal': 1,
'realestate': 2,
'casualty': 3,
'misc': 4,
'interest': 5,
'charity': 6
}

@staticmethod
def translate_json_reform_suffixes(jsonstr):
def translate_json_reform_suffixes(indict):
"""
Replace any parameters with suffixes with array parameters
and return the consolidated JSON string
Replace any array parameters with suffixes in the specified
JSON-derived "policy" dictionary, indict, and
return a JSON-equivalent dictionary containing constructed array
parameters and containing no parameters with suffixes, odict.
"""
return jsonstr # TODO temporary code

# define no_suffix function used only in this method
def no_suffix(idict):
"""
Return param_base:year dictionary having only no-suffix parameters.
"""
odict = dict()
suffixes = Policy.JSON_REFORM_SUFFIXES.keys()
for param in idict.keys():
param_pieces = param.split('_')
suffix = param_pieces[-1]
if suffix not in suffixes:
odict[param] = idict[param]
return odict

# define group_dict function used only in this method
def suffix_group_dict(idict):
"""
Return param_base:year:suffix dictionary with each idict value.
"""
gdict = dict()
suffixes = Policy.JSON_REFORM_SUFFIXES.keys()
for param in idict.keys():
param_pieces = param.split('_')
suffix = param_pieces[-1]
if suffix in suffixes:
del param_pieces[-1]
param_base = '_'.join(param_pieces)
if param_base not in gdict:
gdict[param_base] = dict()
for year in sorted(idict[param].keys()):
if year not in gdict[param_base]:
gdict[param_base][year] = dict()
gdict[param_base][year][suffix] = idict[param][year][0]
return gdict

# define with_suffix function used only in this method
def with_suffix(gdict):
"""
Return param_base:year dictionary having only suffix parameters.
"""
pol = Policy()
odict = dict()
for param in gdict.keys():
odict[param] = dict()
for year in sorted(gdict[param].keys()):
odict[param][year] = dict()
for suffix in gdict[param][year].keys():
plist = getattr(pol, param).tolist()
dvals = plist[int(year) - Policy.JSON_START_YEAR]
odict[param][year] = [dvals]
idx = Policy.JSON_REFORM_SUFFIXES[suffix]
odict[param][year][0][idx] = gdict[param][year][suffix]
udict = {int(year): {param: odict[param][year]}}
pol.implement_reform(udict)
return odict

# high-level logic of translate_json_reform_suffixes method:
# - construct odict containing just parameters without a suffix
odict = no_suffix(indict)
# - group params with suffix into param_base:year:suffix dictionary
gdict = suffix_group_dict(indict)
# - add to odict the consolidated values for parameters with a suffix
if len(gdict) > 0:
odict.update(with_suffix(gdict))
# - return policy dictionary containing constructed parameter arrays
return odict

# ----- begin private methods of Policy class -----

Expand Down
110 changes: 110 additions & 0 deletions taxcalc/tests/test_calculate.py
Original file line number Diff line number Diff line change
Expand Up @@ -688,3 +688,113 @@ def test_calc_all(reform_file, rawinputfile):
sync_years=False) # keeps raw data unchanged
assert calc.current_year == cyr
calc.calc_all()


def test_translate_json_reform_suffixes_mars():
# test read_json_param_files(...) using MARS-indexed parameter suffixes
json1 = """{"policy": {
"_II_em": {"2020": [20000], "2015": [15000]},
"_STD_single": {"2018": [18000], "2016": [16000]},
"_STD_widow": {"2017": [17000], "2019": [19000]}
}}"""
pdict1 = Calculator.read_json_param_files(reform_filename=json1,
assump_filename=None,
arrays_not_lists=True)
rdict1 = pdict1['policy']
json2 = """{"policy": {
"_STD": {"2016": [[16000.00, 12600.00, 6300.00, 9300.00, 12600.00]],
"2017": [[16364.80, 12887.28, 6443.64, 9512.04, 17000.00]],
"2018": [[18000.00, 13173.38, 6586.69, 9723.21, 17377.40]],
"2019": [[18412.20, 13475.05, 6737.52, 9945.87, 19000.00]]},
"_II_em": {"2020": [20000], "2015": [15000]}
}}"""
pdict2 = Calculator.read_json_param_files(reform_filename=json2,
assump_filename=None,
arrays_not_lists=True)
rdict2 = pdict2['policy']
assert len(rdict2) == len(rdict1)
for year in rdict2.keys():
if '_II_em' in rdict2[year].keys():
assert np.allclose(rdict1[year]['_II_em'],
rdict2[year]['_II_em'],
atol=0.01, rtol=0.0)
if '_STD' in rdict2[year].keys():
assert np.allclose(rdict1[year]['_STD'],
rdict2[year]['_STD'],
atol=0.01, rtol=0.0)


def test_translate_json_reform_suffixes_eic():
# test read_json_param_files(...) using EIC-indexed parameter suffixes
json1 = """{"policy": {
"_II_em": {"2020": [20000], "2015": [15000]},
"_EITC_c_0kids": {"2018": [510], "2019": [510]},
"_EITC_c_1kid": {"2019": [3400], "2018": [3400]},
"_EITC_c_2kids": {"2018": [5616], "2019": [5616]},
"_EITC_c_3+kids": {"2019": [6318], "2018": [6318]}
}}"""
pdict1 = Calculator.read_json_param_files(reform_filename=json1,
assump_filename=None,
arrays_not_lists=True)
rdict1 = pdict1['policy']
json2 = """{"policy": {
"_EITC_c": {"2019": [[510, 3400, 5616, 6318]],
"2018": [[510, 3400, 5616, 6318]]},
"_II_em": {"2020": [20000], "2015": [15000]}
}}"""
pdict2 = Calculator.read_json_param_files(reform_filename=json2,
assump_filename=None,
arrays_not_lists=True)
rdict2 = pdict2['policy']
assert len(rdict2) == len(rdict1)
for year in rdict2.keys():
if '_II_em' in rdict2[year].keys():
assert np.allclose(rdict1[year]['_II_em'],
rdict2[year]['_II_em'],
atol=0.01, rtol=0.0)
if '_EITC_c' in rdict2[year].keys():
assert np.allclose(rdict1[year]['_EITC_c'],
rdict2[year]['_EITC_c'],
atol=0.01, rtol=0.0)


def test_translate_json_reform_suffixes_idedtype():
# test read_json_param_files(...) using idedtype-indexed parameter suffixes
json1 = """{"policy": {
"_ID_BenefitCap_rt": {"2019": [0.2]},
"_ID_BenefitCap_Switch_medical": {"2019": [false]},
"_ID_BenefitCap_Switch_casualty": {"2019": [false]},
"_ID_BenefitCap_Switch_misc": {"2019": [false]},
"_ID_BenefitCap_Switch_interest": {"2019": [false]},
"_ID_BenefitCap_Switch_charity": {"2019": [false]},
"_II_em": {"2020": [20000], "2015": [15000]}
}}"""
pdict1 = Calculator.read_json_param_files(reform_filename=json1,
assump_filename=None,
arrays_not_lists=True)
rdict1 = pdict1['policy']
json2 = """{"policy": {
"_II_em": {"2020": [20000], "2015": [15000]},
"_ID_BenefitCap_Switch": {
"2019": [[false, true, true, false, false, false, false]]
},
"_ID_BenefitCap_rt": {"2019": [0.2]}
}}"""
pdict2 = Calculator.read_json_param_files(reform_filename=json2,
assump_filename=None,
arrays_not_lists=True)
rdict2 = pdict2['policy']
assert len(rdict2) == len(rdict1)
for year in rdict2.keys():
if '_II_em' in rdict2[year].keys():
assert np.allclose(rdict1[year]['_II_em'],
rdict2[year]['_II_em'],
atol=0.01, rtol=0.0)
if '_ID_BenefitCap_rt' in rdict2[year].keys():
assert np.allclose(rdict1[year]['_ID_BenefitCap_rt'],
rdict2[year]['_ID_BenefitCap_rt'],
atol=0.01, rtol=0.0)
if '_ID_BenefitCap_Switch' in rdict2[year].keys():
assert np.allclose(rdict1[year]['_ID_BenefitCap_Switch'],
rdict2[year]['_ID_BenefitCap_Switch'],
atol=0.01, rtol=0.0)
9 changes: 6 additions & 3 deletions taxcalc/tests/test_policy.py
Original file line number Diff line number Diff line change
Expand Up @@ -735,17 +735,20 @@ def test_json_reform_suffixes(tests_path):
clpdict = json.load(clpfile)
clpfile.close()
# create set of suffixes in the clpdict "col_label" lists
suffixes = set()
json_suffixes = Policy.JSON_REFORM_SUFFIXES.keys()
clp_suffixes = set()
for param in clpdict:
suffix = param.split('_')[-1]
assert suffix not in json_suffixes
col_var = clpdict[param]['col_var']
col_label = clpdict[param]['col_label']
if col_var == '':
assert col_label == ''
continue
assert isinstance(col_label, list)
suffixes.update(col_label)
clp_suffixes.update(col_label)
# check that suffixes set is same as Policy.JSON_REFORM_SUFFIXES set
unmatched = suffixes ^ Policy.JSON_REFORM_SUFFIXES
unmatched = clp_suffixes ^ set(json_suffixes)
if len(unmatched) != 0:
assert unmatched == 'UNMATCHED SUFFIXES'

Expand Down