From 7559b1e5fbb2503925e9c0eec6b9c4ea1c446907 Mon Sep 17 00:00:00 2001 From: Martin Holmer Date: Sun, 13 Jan 2019 16:35:21 -0500 Subject: [PATCH 1/2] Fix bottom decile sub-categories in distribution table --- taxcalc/utils.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/taxcalc/utils.py b/taxcalc/utils.py index ecc3d46ac..12b575a3c 100644 --- a/taxcalc/utils.py +++ b/taxcalc/utils.py @@ -187,8 +187,13 @@ def add_quantile_table_row_variable(dframe, income_measure, num_quantiles, num_bins = num_quantiles if decile_details: assert bin_edges[1] > 1e-9 # bin_edges[1] is top of bottom decile - bin_edges.insert(1, 1e-9) # top of zeros - bin_edges.insert(1, -1e-9) # top of negatives + neg_im = np.less_equal(dframe[income_measure], -1e-9) + neg_wght = dframe['s006'][neg_im].sum() + zer_im = np.logical_and(np.greater(dframe[income_measure], -1e-9), + np.less(dframe[income_measure], 1e-9)) + zer_wght = dframe['s006'][zer_im].sum() + bin_edges.insert(1, neg_wght + zer_wght) # top of zeros + bin_edges.insert(1, neg_wght) # top of negatives bin_edges.insert(-1, bin_edges[-2] + 0.5 * bin_width) # top of 90-95 bin_edges.insert(-1, bin_edges[-2] + 0.4 * bin_width) # top of 95-99 num_bins += 4 From 2f83a9d3ea4d9b0f577c84413fb4ef304234980b Mon Sep 17 00:00:00 2001 From: Martin Holmer Date: Sun, 13 Jan 2019 17:13:30 -0500 Subject: [PATCH 2/2] Update test results --- taxcalc/tests/tbi_cps_expect.txt | 174 +++++++++---------- taxcalc/tests/tbi_puf_expect.txt | 280 +++++++++++++++---------------- taxcalc/utils.py | 1 + 3 files changed, 228 insertions(+), 227 deletions(-) diff --git a/taxcalc/tests/tbi_cps_expect.txt b/taxcalc/tests/tbi_cps_expect.txt index 298b32534..9c8d98be0 100644 --- a/taxcalc/tests/tbi_cps_expect.txt +++ b/taxcalc/tests/tbi_cps_expect.txt @@ -218,6 +218,7 @@ TABLE diff_comb_xbin RESULTS: TABLE diff_comb_xdec RESULTS: { "0-10n_2": [ + "0.03", "0.00", "0.00", "0.00", @@ -226,13 +227,12 @@ TABLE diff_comb_xdec RESULTS: "0.00", "0.00", "0.00", - "0.00", - "0.00", - "0.00", - "nan" + "-1.73", + "-1.73", + "0.00" ], "0-10p_2": [ - "16.72", + "15.93", "0.00", "0.00", "0.00", @@ -241,11 +241,12 @@ TABLE diff_comb_xdec RESULTS: "0.00", "0.00", "0.00", - "-283.69", - "-283.69", + "-281.95", + "-281.95", "0.00" ], "0-10z_2": [ + "0.76", "0.00", "0.00", "0.00", @@ -256,8 +257,7 @@ TABLE diff_comb_xdec RESULTS: "0.00", "0.00", "0.00", - "0.00", - "nan" + "0.00" ], "10-20_2": [ "16.78", @@ -644,6 +644,7 @@ TABLE diff_itax_xbin RESULTS: TABLE diff_itax_xdec RESULTS: { "0-10n_2": [ + "0.03", "0.00", "0.00", "0.00", @@ -652,13 +653,12 @@ TABLE diff_itax_xdec RESULTS: "0.00", "0.00", "0.00", - "0.00", - "0.00", - "0.00", - "nan" + "-1.04", + "-1.04", + "0.00" ], "0-10p_2": [ - "16.72", + "15.93", "0.00", "0.00", "0.00", @@ -667,11 +667,12 @@ TABLE diff_itax_xdec RESULTS: "0.00", "0.00", "0.00", - "-170.21", - "-170.21", + "-169.17", + "-169.17", "0.00" ], "0-10z_2": [ + "0.76", "0.00", "0.00", "0.00", @@ -682,8 +683,7 @@ TABLE diff_itax_xdec RESULTS: "0.00", "0.00", "0.00", - "0.00", - "nan" + "0.00" ], "10-20_2": [ "16.78", @@ -1070,7 +1070,7 @@ TABLE diff_ptax_xbin RESULTS: TABLE diff_ptax_xdec RESULTS: { "0-10n_2": [ - "0.00", + "0.03", "0.00", "0.00", "0.00", @@ -1079,12 +1079,12 @@ TABLE diff_ptax_xdec RESULTS: "0.00", "nan", "0.00", - "0.00", - "0.00", - "nan" + "-1.39", + "-1.39", + "0.00" ], "0-10p_2": [ - "16.72", + "15.93", "0.00", "0.00", "0.00", @@ -1093,12 +1093,12 @@ TABLE diff_ptax_xdec RESULTS: "0.00", "nan", "0.00", - "-226.95", - "-226.95", + "-225.56", + "-225.56", "0.00" ], "0-10z_2": [ - "0.00", + "0.76", "0.00", "0.00", "0.00", @@ -1109,7 +1109,7 @@ TABLE diff_ptax_xdec RESULTS: "0.00", "0.00", "0.00", - "nan" + "0.00" ], "10-20_2": [ "16.78", @@ -1664,42 +1664,42 @@ TABLE dist1_xbin RESULTS: TABLE dist1_xdec RESULTS: { "0-10n_2": [ + "0.03", + "-4.45", + "0.03", + "0.62", "0.00", "0.00", "0.00", "0.00", "0.00", + "-4.45", "0.00", "0.00", "0.00", "0.00", "0.00", "0.00", + "-0.00", + "0.02", + "0.02", "0.00", - "0.00", - "0.00", - "0.00", - "0.00", - "0.00", - "0.00", - "0.00", - "0.00", - "0.00", - "0.00", - "0.00", - "0.00" + "0.35", + "0.35", + "-4.01", + "-4.03" ], "0-10p_2": [ - "16.72", - "131.93", - "16.59", - "202.20", + "15.93", + "136.38", + "15.80", + "188.93", "0.13", "1.83", "0.00", "20.89", "1.89", - "130.38", + "134.83", "0.01", "0.00", "1.89", @@ -1707,20 +1707,25 @@ TABLE dist1_xdec RESULTS: "0.00", "6.08", "-4.35", - "19.32", - "14.96", + "19.30", + "14.95", "0.00", - "56.74", - "56.74", - "195.02", - "180.06" + "56.39", + "56.39", + "199.04", + "184.09" ], "0-10z_2": [ + "0.76", + "-0.01", + "0.76", + "12.66", "0.00", "0.00", "0.00", "0.00", "0.00", + "-0.01", "0.00", "0.00", "0.00", @@ -1733,13 +1738,8 @@ TABLE dist1_xdec RESULTS: "0.00", "0.00", "0.00", - "0.00", - "0.00", - "0.00", - "0.00", - "0.00", - "0.00", - "0.00" + "-0.01", + "-0.01" ], "10-20_2": [ "16.78", @@ -2450,42 +2450,42 @@ TABLE dist2_xbin RESULTS: TABLE dist2_xdec RESULTS: { "0-10n_2": [ + "0.03", + "-4.45", + "0.03", + "0.62", "0.00", "0.00", "0.00", "0.00", "0.00", + "-4.45", "0.00", "0.00", "0.00", "0.00", "0.00", "0.00", + "-0.00", + "0.02", + "0.02", "0.00", - "0.00", - "0.00", - "0.00", - "0.00", - "0.00", - "0.00", - "0.00", - "0.00", - "0.00", - "0.00", - "0.00", - "0.00" + "-0.69", + "-0.69", + "-4.01", + "-4.03" ], "0-10p_2": [ - "16.72", - "131.93", - "16.59", - "202.20", + "15.93", + "136.38", + "15.80", + "188.93", "0.13", "1.83", "0.00", "20.89", "1.89", - "130.38", + "134.83", "0.01", "0.00", "1.89", @@ -2493,20 +2493,25 @@ TABLE dist2_xdec RESULTS: "0.00", "6.08", "-4.35", - "19.32", - "14.96", + "19.30", + "14.95", "0.00", - "-113.47", - "-113.47", - "195.02", - "180.06" + "-112.78", + "-112.78", + "199.04", + "184.09" ], "0-10z_2": [ + "0.76", + "-0.01", + "0.76", + "12.66", "0.00", "0.00", "0.00", "0.00", "0.00", + "-0.01", "0.00", "0.00", "0.00", @@ -2519,13 +2524,8 @@ TABLE dist2_xdec RESULTS: "0.00", "0.00", "0.00", - "0.00", - "0.00", - "0.00", - "0.00", - "0.00", - "0.00", - "0.00" + "-0.01", + "-0.01" ], "10-20_2": [ "16.78", diff --git a/taxcalc/tests/tbi_puf_expect.txt b/taxcalc/tests/tbi_puf_expect.txt index 17e7ba260..92cdca72f 100644 --- a/taxcalc/tests/tbi_puf_expect.txt +++ b/taxcalc/tests/tbi_puf_expect.txt @@ -218,10 +218,21 @@ TABLE diff_comb_xbin RESULTS: TABLE diff_comb_xdec RESULTS: { "0-10n_2": [ + "0.90", "0.00", "0.00", "0.00", + "8.00", + "557.80", + "0.50", + "0.20", "0.00", + "-8.58", + "-8.58", + "0.30" + ], + "0-10p_2": [ + "13.36", "0.00", "0.00", "0.00", @@ -229,23 +240,13 @@ TABLE diff_comb_xdec RESULTS: "0.00", "0.00", "0.00", - "nan" - ], - "0-10p_2": [ - "17.73", - "0.00", "0.00", - "0.00", - "0.40", - "28.40", - "0.50", - "0.20", - "0.00", - "-23.53", - "-23.53", - "0.40" + "-14.96", + "-14.96", + "0.00" ], "0-10z_2": [ + "3.46", "0.00", "0.00", "0.00", @@ -256,8 +257,7 @@ TABLE diff_comb_xdec RESULTS: "0.00", "0.00", "0.00", - "0.00", - "nan" + "0.00" ], "10-20_2": [ "17.76", @@ -644,34 +644,35 @@ TABLE diff_itax_xbin RESULTS: TABLE diff_itax_xdec RESULTS: { "0-10n_2": [ + "0.90", "0.00", "0.00", "0.00", + "8.00", + "557.80", + "0.50", + "0.20", "0.00", "0.00", "0.00", + "0.30" + ], + "0-10p_2": [ + "13.36", "0.00", "0.00", "0.00", "0.00", "0.00", - "nan" - ], - "0-10p_2": [ - "17.73", - "0.00", "0.00", "0.00", - "0.40", - "28.40", - "0.50", - "0.20", "0.00", "0.00", "0.00", - "0.40" + "0.00" ], "0-10z_2": [ + "3.46", "0.00", "0.00", "0.00", @@ -682,8 +683,7 @@ TABLE diff_itax_xdec RESULTS: "0.00", "0.00", "0.00", - "0.00", - "nan" + "0.00" ], "10-20_2": [ "17.76", @@ -1070,7 +1070,7 @@ TABLE diff_ptax_xbin RESULTS: TABLE diff_ptax_xdec RESULTS: { "0-10n_2": [ - "0.00", + "0.90", "0.00", "0.00", "0.00", @@ -1079,12 +1079,12 @@ TABLE diff_ptax_xdec RESULTS: "0.00", "nan", "0.00", - "0.00", - "0.00", - "nan" + "-4.29", + "-4.29", + "0.30" ], "0-10p_2": [ - "17.73", + "13.36", "0.00", "0.00", "0.00", @@ -1093,12 +1093,12 @@ TABLE diff_ptax_xdec RESULTS: "0.00", "nan", "0.00", - "-11.77", - "-11.77", - "0.40" + "-7.48", + "-7.48", + "0.00" ], "0-10z_2": [ - "0.00", + "3.46", "0.00", "0.00", "0.00", @@ -1109,7 +1109,7 @@ TABLE diff_ptax_xdec RESULTS: "0.00", "0.00", "0.00", - "nan" + "0.00" ], "10-20_2": [ "17.76", @@ -1664,82 +1664,82 @@ TABLE dist1_xbin RESULTS: TABLE dist1_xdec RESULTS: { "0-10n_2": [ + "0.90", + "-8.91", + "0.84", + "14.78", + "0.06", + "3.00", "0.00", + "22.12", + "4.29", + "-12.81", "0.00", + "0.16", + "4.46", + "0.18", + "0.24", + "0.10", + "4.42", + "2.26", + "6.68", "0.00", - "0.00", - "0.00", - "0.00", - "0.00", - "0.00", - "0.00", - "0.00", - "0.00", - "0.00", - "0.00", - "0.00", - "0.00", - "0.00", - "0.00", - "0.00", - "0.00", - "0.00", - "0.00", - "0.00", - "0.00", - "0.00" + "4.29", + "4.29", + "-143.77", + "-150.45" ], "0-10p_2": [ - "17.73", - "25.01", - "17.32", - "192.66", - "0.40", - "3.92", + "13.36", + "34.55", + "13.01", + "124.43", + "0.35", + "0.92", "0.00", - "25.03", - "4.50", - "20.94", + "2.91", + "0.21", + "34.38", "0.00", - "0.20", - "4.70", - "0.25", + "0.03", "0.24", - "3.08", - "1.61", - "7.73", - "9.34", + "0.07", + "0.00", + "2.56", + "-2.39", + "4.82", + "2.43", "0.00", - "11.77", - "11.77", - "-105.32", - "-114.66" + "7.48", + "7.48", + "38.46", + "36.03" ], "0-10z_2": [ + "3.46", + "-0.63", + "3.46", + "53.45", "0.00", "0.00", "0.00", "0.00", "0.00", + "-0.63", "0.00", "0.00", "0.00", "0.00", "0.00", + "0.42", + "-0.42", + "0.65", + "0.23", "0.00", "0.00", "0.00", "0.00", - "0.00", - "0.00", - "0.00", - "0.00", - "0.00", - "0.00", - "0.00", - "0.00", - "0.00", - "0.00" + "-0.23" ], "10-20_2": [ "17.76", @@ -2450,82 +2450,82 @@ TABLE dist2_xbin RESULTS: TABLE dist2_xdec RESULTS: { "0-10n_2": [ + "0.90", + "-8.91", + "0.85", + "14.83", + "0.05", + "2.95", "0.00", + "22.11", + "4.81", + "-12.78", "0.00", + "0.15", + "4.96", + "0.18", + "0.24", + "0.10", + "4.92", + "2.26", + "7.18", "0.00", - "0.00", - "0.00", - "0.00", - "0.00", - "0.00", - "0.00", - "0.00", - "0.00", - "0.00", - "0.00", - "0.00", - "0.00", - "0.00", - "0.00", - "0.00", - "0.00", - "0.00", - "0.00", - "0.00", - "0.00", - "0.00" + "4.29", + "4.29", + "-143.77", + "-150.96" ], "0-10p_2": [ - "17.73", - "25.01", - "17.32", - "192.71", - "0.40", - "3.87", + "13.36", + "34.55", + "13.01", + "124.43", + "0.35", + "0.92", "0.00", - "25.02", - "5.02", - "20.97", + "2.91", + "0.21", + "34.38", "0.00", - "0.19", - "5.21", - "0.25", + "0.03", "0.24", - "3.08", - "2.12", - "7.73", - "9.85", + "0.07", + "0.00", + "2.56", + "-2.39", + "4.82", + "2.43", "0.00", - "11.77", - "11.77", - "-105.32", - "-115.16" + "7.48", + "7.48", + "38.46", + "36.03" ], "0-10z_2": [ + "3.46", + "-0.63", + "3.46", + "53.45", "0.00", "0.00", "0.00", "0.00", "0.00", + "-0.63", "0.00", "0.00", "0.00", "0.00", "0.00", + "0.42", + "-0.42", + "0.65", + "0.23", "0.00", "0.00", "0.00", "0.00", - "0.00", - "0.00", - "0.00", - "0.00", - "0.00", - "0.00", - "0.00", - "0.00", - "0.00", - "0.00" + "-0.23" ], "10-20_2": [ "17.76", diff --git a/taxcalc/utils.py b/taxcalc/utils.py index 12b575a3c..1f9ffb2e6 100644 --- a/taxcalc/utils.py +++ b/taxcalc/utils.py @@ -163,6 +163,7 @@ def add_quantile_table_row_variable(dframe, income_measure, num_quantiles, and pos income_measure ) and the top decile is broken into three subgroups (90-95, 95-99, and top 1%). """ + # pylint: disable=too-many-locals assert isinstance(dframe, pd.DataFrame) assert income_measure in dframe if decile_details and num_quantiles != 10: