Resolve more backwards compatibility issues related to old taxcalc tables

webapp/apps/conftest.py

@@ -274,3 +274,19 @@ def no_assumptions_text():
 @pytest.fixture()
 def no_assumptions_text_json(no_assumptions_text):
     return json.loads(no_assumptions_text)
+
+
+@pytest.fixture()
+@set_fixture_prop
+def skelaton_res_lt_0130(request):
+    _path = os.path.join(CUR_PATH, "skelaton_res_lt_0130.json")
+    with open(_path) as js:
+        return json.loads(js.read())
+
+
+@pytest.fixture()
+@set_fixture_prop
+def skelaton_res_gt_0130(request):
+    _path = os.path.join(CUR_PATH, "skelaton_res_gt_0130.json")
+    with open(_path) as js:
+        return json.loads(js.read())

webapp/apps/constants.py

@@ -18,3 +18,5 @@
 
 START_YEARS = ('2013', '2014', '2015', '2016', '2017', '2018')
 START_YEAR = os.environ.get('START_YEAR', '2017')
+
+TAXCALC_VERS_RESULTS_BACKWARDS_INCOMPATIBLE = "0.13.0"

webapp/apps/dynamic/models.py

@@ -1,4 +1,5 @@
 import re
+from distutils.version import LooseVersion
 
 from django.db import models
 from django.core import validators
@@ -13,6 +14,9 @@
 
 from ..taxbrain.models import (CommaSeparatedField, SeparatedValuesField,
                                TaxSaveInputs, OutputUrl)
+
+from ..taxbrain import helpers as taxbrain_helpers
+
 import datetime
 
 
@@ -87,6 +91,18 @@ class DynamicBehaviorSaveInputs(models.Model):
     micro_sim = models.ForeignKey(OutputUrl, blank=True, null=True,
                                   on_delete=models.SET_NULL)
 
+    def get_tax_result(self):
+        """
+        If taxcalc version is greater than or equal to 0.13.0, return table
+        If taxcalc version is less than 0.13.0, then rename keys to new names
+        and then return table
+        """
+        return taxbrain_helpers.get_tax_result(
+            DynamicBehaviorOutputUrl,
+            self.pk,
+            self.tax_result
+        )
+
     class Meta:
         permissions = (
             ("view_inputs", "Allowed to view Taxbrain."),

webapp/apps/dynamic/tests/test_models.py

@@ -0,0 +1,19 @@
+from django.test import TestCase
+import pytest
+
+from ...test_assets.utils import get_taxbrain_model
+from ...test_assets.test_models import TaxBrainTableResults
+
+from ...dynamic.models import DynamicBehaviorOutputUrl
+from ...dynamic.forms import DynamicBehavioralInputsModelForm
+
+
+class TaxBrainDynamicResultsTest(TaxBrainTableResults, TestCase):
+
+    def test_dynamic_tc_lt_0130(self):
+        self.tc_lt_0130(Form=DynamicBehavioralInputsModelForm,
+                        UrlModel=DynamicBehaviorOutputUrl)
+
+    def test_dynamic_tc_gt_0130(self):
+        self.tc_gt_0130(Form=DynamicBehavioralInputsModelForm,
+                        UrlModel=DynamicBehaviorOutputUrl)

webapp/apps/dynamic/views.py

@@ -718,7 +718,7 @@ def behavior_results(request, pk):
     model = url.unique_inputs
     if model.tax_result:
 
-        output = model.tax_result
+        output = model.get_tax_result()
         first_year = model.first_year
         created_on = model.creation_date
         if 'fiscal_tots' in output:

webapp/apps/taxbrain/helpers.py

@@ -1,4 +1,5 @@
 from collections import namedtuple
+from distutils.version import LooseVersion
 import numbers
 import os
 import pandas as pd
@@ -13,7 +14,8 @@
 from mock import Mock
 import sys
 
-from ..constants import START_YEAR
+from ..constants import (START_YEAR,
+                         TAXCALC_VERS_RESULTS_BACKWARDS_INCOMPATIBLE)
 
 import taxcalc
 from taxcalc import Policy
@@ -356,6 +358,9 @@ def default_taxcalc_data(cls, start_year, metadata=False):
     'combined_tax':'Combined Payroll and Individual Income Tax Liability Change',
 }
 
+REORDER_LT_TC_0130_DIFF_LIST = [1, 3, 0, 5, 6, 4, 2, 7]
+DIFF_TABLE_IDs = ['diff_itax_xdec', 'diff_ptax_xdec', 'diff_comb_xdec',
+                  'diff_itax_xbin', 'diff_ptax_xbin', 'diff_comb_xbin']
 
 def get_default_policy_param_name(param, default_params):
     """
@@ -1101,6 +1106,30 @@ def rename_keys(rename_dict, map_dict):
     return rename_dict
 
 
+def reorder_lists(results, reorder_ix_map, table_names):
+    """
+    Reorder lists in `results[table_id][bin_label]`. Required for difference
+    tables calculated with Tax-Calculator version <0.13.0
+
+    returns: results table with reordered lists in selected tables
+    """
+
+    def reorder(disordered):
+        reordered = disordered[:]
+        for ix in range(len(reorder_ix_map)):
+            reordered[reorder_ix_map[ix]] = disordered[ix]
+        return reordered
+
+    for table_name in table_names:
+        bins = list(results[table_name].keys())
+        for ix in range(len(bins)):
+            results[table_name][bins[ix]] = reorder(
+                results[table_name][bins[ix]]
+            )
+
+    return results
+
+
 def taxcalc_results_to_tables(results, first_budget_year):
     """
     Take various results from dropq, i.e. mY_dec, mX_bin, df_dec, etc
@@ -1149,6 +1178,9 @@ def taxcalc_results_to_tables(results, first_budget_year):
             multi_year_cells = True
 
         elif table_id in ['diff_itax_xbin', 'diff_ptax_xbin', 'diff_comb_xbin']:
+            if table_id == "diff_comb_xbin":
+                print(results[table_id]['<$10K_0'])
+
             row_keys = TAXCALC_RESULTS_BIN_ROW_KEYS
             row_labels = TAXCALC_RESULTS_BIN_ROW_KEY_LABELS
             col_labels = TAXCALC_RESULTS_DFTABLE_COL_LABELS
@@ -1394,3 +1426,30 @@ def format_csv(tax_results, url_id, first_budget_year):
                 res.append(dfb[row+"_" + str(count)])
 
     return res
+
+
+def get_tax_result(OutputUrlCls, pk, tax_result):
+    """
+    If taxcalc version is greater than or equal to 0.13.0, return table
+    If taxcalc version is less than 0.13.0, then rename keys to new names
+    and then return table
+    """
+    outputurl = OutputUrlCls.objects.get(unique_inputs__pk=pk)
+    taxcalc_vers = outputurl.taxcalc_vers
+    # only the older (pre 0.13.0) taxcalc versions are null
+    if taxcalc_vers:
+        taxcalc_vers = LooseVersion(taxcalc_vers)
+    else:
+        return rename_keys(tax_result, PRE_TC_0130_RES_MAP)
+
+    # older PB versions stored commit reference too
+    # e.g. taxcalc_vers = "0.9.0.d79abf"
+    backwards_incompatible = LooseVersion(
+        TAXCALC_VERS_RESULTS_BACKWARDS_INCOMPATIBLE
+    )
+    if taxcalc_vers >= backwards_incompatible:
+        return tax_result
+    else:
+        renamed = rename_keys(tax_result, PRE_TC_0130_RES_MAP)
+        return reorder_lists(renamed, REORDER_LT_TC_0130_DIFF_LIST,
+                             DIFF_TABLE_IDs)

webapp/apps/taxbrain/migrations/0056_auto_20171120_2123.py

@@ -13,10 +13,6 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
-        migrations.RemoveField(
-            model_name='taxsaveinputs',
-            name='tax_result',
-        ),
         migrations.AddField(
             model_name='taxsaveinputs',
             name='CTC_new_for_all',
@@ -177,11 +173,6 @@ class Migration(migrations.Migration):
             name='PT_wages_active_income',
             field=models.CharField(default=b'False', max_length=50, null=True, blank=True),
         ),
-        migrations.AddField(
-            model_name='taxsaveinputs',
-            name='_tax_result',
-            field=jsonfield.fields.JSONField(default=None, null=True, db_column=b'tax_result', blank=True),
-        ),
         migrations.AddField(
             model_name='taxsaveinputs',
             name='cpi_offset',

webapp/apps/taxbrain/models.py

@@ -1,4 +1,5 @@
 import re
+from distutils.version import LooseVersion
 
 from django.db import models
 from django.core import validators
@@ -11,7 +12,7 @@
 from jsonfield import JSONField
 import datetime
 
-from helpers import rename_keys, PRE_TC_0130_RES_MAP
+import helpers
 
 def convert_to_floats(tsi):
     """
@@ -757,7 +758,7 @@ class TaxSaveInputs(models.Model):
     """
 
     # Result
-    _tax_result = JSONField(default=None, blank=True, null=True, db_column='tax_result')
+    tax_result = JSONField(default=None, blank=True, null=True)
 
     # JSON input text
     json_text = models.ForeignKey(JSONReformTaxCalculator, null=True, default=None, blank=True)
@@ -768,25 +769,17 @@ class TaxSaveInputs(models.Model):
     # Creation DateTime
     creation_date = models.DateTimeField(default=datetime.datetime(2015, 1, 1))
 
-
-    @property
-    def tax_result(self):
+    def get_tax_result(self):
         """
         If taxcalc version is greater than or equal to 0.13.0, return table
         If taxcalc version is less than 0.13.0, then rename keys to new names
         and then return table
         """
-        outputurl = OutputUrl.objects.get(unique_inputs__pk=self.pk)
-        taxcalc_vers = outputurl.taxcalc_vers
-        taxcalc_vers = tuple(map(int, taxcalc_vers.split('.')))
-        if taxcalc_vers >= (0, 13, 0):
-            return self._tax_result
-        else:
-            return rename_keys(self._tax_result, PRE_TC_0130_RES_MAP)
-
-    @tax_result.setter
-    def tax_result(self, result):
-        self._tax_result = result
+        return helpers.get_tax_result(
+            OutputUrl,
+            self.pk,
+            self.tax_result
+        )
 
     class Meta:
         permissions = (