fix conflicts

cps_data/README.md

@@ -3,10 +3,12 @@ About cps_data
 
 This directory contains the following script:
 
-* Python script **....py**, which reads/writes:
+* Python script **finalprep.py**, which reads/writes:
 
   Input files:
-    - ...
+    - cps_raw.csv.gz
+    - adjustment_targets.csv
+    - benefitprograms.csv
 
   Output files:
     - cps.csv
@@ -16,3 +18,7 @@ Documentation
 -------------
 
 **all documentation files go in a `doc` subdirectory of this directory**
+
+All of the benefit costs listed in `benefitprograms.csv` can be found in tables
+3.2 and 11.3 of the archived [Historical tables](https://obamawhitehouse.archives.gov/omb/budget/Historicals)
+of the Office of Management and Budget. All costs are in millions of dollars.

cps_data/benefitprograms.csv

@@ -0,0 +1,30 @@
+Program,Table,Row,2014_cost,2015_cost,2016_cost,2017_cost,2018_cost,2019_cost,2020_cost,2021_cost
+Unemployment Assistance,11.3,16,43504,32671,33137,35585,39019,40216,42600,44422
+Children's health insurance,11.3,20,9317,9242,14479,15195,16455,17553,6050,5870
+Indian health,11.3,22,4510,4550,4356,4460,4599,4697,4787,4874
+Health resources and services,11.3,24,7604,7670,8681,9952,9525,9761,8025,6334
+Substance abuse and mental health services,11.3,25,3193,3141,3810,3701,4022,3946,3714,3733
+Center for Medicare and Medicaid Innovation,11.3,27,997,971,1408,1595,1514,1402,1220,1367
+Refundable Premium Tax Credit and Cost Sharing Reductions,11.3,28,13068,27213,39285,57700,78440,93693,105713,112410
+Other,11.3,30,12834,12718,14461,14279,15109,16073,17127,18404
+Student assistance--Department of Education and other,11.3,34,56337,61103,45699,33043,40365,44151,45592,47634
+Housing assistance,11.3,36,46600,46745,48587,49963,49247,49649,49325,49766
+Child nutrition and special milk programs,11.3,39,19490,21023,22150,23268,24797,25787,26789,27823
+Supplemental feeding programs (WIC and CSFP),11.3,40,6266,6349,6389,6340,6448,6577,6708,6843
+Commodity donations and other,11.3,41,823,1126,1298,1296,1514,1708,1750,1784
+Family support payments to States and TANF,11.3,45,20378,19982,20560,21353,21967,22874,23773,23732
+Low income home energy assistance,11.3,46,3537,3437,3213,3365,3251,3111,3159,3213
+Payments to States for daycare assistance,11.3,48,5064,5134,5747,8846,9821,10636,11643,12745
+Payments to States--Foster Care/Adoption Assist.,11.3,50,6868,7314,7478,8058,8470,8825,9215,9587
+Other public assistance,11.3,53,1071,1154,928,1904,2481,2903,3003,2423
+Coal miners and black lung benefits,11.3,56,426,437,438,448,478,372,401,424
+Aging services programs,11.3,58,1462,1680,2208,1929,1976,2019,2071,2109
+Energy employees compensation fund,11.3,59,1052,1010,977,964,945,930,914,903
+September 11th victim compensation,11.3,60,49,121,2512,35,0,0,0,0
+Refugee assistance and other,11.3,61,4403,3227,3622,3507,3347,3489,3586,3668
+351 Farm income stabilization,3.2,42,20012,13424,19355,20393,20099,15474,14102,17687
+352 Agricultural research and services,3.2,43,4374,5076,6219,5771,5582,5750,5714,5776
+451 Community development,3.2,62,7896,7817,8502,8169,7735,6174,5466,5327
+452 Area and regional development,3.2,63,3027,3861,3303,3589,3298,3520,3744,3975
+453 Disaster relief and insurance,3.2,64,9747,8992,16047,9360,7897,8438,6808,5081
+506 Social services,3.2,72,17299,18303,20128,20408,21155,21490,21853,22248

cps_data/finalprep.py

@@ -10,7 +10,7 @@ def main():
     # Import CPS data file
     data = pd.read_csv('cps_raw.csv.gz', compression='gzip')
     adj_targets = pd.read_csv('adjustment_targets.csv')
-    # other_ben = pd.read_csv('benefitprograms.csv')
+    other_ben = pd.read_csv('benefitprograms.csv', index_col='Program')
 
     # Rename specified variables
     renames = {
@@ -25,7 +25,6 @@ def main():
         'JCPS28': 'e02100p',
         'JCPS38': 'e02100s',
         'UCOMP': 'e02300',
-        'SOCSEC': 'e02400',
         'SEHEALTH': 'e03270',
         'DPAD': 'e03240',
         'MEDICALEXP': 'e17500',
@@ -42,19 +41,22 @@ def main():
         'KEOGH': 'e03300',
         'TIRAD': 'e01400',
         'NU18': 'nu18',
-        'N1821': 'n1821',
+        'N1821': 'n1820',
         'N21': 'n21',
         'CGAGIX': 'e01100',
         'BLIND_HEAD': 'blind_head',
         'BLIND_SPOUSE': 'blind_spouse',
         'HMIE': 'e19200',
+        'SS': 'e02400',
+        'VB': 'vet_ben',
+        'MEDICARE': 'mcare_ben',
+        'MEDICAID': 'mcaid_ben',
         'SSI': 'ssi_ben',
-        'vb_ben': 'vet_ben',
-        'medicare_ben': 'mcare_ben',
-        'medicaid_ben': 'mcaid_ben',
-        'SS': 'ss_ben',
         'SNAP': 'snap_ben',
-        'SLTX': 'e18400'
+        'SLTX': 'e18400',
+        'XHID': 'h_seq',
+        'XFID': 'ffpos',
+        'XSTATE': 'fips'
     }
     data = data.rename(columns=renames)
     data['MARS'] = np.where(data.JS == 3, 4, data.JS)
@@ -84,8 +86,8 @@ def main():
     data = add_agi_bin(data, 'INCOME')
     print 'Adjusting distribution'
     data = adjust(data, adj_targets)
-    # print 'Adding Benefits Data'
-    # data = benefits(data, other_ben)
+    print 'Adding Benefits Data'
+    data = benefits(data, other_ben)
     print 'Dropping unused variables'
     data = drop_vars(data)
 
@@ -195,9 +197,10 @@ def drop_vars(data):
         'e18400', 'e18500', 'e19200', 'e19800', 'e20100', 'e20400', 'g20500',
         'e24515', 'e24518', 'e26270', 'e27200', 'e32800', 'e58990', 'e62900',
         'e87530', 'elderly_dependent', 'f2441', 'f6251', 'filer', 'n24',
-        'nu05', 'nu13', 'nu18', 'n1821', 'n21', 'p08000', 'p22250', 'p23250',
+        'nu05', 'nu13', 'nu18', 'n1820', 'n21', 'p08000', 'p22250', 'p23250',
         'p25470', 'p87521', 's006', 'e03210', 'ssi_ben', 'snap_ben',
-        'vet_ben', 'mcare_ben', 'mcaid_ben', 'ss_ben', 'other_ben', 'total_ben'
+        'vet_ben', 'mcare_ben', 'mcaid_ben', 'oasdi_ben', 'other_ben',
+        'h_seq', 'ffpos', 'fips'
     ]
 
     drop_vars = []
@@ -357,15 +360,31 @@ def benefits(data, other_ben):
     Distribute benefits from non-models benefit programs and create total
     benefits variable
     """
+    other_ben['2014_cost'] *= 1e6
+    # Adjust unemployment compensation
+    ucomp_ratio = (other_ben['2014_cost']['Unemployment Assistance'] /
+                   (data['e02300'] * data['s006']).sum())
+    data['e02300'] *= ucomp_ratio
+    other_ben.drop('Unemployment Assistance', inplace=True)
     # Distribute other benefits
     data['dist_ben'] = (data['mcaid_ben'] + data['ssi_ben'] +
                         data['snap_ben'] + data['vet_ben'])
     data['ratio'] = (data['dist_ben'] * data['s006'] /
-                     (data['dist_ben'] + data['s006']).sum())
-    data['other_ben'] = data['ratio'] * other_ben['Cost'].sum() / data['s006']
-    data['total_ben'] = (data['mcaid_ben'] + data['mcare_ben'] +
-                         data['ssi_ben'] + data['snap_ben'] + data['ss_ben'] +
-                         data['vet_ben'] + data['other_ben'])
+                     (data['dist_ben'] * data['s006']).sum())
+    # divide by the weight to account for weighting in Tax-Calculator
+    data['other_ben'] = (data['ratio'] * other_ben['2014_cost'].sum() /
+                         data['s006'])
+
+    # Convert benefit data to integers
+    data['mcaid_ben'] = data['mcaid_ben'].astype(np.int32)
+    data['mcare_ben'] = data['mcare_ben'].astype(np.int32)
+    data['ssi_ben'] = data['ssi_ben'].astype(np.int32)
+    data['snap_ben'] = data['snap_ben'].astype(np.int32)
+    data['vet_ben'] = data['vet_ben'].astype(np.int32)
+    data['e02400'] = data['e02400'].astype(np.int32)
+    data['e02300'] = data['e02300'].astype(np.int32)
+    data['other_ben'] = data['other_ben'].astype(np.int32)
+
     return data