diff --git a/distributed/api/celery_tasks.py b/distributed/api/celery_tasks.py index 165d2f73..3328f088 100644 --- a/distributed/api/celery_tasks.py +++ b/distributed/api/celery_tasks.py @@ -1,5 +1,6 @@ import json import os +from io import BytesIO from celery import Celery @@ -129,3 +130,19 @@ def btax_async(user_mods, start_year): results['btax_version'] = binfo['version'] json_res = json.dumps(results) return json_res + + +@celery_app.task(name='api.celery_tasks.file_upload_test') +def file_upload_test(data, compression): + import pandas as pd + df = pd.read_csv(BytesIO(data), compression='gzip') + desc = df.describe() + formatted = {'outputs': [], 'aggr_outputs': []} + formatted['aggr_outputs'].append({ + 'tags': {'default': 'default'}, + 'title': 'desc', + 'downloadable': [{'filename': 'desc' + '.csv', + 'text': desc.to_csv()}], + 'renderable': desc.to_html() + }) + return json.dumps(formatted) diff --git a/distributed/api/endpoints.py b/distributed/api/endpoints.py index 5d9dbbba..3b0cfa5f 100644 --- a/distributed/api/endpoints.py +++ b/distributed/api/endpoints.py @@ -12,7 +12,7 @@ dropq_task_async, dropq_task_small_async, taxbrain_elast_async, - btax_async) + btax_async, file_upload_test) bp = Blueprint('endpoints', __name__) @@ -48,6 +48,17 @@ def endpoint(task): return json.dumps(data) +def file_test_endpoint(task): + print('file test endpoint') + data = request.get_data() + inputs = msgpack.loads(data, encoding='utf8', + use_list=True) + result = task.apply_async(kwargs=inputs[0], serializer='msgpack') + length = client.llen(queue_name) + 1 + data = {'job_id': str(result), 'qlength': length} + return json.dumps(data) + + @bp.route("/dropq_start_job", methods=['POST']) def dropq_endpoint_full(): return aggr_endpoint(dropq_task_async, taxbrain_postprocess) @@ -68,6 +79,11 @@ def elastic_endpoint(): return aggr_endpoint(taxbrain_elast_async, taxbrain_elast_postprocess) +@bp.route("/file_upload_test", methods=['POST']) +def file_upload_test_endpoint(): + return file_test_endpoint(file_upload_test) + + @bp.route("/dropq_get_result", methods=['GET']) def dropq_results(): job_id = request.args.get('job_id', '') diff --git a/templates/fileuploadex/file.html b/templates/fileuploadex/file.html new file mode 100644 index 00000000..6fed8b0c --- /dev/null +++ b/templates/fileuploadex/file.html @@ -0,0 +1,20 @@ +{% extends 'taxbrain/input_form_section.html' %} + +{% block input_section_shortname %}payroll-taxes{% endblock %} +{% block input_section_title %}Payroll Taxes{% endblock %} + +{% load flatblocks %} + +{% block content %} +
+

Data File: A CSV file to describe

+ {% block section_warnings %} + {% endblock %} +
+ +

+

+ +{% endblock %} + +{% block provide_if_no_continue %}{% endblock %} diff --git a/templates/fileuploadex/input.html b/templates/fileuploadex/input.html new file mode 100644 index 00000000..084b3bc8 --- /dev/null +++ b/templates/fileuploadex/input.html @@ -0,0 +1,176 @@ +{% extends 'taxbrain/input_base.html' %} + +{% load staticfiles %} + +{% load flatblocks %} + +{% block content %} +
+
+ + + + + +
+
+ +

A platform for accessing open-source tax models.

+ Start Exploring +
+ +
+
+
+ +
+
+
+

About TaxBrain

+

TaxBrain is an interface to open source economic models for tax policy analysis. The code for the TaxBrain webapp interface is itself open source.

+ +
    +
  • Step 1. Create a policy reform by modifying tax law parameters such as rates and deductions, adjust the economic baseline, and request the static result. You can do so with the graphical user interface below or by uploading a policy reform file from this page.
    • +
  • Step 2. Review your static output carefully. Ask questions.
    • +
  • Step 3. Choose a dynamic modeling approach. Because different approaches generally lead to different estimates, you may want to compare several approaches.
    • +
  • Step 4. Adjust economic assumptions and request the dynamic analysis.
    • +
  • Step 5. Review your dynamic output carefully. Ask questions.
    • +
  • Step 6. Share your results! The link to every results page is static and will never change. Send them around.
    • +
+ +

Throughout this process, if you have a question about how to use TaxBrain or interpret the results, if you want to make a suggestion for making the interface or underlying models better, or if you discover a bug, please send a message to our mailing list, which you can join at list.ospc.org/mailman/listinfo/users_list.ospc.org.

+ +

Disclaimer + +

Proper use of this tool and description of that use is ultimately your responsibility. If you plan on publishing your results, I highly recommend that you confirm with the community that you are using the tools properly and interpreting the results correctly before you publish them. If you have a compelling reason not to leave a public note on the mailing list, email me at matt.jensen@aei.org.

+ +

Results will change as the underlying models improve. A fundamental reason for adopting open source methods in this project is to let people from all backgrounds contribute to the models that our society uses to assess economic policy; when community-contributed improvements are incorporated, the models will produce different results.

+ +

Neither the Open Source Policy Center nor the American Enterprise Institute maintain institutional positions, and the results from models accessible via the TaxBrain interface should not be attributed directly to OSPC or AEI. A suggested acknowledgement is, "We thank AEI for making TaxBrain available, but we bear sole responsibility for the use of the models and any conclusions drawn."

+ +

- Matt Jensen, managing director and founder of the Open Source Policy Center

+ +
+ +

Static modeling (Step 1)

+ +

Static tax analysis entails computing individuals' tax changes under the assumption that behavior does not change in response to tax policy. Static analyses are useful for understanding the mechanistic effects of tax policy changes, and they form the basis to which behavior is applied for dynamic analyses. + +

TaxBrain's static modeling capabilities rely on several open source economic models and other packages:

+
    +
  • Tax-Calculator computes federal individual income taxes and Federal Insurance Contribution Act(FICA) taxes for a sample of tax filing units in years beginning with 2013 .
    • +
  • TaxData creates a microdataset that closely reproduces the multivariate distribution of income, deduction and credit items in 2009, extrapolated to 2015-2026 levels in accordance with Congressional Budget Office forecasts available in spring 2016. It is intended to match similar but confidential data used by the Congressional Joint Committee on Taxation. The underlying dataset must be purchased from the Statistics of Income division of the Internal Revenue Service. Additional information on non-filers is taken from the March 2013 Current Population Survey.
    • +
  • DropQ implements a disclosure avoidance algorithm initially proposed by the U.S. Census Bureau to protect confidential data from differencing attacks. The code is undergoing security review before it is open sourced.
    • +
  • TaxBrain itself is an open source project. The underlying models are deployed to TaxBrain using conda, a free and open source package management system supported by Continuum Analytics
    • +
+ +

Transparency and Replicability

+ +

In addition to relying on open source models, we are devoted to making it easy for reviewers to understand the models even if they can't understand the source code or don't have access to the underlying data. Toward that end we produce several additional reports to enhance transparency, peer review, collaboration and a scientific advancement.

+ +

Note that these reports currently rely on the latest versions of tax-calculator and TaxData, which might not correspond perfectly to TaxBrain.

+ + + +

Accuracy notes

+ +

The Python code that performs the tax calculations has been validated in a + number of ways. First, Tax-Calculator results for a number of tax filing + units have been compared to hand calculations performed using IRS tax + forms. Second, Tax-Calculator results for a large sample of tax filing units + have been compared to results for the same sample generated by a + detailed SAS program developed by Dan + Feenberg and Ina Shapiro of NBER. Third, a subset of input variables has been used to compare the results of Tax-Calculator to Internet TAXSIM as well as against the Policy Simulation Group's PENSIM tax module

+ +

Bugs aside, the results from TaxBrain might differ in comparison to those produced by Congress or the Administration for other reasons. Modeling requires many assumptions, and neither Congress nor the executive branch publicize all of their assumptions. For example, the distribution of wages in TaxData is assumed to stay the same in real terms for all years after the last year we have available data (2013). We know that Congress assumes this distribution changes over time, but it doesn't publish by how much. These assumptions are all flexible in TaxData, so please conduct sensitivity analyses. Other assumptions can be made flexible in TaxBrain based on user requests.

+ +

+ Core Maintainers (static modeling)*: +

+

+

These members have "write access" to one or both of the core static modeling repositories, Tax-Calculator and TaxData, and work as a team to determine which open source contributions are accepted.

+
+
+

PolicyBrain Code Build

+

Version {{ webapp_version }} - GitHub

+
+
+

Tax-Calculator Code Build

+

Version {{ taxcalc_version }} - GitHub

+
+
+
+
+
+ {% csrf_token %} + + + +
+
+
+
+
+ + +
+
+ + +
+ +
+
+
+
+ + {% for error in errors %} + + {% endfor %} + +
+
+

Get Started

+
+
+ + {% include 'fileuploadex/file.html' %} +
+
+
+
+
+
+
+ +{% endblock %} diff --git a/webapp/apps/core/compute.py b/webapp/apps/core/compute.py index 1b66056f..93c4bead 100644 --- a/webapp/apps/core/compute.py +++ b/webapp/apps/core/compute.py @@ -26,7 +26,7 @@ def remote_submit_job( data, timeout=TIMEOUT_IN_SECONDS, headers=None): - print(theurl, data) + # print(theurl, data) if headers is not None: response = requests.post(theurl, data=data, @@ -56,13 +56,17 @@ def submit_elastic_calculation(self, data): url_template = "http://{hn}/elastic_gdp_start_job" return self.submit(data, url_template) + def submit_file_upload_test(self, data): + url_template = "http://{hn}/file_upload_test" + return self.submit(data, url_template) + def submit(self, data_list, url_template, increment_counter=True, use_wnc_offset=True): print("hostnames: ", WORKER_HN) - print("submitting data: ", data_list) + # print("submitting data: ", data_list) queue_length = 0 submitted = False attempts = 0 @@ -80,7 +84,7 @@ def submit(self, job_id = response_d['job_id'] queue_length = response_d['qlength'] else: - print("FAILED: ", data_list, WORKER_HN) + print("FAILED: ", WORKER_HN) attempts += 1 except Timeout: print("Couldn't submit to: ", WORKER_HN) @@ -124,7 +128,7 @@ def _get_results_base(self, job_id, job_failure=False): msg = 'PROBLEM WITH RESPONSE. TEXT RECEIVED: {}' raise ValueError(msg) else: - raise # TODO + raise IOError() def get_results(self, job_id, job_failure=False): if job_failure: diff --git a/webapp/apps/core/views.py b/webapp/apps/core/views.py index dce6b39d..d19a65f5 100644 --- a/webapp/apps/core/views.py +++ b/webapp/apps/core/views.py @@ -92,7 +92,12 @@ def dispatch(self, request, *args, **kwargs): return self.fail() if job_ready == 'YES': - results = self.dropq_compute.get_results(job_id) + try: + results = self.dropq_compute.get_results(job_id) + except Exception as e: + self.object.error_text = str(e) + self.object.save() + return self.fail() self.object.outputs = results['outputs'] self.object.aggr_outputs = results['aggr_outputs'] self.object.creation_date = timezone.now() diff --git a/webapp/apps/fileuploadex/__init__.py b/webapp/apps/fileuploadex/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/webapp/apps/fileuploadex/admin.py b/webapp/apps/fileuploadex/admin.py new file mode 100644 index 00000000..8c38f3f3 --- /dev/null +++ b/webapp/apps/fileuploadex/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/webapp/apps/fileuploadex/apps.py b/webapp/apps/fileuploadex/apps.py new file mode 100644 index 00000000..e29b4b20 --- /dev/null +++ b/webapp/apps/fileuploadex/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class FileuploadexConfig(AppConfig): + name = 'fileuploadex' diff --git a/webapp/apps/fileuploadex/migrations/0001_initial.py b/webapp/apps/fileuploadex/migrations/0001_initial.py new file mode 100644 index 00000000..19716c8f --- /dev/null +++ b/webapp/apps/fileuploadex/migrations/0001_initial.py @@ -0,0 +1,60 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9 on 2018-08-23 21:12 +from __future__ import unicode_literals + +import datetime +from django.conf import settings +import django.contrib.postgres.fields.jsonb +import django.core.validators +from django.db import migrations, models +import django.db.models.deletion +from django.utils.timezone import utc +import re +import uuid + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='FileInput', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('raw_gui_field_inputs', django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=None, null=True)), + ('gui_field_inputs', django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=None, null=True)), + ('inputs_file', django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict, null=True)), + ('errors_warnings_text', django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=None, null=True)), + ('upstream_parameters', django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict, null=True)), + ('first_year', models.IntegerField(default=None, null=True)), + ('years_n', models.CharField(max_length=300, validators=[django.core.validators.RegexValidator(re.compile('^\\d+(?:\\,\\d+)*\\Z', 32), code='invalid', message='Enter only digits separated by commas.')])), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='FileOutput', + fields=[ + ('outputs', django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=None, null=True)), + ('aggr_outputs', django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=None, null=True)), + ('uuid', models.UUIDField(default=uuid.uuid1, editable=False, primary_key=True, serialize=False, unique=True)), + ('error_text', models.CharField(blank=True, default=None, max_length=4000, null=True)), + ('creation_date', models.DateTimeField(default=datetime.datetime(2015, 1, 1, 0, 0, tzinfo=utc))), + ('exp_comp_datetime', models.DateTimeField(default=datetime.datetime(2015, 1, 1, 0, 0, tzinfo=utc))), + ('job_id', models.UUIDField(blank=True, default=None, null=True)), + ('upstream_vers', models.CharField(blank=True, default=None, max_length=50, null=True)), + ('webapp_vers', models.CharField(blank=True, default=None, max_length=50, null=True)), + ('inputs', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='outputs', to='fileuploadex.FileInput')), + ('user', models.ForeignKey(default=None, null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + options={ + 'abstract': False, + }, + ), + ] diff --git a/webapp/apps/fileuploadex/migrations/__init__.py b/webapp/apps/fileuploadex/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/webapp/apps/fileuploadex/models.py b/webapp/apps/fileuploadex/models.py new file mode 100644 index 00000000..1f915290 --- /dev/null +++ b/webapp/apps/fileuploadex/models.py @@ -0,0 +1,26 @@ +from django.db import models +from django.core.urlresolvers import reverse +from ..core.models import CoreInputs, CoreRun + + +class FileInput(CoreInputs): + pass + + +class FileOutput(CoreRun): + inputs = models.OneToOneField(FileInput, related_name='outputs') + + def get_absolute_url(self): + kwargs = { + 'pk': self.pk + } + return reverse('file_results', kwargs=kwargs) + + def get_absolute_download_url(self): + kwargs = { + 'pk': self.pk + } + return reverse('fileinput_download', kwargs=kwargs) + + def zip_filename(self): + return 'taxbrain.zip' diff --git a/webapp/apps/fileuploadex/tests.py b/webapp/apps/fileuploadex/tests.py new file mode 100644 index 00000000..7ce503c2 --- /dev/null +++ b/webapp/apps/fileuploadex/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/webapp/apps/fileuploadex/urls.py b/webapp/apps/fileuploadex/urls.py new file mode 100644 index 00000000..56517893 --- /dev/null +++ b/webapp/apps/fileuploadex/urls.py @@ -0,0 +1,15 @@ + + +from django.conf.urls import url + +from .views import (FileInputView, FileRunDetailView, FileRunDownloadView) + + +urlpatterns = [ + url(r'^$', FileInputView.as_view(), + name='fileinput'), + url(r'^(?P[-\d\w]+)/download/?$', FileRunDetailView.as_view(), + name='file_results'), + url(r'^(?P[-\d\w]+)/', FileRunDownloadView.as_view(), + name='fileinput_download') +] diff --git a/webapp/apps/fileuploadex/views.py b/webapp/apps/fileuploadex/views.py new file mode 100644 index 00000000..fb3617f7 --- /dev/null +++ b/webapp/apps/fileuploadex/views.py @@ -0,0 +1,59 @@ +import datetime + +from django.utils import timezone +from django.shortcuts import render, redirect +from django.views.generic import View + +# Create your views here. +from ..core.models import Tag, TagOption +from ..core.views import CoreRunDetailView, CoreRunDownloadView +from ..core.compute import Compute + +from .models import FileInput, FileOutput + + +class FileInputView(View): + model = FileInput + result_header = "Describe" + template_name = 'fileuploadex/input.html' + compute = Compute() + + def get(self, request, *args, **kwargs): + return render(request, self.template_name) + + def post(self, request, *args, **kwargs): + tmpfile = request.FILES['datafile'] + data_list = [{'data': tmpfile.read(), 'compression': 'gzip'}] + submitted_id, max_q_length = ( + self.compute.submit_file_upload_test(data_list)) + fi = FileInput() + fi.save() + fo = FileOutput() + fo.inputs = fi + fo.job_id = submitted_id + delta = datetime.timedelta(seconds=20) + fo.exp_comp_datetime = timezone.now() + delta + fo.save() + return redirect(fo) + + +class FileRunDetailView(CoreRunDetailView): + model = FileOutput + + result_header = "Static Results" + + tags = [] + aggr_tags = [ + Tag(key="default", + values=[ + TagOption( + value="default", + title="Descriptive"), + ])] + + def has_link_to_dyn(self): + return False + + +class FileRunDownloadView(CoreRunDownloadView): + model = FileOutput diff --git a/webapp/settings.py b/webapp/settings.py index ce5aa2b1..e327268c 100644 --- a/webapp/settings.py +++ b/webapp/settings.py @@ -78,6 +78,7 @@ 'webapp.apps.pages', 'webapp.apps.register', 'webapp.apps.btax', + 'webapp.apps.fileuploadex', # Third party apps 'flatblocks', diff --git a/webapp/urls.py b/webapp/urls.py index 2563cdc8..e1d4f99c 100644 --- a/webapp/urls.py +++ b/webapp/urls.py @@ -18,6 +18,7 @@ url(r'^taxbrain/', include("webapp.apps.taxbrain.urls")), url(r'^dynamic/', include("webapp.apps.dynamic.urls")), url(r'^ccc/', include("webapp.apps.btax.urls")), + url(r'^upload/', include('webapp.apps.fileuploadex.urls')), # Login & Registration URL Confs url(r'^accounts/login/$', login), url(r'^logout/$', logout),