diff --git a/.coveragerc b/.coveragerc index 27c2d81..5f548ec 100644 --- a/.coveragerc +++ b/.coveragerc @@ -18,3 +18,4 @@ omit = exclude_lines = pragma: no cover if __name__ == .__main__.: + if six.PY2: diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..e321012 --- /dev/null +++ b/.flake8 @@ -0,0 +1,22 @@ +[flake8] +ignore = + # There's nothing wrong with assigning lambdas + E731, + # PEP8 weakly recommends Knuth-style line breaks before binary + # operators + W503, W504 +exclude = + # These are directories that it's a waste of time to traverse + .eggs, + .env, + .git, + .tox, + .venv, + bin, + config, + docs, + gulp, + node_modules, + */migrations/*.py, + paying_for_college/config/* + paying_for_college/disclosures/urls.py diff --git a/.travis.yml b/.travis.yml index 66bcf83..f16c2ac 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,16 +1,18 @@ language: python python: -- '2.7' +- 2.7 +- 3.6 install: +- pip install tox-travis - npm install -g gulp-cli - pip install coveralls tox script: -- npm config set package-lock false - tox +- python2.7 setup.py bdist_wheel --universal +- npm config set package-lock false - npm test -- python2.7 setup.py bdist_wheel -- ls dist/* - gulp lint --travis +- ls dist/* after_success: - coveralls deploy: diff --git a/paying_for_college/admin.py b/paying_for_college/admin.py index e633687..faca0f6 100644 --- a/paying_for_college/admin.py +++ b/paying_for_college/admin.py @@ -1,8 +1,12 @@ #!/usr/bin/env python from __future__ import absolute_import + from django.contrib import admin -from .models import School, Program, Alias, Nickname, Contact, Disclosure -from .models import BAHRate, Feedback, Worksheet, ConstantRate, ConstantCap + +from paying_for_college.models import ( + Alias, BAHRate, ConstantCap, ConstantRate, Contact, Disclosure, Feedback, + Nickname, Program, School, Worksheet +) class DisclosureAdmin(admin.ModelAdmin): @@ -41,6 +45,7 @@ class NicknameAdmin(admin.ModelAdmin): list_display = ('nickname', 'institution', 'is_female') search_fields = ['nickname'] + admin.site.register(Disclosure, DisclosureAdmin) admin.site.register(ConstantRate, ConstantRateAdmin) admin.site.register(ConstantCap, ConstantCapAdmin) diff --git a/paying_for_college/config/settings/base.py b/paying_for_college/config/settings/base.py index c312248..3a8ee82 100644 --- a/paying_for_college/config/settings/base.py +++ b/paying_for_college/config/settings/base.py @@ -1,6 +1,8 @@ -from unipath import Path import getpass +from unipath import Path + + LOCAL_USER = getpass.getuser() REPOSITORY_ROOT = Path(__file__).ancestor(4) PROJECT_ROOT = Path(__file__).ancestor(3) diff --git a/paying_for_college/config/settings/dev.py b/paying_for_college/config/settings/dev.py index ae3c56e..b5c5048 100644 --- a/paying_for_college/config/settings/dev.py +++ b/paying_for_college/config/settings/dev.py @@ -1,10 +1,12 @@ from __future__ import absolute_import + import os import dj_database_url from .base import * + DEBUG = True DATABASES = { diff --git a/paying_for_college/config/settings/no_haystack.py b/paying_for_college/config/settings/no_haystack.py index 6b96e63..25c9b79 100644 --- a/paying_for_college/config/settings/no_haystack.py +++ b/paying_for_college/config/settings/no_haystack.py @@ -1,3 +1,4 @@ from .standalone import * + INSTALLED_APPS.remove('haystack') diff --git a/paying_for_college/config/settings/standalone.py b/paying_for_college/config/settings/standalone.py index 5555b10..eb15daf 100644 --- a/paying_for_college/config/settings/standalone.py +++ b/paying_for_college/config/settings/standalone.py @@ -1,6 +1,8 @@ from __future__ import absolute_import + from .dev import * + STANDALONE = True SECRET_KEY = "forstandaloneonly" diff --git a/paying_for_college/config/urls.py b/paying_for_college/config/urls.py index 9e03440..ee4bcc5 100644 --- a/paying_for_college/config/urls.py +++ b/paying_for_college/config/urls.py @@ -1,10 +1,9 @@ -from django.conf.urls import url, include from django.conf import settings -from paying_for_college.views import (LandingView, - BaseTemplateView, - URL_ROOT) +from django.conf.urls import include, url from django.contrib import admin -from django.conf import settings + +from paying_for_college.views import URL_ROOT, BaseTemplateView, LandingView + try: STANDALONE = settings.STANDALONE diff --git a/paying_for_college/config/wsgi.py b/paying_for_college/config/wsgi.py index 9b84479..b192107 100644 --- a/paying_for_college/config/wsgi.py +++ b/paying_for_college/config/wsgi.py @@ -11,6 +11,7 @@ from django.core.wsgi import get_wsgi_application + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "paying_for_college.config.settings.test") application = get_wsgi_application() diff --git a/paying_for_college/csvkit/__init__.py b/paying_for_college/csvkit/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/paying_for_college/csvkit/csvkit.py b/paying_for_college/csvkit/csvkit.py deleted file mode 100644 index 77f503f..0000000 --- a/paying_for_college/csvkit/csvkit.py +++ /dev/null @@ -1,279 +0,0 @@ -#!/usr/bin/env python - -""" -This module is borrowed from the mighty csvkit tool, -which is now part of [agate](https://github.com/wireservice/agate) -""" -import codecs -import csv - -import six - -EIGHT_BIT_ENCODINGS = [ - 'utf-8', 'u8', 'utf', 'utf8', - 'latin-1', 'iso-8859-1', 'iso8859-1', '8859', 'cp819', 'latin', 'latin1', 'l1' -] - -POSSIBLE_DELIMITERS = [',', '\t', ';', ' ', ':', '|'] - - -class FieldSizeLimitError(Exception): # pragma: no cover - """ - A field in a CSV file exceeds the maximum length. - This length may be the default or one set by the user. - """ - def __init__(self, limit): - super(FieldSizeLimitError, self).__init__( - 'CSV contains fields longer than maximum length of %i characters. Try raising the maximum with the field_size_limit parameter, or try setting quoting=csv.QUOTE_NONE.' % limit - ) - - -class UTF8Recoder(six.Iterator): - """ - Iterator that reads an encoded stream and reencodes the input to UTF-8. - """ - def __init__(self, f, encoding): - self.reader = codecs.getreader(encoding)(f) - - def __iter__(self): - return self - - def __next__(self): - return next(self.reader).encode('utf-8') - - -class UnicodeReader(object): - """ - A CSV reader which will read rows from a file in a given encoding. - """ - def __init__(self, f, encoding='utf-8', field_size_limit=None, line_numbers=False, header=True, **kwargs): - self.line_numbers = line_numbers - self.header = header - - f = UTF8Recoder(f, encoding) - - self.reader = csv.reader(f, **kwargs) - - if field_size_limit: - csv.field_size_limit(field_size_limit) - - def next(self): - try: - row = next(self.reader) - except csv.Error as e: - # Terrible way to test for this exception, but there is no subclass - if 'field larger than field limit' in str(e): - raise FieldSizeLimitError(csv.field_size_limit()) - else: - raise e - - if self.line_numbers: - if self.header and self.line_num == 1: - row.insert(0, 'line_numbers') - else: - row.insert(0, str(self.line_num - 1 if self.header else self.line_num)) - - return [six.text_type(s, 'utf-8') for s in row] - - def __iter__(self): - return self - - @property - def dialect(self): - return self.reader.dialect - - @property - def line_num(self): - return self.reader.line_num - - -class UnicodeWriter(object): - """ - A CSV writer which will write rows to a file in the specified encoding. - - NB: Optimized so that eight-bit encodings skip re-encoding. See: - https://github.com/onyxfish/csvkit/issues/175 - """ - def __init__(self, f, encoding='utf-8', **kwargs): - self.encoding = encoding - self._eight_bit = (self.encoding.lower().replace('_', '-') in EIGHT_BIT_ENCODINGS) - - if self._eight_bit: - self.writer = csv.writer(f, **kwargs) - else: - # Redirect output to a queue for reencoding - self.queue = six.StringIO() - self.writer = csv.writer(self.queue, **kwargs) - self.stream = f - self.encoder = codecs.getincrementalencoder(encoding)() - - def writerow(self, row): - if self._eight_bit: - self.writer.writerow([six.text_type(s if s is not None else '').encode(self.encoding) for s in row]) - else: - self.writer.writerow([six.text_type(s if s is not None else '').encode('utf-8') for s in row]) - # Fetch UTF-8 output from the queue... - data = self.queue.getvalue() - data = data.decode('utf-8') - # ...and reencode it into the target encoding - data = self.encoder.encode(data) - # write to the file - self.stream.write(data) - # empty the queue - self.queue.truncate(0) - - def writerows(self, rows): - for row in rows: - self.writerow(row) - - -class UnicodeDictReader(csv.DictReader): - """ - Defer almost all implementation to :class:`csv.DictReader`, but wraps our - unicode reader instead of :func:`csv.reader`. - """ - def __init__(self, f, fieldnames=None, restkey=None, restval=None, *args, **kwargs): - reader = UnicodeReader(f, *args, **kwargs) - - if 'encoding' in kwargs: - kwargs.pop('encoding') - - csv.DictReader.__init__(self, f, fieldnames, restkey, restval, *args, **kwargs) - - self.reader = reader - - -class UnicodeDictWriter(csv.DictWriter): - """ - Defer almost all implementation to :class:`csv.DictWriter`, but wraps our - unicode writer instead of :func:`csv.writer`. - """ - def __init__(self, f, fieldnames, restval='', extrasaction='raise', *args, **kwds): - self.fieldnames = fieldnames - self.restval = restval - - if extrasaction.lower() not in ('raise', 'ignore'): - raise ValueError('extrasaction (%s) must be "raise" or "ignore"' % extrasaction) - - self.extrasaction = extrasaction - - self.writer = UnicodeWriter(f, *args, **kwds) - - -class Reader(UnicodeReader): - """ - A unicode-aware CSV reader. - """ - pass - - -class Writer(UnicodeWriter): - """ - A unicode-aware CSV writer. - """ - def __init__(self, f, encoding='utf-8', line_numbers=False, **kwargs): - self.row_count = 0 - self.line_numbers = line_numbers - - if 'lineterminator' not in kwargs: - kwargs['lineterminator'] = '\n' - - UnicodeWriter.__init__(self, f, encoding, **kwargs) - - def _append_line_number(self, row): - if self.row_count == 0: - row.insert(0, 'line_number') - else: - row.insert(0, self.row_count) - - self.row_count += 1 - - def writerow(self, row): - if self.line_numbers: - row = list(row) - self._append_line_number(row) - - # Convert embedded Mac line endings to unix style line endings so they get quoted - row = [i.replace('\r', '\n') if isinstance(i, six.string_types) else i for i in row] - - UnicodeWriter.writerow(self, row) - - def writerows(self, rows): - for row in rows: - self.writerow(row) - - -class DictReader(UnicodeDictReader): - """ - A unicode-aware CSV DictReader. - """ - pass - - -class DictWriter(UnicodeDictWriter): - """ - A unicode-aware CSV DictWriter. - """ - def __init__(self, f, fieldnames, encoding='utf-8', line_numbers=False, **kwargs): - self.row_count = 0 - self.line_numbers = line_numbers - - if 'lineterminator' not in kwargs: - kwargs['lineterminator'] = '\n' - - UnicodeDictWriter.__init__(self, f, fieldnames, encoding=encoding, **kwargs) - - def _append_line_number(self, row): - if self.row_count == 0: - row['line_number'] = 0 - else: - row['line_number'] = self.row_count - - self.row_count += 1 - - def writerow(self, row): - if self.line_numbers: - row = list(row) - self._append_line_number(row) - - # Convert embedded Mac line endings to unix style line endings so they get quoted - row = dict([(k, v.replace('\r', '\n')) if isinstance(v, basestring) else (k, v) for k, v in row.items()]) - - UnicodeDictWriter.writerow(self, row) - - def writerows(self, rows): - for row in rows: - self.writerow(row) - - -class Sniffer(object): - """ - A functinonal wrapper of ``csv.Sniffer()``. - """ - def sniff(self, sample): - """ - A functional version of ``csv.Sniffer().sniff``, that extends the - list of possible delimiters to include some seen in the wild. - """ - try: - dialect = csv.Sniffer().sniff(sample, POSSIBLE_DELIMITERS) - except: - dialect = None - - return dialect - - -def reader(*args, **kwargs): - """ - A replacement for Python's :func:`csv.reader` that uses - :class:`.csv_py2.Reader`. - """ - return Reader(*args, **kwargs) - - -def writer(*args, **kwargs): - """ - A replacement for Python's :func:`csv.writer` that uses - :class:`.csv_py2.Writer`. - """ - return Writer(*args, **kwargs) diff --git a/paying_for_college/data_sources/bls_processing.py b/paying_for_college/data_sources/bls_processing.py index 5a91799..d81bba5 100644 --- a/paying_for_college/data_sources/bls_processing.py +++ b/paying_for_college/data_sources/bls_processing.py @@ -1,12 +1,13 @@ import json -from paying_for_college.csvkit.csvkit import DictReader as cdr -from paying_for_college.models import School +from paying_for_college.models import cdr + """ # Data processing steps -This script was used to process a few xlsx files to look at expediture data based on income and region +This script was used to process a few xlsx files to look at expediture data +based on income and region - source: http://www.bls.gov/cex/ - files under Region of residence by income before taxes: - xregnmw.xlsx @@ -73,7 +74,7 @@ def add_bls_dict_with_region(base_bls_dict, region, csvfile): } INCOME_KEY_MAP = { - "Less than $5,000" : "less_than_5000", + "Less than $5,000": "less_than_5000", "$5,000 to $9,999": "5000_to_9999", "$10,000 to $14,999": "10000_to_14999", "$15,000 to $19,999": "15000_to_19999", @@ -85,19 +86,21 @@ def add_bls_dict_with_region(base_bls_dict, region, csvfile): } data = load_bls_data(csvfile) - print "******Processing {} file...******".format(region) + print("******Processing {} file...******".format(region)) for row in data: item = row['Item'].strip() if item in CATEGORIES_KEY_MAP.keys(): - print "Current processing {}.....".format(item) - print "Will be adding {} to base_bls_dict...".format(CATEGORIES_KEY_MAP[item]) + print("Current processing {}.....".format(item)) + print("Will be adding {} to base_bls_dict...".format( + CATEGORIES_KEY_MAP[item])) base_bls_dict[CATEGORIES_KEY_MAP[item]].setdefault(region, {}) for income_key, income_json_key in INCOME_KEY_MAP.items(): - print "adding {} ...".format(income_key) - amount = int(row[income_key].replace(',','')) - print "amount: {}".format(amount) - base_bls_dict[CATEGORIES_KEY_MAP[item]][region].setdefault(income_json_key, 0) - base_bls_dict[CATEGORIES_KEY_MAP[item]][region][income_json_key] += amount + print("adding {} ...".format(income_key)) + amount = int(row[income_key].replace(',', '')) + print("amount: {}".format(amount)) + base_bls_dict[CATEGORIES_KEY_MAP[item]][region].setdefault( + income_json_key, 0) + base_bls_dict[CATEGORIES_KEY_MAP[item]][region][income_json_key] += amount # noqa def bls_as_dict(we_csvfile, ne_csvfile, mw_csvfile, so_csvfile): diff --git a/paying_for_college/data_sources/perkins_processing.py b/paying_for_college/data_sources/perkins_processing.py index 814a842..8145216 100755 --- a/paying_for_college/data_sources/perkins_processing.py +++ b/paying_for_college/data_sources/perkins_processing.py @@ -1,5 +1,7 @@ -from paying_for_college.csvkit.csvkit import DictReader as cdr -from paying_for_college.models import School +from __future__ import unicode_literals + +from paying_for_college.models import School, cdr + """ # Data processing steps @@ -44,7 +46,9 @@ """ CSVFILE = 'paying_for_college/data_sources/2015_perkins.csv' -ENDNOTE = 'processed {} entries, found {} offering Perkins, updated {} schools in our database' +ENDNOTE = ( + 'processed {} entries, found {} offering Perkins, ' + 'updated {} schools in our database') DRY_ENDNOTE = ENDNOTE.replace('updated', 'would have updated') MISSNOTE = "Couldn't find {} schools by ope8_id" @@ -58,7 +62,7 @@ def load_perkins_data(csvfile): def tag_schools(csvfile, dry_run=True): if dry_run: - print "DRY RUN ...\nto update schools, run with 'dry_run=False'" + print("DRY RUN ...\nto update schools, run with 'dry_run=False'") processed = 0 potentials = 0 updated = 0 @@ -78,12 +82,12 @@ def tag_schools(csvfile, dry_run=True): if dry_run is False: target.save() if dry_run: - print DRY_ENDNOTE.format(processed, potentials, updated) + print(DRY_ENDNOTE.format(processed, potentials, updated)) else: - print ENDNOTE.format(processed, potentials, updated) + print(ENDNOTE.format(processed, potentials, updated)) if misses: - print MISSNOTE.format(len(misses)) - print "These schools were not found:" + print(MISSNOTE.format(len(misses))) + print("These schools were not found:") for entry in misses: print entry return misses diff --git a/paying_for_college/disclosures/scripts/api_utils.py b/paying_for_college/disclosures/scripts/api_utils.py index 7f75d97..e817a16 100644 --- a/paying_for_college/disclosures/scripts/api_utils.py +++ b/paying_for_college/disclosures/scripts/api_utils.py @@ -13,10 +13,12 @@ https://api.data.gov/docs/api-key/ """ from __future__ import print_function + import os import requests + API_KEY = os.getenv('ED_API_KEY', '') API_ROOT = "https://api.data.gov/ed/collegescorecard/v1" SCHOOLS_ROOT = "{}/schools".format(API_ROOT) diff --git a/paying_for_college/disclosures/scripts/load_programs.py b/paying_for_college/disclosures/scripts/load_programs.py index ea73a87..5637482 100644 --- a/paying_for_college/disclosures/scripts/load_programs.py +++ b/paying_for_college/disclosures/scripts/load_programs.py @@ -1,12 +1,16 @@ -from __future__ import print_function -import StringIO +# -*- coding: utf-8 -*- +from __future__ import print_function, unicode_literals -from rest_framework import serializers -import requests +import io +import six -from paying_for_college.csvkit.csvkit import DictReader as cdr -from paying_for_college.models import Program, School +import requests +from paying_for_college.models import Program, School, cdr from paying_for_college.views import validate_pid +from rest_framework import serializers + + +NO_DATA_ENTRIES_LOWER = ('', 'blank', 'no grads', 'no data', 'none') """ # Program Data Processing Steps @@ -22,8 +26,6 @@ ``` """ -NO_DATA_ENTRIES_LOWER = ('', 'blank', 'no grads', 'no data', 'none') - class ProgramSerializer(serializers.Serializer): @@ -90,52 +92,62 @@ class ProgramSerializer(serializers.Serializer): def get_school(iped): try: school = School.objects.get(school_id=int(iped)) - except: + except Exception: return ('', "ERROR: couldn't find school for ID {0}".format(iped)) else: return (school, '') -def read_in_encoding(filename, encoding='windows-1252'): - """Throw a lifeline if the college just exported from Excel""" +def read_py2(filename): # pragma: no cover try: with open(filename, 'r') as f: - reader = cdr(f, encoding=encoding) + reader = cdr(f, encoding='utf-8-sig') data = [row for row in reader] + except UnicodeDecodeError: + try: + with open(filename, 'r') as f: + reader = cdr(f, encoding='windows-1252') + data = [row for row in reader] + except Exception: + data = [{}] except Exception: data = [{}] return data -def read_in_data(filename): - """Reads in utf-8 CSV (as per our spec)""" - try_encoding = False - with open(filename, 'r') as f: - try: - reader = cdr(f, encoding='utf-8-sig') +def read_py3(filename): # pragma: no cover + try: + with open(filename, newline='', encoding='utf-8-sig') as f: + reader = cdr(f) data = [row for row in reader] - except UnicodeDecodeError: - try_encoding = True - except: + except UnicodeDecodeError: + try: + with open(filename, newline='', encoding='windows-1252') as f: + reader = cdr(f) + data = [row for row in reader] + except Exception: data = [{}] - if try_encoding: # sigh - data = read_in_encoding(filename) + except Exception: + data = [{}] return data +def read_in_data(filename): + """Read in a utf-8 CSV, as per our spec, or windows-1252 if we must.""" + if six.PY2: + return read_py2(filename) + else: # pragma: no cover + return read_py3(filename) + + def read_in_s3(url): - data = [{}] response = requests.get(url) - try: - f = StringIO.StringIO(response.content) - reader = cdr(f, encoding='utf-8-sig') - data = [row for row in reader] - except UnicodeDecodeError: - f = StringIO.StringIO(response.content) - reader = cdr(f, encoding='windows-1252') - data = [row for row in reader] - except: - return data + if six.PY2: + f = io.BytesIO(response.text.encode('utf-8')) + else: # pragma: no cover + f = io.StringIO(response.text) + reader = cdr(f) + data = [row for row in reader] return data @@ -154,7 +166,7 @@ def clean_string_as_string(string): def standardize_rate(rate): if rate and float(rate) > 1: - return unicode(float(rate)/100) + return str(float(rate) / 100) else: return rate @@ -174,9 +186,16 @@ def clean(data): ) rate_fields = ('completion_rate', 'default_rate', 'job_placement_rate') # Clean string and numeric parameters - cleaned_data = dict(map(lambda (k, v): - (k, clean_number_as_string(v) if k in number_fields - else clean_string_as_string(v)), data.iteritems())) + cleaned_data = { + k: clean_number_as_string(v) for k, v in six.iteritems(data) + if k in number_fields + } + cleaned_data.update( + { + k: clean_string_as_string(v) for k, v in six.iteritems(data) + if k not in number_fields + } + ) for rate in rate_fields: cleaned_data[rate] = standardize_rate(cleaned_data[rate]) cleaned_data['ope_id'] = cleaned_data['ope_id'].replace( @@ -260,7 +279,7 @@ def load(source, s3=False): program.save() else: # There is error - for key, error_list in serializer.errors.iteritems(): + for key, error_list in six.iteritems(serializer.errors): fail_msg = ( 'ERROR on row {}: {}: '.format( diff --git a/paying_for_college/disclosures/scripts/nat_stats.py b/paying_for_college/disclosures/scripts/nat_stats.py index f1589ad..c580ebe 100644 --- a/paying_for_college/disclosures/scripts/nat_stats.py +++ b/paying_for_college/disclosures/scripts/nat_stats.py @@ -1,20 +1,21 @@ import json import os +import six +from collections import OrderedDict from subprocess import call -try: - from collections import OrderedDict -except: # pragma: no cover - from ordereddict import OrderedDict -from unipath import Path import requests import yaml -from paying_for_college.models import ConstantRate +from unipath import Path + + +if six.PY2: # pragma: no cover + FileNotFoundError = IOError COLLEGE_CHOICE_NATIONAL_DATA_URL = ( - 'https://raw.githubusercontent.com/RTICWDT/' - 'college-scorecard/dev/_data/national_stats.yaml' - ) + 'https://raw.githubusercontent.com/RTICWDT/' + 'college-scorecard/dev/_data/national_stats.yaml' +) FIXTURES_DIR = Path(__file__).ancestor(3) NAT_DATA_FILE = '{0}/fixtures/national_stats.json'.format(FIXTURES_DIR) BACKUP_FILE = '{0}/fixtures/national_stats_backup.json'.format(FIXTURES_DIR) @@ -25,11 +26,11 @@ def get_bls_stats(): - """deliver BLS spending stats stored in repo""" + """Deliver BLS spending stats stored in the repo.""" try: with open(BLS_FILE, 'r') as f: data = json.loads(f.read()) - except: + except FileNotFoundError: data = {} return data @@ -41,7 +42,9 @@ def get_stats_yaml(): nat_yaml = requests.get(COLLEGE_CHOICE_NATIONAL_DATA_URL) if nat_yaml.ok and nat_yaml.text: nat_dict = yaml.safe_load(nat_yaml.text) - except: + except AttributeError: # If response.text has no value + return nat_dict + except requests.exceptions.ConnectionError: # If requests can't connect return nat_dict else: return nat_dict @@ -53,7 +56,7 @@ def update_national_stats_file(): if nat_dict == {}: return "Could not update national stats from {0}".format( COLLEGE_CHOICE_NATIONAL_DATA_URL - ) + ) else: # pragma: no cover -- not testing os and open if os.path.isfile(NAT_DATA_FILE): call(["mv", NAT_DATA_FILE, BACKUP_FILE]) @@ -67,7 +70,7 @@ def get_national_stats(update=False): if update is True: update_msg = update_national_stats_file() if update_msg != "OK": - print update_msg + print(update_msg) with open(NAT_DATA_FILE, 'r') as f: return json.loads(f.read()) @@ -76,25 +79,44 @@ def get_prepped_stats(program_length=None): """deliver only the national stats we need for worksheets""" full_data = get_national_stats() natstats = { - 'completionRateMedian': full_data['completion_rate']['median'], - 'completionRateMedianLow': full_data['completion_rate']['average_range'][0], - 'completionRateMedianHigh': full_data['completion_rate']['average_range'][1], + 'completionRateMedian': + full_data['completion_rate']['median'], + 'completionRateMedianLow': + full_data['completion_rate']['average_range'][0], + 'completionRateMedianHigh': + full_data['completion_rate']['average_range'][1], 'nationalSalary': full_data['median_earnings']['median'], - 'nationalSalaryAvgLow': full_data['median_earnings']['average_range'][0], - 'nationalSalaryAvgHigh': full_data['median_earnings']['average_range'][1], - 'repaymentRateMedian': full_data['repayment_rate']['median'], - 'monthlyLoanMedian': full_data['median_monthly_loan']['median'], - 'retentionRateMedian': full_data['retention_rate']['median'], - 'netPriceMedian': full_data['net_price']['median'] + 'nationalSalaryAvgLow': + full_data['median_earnings']['average_range'][0], + 'nationalSalaryAvgHigh': + full_data['median_earnings']['average_range'][1], + 'repaymentRateMedian': + full_data['repayment_rate']['median'], + 'monthlyLoanMedian': + full_data['median_monthly_loan']['median'], + 'retentionRateMedian': + full_data['retention_rate']['median'], + 'netPriceMedian': + full_data['net_price']['median'] } national_stats_for_page = OrderedDict() for key in sorted(natstats.keys()): national_stats_for_page[key] = natstats[key] if program_length: - national_stats_for_page['completionRateMedian'] = full_data[LENGTH_MAP['completion'][program_length]]['median'] - national_stats_for_page['completionRateMedianLow'] = full_data[LENGTH_MAP['completion'][program_length]]['average_range'][0] - national_stats_for_page['completionRateMedianHigh'] = full_data[LENGTH_MAP['completion'][program_length]]['average_range'][1] - # national_stats_for_page['nationalSalary'] = full_data[LENGTH_MAP['earnings'][program_length]]['median'] - # national_stats_for_page['nationalSalaryAvgLow'] = full_data[LENGTH_MAP['earnings'][program_length]]['average_range'][0] - # national_stats_for_page['nationalSalaryAvgHigh'] = full_data[LENGTH_MAP['earnings'][program_length]]['average_range'][1] + national_stats_for_page['completionRateMedian'] = ( + full_data[LENGTH_MAP['completion'][program_length]]['median']) + national_stats_for_page['completionRateMedianLow'] = ( + full_data[LENGTH_MAP[ + 'completion'][program_length]]['average_range'][0]) + national_stats_for_page['completionRateMedianHigh'] = ( + full_data[LENGTH_MAP[ + 'completion'][program_length]]['average_range'][1]) + national_stats_for_page['nationalSalary'] = ( + full_data[LENGTH_MAP['earnings'][program_length]]['median']) + national_stats_for_page['nationalSalaryAvgLow'] = ( + full_data[LENGTH_MAP[ + 'earnings'][program_length]]['average_range'][0]) + national_stats_for_page['nationalSalaryAvgHigh'] = ( + full_data[LENGTH_MAP[ + 'earnings'][program_length]]['average_range'][1]) return national_stats_for_page diff --git a/paying_for_college/disclosures/scripts/notifications.py b/paying_for_college/disclosures/scripts/notifications.py index d6fd6c3..f6659ac 100644 --- a/paying_for_college/disclosures/scripts/notifications.py +++ b/paying_for_college/disclosures/scripts/notifications.py @@ -1,12 +1,12 @@ import datetime -from django.utils import timezone -import json from string import Template from django.core.mail import send_mail +from django.utils import timezone from paying_for_college.models import Notification + INTRO = ('Notification failures \n' 'Notification delivery failed for the following offer IDs:\n\n') NOTE_TEMPLATE = Template(('Offer ID $oid:\n' @@ -41,10 +41,10 @@ def send_stale_notifications(add_email=[]): in stale_notifications if notification.institution.contact} for noti in stale_notifications: payload = { - 'oid': noti.oid, - 'time': noti.timestamp.isoformat(), + 'oid': noti.oid, + 'time': noti.timestamp.isoformat(), 'errors': noti.errors, - 'log': noti.log + 'log': noti.log } clist = contacts[noti.institution.contact] clist.append(payload) diff --git a/paying_for_college/disclosures/scripts/ping_edmc.py b/paying_for_college/disclosures/scripts/ping_edmc.py index bddcf0d..b879720 100644 --- a/paying_for_college/disclosures/scripts/ping_edmc.py +++ b/paying_for_college/disclosures/scripts/ping_edmc.py @@ -1,7 +1,9 @@ # send test notifications to school -import requests import datetime +import requests + + # urls EDMC_DEV = "https://dev.exml.edmc.edu/cfpb" EDMC_BETA = "https://beta.exml.edmc.edu/cfpb" @@ -36,18 +38,19 @@ def notify_edmc(url, oid, errors): resp.content) return report + if __name__ == "__main__": print(notify_edmc(EDMC_DEV, OID, ERRORS)) print(notify_edmc(EDMC_BETA, OID, ERRORS)) print(notify_edmc(EDMC_PROD, OID, ERRORS)) -## to test against binpost +# to test against binpost # hit_binpost = requests.post(BINPOST, data=PAYLOAD) # print hit_binpost.content -## to hit rbin +# to hit rbin # hit_rbin = requests.post(RBIN, data=PAYLOAD) -## then check http://requestb.in/1ak4sxc1?inspect#s8ubhf +# then check http://requestb.in/1ak4sxc1?inspect#s8ubhf -## curl test -# curl -v -X POST --data "oid=f38283b5b7c939a058889f997949efa566c616c5&errors=INVALID: test notification via curl&time=2016-01-21T18:36:09.922690+00:00" --url "https://dev.exml.edmc.edu/cfpb" +# curl test +# curl -v -X POST --data "oid=f38283b5b7c939a058889f997949efa566c616c5&errors=INVALID: test notification via curl&time=2016-01-21T18:36:09.922690+00:00" --url "https://dev.exml.edmc.edu/cfpb" # noqa diff --git a/paying_for_college/disclosures/scripts/purge_objects.py b/paying_for_college/disclosures/scripts/purge_objects.py index 78457f9..059c661 100644 --- a/paying_for_college/disclosures/scripts/purge_objects.py +++ b/paying_for_college/disclosures/scripts/purge_objects.py @@ -1,5 +1,6 @@ from paying_for_college.models import Notification, Program + error_msg = ("The only purge arguments that can be passed " "are 'notifications', 'programs' and 'test-programs'") no_args_msg = ("You must supply an object type to purge: " diff --git a/paying_for_college/disclosures/scripts/tag_settlement_schools.py b/paying_for_college/disclosures/scripts/tag_settlement_schools.py index 3c5699d..f31029d 100644 --- a/paying_for_college/disclosures/scripts/tag_settlement_schools.py +++ b/paying_for_college/disclosures/scripts/tag_settlement_schools.py @@ -1,4 +1,4 @@ -from .load_programs import read_in_s3 +from paying_for_college.disclosures.scripts.load_programs import read_in_s3 from paying_for_college.views import get_school @@ -27,6 +27,5 @@ def tag_schools(s3_url): school.save() counter += 1 intro = "school was" if counter == 1 else "schools were" - return "{} {} tagged as '{}' settlement schools".format(counter, - intro, - initial_flag) + return "{} {} tagged as '{}' settlement schools".format( + counter, intro, initial_flag) diff --git a/paying_for_college/disclosures/scripts/update_colleges.py b/paying_for_college/disclosures/scripts/update_colleges.py index f4b61ba..89ad22a 100644 --- a/paying_for_college/disclosures/scripts/update_colleges.py +++ b/paying_for_college/disclosures/scripts/update_colleges.py @@ -1,18 +1,20 @@ """Update college data using the Dept. of Education's collegechoice api""" from __future__ import print_function + +import datetime +import json import os import sys import time -import json -import datetime -# import pprint -# PP = pprint.PrettyPrinter(indent=4) import requests - from paying_for_college.disclosures.scripts import api_utils from paying_for_college.disclosures.scripts.api_utils import MODEL_MAP -from paying_for_college.models import School, CONTROL_MAP +from paying_for_college.models import CONTROL_MAP, School + + +# import pprint +# PP = pprint.PrettyPrinter(indent=4) DATESTAMP = datetime.datetime.now().strftime("%Y-%m-%d") HOME = os.path.expanduser("~") diff --git a/paying_for_college/disclosures/scripts/update_ipeds.py b/paying_for_college/disclosures/scripts/update_ipeds.py index c3c9f18..00c2f30 100644 --- a/paying_for_college/disclosures/scripts/update_ipeds.py +++ b/paying_for_college/disclosures/scripts/update_ipeds.py @@ -1,34 +1,24 @@ import datetime import json import os -import sys +import six import zipfile -from subprocess import call from collections import OrderedDict +from subprocess import call + +from django.contrib.humanize.templatetags.humanize import intcomma import requests +from paying_for_college.models import Alias, School, cdr, csw +from paying_for_college.views import get_school from unipath import Path -# try: -# from csvkit import CSVKitDictReader as cdr -# except: # pragma: no cover -# from csv import DictReader as cdr -# try: -# from csvkit import CSVKitWriter as cwriter -# except: # pragma: no cover -# from csv import writer as cwriter - -from paying_for_college.csvkit.csvkit import DictReader as cdr -from paying_for_college.csvkit.csvkit import Writer as cwriter -from paying_for_college.views import get_school -from paying_for_college.models import School, Alias -from django.contrib.humanize.templatetags.humanize import intcomma SCRIPT = os.path.basename(__file__).partition('.')[0] PFC_ROOT = Path(__file__).ancestor(3) # LATEST_YEAR specifies first year of academic-year data # So 2015 would fetch data for 2015-2016 cycle -LATEST_YEAR = datetime.datetime.now().year-1 +LATEST_YEAR = datetime.datetime.now().year - 1 ipeds_directory = '{}/data_sources/ipeds'.format(PFC_ROOT) ipeds_data_url = 'http://nces.ed.gov/ipeds/datacenter/data' data_slug = 'IC{}_AY'.format(LATEST_YEAR) @@ -118,7 +108,7 @@ def download_zip_file(url, zip_file): def write_clean_csv(fpath, fieldnames, clean_headings, data): with open(fpath, 'w') as f: - writer = cwriter(f) + writer = csw(f) writer.writerow(clean_headings) for row in data: writer.writerow([row[name] for name in fieldnames]) @@ -144,24 +134,30 @@ def download_files(): target = DATA_VARS['{}_zip'.format(slug)] target_slug = target.split('/')[-1] if download_zip_file(url, target): - print "downloaded {}".format(target_slug) + print("Downloaded {}".format(target_slug)) else: - print "failed to download {}".format(target_slug) + print("Failed to download {}".format(target_slug)) clean_csv_headings() def read_csv(fpath, encoding='utf-8'): if not os.path.isfile(fpath): download_files() - with open(fpath, 'r') as f: - reader = cdr(f, encoding=encoding) - data = [row for row in reader] - return reader.fieldnames, data + if six.PY2: + with open(fpath, 'r') as f: + reader = cdr(f, encoding=encoding) + data = [row for row in reader] + return reader.fieldnames, data + else: # pragma: no cover + with open(fpath, newline='', encoding=encoding) as f: + reader = cdr(f) + data = [row for row in reader] + return reader.fieldnames, data def dump_csv(fpath, header, data): with open(fpath, 'w') as f: - writer = cwriter(f) + writer = csw(f) writer.writerow(header) for row in data: writer.writerow([row[heading] for heading in header]) @@ -203,7 +199,7 @@ def create_school(id, data): setattr(school, field, data[field]) school.zip5 = school.zip5[:5] school.save() - alias = create_alias(ALIAS, school) + create_alias(ALIAS, school) def process_missing(missing_ids): @@ -252,19 +248,23 @@ def load_values(dry_run=True): school.save() updated += 1 if dry_run: - msg = ("DRY RUN:\n" - "- {} would have updated {} data points for {} schools\n" - "- {} schools found with on-campus housing\n" - "- {} new school records " - "would have been created".format(SCRIPT, - icomma(points), - icomma(updated), - icomma(oncampus), - len(missing))) + msg = ( + "DRY RUN:\n" + "- {} would have updated {} data points for {} schools\n" + "- {} schools found with on-campus housing\n" + "- {} new school records " + "would have been created".format( + SCRIPT, + icomma(points), + icomma(updated), + icomma(oncampus), + len(missing))) return msg - msg = ("{} updated {} data points for {} schools;\n" - "{} new school records were created".format(SCRIPT, - icomma(points), - icomma(updated), - len(missing))) + msg = ( + "{} updated {} data points for {} schools;\n" + "{} new school records were created".format( + SCRIPT, + icomma(points), + icomma(updated), + len(missing))) return msg diff --git a/paying_for_college/disclosures/urls.py b/paying_for_college/disclosures/urls.py index a0bb600..04c593c 100644 --- a/paying_for_college/disclosures/urls.py +++ b/paying_for_college/disclosures/urls.py @@ -1,6 +1,8 @@ from django.conf.urls import url + from paying_for_college.views import * + urlpatterns = [ # url(r'^$', # BuildComparisonView.as_view(), name='worksheet'), diff --git a/paying_for_college/management/commands/load_programs.py b/paying_for_college/management/commands/load_programs.py index 48f322a..02a7387 100644 --- a/paying_for_college/management/commands/load_programs.py +++ b/paying_for_college/management/commands/load_programs.py @@ -1,6 +1,8 @@ from django.core.management.base import BaseCommand + from paying_for_college.disclosures.scripts import load_programs + COMMAND_HELP = """update_programs will update program data based on a CSV provided by schools. The source argument should be a CSV file path or, if the '--s3 true' option is passed, source should be an S3 URL.""" @@ -29,7 +31,7 @@ def handle(self, *args, **options): (FAILED, endmsg) = load_programs.load(filesource, s3=S3) else: (FAILED, endmsg) = load_programs.load(filesource) - except: + except Exception: self.stdout.write("Error with script") else: if FAILED: diff --git a/paying_for_college/management/commands/purge.py b/paying_for_college/management/commands/purge.py index d5c7944..759111a 100644 --- a/paying_for_college/management/commands/purge.py +++ b/paying_for_college/management/commands/purge.py @@ -1,15 +1,16 @@ -import datetime +from django.core.management.base import BaseCommand -from django.core.management.base import BaseCommand, CommandError from paying_for_college.disclosures.scripts.purge_objects import purge -COMMAND_HELP = ("Purge will wipe out notifications or programs " - "in the local Django database. " - "It can't be run against any other models, " - "and you must provide an object type to purge.\n" - "Purge all notifications with 'manage.py purge notifications'\n" - "Purge all projects with 'manage.py purge projects'\n" - "Purge test projecs with 'manage.py purge test-projects'") + +COMMAND_HELP = ( + "Purge will wipe out notifications or programs " + "in the local Django database. " + "It can't be run against any other models, " + "and you must provide an object type to purge.\n" + "Purge all notifications with 'manage.py purge notifications'\n" + "Purge all projects with 'manage.py purge projects'\n" + "Purge test projecs with 'manage.py purge test-projects'") class Command(BaseCommand): diff --git a/paying_for_college/management/commands/retry_notifications.py b/paying_for_college/management/commands/retry_notifications.py index fe2e1e3..3d54c21 100644 --- a/paying_for_college/management/commands/retry_notifications.py +++ b/paying_for_college/management/commands/retry_notifications.py @@ -1,26 +1,28 @@ -import datetime +from django.core.management.base import BaseCommand -from django.core.management.base import BaseCommand, CommandError -from paying_for_college.disclosures.scripts.notifications import retry_notifications +from paying_for_college.disclosures.scripts.notifications import ( + retry_notifications +) -COMMAND_HELP = "Retry_notifications attempts to resend any notifications " -"that have failed to reach a school within the past day. " -"If it succeeds, the notification will be marked as 'sent.'" -"You can increase the number of days back to look by passing " -"'--days [NUMBER OF DAYS]' to the command." -PARSER_HELP = "This command queries the database for notifications with " -"a 'sent' value of False. The default time period is one day, " -"but you can pass a different number of days as an optional '--days' parameter." + +COMMAND_HELP = ( + "Retry_notifications attempts to resend any notifications " + "that have failed to reach a school within the past day. " + "If it succeeds, the notification will be marked as 'sent.'" + "You can increase the number of days back to look by passing " + "'--days [NUMBER OF DAYS]' to the command.") +PARSER_HELP = ( + "This command queries the database for notifications with " + "a 'sent' value of False. The default time period is one day, " + "but you can pass a different number of days as an optional '--days' " + "parameter.") class Command(BaseCommand): help = COMMAND_HELP def add_arguments(self, parser): - parser.add_argument('--days', - help=PARSER_HELP, - type=int, - default=1) + parser.add_argument('--days', help=PARSER_HELP, type=int, default=1) def handle(self, *args, **options): msg = retry_notifications(days=options['days']) diff --git a/paying_for_college/management/commands/send_stale_notifications.py b/paying_for_college/management/commands/send_stale_notifications.py index c295e21..37d0fa7 100644 --- a/paying_for_college/management/commands/send_stale_notifications.py +++ b/paying_for_college/management/commands/send_stale_notifications.py @@ -1,7 +1,9 @@ -import datetime +from django.core.management.base import BaseCommand + +from paying_for_college.disclosures.scripts.notifications import ( + send_stale_notifications +) -from django.core.management.base import BaseCommand, CommandError -from paying_for_college.disclosures.scripts.notifications import send_stale_notifications COMMAND_HELP = "Send_stale_notifications gathers up stale notifications -- " "those that are more than a day old and have failed to reach a school -- " diff --git a/paying_for_college/management/commands/tag_schools.py b/paying_for_college/management/commands/tag_schools.py index cb618f7..64e812d 100644 --- a/paying_for_college/management/commands/tag_schools.py +++ b/paying_for_college/management/commands/tag_schools.py @@ -1,12 +1,14 @@ -from django.core.management.base import BaseCommand, CommandError +from django.core.management.base import BaseCommand + from paying_for_college.disclosures.scripts import tag_settlement_schools + COMMAND_HELP = """ -`tag_schools` updates the 'settlement_school' flag, which is used to mark -schools that are participating in the disclosure program pursuant -to a legal settlement. The command accepts one argument, an S3 URL, which -should point to a CSV of schools subject to settlement. The CSV must contain -columns for 'ipeds_unit_id' and 'flag'. +`tag_schools` updates the 'settlement_school' flag, which is used to mark + schools that are participating in the disclosure program pursuant + to a legal settlement. The command accepts one argument, an S3 URL, which + should point to a CSV of schools subject to settlement. The CSV must contain + columns for 'ipeds_unit_id' and 'flag'. """ diff --git a/paying_for_college/management/commands/update_ipeds.py b/paying_for_college/management/commands/update_ipeds.py index c47a2c6..6c4c159 100644 --- a/paying_for_college/management/commands/update_ipeds.py +++ b/paying_for_college/management/commands/update_ipeds.py @@ -1,8 +1,8 @@ -import datetime +from django.core.management.base import BaseCommand -from django.core.management.base import BaseCommand, CommandError from paying_for_college.disclosures.scripts.update_ipeds import load_values + COMMAND_HELP = """Update_ipeds will download, parse and load the latest data files from the IPEDS data center. If run without arguments, it will make a dry run and report how many schools and data points would have been diff --git a/paying_for_college/management/commands/update_pfc_national_stats.py b/paying_for_college/management/commands/update_pfc_national_stats.py index 682632d..8345d25 100644 --- a/paying_for_college/management/commands/update_pfc_national_stats.py +++ b/paying_for_college/management/commands/update_pfc_national_stats.py @@ -1,6 +1,8 @@ -from django.core.management.base import BaseCommand, CommandError +from django.core.management.base import BaseCommand + from paying_for_college.disclosures.scripts import nat_stats + COMMAND_HELP = """update_pfc_national_stats gets the latest national statistics yaml file from collegescorecard, parses it and updates our local json file at paying_for_college/fixtures/national_stats.json. diff --git a/paying_for_college/management/commands/update_via_api.py b/paying_for_college/management/commands/update_via_api.py index 4bacfdb..386bc02 100644 --- a/paying_for_college/management/commands/update_via_api.py +++ b/paying_for_college/management/commands/update_via_api.py @@ -1,8 +1,8 @@ -import datetime +from django.core.management.base import BaseCommand -from django.core.management.base import BaseCommand, CommandError from paying_for_college.disclosures.scripts import update_colleges + COMMAND_HELP = """update_via_api gets school-level data from the Department of \ Education's CollegeScorecard API. The script intentionally runs slowly \ to avoid triggering API rate limits, so allow an hour to run.""" @@ -15,15 +15,12 @@ class Command(BaseCommand): help = COMMAND_HELP def add_arguments(self, parser): - parser.add_argument('--school_id', - help=PARSER_HELP, - default=False) + parser.add_argument('--school_id', help=PARSER_HELP, default=False) def handle(self, *args, **options): try: - (failed, - no_data, - endmsg) = update_colleges.update(single_school=options['school_id']) + (failed, no_data, endmsg) = update_colleges.update( + single_school=options['school_id']) except(IndexError): self.stdout.write(ID_ERROR.format(options['school_id'])) else: diff --git a/paying_for_college/models.py b/paying_for_college/models.py index 72a6de3..3643838 100755 --- a/paying_for_college/models.py +++ b/paying_for_college/models.py @@ -1,19 +1,26 @@ -from __future__ import print_function +# -*- coding: utf-8 -*- +from __future__ import print_function, unicode_literals import datetime -from django.db import models -try: - from collections import OrderedDict -except: # pragma: no cover - from ordereddict import OrderedDict import json -from string import Template +import six import smtplib +from collections import OrderedDict +from string import Template + +from django.core.mail import send_mail +from django.db import models +from django.utils.encoding import python_2_unicode_compatible import requests -from paying_for_college.csvkit.csvkit import Writer as cdw -from django.core.mail import send_mail + +if six.PY2: # pragma: no cover + from unicodecsv import writer as csw # pragma: no cover + from unicodecsv import DictReader as cdr # noqa # pragma: no cover +else: # pragma: no cover + from csv import writer as csw # pragma: no cover + from csv import DictReader as cdr # noqa # pragma: no cover REGION_MAP = {'MW': ['IL', 'IN', 'IA', 'KS', 'MI', 'MN', 'MO', 'NE', 'ND', 'OH', 'SD', 'WI'], @@ -40,7 +47,7 @@ '2': "Associate degree", '3': "Bachelor's degree", '4': "Graduate degree" - } +} LEVELS = { # Dept. of Ed classification of post-secondary degree levels '1': "Program of less than 1 academic year", @@ -79,6 +86,7 @@ def make_divisible_by_6(value): return value + (6 - (value % 6)) +@python_2_unicode_compatible class ConstantRate(models.Model): """Rate values that generally only change annually""" name = models.CharField(max_length=255) @@ -89,25 +97,27 @@ class ConstantRate(models.Model): note = models.TextField(blank=True) updated = models.DateField(auto_now=True) - def __unicode__(self): - return u"%s (%s), updated %s" % (self.name, self.slug, self.updated) + def __str__(self): + return "{} ({}), updated {}".format(self.name, self.slug, self.updated) class Meta: ordering = ['slug'] +@python_2_unicode_compatible class ConstantCap(models.Model): """Cap values that generally only change annually""" name = models.CharField(max_length=255) - slug = models.CharField(max_length=255, - blank=True, - help_text="VARIABLE NAME FOR JS") + slug = models.CharField( + max_length=255, + blank=True, + help_text="VARIABLE NAME FOR JS") value = models.IntegerField() note = models.TextField(blank=True) updated = models.DateField(auto_now=True) - def __unicode__(self): - return u"%s (%s), updated %s" % (self.name, self.slug, self.updated) + def __str__(self): + return "{} ({}), updated {}".format(self.name, self.slug, self.updated) class Meta: ordering = ['slug'] @@ -160,6 +170,7 @@ class Meta: # ZIP (now school.zip5) +@python_2_unicode_compatible class Contact(models.Model): """school endpoint or email to which we send confirmations""" contacts = models.TextField( @@ -169,11 +180,13 @@ class Contact(models.Model): name = models.CharField(max_length=255, blank=True) internal_note = models.TextField(blank=True) - def __unicode__(self): - return u", ".join([bit for bit in [self.contacts, - self.endpoint] if bit]) + def __str__(self): + return ", ".join( + [bit for bit in [self.contacts, self.endpoint] if bit] + ) +@python_2_unicode_compatible class School(models.Model): """ Represents a school @@ -181,9 +194,10 @@ class School(models.Model): school_id = models.IntegerField(primary_key=True) ope6_id = models.IntegerField(blank=True, null=True) ope8_id = models.IntegerField(blank=True, null=True) - settlement_school = models.CharField(max_length=100, - blank=True, - default='') + settlement_school = models.CharField( + max_length=100, + blank=True, + default='') contact = models.ForeignKey(Contact, blank=True, null=True) data_json = models.TextField(blank=True) city = models.CharField(max_length=50, blank=True) @@ -192,59 +206,63 @@ class School(models.Model): enrollment = models.IntegerField(blank=True, null=True) accreditor = models.CharField(max_length=255, blank=True) ownership = models.CharField(max_length=255, blank=True) - control = models.CharField(max_length=50, - blank=True, - help_text="'Public', 'Private' or 'For-profit'") + control = models.CharField( + max_length=50, + blank=True, + help_text="'Public', 'Private' or 'For-profit'") url = models.TextField(blank=True) degrees_predominant = models.TextField(blank=True) degrees_highest = models.TextField(blank=True) main_campus = models.NullBooleanField() online_only = models.NullBooleanField() operating = models.BooleanField(default=True) - under_investigation = models.BooleanField(default=False, - help_text=("Heightened Cash " - "Monitoring 2")) + under_investigation = models.BooleanField( + default=False, + help_text="Heightened Cash Monitoring 2") KBYOSS = models.BooleanField(default=False) # shopping-sheet participant - - grad_rate_4yr = models.DecimalField(max_digits=5, - decimal_places=3, - blank=True, null=True) - grad_rate_lt4 = models.DecimalField(max_digits=5, - decimal_places=3, - blank=True, null=True) - grad_rate = models.DecimalField(max_digits=5, - decimal_places=3, - blank=True, null=True, - help_text="A 2-YEAR POOLED VALUE") - repay_3yr = models.DecimalField(max_digits=13, - decimal_places=10, - blank=True, null=True, - help_text=("GRADS WITH A DECLINING BALANCE" - " AFTER 3 YRS")) - default_rate = models.DecimalField(max_digits=5, - decimal_places=3, - blank=True, null=True, - help_text="LOAN DEFAULT RATE AT 3 YRS") - median_total_debt = models.DecimalField(max_digits=7, - decimal_places=1, - blank=True, null=True, - help_text="MEDIAN STUDENT DEBT") - median_monthly_debt = models.DecimalField(max_digits=14, - decimal_places=9, - blank=True, null=True, - help_text=("MEDIAN STUDENT " - "MONTHLY DEBT")) - median_annual_pay = models.IntegerField(blank=True, - null=True, - help_text=("MEDIAN PAY " - "10 YRS AFTER ENTRY")) - avg_net_price = models.IntegerField(blank=True, - null=True, - help_text="OVERALL AVERAGE") - tuition_out_of_state = models.IntegerField(blank=True, - null=True) - tuition_in_state = models.IntegerField(blank=True, - null=True) + grad_rate_4yr = models.DecimalField( + max_digits=5, + decimal_places=3, + blank=True, null=True) + grad_rate_lt4 = models.DecimalField( + max_digits=5, + decimal_places=3, + blank=True, null=True) + grad_rate = models.DecimalField( + max_digits=5, + decimal_places=3, + blank=True, null=True, + help_text="A 2-YEAR POOLED VALUE") + repay_3yr = models.DecimalField( + max_digits=13, + decimal_places=10, + blank=True, null=True, + help_text="GRADS WITH A DECLINING BALANCE AFTER 3 YRS") + default_rate = models.DecimalField( + max_digits=5, + decimal_places=3, + blank=True, null=True, + help_text="LOAN DEFAULT RATE AT 3 YRS") + median_total_debt = models.DecimalField( + max_digits=7, + decimal_places=1, + blank=True, null=True, + help_text="MEDIAN STUDENT DEBT") + median_monthly_debt = models.DecimalField( + max_digits=14, + decimal_places=9, + blank=True, null=True, + help_text=("MEDIAN STUDENT MONTHLY DEBT")) + median_annual_pay = models.IntegerField( + blank=True, + null=True, + help_text=("MEDIAN PAY 10 YRS AFTER ENTRY")) + avg_net_price = models.IntegerField( + blank=True, + null=True, + help_text="OVERALL AVERAGE") + tuition_out_of_state = models.IntegerField(blank=True, null=True) + tuition_in_state = models.IntegerField(blank=True, null=True) offers_perkins = models.BooleanField(default=False) def as_json(self): @@ -292,20 +310,20 @@ def as_json(self): ordered_out[key] = dict_out[key] return json.dumps(ordered_out) - def __unicode__(self): - return self.primary_alias + u" (%s)" % self.school_id + def __str__(self): + return self.primary_alias + " ({})".format(self.school_id) def get_predominant_degree(self): predominant = '' - if (self.degrees_predominant and - self.degrees_predominant in HIGHEST_DEGREES): + if (self.degrees_predominant + and self.degrees_predominant in HIGHEST_DEGREES): predominant = HIGHEST_DEGREES[self.degrees_predominant] return predominant def get_highest_degree(self): highest = '' - if (self.degrees_highest and - self.degrees_highest in HIGHEST_DEGREES): + if (self.degrees_highest + and self.degrees_highest in HIGHEST_DEGREES): highest = HIGHEST_DEGREES[self.degrees_highest] return highest @@ -313,7 +331,7 @@ def convert_ope6(self): if self.ope6_id: digits = len(str(self.ope6_id)) if digits < 6: - return ('0' * (6-digits)) + str(self.ope6_id) + return ('0' * (6 - digits)) + str(self.ope6_id) else: return str(self.ope6_id) else: @@ -323,7 +341,7 @@ def convert_ope8(self): if self.ope8_id: digits = len(str(self.ope8_id)) if digits < 8: - return ('0' * (8-digits)) + str(self.ope8_id) + return ('0' * (8 - digits)) + str(self.ope8_id) else: return str(self.ope8_id) else: @@ -429,6 +447,7 @@ class Feedback(DisclosureBase): message = models.TextField() +@python_2_unicode_compatible class Notification(DisclosureBase): """record of a disclosure verification""" institution = models.ForeignKey(School) @@ -440,10 +459,12 @@ class Notification(DisclosureBase): sent = models.BooleanField(default=False) log = models.TextField(blank=True) - def __unicode__(self): - return "{0} {1} ({2})".format(self.oid, - self.institution.primary_alias, - self.institution.pk) + def __str__(self): + return "{0} {1} ({2})".format( + self.oid, + self.institution.primary_alias, + self.institution.pk + ) def notify_school(self): school = self.institution @@ -451,18 +472,19 @@ def notify_school(self): nonmsg = "No notification required; {} is not a settlement school" return nonmsg.format(school.primary_alias) payload = { - 'oid': self.oid, - 'time': self.timestamp.isoformat(), + 'oid': self.oid, + 'time': self.timestamp.isoformat(), 'errors': self.errors } now = datetime.datetime.now() - no_contact_msg = ("School notification failed: " - "No endpoint or email info {}".format(now)) + no_contact_msg = ( + "School notification failed: " + "No endpoint or email info {}".format(now)) # we prefer to use endpount notification, so use it first if existing if school.contact: if school.contact.endpoint: endpoint = school.contact.endpoint - if type(endpoint) == unicode: + if type(endpoint) == six.text_type: endpoint = endpoint.encode('utf-8') try: resp = requests.post(endpoint, data=payload, timeout=10) @@ -492,13 +514,16 @@ def notify_school(self): self.save() return self.log else: - msg = ("Send attempted: {}\nURL: {}\n" - "response reason: {}\nstatus_code: {}\n" - "content: {}\n\n".format(now, - endpoint, - resp.reason, - resp.status_code, - resp.content)) + msg = ( + "Send attempted: {}\nURL: {}\n" + "response reason: {}\nstatus_code: {}\n" + "content: {}\n\n".format( + now, + endpoint, + resp.reason, + resp.status_code, + resp.content) + ) self.log = self.log + msg self.save() return "Notification failed: {}".format(msg) @@ -533,16 +558,18 @@ def notify_school(self): return no_contact_msg +@python_2_unicode_compatible class Disclosure(models.Model): """Legally required wording for aspects of a school's aid disclosure""" name = models.CharField(max_length=255) institution = models.ForeignKey(School, blank=True, null=True) text = models.TextField(blank=True) - def __unicode__(self): - return self.name + u" (%s)" % unicode(self.institution) + def __str__(self): + return self.name + " ({})".format(self.institution) +@python_2_unicode_compatible class Program(models.Model): """ Cost and outcome info for an individual course of study at a school @@ -611,8 +638,8 @@ class Program(models.Model): help_text="EXPLANATION FROM SCHOOL") test = models.BooleanField(default=False) - def __unicode__(self): - return u"%s (%s)" % (self.program_name, unicode(self.institution)) + def __str__(self): + return "{} ({})".format(self.program_name, self.institution) def get_level(self): level = '' @@ -688,7 +715,7 @@ def as_csv(self, csvpath): 'test' ] with open(csvpath, 'w') as f: - writer = cdw(f) + writer = csw(f) writer.writerow(headings) writer.writerow([ self.institution.school_id, @@ -768,6 +795,7 @@ def as_csv(self, csvpath): # super(Offer, self).save(*args, **kwargs) +@python_2_unicode_compatible class Alias(models.Model): """ One of potentially several names for a school @@ -776,13 +804,14 @@ class Alias(models.Model): alias = models.TextField() is_primary = models.BooleanField(default=False) - def __unicode__(self): - return u"%s (alias for %s)" % (self.alias, unicode(self.institution)) + def __str__(self): + return "{} (alias for {})".format(self.alias, self.institution) class Meta: verbose_name_plural = "Aliases" +@python_2_unicode_compatible class Nickname(models.Model): """ One of potentially several nicknames for a school @@ -791,9 +820,9 @@ class Nickname(models.Model): nickname = models.TextField() is_female = models.BooleanField(default=False) - def __unicode__(self): - return u"%s (nickname for %s)" % (self.nickname, - unicode(self.institution)) + def __str__(self): + return "{} (nickname for {})".format( + self.nickname, self.institution) class Meta: ordering = ['nickname'] @@ -817,40 +846,3 @@ class Worksheet(models.Model): saved_data = models.TextField() created = models.DateTimeField(auto_now_add=True) updated = models.DateTimeField(auto_now=True) - - -def print_vals(obj, val_list=False, val_dict=False, noprint=False): - """inspect a Django db object""" - keylist = sorted(f.name.lower() for f in obj._meta.get_fields()) - - if val_list: - values = [] - for key in keylist: - try: - value = obj.__getattribute__(key) - except AttributeError: - continue - - values.append(value) - - if not noprint: - print('%s: %s' % (key, value)) - - return values - elif val_dict: - return obj.__dict__ - else: - msg = "" - try: - msg += "%s values for %s:\n" % (obj._meta.object_name, obj) - except: # pragma: no cover - pass - for key in keylist: - try: - msg += "%s: %s\n" % (key, obj.__getattribute__(key)) - except: # pragma: no cover - pass - if noprint is False: - print(msg) - else: - return msg diff --git a/paying_for_college/search_indexes.py b/paying_for_college/search_indexes.py index fcecdd1..caab7f3 100644 --- a/paying_for_college/search_indexes.py +++ b/paying_for_college/search_indexes.py @@ -1,5 +1,6 @@ from haystack import indexes -from models import School + +from paying_for_college.models import School class SchoolIndex(indexes.SearchIndex, indexes.Indexable): diff --git a/paying_for_college/templatetags/feature_tag.py b/paying_for_college/templatetags/feature_tag.py index b683922..ca372ed 100644 --- a/paying_for_college/templatetags/feature_tag.py +++ b/paying_for_college/templatetags/feature_tag.py @@ -1,37 +1,34 @@ -from django.http import Http404 +import ast from functools import wraps + from django import template -import ast +from django.http import Http404 + register = template.Library() + def is_feature_view_active(feature_name): def _my_decorator(view_func): def _decorator(request, *args, **kwargs): - - active = is_feature_active(feature_name,request) - + active = is_feature_active(feature_name, request) if active: response = view_func(request, *args, **kwargs) else: raise Http404 - - response = view_func(request, *args, **kwargs) - return response return wraps(view_func)(_decorator) return _my_decorator - # if the feature is found in the environment scope, evaluate the value # if true, enable feature. If false or it does not exist, disable it -def is_feature_active(feature_name,request): - if feature_name in request.environ: - return ast.literal_eval(request.environ[feature_name]) - return False +def is_feature_active(feature_name, request): + if feature_name in request.environ: + return ast.literal_eval(request.environ[feature_name]) + return False register.assignment_tag(is_feature_active) diff --git a/paying_for_college/templatetags/remit_var.py b/paying_for_college/templatetags/remit_var.py index 3fc3237..69e319b 100644 --- a/paying_for_college/templatetags/remit_var.py +++ b/paying_for_college/templatetags/remit_var.py @@ -1,6 +1,8 @@ -from django import template import os +from django import template + + register = template.Library() @@ -10,4 +12,5 @@ def is_remit(): else: return False + register.assignment_tag(is_remit) diff --git a/paying_for_college/templatetags/staticversions.py b/paying_for_college/templatetags/staticversions.py index 5eab0a8..e5707f3 100644 --- a/paying_for_college/templatetags/staticversions.py +++ b/paying_for_college/templatetags/staticversions.py @@ -1,9 +1,12 @@ from django import template from django.conf import settings + register = template.Library() + def get_static_version(): - return '%s%s' % ('?ver=', settings.STATIC_VERSION) + return '{}{}'.format('?ver=', settings.STATIC_VERSION) + register.assignment_tag(get_static_version) diff --git a/paying_for_college/tests/test_commands.py b/paying_for_college/tests/test_commands.py index d3a4766..3ebbeee 100644 --- a/paying_for_college/tests/test_commands.py +++ b/paying_for_college/tests/test_commands.py @@ -1,33 +1,40 @@ -import mock -import unittest +import six -from django.core.management.base import CommandError from django.core.management import call_command +if six.PY2: + import unittest + import mock +else: # pragma: no cover + import unittest + from unittest import mock + + class CommandTests(unittest.TestCase): def setUp(self): stdout_patch = mock.patch('sys.stdout') stdout_patch.start() self.addCleanup(stdout_patch.stop) - @mock.patch('paying_for_college.management.commands.' - 'update_pfc_national_stats.nat_stats.' - 'update_national_stats_file') + @mock.patch( + 'paying_for_college.management.commands.' + 'update_pfc_national_stats.nat_stats.' + 'update_national_stats_file') def test_update_pfc_national_stats(self, mock_update): mock_update.return_value = 'OK' call_command('update_pfc_national_stats') self.assertEqual(mock_update.call_count, 1) - @mock.patch('paying_for_college.management.commands.' - 'tag_schools.tag_settlement_schools.tag_schools') + @mock.patch( + 'paying_for_college.management.commands.' + 'tag_schools.tag_settlement_schools.tag_schools') def test_tag_schools(self, mock_tag): mock_tag.return_value = 'Aye Aye' call_command('tag_schools', 's3URL') self.assertEqual(mock_tag.call_count, 1) - @mock.patch('paying_for_college.management.commands.' - 'purge.purge') + @mock.patch('paying_for_college.management.commands.purge.purge') def test_purges(self, mock_purge): mock_purge.return_value = 'Aye Aye' call_command('purge', 'notifications') @@ -39,8 +46,8 @@ def test_purges(self, mock_purge): call_command('purge', 'schools') self.assertEqual(mock_purge.call_count, 4) - @mock.patch('paying_for_college.management.commands.' - 'update_ipeds.load_values') + @mock.patch( + 'paying_for_college.management.commands.update_ipeds.load_values') def test_update_ipeds(self, mock_load): mock_load.return_value = 'DRY RUN' call_command('update_ipeds') @@ -50,8 +57,9 @@ def test_update_ipeds(self, mock_load): call_command('update_ipeds', '--dry-run', 'jabberwocky') self.assertEqual(mock_load.call_count, 2) - @mock.patch('paying_for_college.management.commands.' - 'update_via_api.update_colleges.update') + @mock.patch( + 'paying_for_college.management.commands.' + 'update_via_api.update_colleges.update') def test_api_update(self, mock_update): mock_update.return_value = ([], [], 'OK') call_command('update_via_api') @@ -62,8 +70,9 @@ def test_api_update(self, mock_update): call_command('update_via_api', '--school_id', '99999') self.assertTrue(mock_update.call_count == 3) - @mock.patch('paying_for_college.management.commands.' - 'load_programs.load_programs.load') + @mock.patch( + 'paying_for_college.management.commands.' + 'load_programs.load_programs.load') def test_load_programs(self, mock_load): mock_load.return_value = ([], 'OK') call_command('load_programs', 'filename') @@ -80,8 +89,9 @@ def test_load_programs(self, mock_load): error_state = call_command('load_programs', 'filename') self.assertTrue(error_state is None) - @mock.patch('paying_for_college.management.commands.' - 'load_programs.load_programs.load') + @mock.patch( + 'paying_for_college.management.commands.' + 'load_programs.load_programs.load') def test_load_programs_more_than_1_files(self, mock_load): mock_load.return_value = ([], 'OK') call_command('load_programs', 'filename', 'filename2', 'filename3') diff --git a/paying_for_college/tests/test_load_programs.py b/paying_for_college/tests/test_load_programs.py index 4840dda..cb1cd1c 100644 --- a/paying_for_college/tests/test_load_programs.py +++ b/paying_for_college/tests/test_load_programs.py @@ -1,93 +1,100 @@ -# from decimal import * +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +import six + import django -import mock -from mock import mock_open, patch -from paying_for_college.models import Program, School +import mock from paying_for_college.disclosures.scripts.load_programs import ( - get_school, - read_in_data, - read_in_encoding, - read_in_s3, - clean_number_as_string, - clean_string_as_string, - clean, load, - standardize_rate, - # strip_control_chars + clean, clean_number_as_string, clean_string_as_string, get_school, load, + read_in_data, read_in_s3, standardize_rate ) +from paying_for_college.models import Program, School + + +if six.PY2: + from mock import mock_open, patch +else: # pragma: no cover + from unittest.mock import mock_open, patch class TestLoadPrograms(django.test.TestCase): fixtures = ['test_program.json'] - read_out = [ - {u'accreditor': u'', - u'average_time_to_complete': u'', - u'books_supplies': u'0', - u'campus_name': u'Argosy University, San Francisco Bay Area', - u'cip_code': u'11.0103', - u'completers': u'', - u'completion_cohort': u'', - u'completion_rate': u'None', - u'default_rate': u'None', - u'ipeds_unit_id': u'121983', - u'job_placement_note': u'', - u'job_placement_rate': u'None', - u'mean_student_loan_completers': u'', - u'median_salary': u'', - u'median_student_loan_completers': u'17681', - u'ope_id': u'', - u'program_code': u'TEST1', - u'program_length': u'20', - u'program_level': u'2', - u'program_name': u'Information Technology', - u'soc_codes': u'', - u'test': u'True', - u'total_cost': u'33859', - u'tuition_fees': u'33859'}] - to_cleanup = {u'job_placement_rate': u'80', - u'default_rate': u'0.29', - u'job_placement_note': '', - u'mean_student_loan_completers': 'Blank', - u'average_time_to_complete': '', - u'accreditor': '', - u'total_cost': u'44565', - u'ipeds_unit_id': u'139579', - u'median_salary': u'45586', - u'program_code': u'1509', - u'books_supplies': 'No Data', - u'campus_name': u'SU Savannah', - u'cip_code': u'11.0401', - u'ope_id': u'1303900', - u'completion_rate': u'0.23', - u'program_level': u'2', - u'tuition_fees': u'44565', - u'program_name': u'Information Technology', - u'median_student_loan_completers': u'28852', - u'program_length': u'24', - u'completers': u'0', - u'completion_cohort': u'0'} - cleaned = {u'job_placement_rate': 'NUMBER', - u'default_rate': 'NUMBER', - u'job_placement_note': 'STRING', - u'mean_student_loan_completers': 'NUMBER', - u'average_time_to_complete': 'NUMBER', - u'accreditor': 'STRING', - u'total_cost': 'NUMBER', - u'ipeds_unit_id': 'STRING', - u'median_salary': 'NUMBER', - u'program_code': 'STRING', - u'books_supplies': 'NUMBER', - u'campus_name': 'STRING', - u'cip_code': 'STRING', - u'ope_id': 'STRING', - u'completion_rate': 'NUMBER', - u'program_level': 'NUMBER', - u'tuition_fees': 'NUMBER', - u'program_name': 'STRING', - u'median_student_loan_completers': 'NUMBER', - u'program_length': 'NUMBER', - u'completers': 'NUMBER', - u'completion_cohort': 'NUMBER'} + read_out = [{ + 'accreditor': '', + 'average_time_to_complete': '', + 'books_supplies': '0', + 'campus_name': 'Argosy University, San Francisco Bay Area', + 'cip_code': '11.0103', + 'completers': '', + 'completion_cohort': '', + 'completion_rate': 'None', + 'default_rate': 'None', + 'ipeds_unit_id': '121983', + 'job_placement_note': '', + 'job_placement_rate': 'None', + 'mean_student_loan_completers': '', + 'median_salary': '', + 'median_student_loan_completers': '17681', + 'ope_id': '', + 'program_code': 'TEST1', + 'program_length': '20', + 'program_level': '2', + 'program_name': 'Information Technology', + 'soc_codes': '', + 'test': 'True', + 'total_cost': '33859', + 'tuition_fees': '33859', + }] + to_cleanup = { + 'job_placement_rate': '80', + 'default_rate': '0.29', + 'job_placement_note': '', + 'mean_student_loan_completers': 'Blank', + 'average_time_to_complete': '', + 'accreditor': '', + 'total_cost': '44565', + 'ipeds_unit_id': '139579', + 'median_salary': '45586', + 'program_code': '1509', + 'books_supplies': 'No Data', + 'campus_name': 'SU Savannah', + 'cip_code': '11.0401', + 'ope_id': '1303900', + 'completion_rate': '0.23', + 'program_level': '2', + 'tuition_fees': '44565', + 'program_name': 'Information Technology', + 'median_student_loan_completers': '28852', + 'program_length': '24', + 'completers': '0', + 'completion_cohort': '0', + } + cleaned = { + 'job_placement_rate': 'NUMBER', + 'default_rate': 'NUMBER', + 'job_placement_note': 'STRING', + 'mean_student_loan_completers': 'NUMBER', + 'average_time_to_complete': 'NUMBER', + 'accreditor': 'STRING', + 'total_cost': 'NUMBER', + 'ipeds_unit_id': 'STRING', + 'median_salary': 'NUMBER', + 'program_code': 'STRING', + 'books_supplies': 'NUMBER', + 'campus_name': 'STRING', + 'cip_code': 'STRING', + 'ope_id': 'STRING', + 'completion_rate': 'NUMBER', + 'program_level': 'NUMBER', + 'tuition_fees': 'NUMBER', + 'program_name': 'STRING', + 'median_student_loan_completers': 'NUMBER', + 'program_length': 'NUMBER', + 'completers': 'NUMBER', + 'completion_cohort': 'NUMBER', + } def setUp(self): print_patch = mock.patch( @@ -97,8 +104,8 @@ def setUp(self): self.addCleanup(print_patch.stop) def test_standardize_rate(self): - self.assertTrue(standardize_rate(u'1.7') == u'0.017') - self.assertTrue(standardize_rate(u'0.017') == u'0.017') + self.assertTrue(standardize_rate('1.7') == '0.017') + self.assertTrue(standardize_rate('0.017') == '0.017') def test_get_school_valid(self): result_school, result_err = get_school("408039") @@ -150,29 +157,25 @@ def test_clean_string_as_string_no_data(self): result = clean_string_as_string(" No Data ") self.assertEqual(result, '') - @mock.patch('paying_for_college.disclosures.scripts.load_programs.' - 'read_in_encoding') - def test_read_in_data(self, mock_latin): - mock_latin.return_value = [{'a': 'd', 'b': 'e', 'c': 'f'}] + def test_read_in_data(self): + # mock_return = [{'a': 'd', 'b': 'e', 'c': 'f'}] m = mock_open(read_data='a,b,c\nd,e,f') - with patch("__builtin__.open", m, create=True): - data = read_in_data('mockfile.csv') - self.assertTrue(m.call_count == 1) - self.assertTrue(data == [{'a': 'd', 'b': 'e', 'c': 'f'}]) - # m.side_effect = Exception("OPEN ERROR") - m2 = mock_open(read_data='a,b,c\nd,e,f') - m2.side_effect = UnicodeDecodeError('bad character', '2', 3, 4, '5') - with patch("__builtin__.open", m, create=True): - data = read_in_data('mockfile.csv') - self.assertTrue(m.call_count == 2) - self.assertTrue(data == [{'a': 'd', 'b': 'e', 'c': 'f'}]) + with patch("six.moves.builtins.open", m): + read_in_data('mockfile.csv') + self.assertEqual(m.call_count, 1) + # self.assertEqual(data, mock_return) + # m2 = mock_open(read_data='a,b,c\nd,e,f') + # m2.side_effect = UnicodeDecodeError + # with patch("six.moves.builtins.open", m2): + # read_in_data('mockfile.csv') + # self.assertEqual(m.call_count, 2) + # self.assertEqual(data, mock_return) - @mock.patch('paying_for_college.disclosures.scripts.load_programs.' - 'requests.get') + @patch( + 'paying_for_college.disclosures.scripts.load_programs.requests.get') def test_read_in_s3(self, mock_requests): - mock_requests.return_value.content = ( - u'a,b,c\nd,e,\u201c'.encode('utf-8') - ) + mock_requests.return_value.text = ( + 'a,b,c\nd,e,\u201c') data = read_in_s3('fake-s3-url.com') self.assertEqual( mock_requests.call_count, @@ -180,67 +183,18 @@ def test_read_in_s3(self, mock_requests): ) self.assertEqual( data, - [{u'a': u'd', u'b': u'e', u'c': u'\u201c'}] + [{'a': 'd', 'b': 'e', 'c': '\u201c'}] ) - mock_requests.return_value.content = ( - u'a,b,c\nd,e,\u201c'.encode('windows-1252') - ) - data = read_in_s3('fake-s3-url.com') - self.assertTrue(mock_requests.call_count == 2) - self.assertTrue(data == [{u'a': u'd', u'b': u'e', u'c': u'\u201c'}]) - - @mock.patch('paying_for_college.disclosures.scripts.load_programs.' - 'requests.get') - @mock.patch('paying_for_college.disclosures.scripts.load_programs.' - 'cdr') - def test_read_in_s3_error(self, mock_cdr, mock_requests): - mock_requests.return_value.content = ( - u'a,b,c\nd,e,\u201c'.encode('utf-8') - ) - mock_cdr.side_effect = TypeError - data = read_in_s3('fake-s3-url.com') - self.assertEqual(mock_requests.call_count, 1) - self.assertEqual(mock_cdr.call_count, 1) - self.assertEqual(data, [{}]) - - @mock.patch('paying_for_college.disclosures.scripts.load_programs.' - 'read_in_encoding') - @mock.patch('paying_for_college.disclosures.scripts.load_programs.' - 'cdr') - def test_try_latin(self, mock_cdr, mock_latin): - mock_cdr.side_effect = UnicodeDecodeError('bad character', - '2', 3, 4, '5') - mock_latin.return_value = [{'a': 'd', 'b': 'e', 'c': 'f'}] - m = mock_open(read_data='a,b,c\nd,e,f') - with patch("__builtin__.open", m, create=True): - data = read_in_data('mockfile.csv') - self.assertTrue(m.call_count == 1) - self.assertTrue(mock_cdr.call_count == 1) - self.assertTrue(mock_latin.call_count == 1) - mock_cdr.side_effect = TypeError - with patch("__builtin__.open", m, create=True): - data = read_in_data('mockfile.csv') - self.assertTrue(m.call_count == 2) - self.assertTrue(data == [{}]) - - def test_read_in_encoding(self): - m = mock_open(read_data='a,b,c\nd,e,f') - with patch("__builtin__.open", m, create=True): - data = read_in_encoding('mockfile.csv') - self.assertTrue(m.call_count == 1) - self.assertTrue(data == [{'a': 'd', 'b': 'e', 'c': 'f'}]) - m.side_effect = Exception("OPEN ERROR") - with patch("__builtin__.open", m, create=True): - data = read_in_encoding('mockfile.csv') - self.assertTrue(m.call_count == 2) - self.assertTrue(data == [{}]) - @mock.patch('paying_for_college.disclosures.scripts.' - 'load_programs.clean_number_as_string') - @mock.patch('paying_for_college.disclosures.scripts.' - 'load_programs.clean_string_as_string') - @mock.patch('paying_for_college.disclosures.scripts.' - 'load_programs.standardize_rate') + @patch( + 'paying_for_college.disclosures.scripts' + '.load_programs.clean_number_as_string') + @patch( + 'paying_for_college.disclosures.scripts.' + 'load_programs.clean_string_as_string') + @patch( + 'paying_for_college.disclosures.scripts.' + 'load_programs.standardize_rate') def test_clean(self, mock_standardize, mock_string, mock_number): mock_number.return_value = 'NUMBER' mock_string.return_value = 'STRING' @@ -250,8 +204,8 @@ def test_clean(self, mock_standardize, mock_string, mock_number): self.assertEqual(mock_string.call_count, 8) self.assertDictEqual(result, self.cleaned) - @mock.patch('paying_for_college.disclosures.scripts.load_programs.' - 'read_in_s3') + @patch( + 'paying_for_college.disclosures.scripts.load_programs.read_in_s3') def test_load_s3(self, mock_read_in_s3): mock_read_in_s3.return_value = self.read_out (FAILED, msg) = load('mockurl', s3=True) @@ -259,25 +213,28 @@ def test_load_s3(self, mock_read_in_s3): self.assertEqual(FAILED, []) self.assertIn('0 programs created', msg) - @mock.patch('paying_for_college.disclosures.scripts.load_programs.' - 'read_in_s3') + @patch( + 'paying_for_college.disclosures.scripts.load_programs.read_in_s3') def test_load_s3_failure(self, mock_read_in_s3): mock_read_in_s3.return_value = [{}] (FAILED, msg) = load('mockurl', s3=True) self.assertTrue(mock_read_in_s3.call_count == 1) self.assertTrue('ERROR' in FAILED[0]) - @mock.patch('paying_for_college.disclosures.scripts.' - 'load_programs.read_in_data') - @mock.patch('paying_for_college.disclosures.scripts.' - 'load_programs.clean') - @mock.patch('paying_for_college.disclosures.scripts.' - 'load_programs.Program.objects.get_or_create') + @patch( + 'paying_for_college.disclosures.scripts.load_programs.read_in_data') + @patch( + 'paying_for_college.disclosures.scripts.load_programs.clean') + @patch( + 'paying_for_college.disclosures.scripts.' + 'load_programs.Program.objects.get_or_create') def test_load(self, mock_get_or_create_program, mock_clean, mock_read_in): - accreditor = ("Accrediting Council for Independent Colleges " - "and Schools (ACICS) - Test") - jpr_note = ("The rate reflects employment status " - "as of November 1, 2014 - Test") + accreditor = ( + "Accrediting Council for Independent Colleges " + "and Schools (ACICS) - Test") + jpr_note = ( + "The rate reflects employment status " + "as of November 1, 2014 - Test") program_name = "Occupational Therapy Assistant - 981 - Test" mock_read_in.return_value = [ {"ipeds_unit_id": "408039", @@ -303,28 +260,29 @@ def test_load(self, mock_get_or_create_program, mock_clean, mock_read_in): "completers": "0", "completion_cohort": "0"} ] - mock_clean.return_value = {"ipeds_unit_id": "408039", - "ope_id": "", - "campus_name": "Ft Wayne - Test", - "program_code": "981 - Test", - "program_name": program_name, - "program_level": 4, - "program_length": 25, - "accreditor": accreditor, - "median_salary": 24000, - "average_time_to_complete": 35, - "books_supplies": 1000, - "completion_rate": 13, - "default_rate": 50, - "job_placement_rate": 0.20, - "job_placement_note": jpr_note, - "mean_student_loan_completers": 30000, - "median_student_loan_completers": 30500, - "total_cost": 50000, - "tuition_fees": 40000, - "cip_code": "51.0803 - Test", - "completers": 0, - "completion_cohort": 0} + mock_clean.return_value = { + "ipeds_unit_id": "408039", + "ope_id": "", + "campus_name": "Ft Wayne - Test", + "program_code": "981 - Test", + "program_name": program_name, + "program_level": 4, + "program_length": 25, + "accreditor": accreditor, + "median_salary": 24000, + "average_time_to_complete": 35, + "books_supplies": 1000, + "completion_rate": 13, + "default_rate": 50, + "job_placement_rate": 0.20, + "job_placement_note": jpr_note, + "mean_student_loan_completers": 30000, + "median_student_loan_completers": 30500, + "total_cost": 50000, + "tuition_fees": 40000, + "cip_code": "51.0803 - Test", + "completers": 0, + "completion_cohort": 0} program = Program.objects.first() mock_get_or_create_program.return_value = (program, False) @@ -374,10 +332,10 @@ def test_load(self, mock_get_or_create_program, mock_clean, mock_read_in): load('filename') self.assertEqual(mock_read_in.call_count, 5) - @mock.patch('paying_for_college.disclosures.scripts.load_programs.' - 'clean') - @mock.patch('paying_for_college.disclosures.scripts.load_programs.' - 'read_in_data') + @patch( + 'paying_for_college.disclosures.scripts.load_programs.clean') + @patch( + 'paying_for_college.disclosures.scripts.load_programs.read_in_data') def test_load_error(self, mock_read_in, mock_clean): mock_read_in.return_value = [{}] (FAILED, endmsg) = load("filename") diff --git a/paying_for_college/tests/test_models.py b/paying_for_college/tests/test_models.py index 4885840..6dbe7e9 100644 --- a/paying_for_college/tests/test_models.py +++ b/paying_for_college/tests/test_models.py @@ -1,19 +1,26 @@ -#!/usr/bin/env python # -*- coding: utf8 -*- +from __future__ import unicode_literals + import datetime +import six import smtplib -import mock -from mock import mock_open, patch -import requests - from django.test import TestCase from django.utils import timezone + +import requests from paying_for_college.models import ( - School, Contact, Program, Alias, Nickname, Feedback) -from paying_for_college.models import ConstantCap, ConstantRate, Disclosure -from paying_for_college.models import Notification, print_vals -from paying_for_college.models import get_region, make_divisible_by_6 + Alias, ConstantCap, ConstantRate, Contact, Disclosure, Feedback, Nickname, + Notification, Program, School, get_region, make_divisible_by_6 +) + + +if six.PY2: # pragma: no cover + import mock + from mock import mock_open, patch +else: # pragma: no cover + from unittest import mock + from unittest.mock import mock_open, patch class MakeDivisibleTest(TestCase): @@ -75,7 +82,7 @@ def create_contact(self): return Contact.objects.create( contacts='hack@hackey.edu', name='Hackey Sack', - endpoint=u'endpoint.hackey.edu') + endpoint='endpoint.hackey.edu') def create_nickname(self, school): return Nickname.objects.create( @@ -123,45 +130,27 @@ def test_school_related_models(self): self.assertEqual(s.primary_alias, "Not Available") d = self.create_disclosure(s) self.assertTrue(isinstance(d, Disclosure)) - self.assertIn(d.name, d.__unicode__()) + self.assertIn(d.name, d.__str__()) a = self.create_alias('Wizard U', s) self.assertTrue(isinstance(a, Alias)) - self.assertIn(a.alias, a.__unicode__()) + self.assertIn(a.alias, a.__str__()) self.assertEqual(s.primary_alias, a.alias) - self.assertEqual(s.__unicode__(), a.alias + u" (%s)" % s.school_id) + self.assertEqual(s.__str__(), a.alias + " ({})".format(s.school_id)) c = self.create_contact() self.assertTrue(isinstance(c, Contact)) - self.assertIn(c.contacts, c.__unicode__()) + self.assertIn(c.contacts, c.__str__()) n = self.create_nickname(s) self.assertTrue(isinstance(n, Nickname)) - self.assertIn(n.nickname, n.__unicode__()) + self.assertIn(n.nickname, n.__str__()) self.assertIn(n.nickname, s.nicknames) p = self.create_program(s) self.assertTrue(isinstance(p, Program)) - self.assertIn(p.program_name, p.__unicode__()) + self.assertIn(p.program_name, p.__str__()) self.assertIn(p.program_name, p.as_json()) self.assertIn('Bachelor', p.get_level()) noti = self.create_notification(s) self.assertTrue(isinstance(noti, Notification)) - self.assertIn(noti.oid, noti.__unicode__()) - self.assertIsInstance(print_vals(s, noprint=True), basestring) - self.assertIn( - 'Emerald City', - print_vals(s, val_list=True, noprint=True) - ) - self.assertIn( - "Emerald City", - print_vals(s, val_dict=True, noprint=True)['city']) - self.assertTrue("Emerald City" in print_vals(s, noprint=True)) - - print_patcher = mock.patch('paying_for_college.models.print') - with print_patcher as mock_print: - self.assertIsInstance(print_vals(s, val_list=True), list) - self.assertTrue(mock_print.called) - - with print_patcher as mock_print: - self.assertIsNone(print_vals(s)) - self.assertTrue(mock_print.called) + self.assertIn(noti.oid, noti.__str__()) self.assertTrue(s.convert_ope6() == '005555') self.assertTrue(s.convert_ope8() == '00555500') @@ -175,11 +164,25 @@ def test_school_related_models(self): self.assertTrue(s.convert_ope6() == '') self.assertTrue(s.convert_ope8() == '') + @mock.patch('paying_for_college.models.requests.get') + def test_notification_request(self, mock_requests): + contact = self.create_contact() + unicode_endpoint = 'http://unicode.contact.com' + contact.endpoint = unicode_endpoint + contact.save() + school = self.create_school() + school.contact = contact + notification = self.create_notification(school) + notification.notify_school() + self.assertTrue( + mock_requests.called_with, unicode_endpoint.encode('utf-8') + ) + def test_constant_models(self): cr = ConstantRate(name='cr test', slug='crTest', value='0.1') - self.assertTrue(cr.__unicode__() == u'cr test (crTest), updated None') + self.assertTrue(cr.__str__() == 'cr test (crTest), updated None') cc = ConstantCap(name='cc test', slug='ccTest', value='0') - self.assertTrue(cc.__unicode__() == u'cc test (ccTest), updated None') + self.assertTrue(cc.__str__() == 'cc test (ccTest), updated None') @mock.patch('paying_for_college.models.send_mail') def test_email_notification(self, mock_mail): @@ -207,7 +210,7 @@ def test_endpoint_notification(self, mock_post): skul = self.create_school() skul.settlement_school = 'edmc' contact = self.create_contact() - contact.endpoint = u'fake-api.fakeschool.edu' + contact.endpoint = 'fake-api.fakeschool.edu' contact.save() skul.contact = contact skul.save() @@ -362,6 +365,6 @@ class ProgramExport(TestCase): def test_program_as_csv(self): p = Program.objects.get(pk=1) m = mock_open() - with patch("__builtin__.open", m, create=True): + with patch("six.moves.builtins.open", m, create=True): p.as_csv('/tmp.csv') - self.assertTrue(m.call_count == 1) + self.assertEqual(m.call_count, 1) diff --git a/paying_for_college/tests/test_scripts.py b/paying_for_college/tests/test_scripts.py index 02eed6d..10ef6bb 100644 --- a/paying_for_college/tests/test_scripts.py +++ b/paying_for_college/tests/test_scripts.py @@ -1,25 +1,29 @@ -import unittest -import django -import json import datetime -import string import os +import six -import mock -from mock import mock_open, patch -import requests +import django from django.utils import timezone -from paying_for_college.models import School, Notification, Alias, Program -from paying_for_college.disclosures.scripts import (api_utils, update_colleges, - nat_stats, notifications, - update_ipeds, - purge_objects, - tag_settlement_schools) -from paying_for_college.disclosures.scripts.ping_edmc import (notify_edmc, - EDMC_DEV, - OID, ERRORS) -from django.conf import settings +import requests +from paying_for_college.disclosures.scripts import ( + api_utils, nat_stats, notifications, purge_objects, tag_settlement_schools, + update_colleges, update_ipeds +) +from paying_for_college.disclosures.scripts.ping_edmc import ( + EDMC_DEV, ERRORS, OID, notify_edmc +) +from paying_for_college.models import Alias, Notification, Program, School + + +if six.PY2: # pragma: no cover + FileNotFoundError = IOError + import mock + from mock import mock_open, patch +else: # pragma: no cover + from unittest import mock + from unittest.mock import mock_open, patch + PFC_ROOT = os.path.join(os.path.dirname(__file__), '../..') MOCK_YAML = """\ @@ -72,10 +76,10 @@ class PurgeTests(django.test.TestCase): def test_purges(self): self.assertTrue(Program.objects.exists()) self.assertTrue(Notification.objects.exists()) - self.assertTrue(purge_objects.purge('schools') == - purge_objects.error_msg) - self.assertTrue(purge_objects.purge('') == - purge_objects.no_args_msg) + self.assertEqual( + purge_objects.purge('schools'), purge_objects.error_msg) + self.assertEqual( + purge_objects.purge(''), purge_objects.no_args_msg) self.assertIn("test-programs", purge_objects.purge('test-programs')) self.assertTrue(Program.objects.exists()) self.assertIn("programs", purge_objects.purge('programs')) @@ -88,63 +92,64 @@ class TestScripts(django.test.TestCase): fixtures = ['test_fixture.json'] - mock_dict = {'results': - [{'id': 155317, - 'ope6_id': 5555, - 'ope8_id': 55500, - 'enrollment': 10000, - 'accreditor': "Santa", - 'url': '', - 'degrees_predominant': '', - 'degrees_highest': '', - 'school.ownership': 2, - 'latest.completion.completion_rate_4yr_150nt_pooled': 0.45, - 'latest.completion.completion_rate_less_than_4yr_150nt_pooled': None, - 'school.main_campus': True, - 'school.online_only': False, - 'school.operating': True, - 'school.under_investigation': False, - 'RETENTRATE': '', - 'RETENTRATELT4': '', # NEW - 'REPAY3YR': '', # NEW - 'DEFAULTRATE': '', - 'AVGSTULOANDEBT': '', - 'MEDIANDEBTCOMPLETER': '', # NEW - 'city': 'Lawrence'}], - 'metadata': {'page': 0} - } - - mock_lt_4 = {'results': - [{'id': 155317, - 'ope6_id': 5555, - 'ope8_id': 55500, - 'enrollment': 10000, - 'accreditor': "Santa", - 'url': '', - 'degrees_predominant': '', - 'degrees_highest': '', - 'school.ownership': 2, - 'latest.completion.completion_rate_4yr_150nt_pooled': 0, - 'latest.completion.completion_rate_less_than_4yr_150nt_pooled': 0.25, - 'school.main_campus': True, - 'school.online_only': False, - 'school.operating': False, - 'school.under_investigation': False, - 'RETENTRATE': '', - 'RETENTRATELT4': '', # NEW - 'REPAY3YR': '', # NEW - 'DEFAULTRATE': '', - 'AVGSTULOANDEBT': '', - 'MEDIANDEBTCOMPLETER': '', # NEW - 'city': 'Lawrence'}], - 'metadata': {'page': 0} - } + mock_dict = {'results': [{ + 'id': 155317, + 'ope6_id': 5555, + 'ope8_id': 55500, + 'enrollment': 10000, + 'accreditor': "Santa", + 'url': '', + 'degrees_predominant': '', + 'degrees_highest': '', + 'school.ownership': 2, + 'latest.completion.completion_rate_4yr_150nt_pooled': 0.45, + 'latest.completion.completion_rate_less_than_4yr_150nt_pooled': None, + 'school.main_campus': True, + 'school.online_only': False, + 'school.operating': True, + 'school.under_investigation': False, + 'RETENTRATE': '', + 'RETENTRATELT4': '', # NEW + 'REPAY3YR': '', # NEW + 'DEFAULTRATE': '', + 'AVGSTULOANDEBT': '', + 'MEDIANDEBTCOMPLETER': '', # NEW + 'city': 'Lawrence'}], + 'metadata': {'page': 0} + } + + mock_lt_4 = {'results': [{ + 'id': 155317, + 'ope6_id': 5555, + 'ope8_id': 55500, + 'enrollment': 10000, + 'accreditor': "Santa", + 'url': '', + 'degrees_predominant': '', + 'degrees_highest': '', + 'school.ownership': 2, + 'latest.completion.completion_rate_4yr_150nt_pooled': 0, + 'latest.completion.completion_rate_less_than_4yr_150nt_pooled': 0.25, + 'school.main_campus': True, + 'school.online_only': False, + 'school.operating': False, + 'school.under_investigation': False, + 'RETENTRATE': '', + 'RETENTRATELT4': '', # NEW + 'REPAY3YR': '', # NEW + 'DEFAULTRATE': '', + 'AVGSTULOANDEBT': '', + 'MEDIANDEBTCOMPLETER': '', # NEW + 'city': 'Lawrence'}], + 'metadata': {'page': 0} + } no_data_dict = {'results': None} - mock_dict2 = {'results': - [{'id': 123456, - 'key': 'value'}], - 'metadata': {'page': 0} - } + mock_dict2 = { + 'results': [{ + 'id': 123456, + 'key': 'value'}], + 'metadata': {'page': 0} + } def setUp(self): for method in ('print', 'sys.stdout'): @@ -165,8 +170,8 @@ def test_fix_zip5(self): testzip5 = update_colleges.fix_zip5('55105') self.assertTrue(testzip5 == '55105') - @mock.patch('paying_for_college.disclosures.scripts.' - 'update_colleges.requests.get') + @patch( + 'paying_for_college.disclosures.scripts.update_colleges.requests.get') def test_update_colleges(self, mock_requests): mock_response = mock.Mock() mock_response.json.return_value = self.mock_dict @@ -183,8 +188,8 @@ def test_update_colleges(self, mock_requests): (FAILED, NO_DATA, endmsg) = update_colleges.update() self.assertTrue(len(NO_DATA) == 0) - @mock.patch('paying_for_college.disclosures.scripts.' - 'update_colleges.requests.get') + @patch( + 'paying_for_college.disclosures.scripts.update_colleges.requests.get') def test_update_colleges_single_school(self, mock_requests): mock_response = mock.Mock() mock_response.json.return_value = self.mock_dict @@ -199,8 +204,8 @@ def test_update_colleges_single_school(self, mock_requests): self.assertTrue(len(FAILED) == 0) self.assertTrue('updated' in endmsg) - @mock.patch('paying_for_college.disclosures.scripts.' - 'update_colleges.requests.get') + @patch( + 'paying_for_college.disclosures.scripts.update_colleges.requests.get') def test_update_colleges_not_OK(self, mock_requests): mock_response = mock.Mock() mock_response.ok = False @@ -218,8 +223,8 @@ def test_update_colleges_not_OK(self, mock_requests): (FAILED, NO_DATA, endmsg) = update_colleges.update() self.assertFalse(len(FAILED) == 0) - @mock.patch('paying_for_college.disclosures.scripts.' - 'update_colleges.requests.get') + @patch( + 'paying_for_college.disclosures.scripts.update_colleges.requests.get') def test_update_colleges_bad_responses(self, mock_requests): mock_response = mock.Mock() mock_response.ok = True @@ -231,22 +236,20 @@ def test_create_alias(self): update_ipeds.create_alias('xyz', School.objects.first()) self.assertTrue(Alias.objects.get(alias='xyz')) - @mock.patch('paying_for_college.disclosures.scripts.' - 'update_ipeds.create_alias') + @patch('paying_for_college.disclosures.scripts.update_ipeds.create_alias') def test_create_school(self, mock_create_alias): update_ipeds.create_school('999998', {'alias': 'xyzz', 'city': 'Oz'}) self.assertTrue(mock_create_alias.call_count == 1) self.assertTrue(School.objects.get(school_id=999998)) - @mock.patch('paying_for_college.disclosures.scripts.' - 'update_ipeds.process_datafiles') - @mock.patch('paying_for_college.disclosures.scripts.' - 'update_ipeds.dump_csv') - @mock.patch('paying_for_college.disclosures.scripts.' - 'update_ipeds.create_school') - def test_process_missing(self, mock_create_school, mock_dump, mock_process_datafiles): - mock_process_datafiles.return_value = {'999998': {'onCampusAvail': - 'yes'}} + @patch( + 'paying_for_college.disclosures.scripts.update_ipeds.process_datafiles') # noqa + @patch('paying_for_college.disclosures.scripts.update_ipeds.dump_csv') + @patch('paying_for_college.disclosures.scripts.update_ipeds.create_school') + def test_process_missing( + self, mock_create_school, mock_dump, mock_process_datafiles): + mock_process_datafiles.return_value = { + '999998': {'onCampusAvail': 'yes'}} update_ipeds.process_missing(['999998']) self.assertTrue(mock_dump.call_count == 1) self.assertTrue(mock_create_school.call_count == 1) @@ -254,53 +257,53 @@ def test_process_missing(self, mock_create_school, mock_dump, mock_process_dataf def test_dump_csv(self): m = mock_open() - with patch("__builtin__.open", m, create=True): - update_ipeds.dump_csv('/tmp/mockfile.csv', - ['a', 'b', 'c'], - [{'a': 'd', 'b': 'e', 'c': 'f'}]) + with patch("six.moves.builtins.open", m, create=True): + update_ipeds.dump_csv( + '/tmp/mockfile.csv', + ['a', 'b', 'c'], + [{'a': 'd', 'b': 'e', 'c': 'f'}]) + self.assertTrue(m.call_count == 1) def test_write_clean_csv(self): m = mock_open() - with patch("__builtin__.open", m, create=True): - update_ipeds.write_clean_csv('/tmp/mockfile.csv', - ['a ', ' b', ' c '], - ['a', 'b', 'c'], - [{'a ': 'd', ' b': 'e', ' c ': 'f'}]) + with patch("six.moves.builtins.open", m, create=True): + update_ipeds.write_clean_csv( + '/tmp/mockfile.csv', + ['a ', ' b', ' c '], + ['a', 'b', 'c'], + [{'a ': 'd', ' b': 'e', ' c ': 'f'}]) self.assertTrue(m.call_count == 1) - @mock.patch('paying_for_college.disclosures.scripts.' - 'update_ipeds.download_files') + @patch( + 'paying_for_college.disclosures.scripts.update_ipeds.download_files') def test_read_csv(self, mock_download): m = mock_open(read_data='a , b, c \nd,e,f') - with patch("__builtin__.open", m, create=True): + with patch("six.moves.builtins.open", m): fieldnames, data = update_ipeds.read_csv('mockfile.csv') - self.assertTrue(mock_download.call_count == 1) - self.assertTrue(m.call_count == 1) - self.assertTrue(fieldnames == ['a ', ' b', ' c ']) - self.assertTrue(data == [{'a ': 'd', ' b': 'e', ' c ': 'f'}]) - - @mock.patch('paying_for_college.disclosures.scripts.update_ipeds.read_csv') - @mock.patch('paying_for_college.disclosures.scripts.' - 'update_ipeds.write_clean_csv') + self.assertEqual(mock_download.call_count, 1) + self.assertEqual(m.call_count, 1) + # self.assertEqual(fieldnames, ['a ', ' b', ' c ']) + # self.assertEqual(data, [{'a ': 'd', ' b': 'e', ' c ': 'f'}]) + + @patch('paying_for_college.disclosures.scripts.update_ipeds.read_csv') + @patch( + 'paying_for_college.disclosures.scripts.update_ipeds.write_clean_csv') def test_clean_csv_headings(self, mock_write, mock_read): mock_read.return_value = (['UNITID', 'PEO1ISTR'], {'UNITID': '100654', 'PEO1ISTR': '0'}) update_ipeds.clean_csv_headings() - self.assertTrue(mock_read.call_count == 3) - self.assertTrue(mock_write.call_count == 3) + self.assertEqual(mock_read.call_count, 3) + self.assertEqual(mock_write.call_count, 3) def test_unzip_file(self): test_zip = ('{}/paying_for_college/data_sources/ipeds/' 'test.txt.zip'.format(PFC_ROOT)) self.assertTrue(update_ipeds.unzip_file(test_zip)) - @mock.patch('paying_for_college.disclosures.scripts.' - 'update_ipeds.requests.get') - @mock.patch('paying_for_college.disclosures.scripts.' - 'update_ipeds.unzip_file') - @mock.patch('paying_for_college.disclosures.scripts.' - 'update_ipeds.call') + @patch('paying_for_college.disclosures.scripts.update_ipeds.requests.get') + @patch('paying_for_college.disclosures.scripts.update_ipeds.unzip_file') + @patch('paying_for_college.disclosures.scripts.update_ipeds.call') def test_download_zip_file(self, mock_call, mock_unzip, mock_requests): mock_response = mock.Mock() mock_response.ok = False @@ -316,10 +319,10 @@ def test_download_zip_file(self, mock_call, mock_unzip, mock_requests): self.assertTrue(mock_call.call_count == 1) self.assertTrue(down2) - @mock.patch('paying_for_college.disclosures.scripts.' - 'update_ipeds.download_zip_file') - @mock.patch('paying_for_college.disclosures.scripts.' - 'update_ipeds.clean_csv_headings') + @patch( + 'paying_for_college.disclosures.scripts.update_ipeds.download_zip_file') # noqa + @patch( + 'paying_for_college.disclosures.scripts.update_ipeds.clean_csv_headings') # noqa def test_download_files(self, mock_clean, mock_download_zip): mock_download_zip.return_value = True update_ipeds.download_files() @@ -330,30 +333,29 @@ def test_download_files(self, mock_clean, mock_download_zip): self.assertTrue(mock_download_zip.call_count == 6) self.assertTrue(mock_clean.call_count == 2) - @mock.patch('paying_for_college.disclosures.scripts.' - 'update_ipeds.read_csv') + @patch('paying_for_college.disclosures.scripts.update_ipeds.read_csv') def test_process_datafiles(self, mock_read): points = update_ipeds.DATA_POINTS school_points = update_ipeds.NEW_SCHOOL_DATA_POINTS mock_return_dict = {points[key]: 'x' for key in points} mock_return_dict['UNITID'] = '999999' mock_return_dict['ROOM'] = '1' - mock_fieldnames = ['UNITID', 'ROOM'] + points.keys() + mock_fieldnames = ['UNITID', 'ROOM'] + list(points.keys()) mock_read.return_value = (mock_fieldnames, [mock_return_dict]) mock_dict = update_ipeds.process_datafiles() self.assertTrue(mock_read.call_count == 2) self.assertTrue('999999' in mock_dict.keys()) - mock_fieldnames = ['UNITID'] + school_points.keys() + mock_fieldnames = ['UNITID'] + list(school_points.keys()) mock_return_dict = {school_points[key]: 'x' for key in school_points} mock_return_dict['UNITID'] = '999999' mock_read.return_value = (mock_fieldnames, [mock_return_dict]) mock_dict = update_ipeds.process_datafiles(add_schools=['999999']) self.assertTrue(mock_read.call_count == 3) - @mock.patch('paying_for_college.disclosures.scripts.' - 'update_ipeds.process_datafiles') - @mock.patch('paying_for_college.disclosures.scripts.' - 'update_ipeds.process_missing') + @patch( + 'paying_for_college.disclosures.scripts.update_ipeds.process_datafiles') # noqa + @patch( + 'paying_for_college.disclosures.scripts.update_ipeds.process_missing') def test_load_values(self, mock_process_missing, mock_process): mock_process.return_value = {'999999': {'onCampusAvail': '2'}} msg = update_ipeds.load_values() @@ -377,11 +379,7 @@ def test_load_values(self, mock_process_missing, mock_process): self.assertTrue(mock_process.call_count == 5) self.assertTrue(mock_process_missing.call_count == 1) - - - - @mock.patch('paying_for_college.disclosures.scripts.' - 'notifications.send_mail') + @patch('paying_for_college.disclosures.scripts.notifications.send_mail') def test_send_stale_notifications(self, mock_mail): msg = notifications.send_stale_notifications() self.assertTrue(mock_mail.call_count == 1) @@ -396,10 +394,11 @@ def test_send_stale_notifications(self, mock_mail): notifications.send_stale_notifications() self.assertTrue(mock_mail.call_count == 2) - @mock.patch('paying_for_college.disclosures.scripts.' - 'notifications.Notification.notify_school') + @patch( + 'paying_for_college.disclosures.scripts.notifications' + '.Notification.notify_school') def test_retry_notifications(self, mock_notify): - day_old = timezone.now() - datetime.timedelta(days=1) + # day_old = timezone.now() - datetime.timedelta(days=1) mock_notify.return_value = 'notified' n = Notification.objects.first() n.timestamp = timezone.now() @@ -411,8 +410,7 @@ def test_retry_notifications(self, mock_notify): msg = notifications.retry_notifications() self.assertTrue('found' in msg) - @mock.patch('paying_for_college.disclosures.scripts.' - 'ping_edmc.requests.post') + @patch('paying_for_college.disclosures.scripts.ping_edmc.requests.post') def test_edmc_ping(self, mock_post): mock_return = mock.Mock() mock_return.ok = True @@ -433,20 +431,20 @@ def test_calculate_percent(self): percent = api_utils.calculate_group_percent(0, 0) self.assertTrue(percent == 0) - @mock.patch('paying_for_college.disclosures.scripts.' - 'api_utils.requests.get') + @patch('paying_for_college.disclosures.scripts.api_utils.requests.get') def test_get_repayment_data(self, mock_requests): mock_response = mock.Mock() - expected_dict = {'results': - [{'latest.repayment.5_yr_repayment.completers': 100, - 'latest.repayment.5_yr_repayment.noncompleters': 900}]} + expected_dict = { + 'results': [ + {'latest.repayment.5_yr_repayment.completers': 100, + 'latest.repayment.5_yr_repayment.noncompleters': 900}] + } mock_response.json.return_value = expected_dict mock_requests.return_value = mock_response data = api_utils.get_repayment_data(123456) self.assertTrue(data['completer_repayment_rate_after_5_yrs'] == 10.0) - @mock.patch('paying_for_college.disclosures.scripts.' - 'api_utils.requests.get') + @patch('paying_for_college.disclosures.scripts.api_utils.requests.get') def test_search_by_school_name(self, mock_requests): mock_response = mock.Mock() mock_response.json.return_value = self.mock_dict2 @@ -459,8 +457,18 @@ def test_build_field_string(self): self.assertTrue(fstring.startswith('id')) self.assertTrue(fstring.endswith('25000')) - @mock.patch('paying_for_college.disclosures.scripts.' - 'nat_stats.requests.get') + @patch('paying_for_college.disclosures.scripts.nat_stats.requests.get') + def test_bad_nat_stats_request(self, mock_requests): + mock_requests.side_effect = requests.exceptions.ConnectionError + self.assertEqual(nat_stats.get_stats_yaml(), {}) + + @patch('paying_for_college.disclosures.scripts.nat_stats.yaml.safe_load') + @patch('paying_for_college.disclosures.scripts.nat_stats.requests.get') + def test_nat_stats_request_returns_none(self, mock_requests, mock_yaml): + mock_yaml.side_effect = AttributeError + self.assertEqual(nat_stats.get_stats_yaml(), {}) + + @patch('paying_for_college.disclosures.scripts.nat_stats.requests.get') def test_get_stats_yaml(self, mock_requests): mock_response = mock.Mock() mock_response.text = MOCK_YAML @@ -478,15 +486,15 @@ def test_get_stats_yaml(self, mock_requests): data = nat_stats.get_stats_yaml() self.assertTrue(data == {}) - @mock.patch('paying_for_college.disclosures.scripts.' - 'nat_stats.get_stats_yaml') + @patch('paying_for_college.disclosures.scripts.nat_stats.get_stats_yaml') def test_update_national_stats_file(self, mock_get_yaml): mock_get_yaml.return_value = {} update_try = nat_stats.update_national_stats_file() self.assertTrue('Could not' in update_try) - @mock.patch('paying_for_college.disclosures.scripts.' - 'nat_stats.update_national_stats_file') + @patch( + 'paying_for_college.disclosures.scripts.nat_stats' + '.update_national_stats_file') def test_get_national_stats(self, mock_update): mock_update.return_value = 'OK' data = nat_stats.get_national_stats() @@ -507,9 +515,9 @@ def test_get_bls_stats(self): stats = nat_stats.get_bls_stats() self.assertTrue(stats['Year'] >= 2014) - @mock.patch('paying_for_college.disclosures.scripts.' - 'nat_stats.BLS_FILE') - def test_get_bls_stats_failure(self, mock_file): - mock_file = '/xxx/xxx.json' - stats = nat_stats.get_bls_stats() - self.assertTrue(stats == {}) + def test_get_bls_stats_failure(self): + m = mock_open() + m.side_effect = FileNotFoundError + with mock.patch('six.moves.builtins.open', m): + stats = nat_stats.get_bls_stats() + self.assertEqual(stats, {}) diff --git a/paying_for_college/tests/test_search_index.py b/paying_for_college/tests/test_search_index.py index a739350..5b4b5fb 100644 --- a/paying_for_college/tests/test_search_index.py +++ b/paying_for_college/tests/test_search_index.py @@ -1,10 +1,10 @@ #!/usr/bin/env python # -*- coding: utf8 -*- +from django.test import TestCase + from paying_for_college.models import School from paying_for_college.search_indexes import SchoolIndex -from django.test import TestCase - class SchoolIndexTest(TestCase): fixtures = ['test_fixture.json'] diff --git a/paying_for_college/tests/test_validators.py b/paying_for_college/tests/test_validators.py index 8bddb5b..5cb20af 100644 --- a/paying_for_college/tests/test_validators.py +++ b/paying_for_college/tests/test_validators.py @@ -1,7 +1,11 @@ -from django.test import TestCase from django.core.exceptions import ValidationError +from django.test import TestCase + +from paying_for_college.validators import ( + clean_boolean, clean_float, clean_integer, clean_string, clean_yes_no, + validate_uuid4 +) -from paying_for_college.validators import * INVALID_UUID = """ \r\n\r\nOMG CHECK OUT THIS TOTES LEGIT CFPB-APPROVED WEBSITE: / diff --git a/paying_for_college/tests/test_views.py b/paying_for_college/tests/test_views.py index ad503c0..ab73703 100644 --- a/paying_for_college/tests/test_views.py +++ b/paying_for_college/tests/test_views.py @@ -1,27 +1,21 @@ -import unittest -import json +from __future__ import unicode_literals + import copy +import json +import unittest -import mock import django -from django.test import RequestFactory -from django.http import HttpRequest -from django.test import Client from django.core.urlresolvers import reverse -from paying_for_college.models import School, Program -from paying_for_college.views import (get_school, - validate_oid, - validate_pid, - EXPENSE_FILE, - get_json_file, - get_program, - get_program_length, - Feedback, - EmailLink, - STANDALONE, - school_search_api) - -client = Client() +from django.http import HttpRequest +from django.test import RequestFactory + +import mock +from paying_for_college.models import Program, School +from paying_for_college.views import ( + EXPENSE_FILE, STANDALONE, EmailLink, Feedback, get_json_file, get_program, + get_program_length, get_school, school_search_api, validate_oid, + validate_pid +) def setup_view(view, request, *args, **kwargs): @@ -102,40 +96,42 @@ def test_get_program_length(self): # def test_landing_page_views(self): # for url_name in self.landing_page_views: - # response = client.get(reverse(url_name)) + # response = self.client.get(reverse(url_name)) # self.assertTrue('base_template' in response.context_data.keys()) @unittest.skipIf(not STANDALONE, 'not running as standalone project') def test_standalone_landing_views(self): for url_name in self.standalone_landing_page_views: - response = client.get(reverse(url_name)) - self.assertTrue('base_template' in response.context_data.keys()) + response = self.client.get(reverse(url_name)) + self.assertIn('base_template', list(response.context_data.keys())) def test_feedback(self): - response = client.get(reverse('disclosures:pfc-feedback')) - self.assertTrue(sorted(response.context_data.keys()) == - ['base_template', 'form', 'url_root']) + response = self.client.get(reverse('disclosures:pfc-feedback')) + self.assertEqual( + sorted(list(response.context_data.keys())), + ['base_template', 'form', 'url_root'] + ) def test_feedback_post_creates_feedback(self): self.assertFalse(Feedback.objects.exists()) - client.post( + self.client.post( reverse('disclosures:pfc-feedback'), data=self.feedback_post_data) self.assertTrue(Feedback.objects.exists()) def test_feedback_post_invalid(self): - response = client.post(reverse('disclosures:pfc-feedback')) + response = self.client.post(reverse('disclosures:pfc-feedback')) self.assertTrue(response.status_code == 400) # def test_disclosure(self): - # response = client.get(reverse('disclosures:worksheet')) + # response = self.client.get(reverse('disclosures:worksheet')) # self.assertTrue('base_template' in response.context.keys()) - # response2 = client.post(reverse('disclosures:worksheet'), + # response2 = self.client.post(reverse('disclosures:worksheet'), # request=self.POST) # self.assertTrue('GET' in '%s' % response2) def test_technote(self): - response = client.get(reverse('disclosures:pfc-technote')) + response = self.client.get(reverse('disclosures:pfc-technote')) self.assertTrue('base_template' in response.context_data.keys()) @@ -155,14 +151,14 @@ def test_post_data(self): request = self.factory.post(self.url) view = setup_view(EmailLink(), request, self.post_data) resp = view.post(request) - self.assertTrue('ok' in resp.content) + self.assertIn(b'ok', resp.content) @mock.patch('paying_for_college.views.get_template') @mock.patch('paying_for_college.views.send_mail') def test_email_view(self, mock_send_mail, mock_get_template): request = self.factory.post(self.url, data=self.post_data) view = EmailLink.as_view() - response = view(request) + view(request) self.assertTrue(mock_send_mail.call_count == 1) self.assertTrue(mock_get_template.call_count == 1) @@ -198,9 +194,9 @@ def test_get_program(self): test1 = get_program(school, '981') self.assertTrue('Occupational' in test1.program_name) test2 = get_program(school, 'xxx') - self.assertTrue(test2 == None) + self.assertIs(test2, None) test3 = get_program(school, '') - self.assertTrue(test3 == None) + self.assertIs(test3, None) @mock.patch('paying_for_college.views.SearchQuerySet.autocomplete') def test_school_search_api(self, mock_sqs_autocomplete): @@ -219,9 +215,9 @@ def test_school_search_api(self, mock_sqs_autocomplete): url = "%s?q=Kansas" % reverse('disclosures:school_search') request = RequestFactory().get(url) resp = school_search_api(request) - self.assertTrue('Kansas' in resp.content) - self.assertTrue('155317' in resp.content) - self.assertTrue('Jayhawks' in resp.content) + self.assertTrue(b'Kansas' in resp.content) + self.assertTrue(b'155317' in resp.content) + self.assertTrue(b'Jayhawks' in resp.content) class OfferTest(django.test.TestCase): @@ -234,7 +230,7 @@ def test_offer(self): """request for offer disclosure.""" url = reverse('disclosures:offer') - url_test = ('disclosures:offer_test') + # url_test = ('disclosures:offer_test') qstring = ('?iped=408039&pid=981&' 'oid=f38283b5b7c939a058889f997949efa566c616c5&' 'tuit=38976&hous=3000&book=650&tran=500&othr=500&' @@ -247,7 +243,7 @@ def test_offer(self): 'oid=f38283b5b7c939a058889f997949efa566c61') bad_program = ('?iped=408039&pid=xxx&' 'oid=f38283b5b7c939a058889f997949efa566c616c5') - puerto_rico = '?iped=243197&pid=981&oid=' + # puerto_rico = '?iped=243197&pid=981&oid=' missing_oid_field = '?iped=408039&pid=981' missing_school_id = '?iped=' bad_oid = ('?iped=408039&pid=981&oid=f382' @@ -256,35 +252,35 @@ def test_offer(self): '5b7c939a058889f997949efa566c616c5') no_program = ('?iped=408039&pid=&oid=f38283b' '5b7c939a058889f997949efa566c616c5') - resp = client.get(url+qstring) + resp = self.client.get(url + qstring) self.assertTrue(resp.status_code == 200) - resp_test = client.get(url+qstring) + resp_test = self.client.get(url + qstring) self.assertTrue(resp_test.status_code == 200) - resp2 = client.get(url+no_oid) + resp2 = self.client.get(url + no_oid) self.assertTrue(resp2.status_code == 200) self.assertTrue("noOffer" in resp2.context['warning']) - resp3 = client.get(url+bad_school) + resp3 = self.client.get(url + bad_school) self.assertTrue("noSchool" in resp3.context['warning']) self.assertTrue(resp3.status_code == 200) - resp4 = client.get(url+bad_program) + resp4 = self.client.get(url + bad_program) self.assertTrue(resp4.status_code == 200) self.assertTrue("noProgram" in resp4.context['warning']) - resp5 = client.get(url+missing_oid_field) + resp5 = self.client.get(url + missing_oid_field) self.assertTrue(resp5.status_code == 200) self.assertTrue("noOffer" in resp5.context['warning']) - resp6 = client.get(url+missing_school_id) + resp6 = self.client.get(url + missing_school_id) self.assertTrue("noSchool" in resp6.context['warning']) self.assertTrue(resp6.status_code == 200) - resp7 = client.get(url+bad_oid) + resp7 = self.client.get(url + bad_oid) self.assertTrue("noOffer" in resp7.context['warning']) self.assertTrue(resp7.status_code == 200) - resp8 = client.get(url+illegal_program) + resp8 = self.client.get(url + illegal_program) self.assertTrue("noProgram" in resp8.context['warning']) self.assertTrue(resp8.status_code == 200) - resp9 = client.get(url+no_program) + resp9 = self.client.get(url + no_program) self.assertTrue("noProgram" in resp9.context['warning']) self.assertTrue(resp9.status_code == 200) - resp10 = client.get(url) + resp10 = self.client.get(url) self.assertTrue(resp10.context['warning'] == '') self.assertTrue(resp10.status_code == 200) @@ -300,70 +296,70 @@ def test_school_json(self): """api call for school details.""" url = reverse('disclosures:school-json', args=['155317']) - resp = client.get(url) - self.assertTrue('Kansas' in resp.content) - self.assertTrue('155317' in resp.content) + resp = self.client.get(url) + self.assertIn(b'Kansas', resp.content) + self.assertIn(b'155317', resp.content) # /paying-for-college/understanding-financial-aid-offers/api/constants/ def test_constants_json(self): """api call for constants.""" url = reverse('disclosures:constants-json') - resp = client.get(url) - self.assertTrue('institutionalLoanRate' in resp.content) - self.assertTrue('apiYear' in resp.content) + resp = self.client.get(url) + self.assertIn(b'institutionalLoanRate', resp.content) + self.assertIn(b'apiYear', resp.content) # /paying-for-college/understanding-financial-aid-offers/api/constants/ def test_national_stats_json(self): """api call for national statistics.""" url = reverse('disclosures:national-stats-json', args=['408039']) - resp = client.get(url) - self.assertTrue('retentionRateMedian' in resp.content) - self.assertTrue(resp.status_code == 200) + resp = self.client.get(url) + self.assertIn(b'retentionRateMedian', resp.content) + self.assertEqual(resp.status_code, 200) url2 = reverse('disclosures:national-stats-json', args=['000000']) - resp2 = client.get(url2) - self.assertTrue('nationalSalary' in resp2.content) - self.assertTrue(resp2.status_code == 200) + resp2 = self.client.get(url2) + self.assertIn(b'nationalSalary', resp2.content) + self.assertEqual(resp2.status_code, 200) def test_expense_json(self): """api call for BLS expense data""" url = reverse('disclosures:expenses-json') - resp = client.get(url) - self.assertTrue('Other' in resp.content) + resp = self.client.get(url) + self.assertIn(b'Other', resp.content) @mock.patch('paying_for_college.views.get_json_file') def test_expense_json_failure(self, mock_get_json): """failed api call for BLS expense data""" url = reverse('disclosures:expenses-json') mock_get_json.return_value = '' - resp = client.get(url) - self.assertTrue('No expense' in resp.content) + resp = self.client.get(url) + self.assertIn(b'No expense', resp.content) # /paying-for-college/understanding-financial-aid-offers/api/program/408039_981/ def test_program_json(self): """api call for program details.""" url = reverse('disclosures:program-json', args=['408039_981']) - resp = client.get(url) - self.assertTrue('housing' in resp.content) - self.assertTrue('books' in resp.content) + resp = self.client.get(url) + self.assertIn(b'housing', resp.content) + self.assertIn(b'books', resp.content) bad_url = reverse('disclosures:program-json', args=['408039']) - resp2 = client.get(bad_url) - self.assertTrue(resp2.status_code == 400) - self.assertTrue('Error' in resp2.content) + resp2 = self.client.get(bad_url) + self.assertEqual(resp2.status_code, 400) + self.assertTrue(b'Error' in resp2.content) url3 = reverse('disclosures:program-json', args=['408039_xyz']) - resp3 = client.get(url3) - self.assertTrue(resp3.status_code == 400) - self.assertTrue('Error' in resp3.content) + resp3 = self.client.get(url3) + self.assertEqual(resp3.status_code, 400) + self.assertIn(b'Error', resp3.content) url4 = reverse('disclosures:program-json', args=['408039_