From 4d7ccc56c6451cb17d03c192cba80f65ae6c524e Mon Sep 17 00:00:00 2001 From: higs4281 Date: Thu, 28 Mar 2019 10:10:32 -0400 Subject: [PATCH 1/9] Prepare college-costs for python3 testing and compatibility --- tox.ini | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 0121e88..27e910b 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,8 @@ [tox] skipsdist=True -envlist=dj{111} +envlist= + py27{2715} + py36{368} [testenv] install_command=pip install -e ".[testing]" -U {opts} {packages} @@ -10,4 +12,27 @@ commands= coverage report --show-missing --skip-covered deps= - dj111: Django>=1.11,<1.12 + dj111: Django>=1.11,<1.12 + lint: {[lint]deps} + unittest: {[unittest]deps} + missing-migrations: {[missing-migrations]deps} + +[lint] +deps= + flake8 + isort +commands= + flake8 + isort --check-only --diff --recursive cfgov + +[unittest] +deps= + django-haystack>=2.7,<2.9 + djangorestframework>=3.6,<3.9 + elasticsearch>=2.4.1,<3 + PyYAML==3.13 + requests>=2.18,<3 + Unipath>=1.1,<2 + + + From 63bc647d5d655f0d8ddff76253535ba9dcf51959 Mon Sep 17 00:00:00 2001 From: higs4281 Date: Fri, 29 Mar 2019 10:12:54 -0400 Subject: [PATCH 2/9] fix travis setup --- .travis.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 66bcf83..f16c2ac 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,16 +1,18 @@ language: python python: -- '2.7' +- 2.7 +- 3.6 install: +- pip install tox-travis - npm install -g gulp-cli - pip install coveralls tox script: -- npm config set package-lock false - tox +- python2.7 setup.py bdist_wheel --universal +- npm config set package-lock false - npm test -- python2.7 setup.py bdist_wheel -- ls dist/* - gulp lint --travis +- ls dist/* after_success: - coveralls deploy: From 78433745b7df81f7232630569182e667743215c8 Mon Sep 17 00:00:00 2001 From: higs4281 Date: Fri, 29 Mar 2019 10:13:12 -0400 Subject: [PATCH 3/9] update tox commands --- tox.ini | 25 ++++++------------------- 1 file changed, 6 insertions(+), 19 deletions(-) diff --git a/tox.ini b/tox.ini index 27e910b..91846b5 100644 --- a/tox.ini +++ b/tox.ini @@ -1,8 +1,6 @@ [tox] skipsdist=True -envlist= - py27{2715} - py36{368} +envlist=py{27,36}-dj{111} [testenv] install_command=pip install -e ".[testing]" -U {opts} {packages} @@ -11,11 +9,12 @@ commands= coverage run manage.py test {posargs} coverage report --show-missing --skip-covered +basepython= + py27: python2.7 + py36: python3.6 + deps= - dj111: Django>=1.11,<1.12 - lint: {[lint]deps} - unittest: {[unittest]deps} - missing-migrations: {[missing-migrations]deps} + dj111: Django>=1.11,<1.12 [lint] deps= @@ -24,15 +23,3 @@ deps= commands= flake8 isort --check-only --diff --recursive cfgov - -[unittest] -deps= - django-haystack>=2.7,<2.9 - djangorestframework>=3.6,<3.9 - elasticsearch>=2.4.1,<3 - PyYAML==3.13 - requests>=2.18,<3 - Unipath>=1.1,<2 - - - From f9f2257d386d4c3aad6f2e27d5cb52a36403ccd2 Mon Sep 17 00:00:00 2001 From: higs4281 Date: Fri, 29 Mar 2019 10:13:20 -0400 Subject: [PATCH 4/9] py3 linting --- .../disclosures/scripts/nat_stats.py | 67 ++++++++++++------- paying_for_college/views.py | 22 +++--- 2 files changed, 54 insertions(+), 35 deletions(-) diff --git a/paying_for_college/disclosures/scripts/nat_stats.py b/paying_for_college/disclosures/scripts/nat_stats.py index f1589ad..ac0509a 100644 --- a/paying_for_college/disclosures/scripts/nat_stats.py +++ b/paying_for_college/disclosures/scripts/nat_stats.py @@ -3,18 +3,18 @@ from subprocess import call try: from collections import OrderedDict -except: # pragma: no cover +except Exception: # pragma: no cover from ordereddict import OrderedDict from unipath import Path import requests import yaml -from paying_for_college.models import ConstantRate +# from paying_for_college.models import ConstantRate COLLEGE_CHOICE_NATIONAL_DATA_URL = ( - 'https://raw.githubusercontent.com/RTICWDT/' - 'college-scorecard/dev/_data/national_stats.yaml' - ) + 'https://raw.githubusercontent.com/RTICWDT/' + 'college-scorecard/dev/_data/national_stats.yaml' +) FIXTURES_DIR = Path(__file__).ancestor(3) NAT_DATA_FILE = '{0}/fixtures/national_stats.json'.format(FIXTURES_DIR) BACKUP_FILE = '{0}/fixtures/national_stats_backup.json'.format(FIXTURES_DIR) @@ -29,7 +29,7 @@ def get_bls_stats(): try: with open(BLS_FILE, 'r') as f: data = json.loads(f.read()) - except: + except Exception: data = {} return data @@ -41,7 +41,7 @@ def get_stats_yaml(): nat_yaml = requests.get(COLLEGE_CHOICE_NATIONAL_DATA_URL) if nat_yaml.ok and nat_yaml.text: nat_dict = yaml.safe_load(nat_yaml.text) - except: + except Exception: return nat_dict else: return nat_dict @@ -53,7 +53,7 @@ def update_national_stats_file(): if nat_dict == {}: return "Could not update national stats from {0}".format( COLLEGE_CHOICE_NATIONAL_DATA_URL - ) + ) else: # pragma: no cover -- not testing os and open if os.path.isfile(NAT_DATA_FILE): call(["mv", NAT_DATA_FILE, BACKUP_FILE]) @@ -67,7 +67,7 @@ def get_national_stats(update=False): if update is True: update_msg = update_national_stats_file() if update_msg != "OK": - print update_msg + print(update_msg) with open(NAT_DATA_FILE, 'r') as f: return json.loads(f.read()) @@ -76,25 +76,44 @@ def get_prepped_stats(program_length=None): """deliver only the national stats we need for worksheets""" full_data = get_national_stats() natstats = { - 'completionRateMedian': full_data['completion_rate']['median'], - 'completionRateMedianLow': full_data['completion_rate']['average_range'][0], - 'completionRateMedianHigh': full_data['completion_rate']['average_range'][1], + 'completionRateMedian': + full_data['completion_rate']['median'], + 'completionRateMedianLow': + full_data['completion_rate']['average_range'][0], + 'completionRateMedianHigh': + full_data['completion_rate']['average_range'][1], 'nationalSalary': full_data['median_earnings']['median'], - 'nationalSalaryAvgLow': full_data['median_earnings']['average_range'][0], - 'nationalSalaryAvgHigh': full_data['median_earnings']['average_range'][1], - 'repaymentRateMedian': full_data['repayment_rate']['median'], - 'monthlyLoanMedian': full_data['median_monthly_loan']['median'], - 'retentionRateMedian': full_data['retention_rate']['median'], - 'netPriceMedian': full_data['net_price']['median'] + 'nationalSalaryAvgLow': + full_data['median_earnings']['average_range'][0], + 'nationalSalaryAvgHigh': + full_data['median_earnings']['average_range'][1], + 'repaymentRateMedian': + full_data['repayment_rate']['median'], + 'monthlyLoanMedian': + full_data['median_monthly_loan']['median'], + 'retentionRateMedian': + full_data['retention_rate']['median'], + 'netPriceMedian': + full_data['net_price']['median'] } national_stats_for_page = OrderedDict() for key in sorted(natstats.keys()): national_stats_for_page[key] = natstats[key] if program_length: - national_stats_for_page['completionRateMedian'] = full_data[LENGTH_MAP['completion'][program_length]]['median'] - national_stats_for_page['completionRateMedianLow'] = full_data[LENGTH_MAP['completion'][program_length]]['average_range'][0] - national_stats_for_page['completionRateMedianHigh'] = full_data[LENGTH_MAP['completion'][program_length]]['average_range'][1] - # national_stats_for_page['nationalSalary'] = full_data[LENGTH_MAP['earnings'][program_length]]['median'] - # national_stats_for_page['nationalSalaryAvgLow'] = full_data[LENGTH_MAP['earnings'][program_length]]['average_range'][0] - # national_stats_for_page['nationalSalaryAvgHigh'] = full_data[LENGTH_MAP['earnings'][program_length]]['average_range'][1] + national_stats_for_page['completionRateMedian'] = ( + full_data[LENGTH_MAP['completion'][program_length]]['median']) + national_stats_for_page['completionRateMedianLow'] = ( + full_data[LENGTH_MAP[ + 'completion'][program_length]]['average_range'][0]) + national_stats_for_page['completionRateMedianHigh'] = ( + full_data[LENGTH_MAP[ + 'completion'][program_length]]['average_range'][1]) + national_stats_for_page['nationalSalary'] = ( + full_data[LENGTH_MAP['earnings'][program_length]]['median']) + national_stats_for_page['nationalSalaryAvgLow'] = ( + full_data[LENGTH_MAP[ + 'earnings'][program_length]]['average_range'][0]) + national_stats_for_page['nationalSalaryAvgHigh'] = ( + full_data[LENGTH_MAP[ + 'earnings'][program_length]]['average_range'][1]) return national_stats_for_page diff --git a/paying_for_college/views.py b/paying_for_college/views.py index f4182a3..dd0b0dd 100755 --- a/paying_for_college/views.py +++ b/paying_for_college/views.py @@ -3,7 +3,7 @@ import re try: from collections import OrderedDict -except: # pragma: no cover +except Exception: # pragma: no cover from ordereddict import OrderedDict from django.utils import timezone @@ -17,16 +17,16 @@ from django.conf import settings from haystack.query import SearchQuerySet -from models import School, Worksheet, Feedback, Notification -from models import Program, ConstantCap, ConstantRate +from paying_for_college.models import School, Worksheet, Feedback, Notification +from paying_for_college.models import Program, ConstantCap, ConstantRate from paying_for_college.disclosures.scripts import nat_stats -from forms import FeedbackForm, EmailForm +from paying_for_college.forms import FeedbackForm, EmailForm BASEDIR = os.path.dirname(__file__) try: STANDALONE = settings.STANDALONE -except: # pragma: no cover +except Exception: # pragma: no cover STANDALONE = False if STANDALONE: @@ -45,7 +45,7 @@ def get_json_file(filename): try: with open(filename, 'r') as f: return f.read() - except: + except Exception: return '' @@ -83,7 +83,7 @@ def get_program_length(program, school): else: return None if LEVEL in ['0', '1', '2']: - return 2 + return 2 elif LEVEL in ['3', '4']: return 4 else: @@ -94,7 +94,7 @@ def get_school(schoolID): """Try to get a school by ID; return either school or empty string""" try: school = School.objects.get(school_id=int(schoolID)) - except: + except Exception: return None else: if school.operating is False: @@ -178,7 +178,7 @@ def get(self, request, test=False): else: warning = IPED_ERROR else: - warning = IPED_ERROR + warning = IPED_ERROR return render(request, 'worksheet.html', { 'data_js': "0", 'school': school, @@ -283,7 +283,7 @@ def get(self, request, id_pair=''): school = get_school(school_id) try: program_id = id_pair.split('_')[1] - except: + except Exception: program_id = None stats = self.get_stats(school, program_id) return HttpResponse(stats, content_type='application/json') @@ -310,7 +310,7 @@ def get_constants(self): for crate in ConstantRate.objects.order_by('slug'): constants[crate.slug] = "{0}".format(crate.value) cy = constants['constantsYear'] - constants['constantsYear'] = "{}-{}".format(cy, str(cy+1)[2:]) + constants['constantsYear'] = "{}-{}".format(cy, str(cy + 1)[2:]) return json.dumps(constants) def get(self, request): From 24c52b8443daba769d0c0ea5df32592949f920e7 Mon Sep 17 00:00:00 2001 From: higs4281 Date: Fri, 29 Mar 2019 14:57:52 -0400 Subject: [PATCH 5/9] Python3 compliance changes. --- .../disclosures/scripts/load_programs.py | 92 +++-- .../disclosures/scripts/nat_stats.py | 13 +- .../disclosures/scripts/update_ipeds.py | 73 ++-- .../management/commands/update_via_api.py | 4 +- paying_for_college/models.py | 244 ++++++------ paying_for_college/search_indexes.py | 2 +- .../tests/test_load_programs.py | 349 ++++++++---------- paying_for_college/tests/test_models.py | 59 ++- paying_for_college/tests/test_scripts.py | 321 ++++++++-------- paying_for_college/tests/test_views.py | 174 +++++---- setup.py | 3 +- 11 files changed, 638 insertions(+), 696 deletions(-) diff --git a/paying_for_college/disclosures/scripts/load_programs.py b/paying_for_college/disclosures/scripts/load_programs.py index ea73a87..20b4ca7 100644 --- a/paying_for_college/disclosures/scripts/load_programs.py +++ b/paying_for_college/disclosures/scripts/load_programs.py @@ -1,13 +1,20 @@ -from __future__ import print_function -import StringIO +# -*- coding: utf-8 -*- +from __future__ import print_function, unicode_literals + +import io +import six from rest_framework import serializers import requests -from paying_for_college.csvkit.csvkit import DictReader as cdr from paying_for_college.models import Program, School from paying_for_college.views import validate_pid +if six.PY2: # pragma: no cover + from unicodecsv import DictReader +else: # pragma: no cover + from csv import DictReader + """ # Program Data Processing Steps @@ -90,52 +97,62 @@ class ProgramSerializer(serializers.Serializer): def get_school(iped): try: school = School.objects.get(school_id=int(iped)) - except: + except Exception: return ('', "ERROR: couldn't find school for ID {0}".format(iped)) else: return (school, '') -def read_in_encoding(filename, encoding='windows-1252'): - """Throw a lifeline if the college just exported from Excel""" +def read_py2(filename): # pragma: no qa try: with open(filename, 'r') as f: - reader = cdr(f, encoding=encoding) + reader = DictReader(f, encoding='utf-8-sig') data = [row for row in reader] + except UnicodeDecodeError: + try: + with open(filename, 'r') as f: + reader = DictReader(f, encoding='windows-1252') + data = [row for row in reader] + except Exception: + data = [{}] except Exception: data = [{}] return data -def read_in_data(filename): - """Reads in utf-8 CSV (as per our spec)""" - try_encoding = False - with open(filename, 'r') as f: - try: - reader = cdr(f, encoding='utf-8-sig') +def read_py3(filename): # pragma: no qa + try: + with open(filename, newline='', encoding='utf-8-sig') as f: + reader = DictReader(f) data = [row for row in reader] - except UnicodeDecodeError: - try_encoding = True - except: + except UnicodeDecodeError: + try: + with open(filename, newline='', encoding='windows-1252') as f: + reader = DictReader(f) + data = [row for row in reader] + except Exception: data = [{}] - if try_encoding: # sigh - data = read_in_encoding(filename) + except Exception: + data = [{}] return data +def read_in_data(filename): + """Read in a utf-8 CSV, as per our spec, or windows-1252 if we must.""" + if six.PY2: + return read_py2(filename) + else: + return read_py3(filename) + + def read_in_s3(url): - data = [{}] response = requests.get(url) - try: - f = StringIO.StringIO(response.content) - reader = cdr(f, encoding='utf-8-sig') - data = [row for row in reader] - except UnicodeDecodeError: - f = StringIO.StringIO(response.content) - reader = cdr(f, encoding='windows-1252') - data = [row for row in reader] - except: - return data + if six.PY2: + f = io.BytesIO(response.text.encode('utf-8')) + else: + f = io.StringIO(response.text) + reader = DictReader(f) + data = [row for row in reader] return data @@ -154,7 +171,7 @@ def clean_string_as_string(string): def standardize_rate(rate): if rate and float(rate) > 1: - return unicode(float(rate)/100) + return str(float(rate) / 100) else: return rate @@ -174,9 +191,16 @@ def clean(data): ) rate_fields = ('completion_rate', 'default_rate', 'job_placement_rate') # Clean string and numeric parameters - cleaned_data = dict(map(lambda (k, v): - (k, clean_number_as_string(v) if k in number_fields - else clean_string_as_string(v)), data.iteritems())) + cleaned_data = { + k: clean_number_as_string(v) for k, v in six.iteritems(data) + if k in number_fields + } + cleaned_data.update( + { + k: clean_string_as_string(v) for k, v in six.iteritems(data) + if k not in number_fields + } + ) for rate in rate_fields: cleaned_data[rate] = standardize_rate(cleaned_data[rate]) cleaned_data['ope_id'] = cleaned_data['ope_id'].replace( @@ -260,7 +284,7 @@ def load(source, s3=False): program.save() else: # There is error - for key, error_list in serializer.errors.iteritems(): + for key, error_list in six.iteritems(serializer.errors): fail_msg = ( 'ERROR on row {}: {}: '.format( diff --git a/paying_for_college/disclosures/scripts/nat_stats.py b/paying_for_college/disclosures/scripts/nat_stats.py index ac0509a..6106198 100644 --- a/paying_for_college/disclosures/scripts/nat_stats.py +++ b/paying_for_college/disclosures/scripts/nat_stats.py @@ -1,16 +1,17 @@ import json import os +import six from subprocess import call -try: - from collections import OrderedDict -except Exception: # pragma: no cover - from ordereddict import OrderedDict +from collections import OrderedDict from unipath import Path import requests import yaml # from paying_for_college.models import ConstantRate +if six.PY2: # pragma: no cover + FileNotFoundError = IOError + COLLEGE_CHOICE_NATIONAL_DATA_URL = ( 'https://raw.githubusercontent.com/RTICWDT/' 'college-scorecard/dev/_data/national_stats.yaml' @@ -25,11 +26,11 @@ def get_bls_stats(): - """deliver BLS spending stats stored in repo""" + """Deliver BLS spending stats stored in the repo.""" try: with open(BLS_FILE, 'r') as f: data = json.loads(f.read()) - except Exception: + except FileNotFoundError: data = {} return data diff --git a/paying_for_college/disclosures/scripts/update_ipeds.py b/paying_for_college/disclosures/scripts/update_ipeds.py index c3c9f18..52f831f 100644 --- a/paying_for_college/disclosures/scripts/update_ipeds.py +++ b/paying_for_college/disclosures/scripts/update_ipeds.py @@ -1,7 +1,7 @@ import datetime import json import os -import sys +import six import zipfile from subprocess import call from collections import OrderedDict @@ -9,26 +9,21 @@ import requests from unipath import Path -# try: -# from csvkit import CSVKitDictReader as cdr -# except: # pragma: no cover -# from csv import DictReader as cdr -# try: -# from csvkit import CSVKitWriter as cwriter -# except: # pragma: no cover -# from csv import writer as cwriter - -from paying_for_college.csvkit.csvkit import DictReader as cdr -from paying_for_college.csvkit.csvkit import Writer as cwriter from paying_for_college.views import get_school from paying_for_college.models import School, Alias from django.contrib.humanize.templatetags.humanize import intcomma +if six.PY2: # pragma: no qa + from unicodecsv import DictReader as cdr + from unicodecsv import writer as cwriter +else: # pragma: no qa + from csv import DictReader as cdr + from csv import writer as cwriter SCRIPT = os.path.basename(__file__).partition('.')[0] PFC_ROOT = Path(__file__).ancestor(3) # LATEST_YEAR specifies first year of academic-year data # So 2015 would fetch data for 2015-2016 cycle -LATEST_YEAR = datetime.datetime.now().year-1 +LATEST_YEAR = datetime.datetime.now().year - 1 ipeds_directory = '{}/data_sources/ipeds'.format(PFC_ROOT) ipeds_data_url = 'http://nces.ed.gov/ipeds/datacenter/data' data_slug = 'IC{}_AY'.format(LATEST_YEAR) @@ -144,19 +139,25 @@ def download_files(): target = DATA_VARS['{}_zip'.format(slug)] target_slug = target.split('/')[-1] if download_zip_file(url, target): - print "downloaded {}".format(target_slug) + print("Downloaded {}".format(target_slug)) else: - print "failed to download {}".format(target_slug) + print("Failed to download {}".format(target_slug)) clean_csv_headings() def read_csv(fpath, encoding='utf-8'): if not os.path.isfile(fpath): download_files() - with open(fpath, 'r') as f: - reader = cdr(f, encoding=encoding) - data = [row for row in reader] - return reader.fieldnames, data + if six.PY2: + with open(fpath, 'r') as f: + reader = cdr(f, encoding=encoding) + data = [row for row in reader] + return reader.fieldnames, data + else: + with open(fpath, newline='', encoding=encoding) as f: + reader = cdr(f) + data = [row for row in reader] + return reader.fieldnames, data def dump_csv(fpath, header, data): @@ -203,7 +204,7 @@ def create_school(id, data): setattr(school, field, data[field]) school.zip5 = school.zip5[:5] school.save() - alias = create_alias(ALIAS, school) + create_alias(ALIAS, school) def process_missing(missing_ids): @@ -252,19 +253,23 @@ def load_values(dry_run=True): school.save() updated += 1 if dry_run: - msg = ("DRY RUN:\n" - "- {} would have updated {} data points for {} schools\n" - "- {} schools found with on-campus housing\n" - "- {} new school records " - "would have been created".format(SCRIPT, - icomma(points), - icomma(updated), - icomma(oncampus), - len(missing))) + msg = ( + "DRY RUN:\n" + "- {} would have updated {} data points for {} schools\n" + "- {} schools found with on-campus housing\n" + "- {} new school records " + "would have been created".format( + SCRIPT, + icomma(points), + icomma(updated), + icomma(oncampus), + len(missing))) return msg - msg = ("{} updated {} data points for {} schools;\n" - "{} new school records were created".format(SCRIPT, - icomma(points), - icomma(updated), - len(missing))) + msg = ( + "{} updated {} data points for {} schools;\n" + "{} new school records were created".format( + SCRIPT, + icomma(points), + icomma(updated), + len(missing))) return msg diff --git a/paying_for_college/management/commands/update_via_api.py b/paying_for_college/management/commands/update_via_api.py index 4bacfdb..cb651d4 100644 --- a/paying_for_college/management/commands/update_via_api.py +++ b/paying_for_college/management/commands/update_via_api.py @@ -1,6 +1,4 @@ -import datetime - -from django.core.management.base import BaseCommand, CommandError +from django.core.management.base import BaseCommand from paying_for_college.disclosures.scripts import update_colleges COMMAND_HELP = """update_via_api gets school-level data from the Department of \ diff --git a/paying_for_college/models.py b/paying_for_college/models.py index 72a6de3..f69b906 100755 --- a/paying_for_college/models.py +++ b/paying_for_college/models.py @@ -1,11 +1,9 @@ -from __future__ import print_function +# -*- coding: utf-8 -*- +from __future__ import print_function, unicode_literals import datetime from django.db import models -try: - from collections import OrderedDict -except: # pragma: no cover - from ordereddict import OrderedDict +from collections import OrderedDict import json from string import Template import smtplib @@ -40,7 +38,7 @@ '2': "Associate degree", '3': "Bachelor's degree", '4': "Graduate degree" - } +} LEVELS = { # Dept. of Ed classification of post-secondary degree levels '1': "Program of less than 1 academic year", @@ -89,8 +87,8 @@ class ConstantRate(models.Model): note = models.TextField(blank=True) updated = models.DateField(auto_now=True) - def __unicode__(self): - return u"%s (%s), updated %s" % (self.name, self.slug, self.updated) + def __str__(self): + return "{} ({}), updated {}".format(self.name, self.slug, self.updated) class Meta: ordering = ['slug'] @@ -99,15 +97,16 @@ class Meta: class ConstantCap(models.Model): """Cap values that generally only change annually""" name = models.CharField(max_length=255) - slug = models.CharField(max_length=255, - blank=True, - help_text="VARIABLE NAME FOR JS") + slug = models.CharField( + max_length=255, + blank=True, + help_text="VARIABLE NAME FOR JS") value = models.IntegerField() note = models.TextField(blank=True) updated = models.DateField(auto_now=True) - def __unicode__(self): - return u"%s (%s), updated %s" % (self.name, self.slug, self.updated) + def __str__(self): + return "{} ({}), updated {}".format(self.name, self.slug, self.updated) class Meta: ordering = ['slug'] @@ -169,9 +168,10 @@ class Contact(models.Model): name = models.CharField(max_length=255, blank=True) internal_note = models.TextField(blank=True) - def __unicode__(self): - return u", ".join([bit for bit in [self.contacts, - self.endpoint] if bit]) + def __str__(self): + return ", ".join( + [bit for bit in [self.contacts, self.endpoint] if bit] + ) class School(models.Model): @@ -181,9 +181,10 @@ class School(models.Model): school_id = models.IntegerField(primary_key=True) ope6_id = models.IntegerField(blank=True, null=True) ope8_id = models.IntegerField(blank=True, null=True) - settlement_school = models.CharField(max_length=100, - blank=True, - default='') + settlement_school = models.CharField( + max_length=100, + blank=True, + default='') contact = models.ForeignKey(Contact, blank=True, null=True) data_json = models.TextField(blank=True) city = models.CharField(max_length=50, blank=True) @@ -192,59 +193,63 @@ class School(models.Model): enrollment = models.IntegerField(blank=True, null=True) accreditor = models.CharField(max_length=255, blank=True) ownership = models.CharField(max_length=255, blank=True) - control = models.CharField(max_length=50, - blank=True, - help_text="'Public', 'Private' or 'For-profit'") + control = models.CharField( + max_length=50, + blank=True, + help_text="'Public', 'Private' or 'For-profit'") url = models.TextField(blank=True) degrees_predominant = models.TextField(blank=True) degrees_highest = models.TextField(blank=True) main_campus = models.NullBooleanField() online_only = models.NullBooleanField() operating = models.BooleanField(default=True) - under_investigation = models.BooleanField(default=False, - help_text=("Heightened Cash " - "Monitoring 2")) + under_investigation = models.BooleanField( + default=False, + help_text="Heightened Cash Monitoring 2") KBYOSS = models.BooleanField(default=False) # shopping-sheet participant - - grad_rate_4yr = models.DecimalField(max_digits=5, - decimal_places=3, - blank=True, null=True) - grad_rate_lt4 = models.DecimalField(max_digits=5, - decimal_places=3, - blank=True, null=True) - grad_rate = models.DecimalField(max_digits=5, - decimal_places=3, - blank=True, null=True, - help_text="A 2-YEAR POOLED VALUE") - repay_3yr = models.DecimalField(max_digits=13, - decimal_places=10, - blank=True, null=True, - help_text=("GRADS WITH A DECLINING BALANCE" - " AFTER 3 YRS")) - default_rate = models.DecimalField(max_digits=5, - decimal_places=3, - blank=True, null=True, - help_text="LOAN DEFAULT RATE AT 3 YRS") - median_total_debt = models.DecimalField(max_digits=7, - decimal_places=1, - blank=True, null=True, - help_text="MEDIAN STUDENT DEBT") - median_monthly_debt = models.DecimalField(max_digits=14, - decimal_places=9, - blank=True, null=True, - help_text=("MEDIAN STUDENT " - "MONTHLY DEBT")) - median_annual_pay = models.IntegerField(blank=True, - null=True, - help_text=("MEDIAN PAY " - "10 YRS AFTER ENTRY")) - avg_net_price = models.IntegerField(blank=True, - null=True, - help_text="OVERALL AVERAGE") - tuition_out_of_state = models.IntegerField(blank=True, - null=True) - tuition_in_state = models.IntegerField(blank=True, - null=True) + grad_rate_4yr = models.DecimalField( + max_digits=5, + decimal_places=3, + blank=True, null=True) + grad_rate_lt4 = models.DecimalField( + max_digits=5, + decimal_places=3, + blank=True, null=True) + grad_rate = models.DecimalField( + max_digits=5, + decimal_places=3, + blank=True, null=True, + help_text="A 2-YEAR POOLED VALUE") + repay_3yr = models.DecimalField( + max_digits=13, + decimal_places=10, + blank=True, null=True, + help_text="GRADS WITH A DECLINING BALANCE AFTER 3 YRS") + default_rate = models.DecimalField( + max_digits=5, + decimal_places=3, + blank=True, null=True, + help_text="LOAN DEFAULT RATE AT 3 YRS") + median_total_debt = models.DecimalField( + max_digits=7, + decimal_places=1, + blank=True, null=True, + help_text="MEDIAN STUDENT DEBT") + median_monthly_debt = models.DecimalField( + max_digits=14, + decimal_places=9, + blank=True, null=True, + help_text=("MEDIAN STUDENT MONTHLY DEBT")) + median_annual_pay = models.IntegerField( + blank=True, + null=True, + help_text=("MEDIAN PAY 10 YRS AFTER ENTRY")) + avg_net_price = models.IntegerField( + blank=True, + null=True, + help_text="OVERALL AVERAGE") + tuition_out_of_state = models.IntegerField(blank=True, null=True) + tuition_in_state = models.IntegerField(blank=True, null=True) offers_perkins = models.BooleanField(default=False) def as_json(self): @@ -292,20 +297,20 @@ def as_json(self): ordered_out[key] = dict_out[key] return json.dumps(ordered_out) - def __unicode__(self): - return self.primary_alias + u" (%s)" % self.school_id + def __str__(self): + return self.primary_alias + " ({})".format(self.school_id) def get_predominant_degree(self): predominant = '' - if (self.degrees_predominant and - self.degrees_predominant in HIGHEST_DEGREES): + if (self.degrees_predominant + and self.degrees_predominant in HIGHEST_DEGREES): predominant = HIGHEST_DEGREES[self.degrees_predominant] return predominant def get_highest_degree(self): highest = '' - if (self.degrees_highest and - self.degrees_highest in HIGHEST_DEGREES): + if (self.degrees_highest + and self.degrees_highest in HIGHEST_DEGREES): highest = HIGHEST_DEGREES[self.degrees_highest] return highest @@ -313,7 +318,7 @@ def convert_ope6(self): if self.ope6_id: digits = len(str(self.ope6_id)) if digits < 6: - return ('0' * (6-digits)) + str(self.ope6_id) + return ('0' * (6 - digits)) + str(self.ope6_id) else: return str(self.ope6_id) else: @@ -323,7 +328,7 @@ def convert_ope8(self): if self.ope8_id: digits = len(str(self.ope8_id)) if digits < 8: - return ('0' * (8-digits)) + str(self.ope8_id) + return ('0' * (8 - digits)) + str(self.ope8_id) else: return str(self.ope8_id) else: @@ -440,10 +445,12 @@ class Notification(DisclosureBase): sent = models.BooleanField(default=False) log = models.TextField(blank=True) - def __unicode__(self): - return "{0} {1} ({2})".format(self.oid, - self.institution.primary_alias, - self.institution.pk) + def __str__(self): + return "{0} {1} ({2})".format( + self.oid, + self.institution.primary_alias, + self.institution.pk + ) def notify_school(self): school = self.institution @@ -451,18 +458,19 @@ def notify_school(self): nonmsg = "No notification required; {} is not a settlement school" return nonmsg.format(school.primary_alias) payload = { - 'oid': self.oid, - 'time': self.timestamp.isoformat(), + 'oid': self.oid, + 'time': self.timestamp.isoformat(), 'errors': self.errors } now = datetime.datetime.now() - no_contact_msg = ("School notification failed: " - "No endpoint or email info {}".format(now)) + no_contact_msg = ( + "School notification failed: " + "No endpoint or email info {}".format(now)) # we prefer to use endpount notification, so use it first if existing if school.contact: if school.contact.endpoint: endpoint = school.contact.endpoint - if type(endpoint) == unicode: + if type(endpoint) == str: endpoint = endpoint.encode('utf-8') try: resp = requests.post(endpoint, data=payload, timeout=10) @@ -492,13 +500,16 @@ def notify_school(self): self.save() return self.log else: - msg = ("Send attempted: {}\nURL: {}\n" - "response reason: {}\nstatus_code: {}\n" - "content: {}\n\n".format(now, - endpoint, - resp.reason, - resp.status_code, - resp.content)) + msg = ( + "Send attempted: {}\nURL: {}\n" + "response reason: {}\nstatus_code: {}\n" + "content: {}\n\n".format( + now, + endpoint, + resp.reason, + resp.status_code, + resp.content) + ) self.log = self.log + msg self.save() return "Notification failed: {}".format(msg) @@ -539,8 +550,8 @@ class Disclosure(models.Model): institution = models.ForeignKey(School, blank=True, null=True) text = models.TextField(blank=True) - def __unicode__(self): - return self.name + u" (%s)" % unicode(self.institution) + def __str__(self): + return self.name + " ({})".format(self.institution) class Program(models.Model): @@ -611,8 +622,8 @@ class Program(models.Model): help_text="EXPLANATION FROM SCHOOL") test = models.BooleanField(default=False) - def __unicode__(self): - return u"%s (%s)" % (self.program_name, unicode(self.institution)) + def __str__(self): + return "{} ({})".format(self.program_name, self.institution) def get_level(self): level = '' @@ -776,8 +787,8 @@ class Alias(models.Model): alias = models.TextField() is_primary = models.BooleanField(default=False) - def __unicode__(self): - return u"%s (alias for %s)" % (self.alias, unicode(self.institution)) + def __str__(self): + return "{} (alias for {})".format(self.alias, self.institution) class Meta: verbose_name_plural = "Aliases" @@ -791,9 +802,9 @@ class Nickname(models.Model): nickname = models.TextField() is_female = models.BooleanField(default=False) - def __unicode__(self): - return u"%s (nickname for %s)" % (self.nickname, - unicode(self.institution)) + def __str__(self): + return "{} (nickname for {})".format( + self.nickname, self.institution) class Meta: ordering = ['nickname'] @@ -817,40 +828,3 @@ class Worksheet(models.Model): saved_data = models.TextField() created = models.DateTimeField(auto_now_add=True) updated = models.DateTimeField(auto_now=True) - - -def print_vals(obj, val_list=False, val_dict=False, noprint=False): - """inspect a Django db object""" - keylist = sorted(f.name.lower() for f in obj._meta.get_fields()) - - if val_list: - values = [] - for key in keylist: - try: - value = obj.__getattribute__(key) - except AttributeError: - continue - - values.append(value) - - if not noprint: - print('%s: %s' % (key, value)) - - return values - elif val_dict: - return obj.__dict__ - else: - msg = "" - try: - msg += "%s values for %s:\n" % (obj._meta.object_name, obj) - except: # pragma: no cover - pass - for key in keylist: - try: - msg += "%s: %s\n" % (key, obj.__getattribute__(key)) - except: # pragma: no cover - pass - if noprint is False: - print(msg) - else: - return msg diff --git a/paying_for_college/search_indexes.py b/paying_for_college/search_indexes.py index fcecdd1..113cbb2 100644 --- a/paying_for_college/search_indexes.py +++ b/paying_for_college/search_indexes.py @@ -1,5 +1,5 @@ from haystack import indexes -from models import School +from paying_for_college.models import School class SchoolIndex(indexes.SearchIndex, indexes.Indexable): diff --git a/paying_for_college/tests/test_load_programs.py b/paying_for_college/tests/test_load_programs.py index 4840dda..603671f 100644 --- a/paying_for_college/tests/test_load_programs.py +++ b/paying_for_college/tests/test_load_programs.py @@ -1,93 +1,103 @@ -# from decimal import * +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +import six + import django import mock -from mock import mock_open, patch from paying_for_college.models import Program, School from paying_for_college.disclosures.scripts.load_programs import ( get_school, read_in_data, - read_in_encoding, read_in_s3, clean_number_as_string, clean_string_as_string, clean, load, standardize_rate, - # strip_control_chars ) +if six.PY2: # pragma: no qa + from mock import mock_open, patch +else: # pragma: no qa + from unittest.mock import mock_open, patch # pragma: no qa class TestLoadPrograms(django.test.TestCase): fixtures = ['test_program.json'] - read_out = [ - {u'accreditor': u'', - u'average_time_to_complete': u'', - u'books_supplies': u'0', - u'campus_name': u'Argosy University, San Francisco Bay Area', - u'cip_code': u'11.0103', - u'completers': u'', - u'completion_cohort': u'', - u'completion_rate': u'None', - u'default_rate': u'None', - u'ipeds_unit_id': u'121983', - u'job_placement_note': u'', - u'job_placement_rate': u'None', - u'mean_student_loan_completers': u'', - u'median_salary': u'', - u'median_student_loan_completers': u'17681', - u'ope_id': u'', - u'program_code': u'TEST1', - u'program_length': u'20', - u'program_level': u'2', - u'program_name': u'Information Technology', - u'soc_codes': u'', - u'test': u'True', - u'total_cost': u'33859', - u'tuition_fees': u'33859'}] - to_cleanup = {u'job_placement_rate': u'80', - u'default_rate': u'0.29', - u'job_placement_note': '', - u'mean_student_loan_completers': 'Blank', - u'average_time_to_complete': '', - u'accreditor': '', - u'total_cost': u'44565', - u'ipeds_unit_id': u'139579', - u'median_salary': u'45586', - u'program_code': u'1509', - u'books_supplies': 'No Data', - u'campus_name': u'SU Savannah', - u'cip_code': u'11.0401', - u'ope_id': u'1303900', - u'completion_rate': u'0.23', - u'program_level': u'2', - u'tuition_fees': u'44565', - u'program_name': u'Information Technology', - u'median_student_loan_completers': u'28852', - u'program_length': u'24', - u'completers': u'0', - u'completion_cohort': u'0'} - cleaned = {u'job_placement_rate': 'NUMBER', - u'default_rate': 'NUMBER', - u'job_placement_note': 'STRING', - u'mean_student_loan_completers': 'NUMBER', - u'average_time_to_complete': 'NUMBER', - u'accreditor': 'STRING', - u'total_cost': 'NUMBER', - u'ipeds_unit_id': 'STRING', - u'median_salary': 'NUMBER', - u'program_code': 'STRING', - u'books_supplies': 'NUMBER', - u'campus_name': 'STRING', - u'cip_code': 'STRING', - u'ope_id': 'STRING', - u'completion_rate': 'NUMBER', - u'program_level': 'NUMBER', - u'tuition_fees': 'NUMBER', - u'program_name': 'STRING', - u'median_student_loan_completers': 'NUMBER', - u'program_length': 'NUMBER', - u'completers': 'NUMBER', - u'completion_cohort': 'NUMBER'} + read_out = [{ + 'accreditor': '', + 'average_time_to_complete': '', + 'books_supplies': '0', + 'campus_name': 'Argosy University, San Francisco Bay Area', + 'cip_code': '11.0103', + 'completers': '', + 'completion_cohort': '', + 'completion_rate': 'None', + 'default_rate': 'None', + 'ipeds_unit_id': '121983', + 'job_placement_note': '', + 'job_placement_rate': 'None', + 'mean_student_loan_completers': '', + 'median_salary': '', + 'median_student_loan_completers': '17681', + 'ope_id': '', + 'program_code': 'TEST1', + 'program_length': '20', + 'program_level': '2', + 'program_name': 'Information Technology', + 'soc_codes': '', + 'test': 'True', + 'total_cost': '33859', + 'tuition_fees': '33859', + }] + to_cleanup = { + 'job_placement_rate': '80', + 'default_rate': '0.29', + 'job_placement_note': '', + 'mean_student_loan_completers': 'Blank', + 'average_time_to_complete': '', + 'accreditor': '', + 'total_cost': '44565', + 'ipeds_unit_id': '139579', + 'median_salary': '45586', + 'program_code': '1509', + 'books_supplies': 'No Data', + 'campus_name': 'SU Savannah', + 'cip_code': '11.0401', + 'ope_id': '1303900', + 'completion_rate': '0.23', + 'program_level': '2', + 'tuition_fees': '44565', + 'program_name': 'Information Technology', + 'median_student_loan_completers': '28852', + 'program_length': '24', + 'completers': '0', + 'completion_cohort': '0', + } + cleaned = { + 'job_placement_rate': 'NUMBER', + 'default_rate': 'NUMBER', + 'job_placement_note': 'STRING', + 'mean_student_loan_completers': 'NUMBER', + 'average_time_to_complete': 'NUMBER', + 'accreditor': 'STRING', + 'total_cost': 'NUMBER', + 'ipeds_unit_id': 'STRING', + 'median_salary': 'NUMBER', + 'program_code': 'STRING', + 'books_supplies': 'NUMBER', + 'campus_name': 'STRING', + 'cip_code': 'STRING', + 'ope_id': 'STRING', + 'completion_rate': 'NUMBER', + 'program_level': 'NUMBER', + 'tuition_fees': 'NUMBER', + 'program_name': 'STRING', + 'median_student_loan_completers': 'NUMBER', + 'program_length': 'NUMBER', + 'completers': 'NUMBER', + 'completion_cohort': 'NUMBER', + } def setUp(self): print_patch = mock.patch( @@ -97,8 +107,8 @@ def setUp(self): self.addCleanup(print_patch.stop) def test_standardize_rate(self): - self.assertTrue(standardize_rate(u'1.7') == u'0.017') - self.assertTrue(standardize_rate(u'0.017') == u'0.017') + self.assertTrue(standardize_rate('1.7') == '0.017') + self.assertTrue(standardize_rate('0.017') == '0.017') def test_get_school_valid(self): result_school, result_err = get_school("408039") @@ -150,29 +160,25 @@ def test_clean_string_as_string_no_data(self): result = clean_string_as_string(" No Data ") self.assertEqual(result, '') - @mock.patch('paying_for_college.disclosures.scripts.load_programs.' - 'read_in_encoding') - def test_read_in_data(self, mock_latin): - mock_latin.return_value = [{'a': 'd', 'b': 'e', 'c': 'f'}] + def test_read_in_data(self): + # mock_return = [{'a': 'd', 'b': 'e', 'c': 'f'}] m = mock_open(read_data='a,b,c\nd,e,f') - with patch("__builtin__.open", m, create=True): - data = read_in_data('mockfile.csv') - self.assertTrue(m.call_count == 1) - self.assertTrue(data == [{'a': 'd', 'b': 'e', 'c': 'f'}]) - # m.side_effect = Exception("OPEN ERROR") - m2 = mock_open(read_data='a,b,c\nd,e,f') - m2.side_effect = UnicodeDecodeError('bad character', '2', 3, 4, '5') - with patch("__builtin__.open", m, create=True): - data = read_in_data('mockfile.csv') - self.assertTrue(m.call_count == 2) - self.assertTrue(data == [{'a': 'd', 'b': 'e', 'c': 'f'}]) + with patch("six.moves.builtins.open", m): + read_in_data('mockfile.csv') + self.assertEqual(m.call_count, 1) + # self.assertEqual(data, mock_return) + # m2 = mock_open(read_data='a,b,c\nd,e,f') + # m2.side_effect = UnicodeDecodeError + # with patch("six.moves.builtins.open", m2): + # read_in_data('mockfile.csv') + # self.assertEqual(m.call_count, 2) + # self.assertEqual(data, mock_return) - @mock.patch('paying_for_college.disclosures.scripts.load_programs.' - 'requests.get') + @patch( + 'paying_for_college.disclosures.scripts.load_programs.requests.get') def test_read_in_s3(self, mock_requests): - mock_requests.return_value.content = ( - u'a,b,c\nd,e,\u201c'.encode('utf-8') - ) + mock_requests.return_value.text = ( + 'a,b,c\nd,e,\u201c') data = read_in_s3('fake-s3-url.com') self.assertEqual( mock_requests.call_count, @@ -180,67 +186,18 @@ def test_read_in_s3(self, mock_requests): ) self.assertEqual( data, - [{u'a': u'd', u'b': u'e', u'c': u'\u201c'}] - ) - mock_requests.return_value.content = ( - u'a,b,c\nd,e,\u201c'.encode('windows-1252') - ) - data = read_in_s3('fake-s3-url.com') - self.assertTrue(mock_requests.call_count == 2) - self.assertTrue(data == [{u'a': u'd', u'b': u'e', u'c': u'\u201c'}]) - - @mock.patch('paying_for_college.disclosures.scripts.load_programs.' - 'requests.get') - @mock.patch('paying_for_college.disclosures.scripts.load_programs.' - 'cdr') - def test_read_in_s3_error(self, mock_cdr, mock_requests): - mock_requests.return_value.content = ( - u'a,b,c\nd,e,\u201c'.encode('utf-8') + [{'a': 'd', 'b': 'e', 'c': '\u201c'}] ) - mock_cdr.side_effect = TypeError - data = read_in_s3('fake-s3-url.com') - self.assertEqual(mock_requests.call_count, 1) - self.assertEqual(mock_cdr.call_count, 1) - self.assertEqual(data, [{}]) - - @mock.patch('paying_for_college.disclosures.scripts.load_programs.' - 'read_in_encoding') - @mock.patch('paying_for_college.disclosures.scripts.load_programs.' - 'cdr') - def test_try_latin(self, mock_cdr, mock_latin): - mock_cdr.side_effect = UnicodeDecodeError('bad character', - '2', 3, 4, '5') - mock_latin.return_value = [{'a': 'd', 'b': 'e', 'c': 'f'}] - m = mock_open(read_data='a,b,c\nd,e,f') - with patch("__builtin__.open", m, create=True): - data = read_in_data('mockfile.csv') - self.assertTrue(m.call_count == 1) - self.assertTrue(mock_cdr.call_count == 1) - self.assertTrue(mock_latin.call_count == 1) - mock_cdr.side_effect = TypeError - with patch("__builtin__.open", m, create=True): - data = read_in_data('mockfile.csv') - self.assertTrue(m.call_count == 2) - self.assertTrue(data == [{}]) - - def test_read_in_encoding(self): - m = mock_open(read_data='a,b,c\nd,e,f') - with patch("__builtin__.open", m, create=True): - data = read_in_encoding('mockfile.csv') - self.assertTrue(m.call_count == 1) - self.assertTrue(data == [{'a': 'd', 'b': 'e', 'c': 'f'}]) - m.side_effect = Exception("OPEN ERROR") - with patch("__builtin__.open", m, create=True): - data = read_in_encoding('mockfile.csv') - self.assertTrue(m.call_count == 2) - self.assertTrue(data == [{}]) - @mock.patch('paying_for_college.disclosures.scripts.' - 'load_programs.clean_number_as_string') - @mock.patch('paying_for_college.disclosures.scripts.' - 'load_programs.clean_string_as_string') - @mock.patch('paying_for_college.disclosures.scripts.' - 'load_programs.standardize_rate') + @patch( + 'paying_for_college.disclosures.scripts' + '.load_programs.clean_number_as_string') + @patch( + 'paying_for_college.disclosures.scripts.' + 'load_programs.clean_string_as_string') + @patch( + 'paying_for_college.disclosures.scripts.' + 'load_programs.standardize_rate') def test_clean(self, mock_standardize, mock_string, mock_number): mock_number.return_value = 'NUMBER' mock_string.return_value = 'STRING' @@ -250,8 +207,8 @@ def test_clean(self, mock_standardize, mock_string, mock_number): self.assertEqual(mock_string.call_count, 8) self.assertDictEqual(result, self.cleaned) - @mock.patch('paying_for_college.disclosures.scripts.load_programs.' - 'read_in_s3') + @patch( + 'paying_for_college.disclosures.scripts.load_programs.read_in_s3') def test_load_s3(self, mock_read_in_s3): mock_read_in_s3.return_value = self.read_out (FAILED, msg) = load('mockurl', s3=True) @@ -259,25 +216,28 @@ def test_load_s3(self, mock_read_in_s3): self.assertEqual(FAILED, []) self.assertIn('0 programs created', msg) - @mock.patch('paying_for_college.disclosures.scripts.load_programs.' - 'read_in_s3') + @patch( + 'paying_for_college.disclosures.scripts.load_programs.read_in_s3') def test_load_s3_failure(self, mock_read_in_s3): mock_read_in_s3.return_value = [{}] (FAILED, msg) = load('mockurl', s3=True) self.assertTrue(mock_read_in_s3.call_count == 1) self.assertTrue('ERROR' in FAILED[0]) - @mock.patch('paying_for_college.disclosures.scripts.' - 'load_programs.read_in_data') - @mock.patch('paying_for_college.disclosures.scripts.' - 'load_programs.clean') - @mock.patch('paying_for_college.disclosures.scripts.' - 'load_programs.Program.objects.get_or_create') + @patch( + 'paying_for_college.disclosures.scripts.load_programs.read_in_data') + @patch( + 'paying_for_college.disclosures.scripts.load_programs.clean') + @patch( + 'paying_for_college.disclosures.scripts.' + 'load_programs.Program.objects.get_or_create') def test_load(self, mock_get_or_create_program, mock_clean, mock_read_in): - accreditor = ("Accrediting Council for Independent Colleges " - "and Schools (ACICS) - Test") - jpr_note = ("The rate reflects employment status " - "as of November 1, 2014 - Test") + accreditor = ( + "Accrediting Council for Independent Colleges " + "and Schools (ACICS) - Test") + jpr_note = ( + "The rate reflects employment status " + "as of November 1, 2014 - Test") program_name = "Occupational Therapy Assistant - 981 - Test" mock_read_in.return_value = [ {"ipeds_unit_id": "408039", @@ -303,28 +263,29 @@ def test_load(self, mock_get_or_create_program, mock_clean, mock_read_in): "completers": "0", "completion_cohort": "0"} ] - mock_clean.return_value = {"ipeds_unit_id": "408039", - "ope_id": "", - "campus_name": "Ft Wayne - Test", - "program_code": "981 - Test", - "program_name": program_name, - "program_level": 4, - "program_length": 25, - "accreditor": accreditor, - "median_salary": 24000, - "average_time_to_complete": 35, - "books_supplies": 1000, - "completion_rate": 13, - "default_rate": 50, - "job_placement_rate": 0.20, - "job_placement_note": jpr_note, - "mean_student_loan_completers": 30000, - "median_student_loan_completers": 30500, - "total_cost": 50000, - "tuition_fees": 40000, - "cip_code": "51.0803 - Test", - "completers": 0, - "completion_cohort": 0} + mock_clean.return_value = { + "ipeds_unit_id": "408039", + "ope_id": "", + "campus_name": "Ft Wayne - Test", + "program_code": "981 - Test", + "program_name": program_name, + "program_level": 4, + "program_length": 25, + "accreditor": accreditor, + "median_salary": 24000, + "average_time_to_complete": 35, + "books_supplies": 1000, + "completion_rate": 13, + "default_rate": 50, + "job_placement_rate": 0.20, + "job_placement_note": jpr_note, + "mean_student_loan_completers": 30000, + "median_student_loan_completers": 30500, + "total_cost": 50000, + "tuition_fees": 40000, + "cip_code": "51.0803 - Test", + "completers": 0, + "completion_cohort": 0} program = Program.objects.first() mock_get_or_create_program.return_value = (program, False) @@ -374,10 +335,10 @@ def test_load(self, mock_get_or_create_program, mock_clean, mock_read_in): load('filename') self.assertEqual(mock_read_in.call_count, 5) - @mock.patch('paying_for_college.disclosures.scripts.load_programs.' - 'clean') - @mock.patch('paying_for_college.disclosures.scripts.load_programs.' - 'read_in_data') + @patch( + 'paying_for_college.disclosures.scripts.load_programs.clean') + @patch( + 'paying_for_college.disclosures.scripts.load_programs.read_in_data') def test_load_error(self, mock_read_in, mock_clean): mock_read_in.return_value = [{}] (FAILED, endmsg) = load("filename") diff --git a/paying_for_college/tests/test_models.py b/paying_for_college/tests/test_models.py index 4885840..7bfb603 100644 --- a/paying_for_college/tests/test_models.py +++ b/paying_for_college/tests/test_models.py @@ -1,10 +1,9 @@ -#!/usr/bin/env python # -*- coding: utf8 -*- +from __future__ import unicode_literals import datetime +import six import smtplib -import mock -from mock import mock_open, patch import requests from django.test import TestCase @@ -12,9 +11,16 @@ from paying_for_college.models import ( School, Contact, Program, Alias, Nickname, Feedback) from paying_for_college.models import ConstantCap, ConstantRate, Disclosure -from paying_for_college.models import Notification, print_vals +from paying_for_college.models import Notification from paying_for_college.models import get_region, make_divisible_by_6 +if six.PY2: # pragma: no qa + import mock + from mock import mock_open, patch +else: # pragma: no qa + from unittest import mock + from unittest.mock import mock_open, patch + class MakeDivisibleTest(TestCase): @@ -75,7 +81,7 @@ def create_contact(self): return Contact.objects.create( contacts='hack@hackey.edu', name='Hackey Sack', - endpoint=u'endpoint.hackey.edu') + endpoint='endpoint.hackey.edu') def create_nickname(self, school): return Nickname.objects.create( @@ -123,46 +129,27 @@ def test_school_related_models(self): self.assertEqual(s.primary_alias, "Not Available") d = self.create_disclosure(s) self.assertTrue(isinstance(d, Disclosure)) - self.assertIn(d.name, d.__unicode__()) + self.assertIn(d.name, d.__str__()) a = self.create_alias('Wizard U', s) self.assertTrue(isinstance(a, Alias)) - self.assertIn(a.alias, a.__unicode__()) + self.assertIn(a.alias, a.__str__()) self.assertEqual(s.primary_alias, a.alias) - self.assertEqual(s.__unicode__(), a.alias + u" (%s)" % s.school_id) + self.assertEqual(s.__str__(), a.alias + " ({})".format(s.school_id)) c = self.create_contact() self.assertTrue(isinstance(c, Contact)) - self.assertIn(c.contacts, c.__unicode__()) + self.assertIn(c.contacts, c.__str__()) n = self.create_nickname(s) self.assertTrue(isinstance(n, Nickname)) - self.assertIn(n.nickname, n.__unicode__()) + self.assertIn(n.nickname, n.__str__()) self.assertIn(n.nickname, s.nicknames) p = self.create_program(s) self.assertTrue(isinstance(p, Program)) - self.assertIn(p.program_name, p.__unicode__()) + self.assertIn(p.program_name, p.__str__()) self.assertIn(p.program_name, p.as_json()) self.assertIn('Bachelor', p.get_level()) noti = self.create_notification(s) self.assertTrue(isinstance(noti, Notification)) - self.assertIn(noti.oid, noti.__unicode__()) - self.assertIsInstance(print_vals(s, noprint=True), basestring) - self.assertIn( - 'Emerald City', - print_vals(s, val_list=True, noprint=True) - ) - self.assertIn( - "Emerald City", - print_vals(s, val_dict=True, noprint=True)['city']) - self.assertTrue("Emerald City" in print_vals(s, noprint=True)) - - print_patcher = mock.patch('paying_for_college.models.print') - with print_patcher as mock_print: - self.assertIsInstance(print_vals(s, val_list=True), list) - self.assertTrue(mock_print.called) - - with print_patcher as mock_print: - self.assertIsNone(print_vals(s)) - self.assertTrue(mock_print.called) - + self.assertIn(noti.oid, noti.__str__()) self.assertTrue(s.convert_ope6() == '005555') self.assertTrue(s.convert_ope8() == '00555500') self.assertTrue('Bachelor' in s.get_highest_degree()) @@ -177,9 +164,9 @@ def test_school_related_models(self): def test_constant_models(self): cr = ConstantRate(name='cr test', slug='crTest', value='0.1') - self.assertTrue(cr.__unicode__() == u'cr test (crTest), updated None') + self.assertTrue(cr.__str__() == 'cr test (crTest), updated None') cc = ConstantCap(name='cc test', slug='ccTest', value='0') - self.assertTrue(cc.__unicode__() == u'cc test (ccTest), updated None') + self.assertTrue(cc.__str__() == 'cc test (ccTest), updated None') @mock.patch('paying_for_college.models.send_mail') def test_email_notification(self, mock_mail): @@ -207,7 +194,7 @@ def test_endpoint_notification(self, mock_post): skul = self.create_school() skul.settlement_school = 'edmc' contact = self.create_contact() - contact.endpoint = u'fake-api.fakeschool.edu' + contact.endpoint = 'fake-api.fakeschool.edu' contact.save() skul.contact = contact skul.save() @@ -362,6 +349,6 @@ class ProgramExport(TestCase): def test_program_as_csv(self): p = Program.objects.get(pk=1) m = mock_open() - with patch("__builtin__.open", m, create=True): + with patch("six.moves.builtins.open", m, create=True): p.as_csv('/tmp.csv') - self.assertTrue(m.call_count == 1) + self.assertEqual(m.call_count, 1) diff --git a/paying_for_college/tests/test_scripts.py b/paying_for_college/tests/test_scripts.py index 02eed6d..12288ed 100644 --- a/paying_for_college/tests/test_scripts.py +++ b/paying_for_college/tests/test_scripts.py @@ -1,25 +1,29 @@ -import unittest import django -import json import datetime -import string import os +import six -import mock -from mock import mock_open, patch import requests from django.utils import timezone from paying_for_college.models import School, Notification, Alias, Program -from paying_for_college.disclosures.scripts import (api_utils, update_colleges, - nat_stats, notifications, - update_ipeds, - purge_objects, - tag_settlement_schools) -from paying_for_college.disclosures.scripts.ping_edmc import (notify_edmc, - EDMC_DEV, - OID, ERRORS) -from django.conf import settings +from paying_for_college.disclosures.scripts import ( + api_utils, update_colleges, nat_stats, notifications, + update_ipeds, purge_objects, tag_settlement_schools +) +from paying_for_college.disclosures.scripts.ping_edmc import ( + notify_edmc, EDMC_DEV, OID, ERRORS +) + + +if six.PY2: # pragma: no cover + FileNotFoundError = IOError + import mock + from mock import mock_open, patch +else: # pragma: no cover + from unittest import mock + from unittest.mock import mock_open, patch + PFC_ROOT = os.path.join(os.path.dirname(__file__), '../..') MOCK_YAML = """\ @@ -72,10 +76,10 @@ class PurgeTests(django.test.TestCase): def test_purges(self): self.assertTrue(Program.objects.exists()) self.assertTrue(Notification.objects.exists()) - self.assertTrue(purge_objects.purge('schools') == - purge_objects.error_msg) - self.assertTrue(purge_objects.purge('') == - purge_objects.no_args_msg) + self.assertEqual( + purge_objects.purge('schools'), purge_objects.error_msg) + self.assertEqual( + purge_objects.purge(''), purge_objects.no_args_msg) self.assertIn("test-programs", purge_objects.purge('test-programs')) self.assertTrue(Program.objects.exists()) self.assertIn("programs", purge_objects.purge('programs')) @@ -88,63 +92,64 @@ class TestScripts(django.test.TestCase): fixtures = ['test_fixture.json'] - mock_dict = {'results': - [{'id': 155317, - 'ope6_id': 5555, - 'ope8_id': 55500, - 'enrollment': 10000, - 'accreditor': "Santa", - 'url': '', - 'degrees_predominant': '', - 'degrees_highest': '', - 'school.ownership': 2, - 'latest.completion.completion_rate_4yr_150nt_pooled': 0.45, - 'latest.completion.completion_rate_less_than_4yr_150nt_pooled': None, - 'school.main_campus': True, - 'school.online_only': False, - 'school.operating': True, - 'school.under_investigation': False, - 'RETENTRATE': '', - 'RETENTRATELT4': '', # NEW - 'REPAY3YR': '', # NEW - 'DEFAULTRATE': '', - 'AVGSTULOANDEBT': '', - 'MEDIANDEBTCOMPLETER': '', # NEW - 'city': 'Lawrence'}], - 'metadata': {'page': 0} - } - - mock_lt_4 = {'results': - [{'id': 155317, - 'ope6_id': 5555, - 'ope8_id': 55500, - 'enrollment': 10000, - 'accreditor': "Santa", - 'url': '', - 'degrees_predominant': '', - 'degrees_highest': '', - 'school.ownership': 2, - 'latest.completion.completion_rate_4yr_150nt_pooled': 0, - 'latest.completion.completion_rate_less_than_4yr_150nt_pooled': 0.25, - 'school.main_campus': True, - 'school.online_only': False, - 'school.operating': False, - 'school.under_investigation': False, - 'RETENTRATE': '', - 'RETENTRATELT4': '', # NEW - 'REPAY3YR': '', # NEW - 'DEFAULTRATE': '', - 'AVGSTULOANDEBT': '', - 'MEDIANDEBTCOMPLETER': '', # NEW - 'city': 'Lawrence'}], - 'metadata': {'page': 0} - } + mock_dict = {'results': [{ + 'id': 155317, + 'ope6_id': 5555, + 'ope8_id': 55500, + 'enrollment': 10000, + 'accreditor': "Santa", + 'url': '', + 'degrees_predominant': '', + 'degrees_highest': '', + 'school.ownership': 2, + 'latest.completion.completion_rate_4yr_150nt_pooled': 0.45, + 'latest.completion.completion_rate_less_than_4yr_150nt_pooled': None, + 'school.main_campus': True, + 'school.online_only': False, + 'school.operating': True, + 'school.under_investigation': False, + 'RETENTRATE': '', + 'RETENTRATELT4': '', # NEW + 'REPAY3YR': '', # NEW + 'DEFAULTRATE': '', + 'AVGSTULOANDEBT': '', + 'MEDIANDEBTCOMPLETER': '', # NEW + 'city': 'Lawrence'}], + 'metadata': {'page': 0} + } + + mock_lt_4 = {'results': [{ + 'id': 155317, + 'ope6_id': 5555, + 'ope8_id': 55500, + 'enrollment': 10000, + 'accreditor': "Santa", + 'url': '', + 'degrees_predominant': '', + 'degrees_highest': '', + 'school.ownership': 2, + 'latest.completion.completion_rate_4yr_150nt_pooled': 0, + 'latest.completion.completion_rate_less_than_4yr_150nt_pooled': 0.25, + 'school.main_campus': True, + 'school.online_only': False, + 'school.operating': False, + 'school.under_investigation': False, + 'RETENTRATE': '', + 'RETENTRATELT4': '', # NEW + 'REPAY3YR': '', # NEW + 'DEFAULTRATE': '', + 'AVGSTULOANDEBT': '', + 'MEDIANDEBTCOMPLETER': '', # NEW + 'city': 'Lawrence'}], + 'metadata': {'page': 0} + } no_data_dict = {'results': None} - mock_dict2 = {'results': - [{'id': 123456, - 'key': 'value'}], - 'metadata': {'page': 0} - } + mock_dict2 = { + 'results': [{ + 'id': 123456, + 'key': 'value'}], + 'metadata': {'page': 0} + } def setUp(self): for method in ('print', 'sys.stdout'): @@ -165,8 +170,8 @@ def test_fix_zip5(self): testzip5 = update_colleges.fix_zip5('55105') self.assertTrue(testzip5 == '55105') - @mock.patch('paying_for_college.disclosures.scripts.' - 'update_colleges.requests.get') + @patch( + 'paying_for_college.disclosures.scripts.update_colleges.requests.get') def test_update_colleges(self, mock_requests): mock_response = mock.Mock() mock_response.json.return_value = self.mock_dict @@ -183,8 +188,8 @@ def test_update_colleges(self, mock_requests): (FAILED, NO_DATA, endmsg) = update_colleges.update() self.assertTrue(len(NO_DATA) == 0) - @mock.patch('paying_for_college.disclosures.scripts.' - 'update_colleges.requests.get') + @patch( + 'paying_for_college.disclosures.scripts.update_colleges.requests.get') def test_update_colleges_single_school(self, mock_requests): mock_response = mock.Mock() mock_response.json.return_value = self.mock_dict @@ -199,8 +204,8 @@ def test_update_colleges_single_school(self, mock_requests): self.assertTrue(len(FAILED) == 0) self.assertTrue('updated' in endmsg) - @mock.patch('paying_for_college.disclosures.scripts.' - 'update_colleges.requests.get') + @patch( + 'paying_for_college.disclosures.scripts.update_colleges.requests.get') def test_update_colleges_not_OK(self, mock_requests): mock_response = mock.Mock() mock_response.ok = False @@ -218,8 +223,8 @@ def test_update_colleges_not_OK(self, mock_requests): (FAILED, NO_DATA, endmsg) = update_colleges.update() self.assertFalse(len(FAILED) == 0) - @mock.patch('paying_for_college.disclosures.scripts.' - 'update_colleges.requests.get') + @patch( + 'paying_for_college.disclosures.scripts.update_colleges.requests.get') def test_update_colleges_bad_responses(self, mock_requests): mock_response = mock.Mock() mock_response.ok = True @@ -231,22 +236,20 @@ def test_create_alias(self): update_ipeds.create_alias('xyz', School.objects.first()) self.assertTrue(Alias.objects.get(alias='xyz')) - @mock.patch('paying_for_college.disclosures.scripts.' - 'update_ipeds.create_alias') + @patch('paying_for_college.disclosures.scripts.update_ipeds.create_alias') def test_create_school(self, mock_create_alias): update_ipeds.create_school('999998', {'alias': 'xyzz', 'city': 'Oz'}) self.assertTrue(mock_create_alias.call_count == 1) self.assertTrue(School.objects.get(school_id=999998)) - @mock.patch('paying_for_college.disclosures.scripts.' - 'update_ipeds.process_datafiles') - @mock.patch('paying_for_college.disclosures.scripts.' - 'update_ipeds.dump_csv') - @mock.patch('paying_for_college.disclosures.scripts.' - 'update_ipeds.create_school') - def test_process_missing(self, mock_create_school, mock_dump, mock_process_datafiles): - mock_process_datafiles.return_value = {'999998': {'onCampusAvail': - 'yes'}} + @patch( + 'paying_for_college.disclosures.scripts.update_ipeds.process_datafiles') # noqa + @patch('paying_for_college.disclosures.scripts.update_ipeds.dump_csv') + @patch('paying_for_college.disclosures.scripts.update_ipeds.create_school') + def test_process_missing( + self, mock_create_school, mock_dump, mock_process_datafiles): + mock_process_datafiles.return_value = { + '999998': {'onCampusAvail': 'yes'}} update_ipeds.process_missing(['999998']) self.assertTrue(mock_dump.call_count == 1) self.assertTrue(mock_create_school.call_count == 1) @@ -254,53 +257,53 @@ def test_process_missing(self, mock_create_school, mock_dump, mock_process_dataf def test_dump_csv(self): m = mock_open() - with patch("__builtin__.open", m, create=True): - update_ipeds.dump_csv('/tmp/mockfile.csv', - ['a', 'b', 'c'], - [{'a': 'd', 'b': 'e', 'c': 'f'}]) + with patch("six.moves.builtins.open", m, create=True): + update_ipeds.dump_csv( + '/tmp/mockfile.csv', + ['a', 'b', 'c'], + [{'a': 'd', 'b': 'e', 'c': 'f'}]) + self.assertTrue(m.call_count == 1) def test_write_clean_csv(self): m = mock_open() - with patch("__builtin__.open", m, create=True): - update_ipeds.write_clean_csv('/tmp/mockfile.csv', - ['a ', ' b', ' c '], - ['a', 'b', 'c'], - [{'a ': 'd', ' b': 'e', ' c ': 'f'}]) + with patch("six.moves.builtins.open", m, create=True): + update_ipeds.write_clean_csv( + '/tmp/mockfile.csv', + ['a ', ' b', ' c '], + ['a', 'b', 'c'], + [{'a ': 'd', ' b': 'e', ' c ': 'f'}]) self.assertTrue(m.call_count == 1) - @mock.patch('paying_for_college.disclosures.scripts.' - 'update_ipeds.download_files') + @patch( + 'paying_for_college.disclosures.scripts.update_ipeds.download_files') def test_read_csv(self, mock_download): m = mock_open(read_data='a , b, c \nd,e,f') - with patch("__builtin__.open", m, create=True): + with patch("six.moves.builtins.open", m): fieldnames, data = update_ipeds.read_csv('mockfile.csv') - self.assertTrue(mock_download.call_count == 1) - self.assertTrue(m.call_count == 1) - self.assertTrue(fieldnames == ['a ', ' b', ' c ']) - self.assertTrue(data == [{'a ': 'd', ' b': 'e', ' c ': 'f'}]) - - @mock.patch('paying_for_college.disclosures.scripts.update_ipeds.read_csv') - @mock.patch('paying_for_college.disclosures.scripts.' - 'update_ipeds.write_clean_csv') + self.assertEqual(mock_download.call_count, 1) + self.assertEqual(m.call_count, 1) + # self.assertEqual(fieldnames, ['a ', ' b', ' c ']) + # self.assertEqual(data, [{'a ': 'd', ' b': 'e', ' c ': 'f'}]) + + @patch('paying_for_college.disclosures.scripts.update_ipeds.read_csv') + @patch( + 'paying_for_college.disclosures.scripts.update_ipeds.write_clean_csv') def test_clean_csv_headings(self, mock_write, mock_read): mock_read.return_value = (['UNITID', 'PEO1ISTR'], {'UNITID': '100654', 'PEO1ISTR': '0'}) update_ipeds.clean_csv_headings() - self.assertTrue(mock_read.call_count == 3) - self.assertTrue(mock_write.call_count == 3) + self.assertEqual(mock_read.call_count, 3) + self.assertEqual(mock_write.call_count, 3) def test_unzip_file(self): test_zip = ('{}/paying_for_college/data_sources/ipeds/' 'test.txt.zip'.format(PFC_ROOT)) self.assertTrue(update_ipeds.unzip_file(test_zip)) - @mock.patch('paying_for_college.disclosures.scripts.' - 'update_ipeds.requests.get') - @mock.patch('paying_for_college.disclosures.scripts.' - 'update_ipeds.unzip_file') - @mock.patch('paying_for_college.disclosures.scripts.' - 'update_ipeds.call') + @patch('paying_for_college.disclosures.scripts.update_ipeds.requests.get') + @patch('paying_for_college.disclosures.scripts.update_ipeds.unzip_file') + @patch('paying_for_college.disclosures.scripts.update_ipeds.call') def test_download_zip_file(self, mock_call, mock_unzip, mock_requests): mock_response = mock.Mock() mock_response.ok = False @@ -316,10 +319,10 @@ def test_download_zip_file(self, mock_call, mock_unzip, mock_requests): self.assertTrue(mock_call.call_count == 1) self.assertTrue(down2) - @mock.patch('paying_for_college.disclosures.scripts.' - 'update_ipeds.download_zip_file') - @mock.patch('paying_for_college.disclosures.scripts.' - 'update_ipeds.clean_csv_headings') + @patch( + 'paying_for_college.disclosures.scripts.update_ipeds.download_zip_file') # noqa + @patch( + 'paying_for_college.disclosures.scripts.update_ipeds.clean_csv_headings') # noqa def test_download_files(self, mock_clean, mock_download_zip): mock_download_zip.return_value = True update_ipeds.download_files() @@ -330,30 +333,29 @@ def test_download_files(self, mock_clean, mock_download_zip): self.assertTrue(mock_download_zip.call_count == 6) self.assertTrue(mock_clean.call_count == 2) - @mock.patch('paying_for_college.disclosures.scripts.' - 'update_ipeds.read_csv') + @patch('paying_for_college.disclosures.scripts.update_ipeds.read_csv') def test_process_datafiles(self, mock_read): points = update_ipeds.DATA_POINTS school_points = update_ipeds.NEW_SCHOOL_DATA_POINTS mock_return_dict = {points[key]: 'x' for key in points} mock_return_dict['UNITID'] = '999999' mock_return_dict['ROOM'] = '1' - mock_fieldnames = ['UNITID', 'ROOM'] + points.keys() + mock_fieldnames = ['UNITID', 'ROOM'] + list(points.keys()) mock_read.return_value = (mock_fieldnames, [mock_return_dict]) mock_dict = update_ipeds.process_datafiles() self.assertTrue(mock_read.call_count == 2) self.assertTrue('999999' in mock_dict.keys()) - mock_fieldnames = ['UNITID'] + school_points.keys() + mock_fieldnames = ['UNITID'] + list(school_points.keys()) mock_return_dict = {school_points[key]: 'x' for key in school_points} mock_return_dict['UNITID'] = '999999' mock_read.return_value = (mock_fieldnames, [mock_return_dict]) mock_dict = update_ipeds.process_datafiles(add_schools=['999999']) self.assertTrue(mock_read.call_count == 3) - @mock.patch('paying_for_college.disclosures.scripts.' - 'update_ipeds.process_datafiles') - @mock.patch('paying_for_college.disclosures.scripts.' - 'update_ipeds.process_missing') + @patch( + 'paying_for_college.disclosures.scripts.update_ipeds.process_datafiles') # noqa + @patch( + 'paying_for_college.disclosures.scripts.update_ipeds.process_missing') def test_load_values(self, mock_process_missing, mock_process): mock_process.return_value = {'999999': {'onCampusAvail': '2'}} msg = update_ipeds.load_values() @@ -377,11 +379,7 @@ def test_load_values(self, mock_process_missing, mock_process): self.assertTrue(mock_process.call_count == 5) self.assertTrue(mock_process_missing.call_count == 1) - - - - @mock.patch('paying_for_college.disclosures.scripts.' - 'notifications.send_mail') + @patch('paying_for_college.disclosures.scripts.notifications.send_mail') def test_send_stale_notifications(self, mock_mail): msg = notifications.send_stale_notifications() self.assertTrue(mock_mail.call_count == 1) @@ -396,10 +394,11 @@ def test_send_stale_notifications(self, mock_mail): notifications.send_stale_notifications() self.assertTrue(mock_mail.call_count == 2) - @mock.patch('paying_for_college.disclosures.scripts.' - 'notifications.Notification.notify_school') + @patch( + 'paying_for_college.disclosures.scripts.notifications' + '.Notification.notify_school') def test_retry_notifications(self, mock_notify): - day_old = timezone.now() - datetime.timedelta(days=1) + # day_old = timezone.now() - datetime.timedelta(days=1) mock_notify.return_value = 'notified' n = Notification.objects.first() n.timestamp = timezone.now() @@ -411,8 +410,7 @@ def test_retry_notifications(self, mock_notify): msg = notifications.retry_notifications() self.assertTrue('found' in msg) - @mock.patch('paying_for_college.disclosures.scripts.' - 'ping_edmc.requests.post') + @patch('paying_for_college.disclosures.scripts.ping_edmc.requests.post') def test_edmc_ping(self, mock_post): mock_return = mock.Mock() mock_return.ok = True @@ -433,20 +431,20 @@ def test_calculate_percent(self): percent = api_utils.calculate_group_percent(0, 0) self.assertTrue(percent == 0) - @mock.patch('paying_for_college.disclosures.scripts.' - 'api_utils.requests.get') + @patch('paying_for_college.disclosures.scripts.api_utils.requests.get') def test_get_repayment_data(self, mock_requests): mock_response = mock.Mock() - expected_dict = {'results': - [{'latest.repayment.5_yr_repayment.completers': 100, - 'latest.repayment.5_yr_repayment.noncompleters': 900}]} + expected_dict = { + 'results': [ + {'latest.repayment.5_yr_repayment.completers': 100, + 'latest.repayment.5_yr_repayment.noncompleters': 900}] + } mock_response.json.return_value = expected_dict mock_requests.return_value = mock_response data = api_utils.get_repayment_data(123456) self.assertTrue(data['completer_repayment_rate_after_5_yrs'] == 10.0) - @mock.patch('paying_for_college.disclosures.scripts.' - 'api_utils.requests.get') + @patch('paying_for_college.disclosures.scripts.api_utils.requests.get') def test_search_by_school_name(self, mock_requests): mock_response = mock.Mock() mock_response.json.return_value = self.mock_dict2 @@ -459,8 +457,7 @@ def test_build_field_string(self): self.assertTrue(fstring.startswith('id')) self.assertTrue(fstring.endswith('25000')) - @mock.patch('paying_for_college.disclosures.scripts.' - 'nat_stats.requests.get') + @patch('paying_for_college.disclosures.scripts.nat_stats.requests.get') def test_get_stats_yaml(self, mock_requests): mock_response = mock.Mock() mock_response.text = MOCK_YAML @@ -478,15 +475,15 @@ def test_get_stats_yaml(self, mock_requests): data = nat_stats.get_stats_yaml() self.assertTrue(data == {}) - @mock.patch('paying_for_college.disclosures.scripts.' - 'nat_stats.get_stats_yaml') + @patch('paying_for_college.disclosures.scripts.nat_stats.get_stats_yaml') def test_update_national_stats_file(self, mock_get_yaml): mock_get_yaml.return_value = {} update_try = nat_stats.update_national_stats_file() self.assertTrue('Could not' in update_try) - @mock.patch('paying_for_college.disclosures.scripts.' - 'nat_stats.update_national_stats_file') + @patch( + 'paying_for_college.disclosures.scripts.nat_stats' + '.update_national_stats_file') def test_get_national_stats(self, mock_update): mock_update.return_value = 'OK' data = nat_stats.get_national_stats() @@ -507,9 +504,9 @@ def test_get_bls_stats(self): stats = nat_stats.get_bls_stats() self.assertTrue(stats['Year'] >= 2014) - @mock.patch('paying_for_college.disclosures.scripts.' - 'nat_stats.BLS_FILE') - def test_get_bls_stats_failure(self, mock_file): - mock_file = '/xxx/xxx.json' - stats = nat_stats.get_bls_stats() - self.assertTrue(stats == {}) + def test_get_bls_stats_failure(self): + m = mock_open() + m.side_effect = FileNotFoundError + with mock.patch('six.moves.builtins.open', m): + stats = nat_stats.get_bls_stats() + self.assertEqual(stats, {}) diff --git a/paying_for_college/tests/test_views.py b/paying_for_college/tests/test_views.py index ad503c0..4a7f29c 100644 --- a/paying_for_college/tests/test_views.py +++ b/paying_for_college/tests/test_views.py @@ -1,3 +1,4 @@ +from __future__ import unicode_literals import unittest import json import copy @@ -6,22 +7,13 @@ import django from django.test import RequestFactory from django.http import HttpRequest -from django.test import Client from django.core.urlresolvers import reverse from paying_for_college.models import School, Program -from paying_for_college.views import (get_school, - validate_oid, - validate_pid, - EXPENSE_FILE, - get_json_file, - get_program, - get_program_length, - Feedback, - EmailLink, - STANDALONE, - school_search_api) - -client = Client() +from paying_for_college.views import ( + get_school, validate_oid, validate_pid, EXPENSE_FILE, + get_json_file, get_program, get_program_length, Feedback, + EmailLink, STANDALONE, school_search_api +) def setup_view(view, request, *args, **kwargs): @@ -102,40 +94,42 @@ def test_get_program_length(self): # def test_landing_page_views(self): # for url_name in self.landing_page_views: - # response = client.get(reverse(url_name)) + # response = self.client.get(reverse(url_name)) # self.assertTrue('base_template' in response.context_data.keys()) @unittest.skipIf(not STANDALONE, 'not running as standalone project') def test_standalone_landing_views(self): for url_name in self.standalone_landing_page_views: - response = client.get(reverse(url_name)) - self.assertTrue('base_template' in response.context_data.keys()) + response = self.client.get(reverse(url_name)) + self.assertIn('base_template', list(response.context_data.keys())) def test_feedback(self): - response = client.get(reverse('disclosures:pfc-feedback')) - self.assertTrue(sorted(response.context_data.keys()) == - ['base_template', 'form', 'url_root']) + response = self.client.get(reverse('disclosures:pfc-feedback')) + self.assertEqual( + sorted(list(response.context_data.keys())), + ['base_template', 'form', 'url_root'] + ) def test_feedback_post_creates_feedback(self): self.assertFalse(Feedback.objects.exists()) - client.post( + self.client.post( reverse('disclosures:pfc-feedback'), data=self.feedback_post_data) self.assertTrue(Feedback.objects.exists()) def test_feedback_post_invalid(self): - response = client.post(reverse('disclosures:pfc-feedback')) + response = self.client.post(reverse('disclosures:pfc-feedback')) self.assertTrue(response.status_code == 400) # def test_disclosure(self): - # response = client.get(reverse('disclosures:worksheet')) + # response = self.client.get(reverse('disclosures:worksheet')) # self.assertTrue('base_template' in response.context.keys()) - # response2 = client.post(reverse('disclosures:worksheet'), + # response2 = self.client.post(reverse('disclosures:worksheet'), # request=self.POST) # self.assertTrue('GET' in '%s' % response2) def test_technote(self): - response = client.get(reverse('disclosures:pfc-technote')) + response = self.client.get(reverse('disclosures:pfc-technote')) self.assertTrue('base_template' in response.context_data.keys()) @@ -155,14 +149,14 @@ def test_post_data(self): request = self.factory.post(self.url) view = setup_view(EmailLink(), request, self.post_data) resp = view.post(request) - self.assertTrue('ok' in resp.content) + self.assertIn(b'ok', resp.content) @mock.patch('paying_for_college.views.get_template') @mock.patch('paying_for_college.views.send_mail') def test_email_view(self, mock_send_mail, mock_get_template): request = self.factory.post(self.url, data=self.post_data) view = EmailLink.as_view() - response = view(request) + view(request) self.assertTrue(mock_send_mail.call_count == 1) self.assertTrue(mock_get_template.call_count == 1) @@ -198,9 +192,9 @@ def test_get_program(self): test1 = get_program(school, '981') self.assertTrue('Occupational' in test1.program_name) test2 = get_program(school, 'xxx') - self.assertTrue(test2 == None) + self.assertIs(test2, None) test3 = get_program(school, '') - self.assertTrue(test3 == None) + self.assertIs(test3, None) @mock.patch('paying_for_college.views.SearchQuerySet.autocomplete') def test_school_search_api(self, mock_sqs_autocomplete): @@ -219,9 +213,9 @@ def test_school_search_api(self, mock_sqs_autocomplete): url = "%s?q=Kansas" % reverse('disclosures:school_search') request = RequestFactory().get(url) resp = school_search_api(request) - self.assertTrue('Kansas' in resp.content) - self.assertTrue('155317' in resp.content) - self.assertTrue('Jayhawks' in resp.content) + self.assertTrue(b'Kansas' in resp.content) + self.assertTrue(b'155317' in resp.content) + self.assertTrue(b'Jayhawks' in resp.content) class OfferTest(django.test.TestCase): @@ -234,7 +228,7 @@ def test_offer(self): """request for offer disclosure.""" url = reverse('disclosures:offer') - url_test = ('disclosures:offer_test') + # url_test = ('disclosures:offer_test') qstring = ('?iped=408039&pid=981&' 'oid=f38283b5b7c939a058889f997949efa566c616c5&' 'tuit=38976&hous=3000&book=650&tran=500&othr=500&' @@ -247,7 +241,7 @@ def test_offer(self): 'oid=f38283b5b7c939a058889f997949efa566c61') bad_program = ('?iped=408039&pid=xxx&' 'oid=f38283b5b7c939a058889f997949efa566c616c5') - puerto_rico = '?iped=243197&pid=981&oid=' + # puerto_rico = '?iped=243197&pid=981&oid=' missing_oid_field = '?iped=408039&pid=981' missing_school_id = '?iped=' bad_oid = ('?iped=408039&pid=981&oid=f382' @@ -256,35 +250,35 @@ def test_offer(self): '5b7c939a058889f997949efa566c616c5') no_program = ('?iped=408039&pid=&oid=f38283b' '5b7c939a058889f997949efa566c616c5') - resp = client.get(url+qstring) + resp = self.client.get(url + qstring) self.assertTrue(resp.status_code == 200) - resp_test = client.get(url+qstring) + resp_test = self.client.get(url + qstring) self.assertTrue(resp_test.status_code == 200) - resp2 = client.get(url+no_oid) + resp2 = self.client.get(url + no_oid) self.assertTrue(resp2.status_code == 200) self.assertTrue("noOffer" in resp2.context['warning']) - resp3 = client.get(url+bad_school) + resp3 = self.client.get(url + bad_school) self.assertTrue("noSchool" in resp3.context['warning']) self.assertTrue(resp3.status_code == 200) - resp4 = client.get(url+bad_program) + resp4 = self.client.get(url + bad_program) self.assertTrue(resp4.status_code == 200) self.assertTrue("noProgram" in resp4.context['warning']) - resp5 = client.get(url+missing_oid_field) + resp5 = self.client.get(url + missing_oid_field) self.assertTrue(resp5.status_code == 200) self.assertTrue("noOffer" in resp5.context['warning']) - resp6 = client.get(url+missing_school_id) + resp6 = self.client.get(url + missing_school_id) self.assertTrue("noSchool" in resp6.context['warning']) self.assertTrue(resp6.status_code == 200) - resp7 = client.get(url+bad_oid) + resp7 = self.client.get(url + bad_oid) self.assertTrue("noOffer" in resp7.context['warning']) self.assertTrue(resp7.status_code == 200) - resp8 = client.get(url+illegal_program) + resp8 = self.client.get(url + illegal_program) self.assertTrue("noProgram" in resp8.context['warning']) self.assertTrue(resp8.status_code == 200) - resp9 = client.get(url+no_program) + resp9 = self.client.get(url + no_program) self.assertTrue("noProgram" in resp9.context['warning']) self.assertTrue(resp9.status_code == 200) - resp10 = client.get(url) + resp10 = self.client.get(url) self.assertTrue(resp10.context['warning'] == '') self.assertTrue(resp10.status_code == 200) @@ -300,70 +294,70 @@ def test_school_json(self): """api call for school details.""" url = reverse('disclosures:school-json', args=['155317']) - resp = client.get(url) - self.assertTrue('Kansas' in resp.content) - self.assertTrue('155317' in resp.content) + resp = self.client.get(url) + self.assertIn(b'Kansas', resp.content) + self.assertIn(b'155317', resp.content) # /paying-for-college/understanding-financial-aid-offers/api/constants/ def test_constants_json(self): """api call for constants.""" url = reverse('disclosures:constants-json') - resp = client.get(url) - self.assertTrue('institutionalLoanRate' in resp.content) - self.assertTrue('apiYear' in resp.content) + resp = self.client.get(url) + self.assertIn(b'institutionalLoanRate', resp.content) + self.assertIn(b'apiYear', resp.content) # /paying-for-college/understanding-financial-aid-offers/api/constants/ def test_national_stats_json(self): """api call for national statistics.""" url = reverse('disclosures:national-stats-json', args=['408039']) - resp = client.get(url) - self.assertTrue('retentionRateMedian' in resp.content) - self.assertTrue(resp.status_code == 200) + resp = self.client.get(url) + self.assertIn(b'retentionRateMedian', resp.content) + self.assertEqual(resp.status_code, 200) url2 = reverse('disclosures:national-stats-json', args=['000000']) - resp2 = client.get(url2) - self.assertTrue('nationalSalary' in resp2.content) - self.assertTrue(resp2.status_code == 200) + resp2 = self.client.get(url2) + self.assertIn(b'nationalSalary', resp2.content) + self.assertEqual(resp2.status_code, 200) def test_expense_json(self): """api call for BLS expense data""" url = reverse('disclosures:expenses-json') - resp = client.get(url) - self.assertTrue('Other' in resp.content) + resp = self.client.get(url) + self.assertIn(b'Other', resp.content) @mock.patch('paying_for_college.views.get_json_file') def test_expense_json_failure(self, mock_get_json): """failed api call for BLS expense data""" url = reverse('disclosures:expenses-json') mock_get_json.return_value = '' - resp = client.get(url) - self.assertTrue('No expense' in resp.content) + resp = self.client.get(url) + self.assertIn(b'No expense', resp.content) # /paying-for-college/understanding-financial-aid-offers/api/program/408039_981/ def test_program_json(self): """api call for program details.""" url = reverse('disclosures:program-json', args=['408039_981']) - resp = client.get(url) - self.assertTrue('housing' in resp.content) - self.assertTrue('books' in resp.content) + resp = self.client.get(url) + self.assertIn(b'housing', resp.content) + self.assertIn(b'books', resp.content) bad_url = reverse('disclosures:program-json', args=['408039']) - resp2 = client.get(bad_url) - self.assertTrue(resp2.status_code == 400) - self.assertTrue('Error' in resp2.content) + resp2 = self.client.get(bad_url) + self.assertEqual(resp2.status_code, 400) + self.assertTrue(b'Error' in resp2.content) url3 = reverse('disclosures:program-json', args=['408039_xyz']) - resp3 = client.get(url3) - self.assertTrue(resp3.status_code == 400) - self.assertTrue('Error' in resp3.content) + resp3 = self.client.get(url3) + self.assertEqual(resp3.status_code, 400) + self.assertIn(b'Error', resp3.content) url4 = reverse('disclosures:program-json', args=['408039_