diff --git a/backend/audit/fixtures/json/federal-awards--test0001test--simple-pass.json b/backend/audit/fixtures/json/federal-awards--test0001test--simple-pass.json index 27e92099d5..36574b0de5 100644 --- a/backend/audit/fixtures/json/federal-awards--test0001test--simple-pass.json +++ b/backend/audit/fixtures/json/federal-awards--test0001test--simple-pass.json @@ -7,17 +7,15 @@ "program": { "federal_agency_prefix": "42", "three_digit_extension": "RD", - "additional_award_identification": 1234, + "additional_award_identification": "1234", "program_name": "FLOWER DREAMING", "is_major": "N", - "audit_report_type": "", "number_of_audit_findings": 0, "amount_expended": 5000000, "federal_program_total": 5000000 }, "loan_or_loan_guarantee": { - "is_guaranteed": "N", - "loan_balance_at_audit_period_end": 0 + "is_guaranteed": "N" }, "direct_or_indirect_award": { "is_direct": "N", diff --git a/backend/audit/models/models.py b/backend/audit/models/models.py index 17f1b021cf..371fd0d2b1 100644 --- a/backend/audit/models/models.py +++ b/backend/audit/models/models.py @@ -8,6 +8,7 @@ from django.conf import settings from django.contrib.auth import get_user_model from django.contrib.postgres.fields import ArrayField +from django.core.exceptions import ValidationError from django.utils.translation import gettext_lazy as _ from django.utils import timezone as django_timezone @@ -15,6 +16,7 @@ from django_fsm import FSMField, RETURN_VALUE, transition import audit.cross_validation +from audit.cross_validation.naming import SECTION_NAMES from audit.intake_to_dissemination import IntakeToDissemination from audit.validators import ( validate_additional_ueis_json, @@ -34,6 +36,7 @@ validate_audit_information_json, validate_component_page_numbers, ) +from audit.utils import FORM_SECTION_HANDLERS from support.cog_over import compute_cog_over, record_cog_assignment from .submission_event import SubmissionEvent @@ -430,18 +433,19 @@ def validate_full(self): """ Full validation, intended for use when the user indicates that the submission is finished. - - Currently a stub, but eventually will call each of the individual - section validation routines and then validate_cross. """ + cross_result = self.validate_cross() + individual_result = self.validate_individually() + full_result = {} - validation_methods = [] - errors = [f(self) for f in validation_methods] - - if errors: - return {"errors": errors} + if "errors" in cross_result: + full_result = cross_result + if "errors" in individual_result: + full_result["errors"].extend(individual_result["errors"]) + elif "errors" in individual_result: + full_result = individual_result - return self.validate_cross() + return full_result def validate_cross(self): """ @@ -469,6 +473,35 @@ def validate_cross(self): return {"errors": errors, "data": shaped_sac} return {} + def validate_individually(self): + """ + Runs the individual workbook validations, returning generic errors as a + list of strings. Ignores workbooks that haven't been uploaded yet. + """ + errors = [] + result = {} + + for section, section_handlers in FORM_SECTION_HANDLERS.items(): + validation_method = section_handlers["validator"] + section_name = section_handlers["field_name"] + audit_data = getattr(self, section_name) + + try: + validation_method(audit_data) + except ValidationError as err: + # err.error_list will be [] if the workbook wasn't uploaded yet + if err.error_list: + errors.append( + { + "error": f"The {SECTION_NAMES[section_name].friendly} workbook contains validation errors and will need to be re-uploaded. This is likely caused by changes made to our validations in the time since it was originally uploaded." + } + ) + + if errors: + result = {"errors": errors} + + return result + @transition( field="submission_status", source=STATUS.IN_PROGRESS, diff --git a/backend/audit/test_views.py b/backend/audit/test_views.py index c1ea421ea4..3e060dcd30 100644 --- a/backend/audit/test_views.py +++ b/backend/audit/test_views.py @@ -293,11 +293,12 @@ def test_post_redirect(self): "audit_information": _fake_audit_information(), "federal_awards": _load_json(AUDIT_JSON_FIXTURES / awardsfile), "general_information": _load_json(AUDIT_JSON_FIXTURES / geninfofile), - "submission_status": STATUSES.AUDITEE_CERTIFIED, + "submission_status": STATUSES.IN_PROGRESS, # Temporarily required for SAR creation below } sac_data["notes_to_sefa"]["NotesToSefa"]["accounting_policies"] = "Exhaustive" sac_data["notes_to_sefa"]["NotesToSefa"]["is_minimis_rate_used"] = "Y" sac_data["notes_to_sefa"]["NotesToSefa"]["rate_explained"] = "At great length" + sac_data["report_id"] = _mock_gen_report_id() user, sac = _make_user_and_sac(**sac_data) required_statuses = ( @@ -311,8 +312,10 @@ def test_post_redirect(self): sac.transition_name.append(rs) sac.transition_date.append(datetime.now(timezone.utc)) - sac.save() + baker.make(SingleAuditReportFile, sac=sac) baker.make(Access, user=user, sac=sac, role="certifying_auditee_contact") + sac.submission_status = STATUSES.AUDITEE_CERTIFIED + sac.save() response = _authed_post( Client(), @@ -373,6 +376,9 @@ def test_ready_for_certification(self): "federal_awards": _load_json(AUDIT_JSON_FIXTURES / awardsfile), "general_information": _load_json(AUDIT_JSON_FIXTURES / geninfofile), } + sac_data["notes_to_sefa"]["NotesToSefa"]["accounting_policies"] = "Exhaustive" + sac_data["notes_to_sefa"]["NotesToSefa"]["is_minimis_rate_used"] = "Y" + sac_data["notes_to_sefa"]["NotesToSefa"]["rate_explained"] = "At great length" sac = SingleAuditChecklist.objects.get(report_id=report_id) for field, value in sac_data.items(): @@ -427,6 +433,9 @@ def test_unlock_after_certification(self): "federal_awards": _load_json(AUDIT_JSON_FIXTURES / awardsfile), "general_information": _load_json(AUDIT_JSON_FIXTURES / geninfofile), } + sac_data["notes_to_sefa"]["NotesToSefa"]["accounting_policies"] = "Exhaustive" + sac_data["notes_to_sefa"]["NotesToSefa"]["is_minimis_rate_used"] = "Y" + sac_data["notes_to_sefa"]["NotesToSefa"]["rate_explained"] = "At great length" sac = SingleAuditChecklist.objects.get(report_id=report_id) for field, value in sac_data.items(): @@ -564,11 +573,12 @@ def test_submission(self): "audit_information": _fake_audit_information(), "federal_awards": _load_json(AUDIT_JSON_FIXTURES / awardsfile), "general_information": _load_json(AUDIT_JSON_FIXTURES / geninfofile), - "submission_status": STATUSES.AUDITEE_CERTIFIED, + "submission_status": STATUSES.IN_PROGRESS, # Temporarily required for SAR creation below } sac_data["notes_to_sefa"]["NotesToSefa"]["accounting_policies"] = "Exhaustive" sac_data["notes_to_sefa"]["NotesToSefa"]["is_minimis_rate_used"] = "Y" sac_data["notes_to_sefa"]["NotesToSefa"]["rate_explained"] = "At great length" + sac_data["report_id"] = _mock_gen_report_id() user, sac = _make_user_and_sac(**sac_data) required_statuses = ( @@ -582,9 +592,10 @@ def test_submission(self): sac.transition_name.append(rs) sac.transition_date.append(datetime.now(timezone.utc)) - sac.save() - + baker.make(SingleAuditReportFile, sac=sac) baker.make(Access, sac=sac, user=user, role="certifying_auditee_contact") + sac.submission_status = STATUSES.AUDITEE_CERTIFIED + sac.save() kwargs = {"report_id": sac.report_id} _authed_post(self.client, user, "audit:Submission", kwargs=kwargs) diff --git a/backend/audit/utils.py b/backend/audit/utils.py index 8997e796d9..e984364e5c 100644 --- a/backend/audit/utils.py +++ b/backend/audit/utils.py @@ -1,5 +1,27 @@ from django.conf import settings +from audit.fixtures.excel import FORM_SECTIONS +from audit.intakelib import ( + extract_additional_ueis, + extract_additional_eins, + extract_federal_awards, + extract_corrective_action_plan, + extract_audit_findings_text, + extract_audit_findings, + extract_secondary_auditors, + extract_notes_to_sefa, +) +from audit.validators import ( + validate_additional_ueis_json, + validate_additional_eins_json, + validate_corrective_action_plan_json, + validate_federal_award_json, + validate_findings_text_json, + validate_findings_uniform_guidance_json, + validate_notes_to_sefa_json, + validate_secondary_auditors_json, +) + class Util: @staticmethod @@ -63,3 +85,47 @@ def __init__( def __str__(self): return f"{self.message} (Error Key: {self.error_key})" + + +FORM_SECTION_HANDLERS = { + FORM_SECTIONS.FEDERAL_AWARDS: { + "extractor": extract_federal_awards, + "field_name": "federal_awards", + "validator": validate_federal_award_json, + }, + FORM_SECTIONS.CORRECTIVE_ACTION_PLAN: { + "extractor": extract_corrective_action_plan, + "field_name": "corrective_action_plan", + "validator": validate_corrective_action_plan_json, + }, + FORM_SECTIONS.FINDINGS_UNIFORM_GUIDANCE: { + "extractor": extract_audit_findings, + "field_name": "findings_uniform_guidance", + "validator": validate_findings_uniform_guidance_json, + }, + FORM_SECTIONS.FINDINGS_TEXT: { + "extractor": extract_audit_findings_text, + "field_name": "findings_text", + "validator": validate_findings_text_json, + }, + FORM_SECTIONS.ADDITIONAL_UEIS: { + "extractor": extract_additional_ueis, + "field_name": "additional_ueis", + "validator": validate_additional_ueis_json, + }, + FORM_SECTIONS.ADDITIONAL_EINS: { + "extractor": extract_additional_eins, + "field_name": "additional_eins", + "validator": validate_additional_eins_json, + }, + FORM_SECTIONS.SECONDARY_AUDITORS: { + "extractor": extract_secondary_auditors, + "field_name": "secondary_auditors", + "validator": validate_secondary_auditors_json, + }, + FORM_SECTIONS.NOTES_TO_SEFA: { + "extractor": extract_notes_to_sefa, + "field_name": "notes_to_sefa", + "validator": validate_notes_to_sefa_json, + }, +} diff --git a/backend/audit/views/views.py b/backend/audit/views/views.py index 32bbbef97e..3ac9ed51f7 100644 --- a/backend/audit/views/views.py +++ b/backend/audit/views/views.py @@ -15,16 +15,7 @@ from audit.fixtures.excel import FORM_SECTIONS, UNKNOWN_WORKBOOK -from audit.intakelib import ( - extract_additional_ueis, - extract_additional_eins, - extract_federal_awards, - extract_corrective_action_plan, - extract_audit_findings_text, - extract_audit_findings, - extract_secondary_auditors, - extract_notes_to_sefa, -) + from audit.forms import ( AuditorCertificationStep1Form, AuditorCertificationStep2Form, @@ -46,17 +37,10 @@ ) from audit.intakelib.exceptions import ExcelExtractionError from audit.validators import ( - validate_additional_ueis_json, - validate_additional_eins_json, validate_auditee_certification_json, validate_auditor_certification_json, - validate_corrective_action_plan_json, - validate_federal_award_json, - validate_findings_text_json, - validate_findings_uniform_guidance_json, - validate_notes_to_sefa_json, - validate_secondary_auditors_json, ) +from audit.utils import FORM_SECTION_HANDLERS from dissemination.remove_workbook_artifacts import remove_workbook_artifacts from dissemination.file_downloads import get_download_url, get_filename @@ -126,49 +110,6 @@ def get(self, request, *args, **kwargs): class ExcelFileHandlerView(SingleAuditChecklistAccessRequiredMixin, generic.View): - FORM_SECTION_HANDLERS = { - FORM_SECTIONS.FEDERAL_AWARDS: { - "extractor": extract_federal_awards, - "field_name": "federal_awards", - "validator": validate_federal_award_json, - }, - FORM_SECTIONS.CORRECTIVE_ACTION_PLAN: { - "extractor": extract_corrective_action_plan, - "field_name": "corrective_action_plan", - "validator": validate_corrective_action_plan_json, - }, - FORM_SECTIONS.FINDINGS_UNIFORM_GUIDANCE: { - "extractor": extract_audit_findings, - "field_name": "findings_uniform_guidance", - "validator": validate_findings_uniform_guidance_json, - }, - FORM_SECTIONS.FINDINGS_TEXT: { - "extractor": extract_audit_findings_text, - "field_name": "findings_text", - "validator": validate_findings_text_json, - }, - FORM_SECTIONS.ADDITIONAL_UEIS: { - "extractor": extract_additional_ueis, - "field_name": "additional_ueis", - "validator": validate_additional_ueis_json, - }, - FORM_SECTIONS.ADDITIONAL_EINS: { - "extractor": extract_additional_eins, - "field_name": "additional_eins", - "validator": validate_additional_eins_json, - }, - FORM_SECTIONS.SECONDARY_AUDITORS: { - "extractor": extract_secondary_auditors, - "field_name": "secondary_auditors", - "validator": validate_secondary_auditors_json, - }, - FORM_SECTIONS.NOTES_TO_SEFA: { - "extractor": extract_notes_to_sefa, - "field_name": "notes_to_sefa", - "validator": validate_notes_to_sefa_json, - }, - } - def _create_excel_file(self, file, sac_id, form_section): excel_file = ExcelFile( **{ @@ -194,7 +135,7 @@ def _event_type(self, form_section): }[form_section] def _extract_and_validate_data(self, form_section, excel_file, auditee_uei): - handler_info = self.FORM_SECTION_HANDLERS.get(form_section) + handler_info = FORM_SECTION_HANDLERS.get(form_section) if handler_info is None: logger.warning("No form section found with name %s", form_section) raise BadRequest() @@ -205,7 +146,7 @@ def _extract_and_validate_data(self, form_section, excel_file, auditee_uei): return audit_data def _save_audit_data(self, sac, form_section, audit_data): - handler_info = self.FORM_SECTION_HANDLERS.get(form_section) + handler_info = FORM_SECTION_HANDLERS.get(form_section) if handler_info is not None: setattr(sac, handler_info["field_name"], audit_data) sac.save() @@ -774,6 +715,16 @@ def post(self, request, *args, **kwargs): try: sac = SingleAuditChecklist.objects.get(report_id=report_id) + errors = sac.validate_full() + if errors: + context = {"report_id": report_id, "errors": errors} + + return render( + request, + "audit/cross-validation/cross-validation-results.html", + context, + ) + sac.transition_to_submitted() sac.save( event_user=request.user, event_type=SubmissionEvent.EventType.SUBMITTED diff --git a/backend/config/settings.py b/backend/config/settings.py index 1d740db156..482f98c77b 100644 --- a/backend/config/settings.py +++ b/backend/config/settings.py @@ -170,11 +170,6 @@ WSGI_APPLICATION = "config.wsgi.application" - -# Database -# https://docs.djangoproject.com/en/4.0/ref/settings/#databases - - POSTGREST = { "URL": env.str("POSTGREST_URL", "http://api:3000"), "LOCAL": env.str("POSTGREST_URL", "http://api:3000"), @@ -232,12 +227,22 @@ # Environment specific configurations DEBUG = False if ENVIRONMENT not in ["DEVELOPMENT", "PREVIEW", "STAGING", "PRODUCTION"]: - DATABASES = { "default": env.dj_db_url( "DATABASE_URL", default="postgres://postgres:password@0.0.0.0/backend" ), } + STORAGES = { + "default": { + "BACKEND": "report_submission.storages.S3PrivateStorage", + }, + "staticfiles": { + "BACKEND": "whitenoise.storage.CompressedManifestStaticFilesStorage" + }, + } + # Per whitenoise docs, insert into middleware list directly after Django + # security middleware: https://whitenoise.readthedocs.io/en/stable/django.html#enable-whitenoise + MIDDLEWARE.insert(1, "whitenoise.middleware.WhiteNoiseMiddleware") # Local environment and Testing environment (CI/CD/GitHub Actions) @@ -248,10 +253,6 @@ CORS_ALLOWED_ORIGINS += ["http://0.0.0.0:8000", "http://127.0.0.1:8000"] - STATICFILES_STORAGE = "whitenoise.storage.CompressedManifestStaticFilesStorage" - MIDDLEWARE.append("whitenoise.middleware.WhiteNoiseMiddleware") - DEFAULT_FILE_STORAGE = "report_submission.storages.S3PrivateStorage" - # Private bucket AWS_PRIVATE_STORAGE_BUCKET_NAME = "gsa-fac-private-s3" @@ -284,8 +285,14 @@ else: # One of the Cloud.gov environments - STATICFILES_STORAGE = "storages.backends.s3boto3.S3ManifestStaticStorage" - DEFAULT_FILE_STORAGE = "report_submission.storages.S3PrivateStorage" + STORAGES = { + "default": { + "BACKEND": "report_submission.storages.S3PrivateStorage", + }, + "staticfiles": { + "BACKEND": "storages.backends.s3boto3.S3ManifestStaticStorage", + }, + } vcap = json.loads(env.str("VCAP_SERVICES")) diff --git a/backend/manifests/vars/vars-staging.yml b/backend/manifests/vars/vars-staging.yml index de419aaac4..f35a00a90b 100644 --- a/backend/manifests/vars/vars-staging.yml +++ b/backend/manifests/vars/vars-staging.yml @@ -1,8 +1,8 @@ -app_name: gsa-fac -mem_amount: 2G -cf_env_name: STAGING -env_name: staging -service_name: staging -endpoint: fac-staging.app.cloud.gov -instances: 1 - +app_name: gsa-fac +mem_amount: 4G +cf_env_name: STAGING +env_name: staging +service_name: staging +endpoint: fac-staging.app.cloud.gov +instances: 1 +