2024-08-15 | MAIN --> PROD | DEV (212a90e) --> STAGING

.github/workflows/testing-from-build.yml

@@ -54,29 +54,61 @@ jobs:
             coverageFile: ./coverage.xml
             token: ${{ secrets.GITHUB_TOKEN }}
 
-  # a11y-testing:
-  #   runs-on: ubuntu-latest
-  #   env:
-  #     ENV: TESTING
-  #     SAM_API_KEY: ${{ secrets.SAM_API_KEY }}
-  #     DJANGO_BASE_URL: "http://localhost:8000"
-  #     DJANGO_SECRET_LOGIN_KEY: ${{ secrets.DJANGO_SECRET_LOGIN_KEY }}
-  #     LOGIN_CLIENT_ID: ${{ secrets.LOGIN_CLIENT_ID }}
-  #     SECRET_KEY: ${{ secrets.SECRET_KEY }}
-  #     ALLOWED_HOSTS: "0.0.0.0 127.0.0.1 localhost"
-  #     DISABLE_AUTH: True
-  #   steps:
-  #     - uses: actions/checkout@v4
-  #     - uses: actions/setup-node@v4
-  #       with:
-  #         node-version: 18
-  #     - name: Start services
-  #       working-directory: ./backend
-  #       run: |
-  #         touch .env
-  #         docker compose -f docker-compose.yml up -d
-  #     - name: Run A11y tests
-  #       working-directory: ./backend
-  #       run: |
-  #         sudo npm ci
-  #         npx cypress run --spec "cypress/e2e/accessibility.cy.js"
+  a11y-testing:
+    runs-on: ubuntu-latest
+    env:
+      ENV: TESTING
+      SAM_API_KEY: ${{ secrets.SAM_API_KEY }}
+      DJANGO_BASE_URL: "http://localhost:8000"
+      DJANGO_SECRET_LOGIN_KEY: ${{ secrets.DJANGO_SECRET_LOGIN_KEY }}
+      LOGIN_CLIENT_ID: ${{ secrets.LOGIN_CLIENT_ID }}
+      SECRET_KEY: ${{ secrets.SECRET_KEY }}
+      ALLOWED_HOSTS: "0.0.0.0 127.0.0.1 localhost"
+      DISABLE_AUTH: True
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+
+      - name: Install Node dependencies
+        working-directory: ./backend
+        run: npm ci
+
+      - name: Create .env file
+        working-directory: ./backend
+        run: touch .env
+
+      - name: Start services
+        working-directory: ./backend
+        run: docker compose -f docker-compose.yml up -d
+
+      - name: Perform first run steps
+        working-directory: ./backend
+        run: make docker-first-run
+
+      - name: Load fixtures
+        working-directory: ./backend
+        run: docker compose run web python manage.py load_fixtures
+
+      - name: Create materialized view
+        working-directory: ./backend
+        run: docker compose run web python manage.py materialized_views --create
+
+      - name: Run A11y tests
+        uses: cypress-io/github-action@v6
+        with:
+          browser: chrome
+          working-directory: ./backend
+          spec: cypress/e2e/accessibility.cy.js
+
+      - uses: actions/upload-artifact@v4
+        # add the line below to store screenshots only on failures
+        if: failure()
+        with:
+          name: cypress-screenshots
+          path: backend/cypress/screenshots
+          if-no-files-found: warn # 'warn' or 'error' are also available, defaults to `warn`

.github/workflows/testing-from-ghcr.yml

@@ -1,8 +1,8 @@
 ---
 name: Run Django and A11y Tests
 on:
-  workflow_dispatch:
   workflow_call:
+  workflow_dispatch:
 
 jobs:
   django-test:
@@ -56,29 +56,61 @@ jobs:
             coverageFile: ./coverage.xml
             token: ${{ secrets.GITHUB_TOKEN }}
 
-  # a11y-testing:
-  #   runs-on: ubuntu-latest
-  #   env:
-  #     ENV: TESTING
-  #     SAM_API_KEY: ${{ secrets.SAM_API_KEY }}
-  #     DJANGO_BASE_URL: "http://localhost:8000"
-  #     DJANGO_SECRET_LOGIN_KEY: ${{ secrets.DJANGO_SECRET_LOGIN_KEY }}
-  #     LOGIN_CLIENT_ID: ${{ secrets.LOGIN_CLIENT_ID }}
-  #     SECRET_KEY: ${{ secrets.SECRET_KEY }}
-  #     ALLOWED_HOSTS: "0.0.0.0 127.0.0.1 localhost"
-  #     DISABLE_AUTH: True
-  #   steps:
-  #     - uses: actions/checkout@v4
-  #     - uses: actions/setup-node@v4
-  #       with:
-  #         node-version: 16
-  #     - name: Start services
-  #       working-directory: ./backend
-  #       run: |
-  #         touch .env
-  #         docker compose -f docker-compose.yml up -d
-  #     - name: Run A11y tests
-  #       working-directory: ./backend
-  #       run: |
-  #         sudo npm ci
-  #         npx cypress run --spec "cypress/e2e/accessibility.cy.js"
+  a11y-testing:
+    runs-on: ubuntu-latest
+    env:
+      ENV: TESTING
+      SAM_API_KEY: ${{ secrets.SAM_API_KEY }}
+      DJANGO_BASE_URL: "http://localhost:8000"
+      DJANGO_SECRET_LOGIN_KEY: ${{ secrets.DJANGO_SECRET_LOGIN_KEY }}
+      LOGIN_CLIENT_ID: ${{ secrets.LOGIN_CLIENT_ID }}
+      SECRET_KEY: ${{ secrets.SECRET_KEY }}
+      ALLOWED_HOSTS: "0.0.0.0 127.0.0.1 localhost"
+      DISABLE_AUTH: True
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+
+      - name: Install Node dependencies
+        working-directory: ./backend
+        run: npm ci
+
+      - name: Create .env file
+        working-directory: ./backend
+        run: touch .env
+
+      - name: Start services
+        working-directory: ./backend
+        run: docker compose -f docker-compose-web.yml up -d
+
+      - name: Perform first run steps
+        working-directory: ./backend
+        run: make docker-first-run
+
+      - name: Load fixtures
+        working-directory: ./backend
+        run: docker compose run web python manage.py load_fixtures
+
+      - name: Create materialized view
+        working-directory: ./backend
+        run: docker compose run web python manage.py materialized_views --create
+
+      - name: Run A11y tests
+        uses: cypress-io/github-action@v6
+        with:
+          browser: chrome
+          working-directory: ./backend
+          spec: cypress/e2e/accessibility.cy.js
+
+      - uses: actions/upload-artifact@v4
+        # add the line below to store screenshots only on failures
+        if: failure()
+        with:
+          name: cypress-screenshots
+          path: backend/cypress/screenshots
+          if-no-files-found: warn # 'warn' or 'error' are also available, defaults to `warn`

backend/audit/templates/audit/upload-report.html

@@ -97,7 +97,7 @@ <h2 class="usa-process-list__heading">{% if already_submitted %}Re-upload{% else
                 </div>
 
                 <button class="usa-button margin-bottom-8 margin-top-5" id="continue">{% if already_submitted %}Re-upload{% else %}Upload{% endif %} Single Audit Package</button>
-                <a class="usa-button usa-button--unstyled margin-left-2" href="{% url 'audit:SubmissionProgress' report_id %}">Cancel</a>
+                <a class="usa-button margin-left-2" href="{% url 'audit:SubmissionProgress' report_id %}">Cancel</a>
             </form>
         </div>
     </div>

backend/audit/views/views.py

@@ -58,6 +58,7 @@
     validate_secondary_auditors_json,
 )
 
+from dissemination.remove_workbook_artifacts import remove_workbook_artifacts
 from dissemination.file_downloads import get_download_url, get_filename
 from dissemination.models import General
 
@@ -741,6 +742,8 @@ def post(self, request, *args, **kwargs):
                     event_user=request.user,
                     event_type=SubmissionEvent.EventType.DISSEMINATED,
                 )
+                # Remove workbook artifacts after the report has been disseminated.
+                remove_workbook_artifacts(sac)
             else:
                 pass
                 # FIXME: We should now provide a reasonable error to the user.

backend/dev-requirements.txt

@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with Python 3.11
+# This file is autogenerated by pip-compile with Python 3.12
 # by the following command:
 #
 #    pip-compile --allow-unsafe --generate-hashes --output-file=dev-requirements.txt ./requirements/dev-requirements.in
@@ -115,9 +115,9 @@ coverage==7.4.0 \
 cssbeautifier==1.14.11 \
     --hash=sha256:40544c2b62bbcb64caa5e7f37a02df95654e5ce1bcacadac4ca1f3dc89c31513
     # via djlint
-django==5.0.4 \
-    --hash=sha256:4bd01a8c830bb77a8a3b0e7d8b25b887e536ad17a81ba2dce5476135c73312bd \
-    --hash=sha256:916423499d75d62da7aa038d19aef23d23498d8df229775eb0a6309ee1013775
+django==5.1 \
+    --hash=sha256:848a5980e8efb76eea70872fb0e4bc5e371619c70fffbe48e3e1b50b2c09455d \
+    --hash=sha256:d3b811bf5371a26def053d7ee42a9df1267ef7622323fe70a601936725aa4557
     # via
     #   -c ./requirements\../requirements.txt
     #   django-debug-toolbar
@@ -317,9 +317,9 @@ pbr==6.0.0 \
     --hash=sha256:4a7317d5e3b17a3dccb6a8cfe67dab65b20551404c52c8ed41279fa4f0cb4cda \
     --hash=sha256:d1377122a5a00e2f940ee482999518efe16d745d423a670c27773dfbc3c9a7d9
     # via stevedore
-pip-tools==7.3.0 \
-    --hash=sha256:8717693288720a8c6ebd07149c93ab0be1fced0b5191df9e9decd3263e20d85e \
-    --hash=sha256:8e9c99127fe024c025b46a0b2d15c7bd47f18f33226cf7330d35493663fc1d1d
+pip-tools==7.4.1 \
+    --hash=sha256:4c690e5fbae2f21e87843e89c26191f0d9454f362d8acdbd695716493ec8b3a9 \
+    --hash=sha256:864826f5073864450e24dbeeb85ce3920cdfb09848a3d69ebf537b521f14bcc9
     # via -r ./requirements/dev-requirements.in
 platformdirs==4.1.0 \
     --hash=sha256:11c8f37bcca40db96d8144522d925583bdb7a31f7b0e37e3ed4318400a8e2380 \
@@ -340,7 +340,9 @@ pygments==2.17.2 \
 pyproject-hooks==1.0.0 \
     --hash=sha256:283c11acd6b928d2f6a7c73fa0d01cb2bdc5f07c57a2eeb6e83d5e56b97976f8 \
     --hash=sha256:f271b298b97f5955d53fb12b72c1fb1948c22c1a6b70b315c54cedaca0264ef5
-    # via build
+    # via
+    #   build
+    #   pip-tools
 python-dateutil==2.9.0.post0 \
     --hash=sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3 \
     --hash=sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427
@@ -592,9 +594,9 @@ pip==23.3.2 \
     --hash=sha256:5052d7889c1f9d05224cd41741acb7c5d6fa735ab34e339624a614eaaa7e7d76 \
     --hash=sha256:7fd9972f96db22c8077a1ee2691b172c8089b17a5652a44494a9ecb0d78f9149
     # via pip-tools
-setuptools==69.4.0 \
-    --hash=sha256:659e902e587e77fab8212358f5b03977b5f0d18d4724310d4a093929fee4ca1a \
-    --hash=sha256:b6df12d754b505e4ca283c61582d5578db83ae2f56a979b3bc9a8754705ae3bf
+setuptools==71.0.2 \
+    --hash=sha256:ca359bea0cd5c8ce267d7463239107e87f312f2e2a11b6ca6357565d82b6c0d7 \
+    --hash=sha256:f6640114f96be808024fbd1f721161215543796d3a68da4524349de700604ce8
     # via
     #   -c ./requirements\../requirements.txt
     #   pip-tools

backend/dissemination/remove_workbook_artifacts.py

@@ -0,0 +1,74 @@
+import logging
+
+from django.conf import settings
+from audit.models.models import ExcelFile
+from boto3 import client as boto3_client
+from botocore.client import ClientError, Config
+
+logger = logging.getLogger(__name__)
+
+
+def remove_workbook_artifacts(sac):
+    """
+    Remove all workbook artifacts associated with the given sac.
+    """
+    try:
+        excel_files = ExcelFile.objects.filter(sac=sac)
+        files = [f"excel/{excel_file.filename}" for excel_file in excel_files]
+
+        if files:
+            # Delete the records from the database
+            count = excel_files.count()
+            excel_files.delete()
+            logger.info(
+                f"Deleted {count} excelfile records from fac-db for report: {sac.report_id}"
+            )
+
+            # Delete the files from S3 in bulk
+            delete_files_in_bulk(files, sac)
+
+    except ExcelFile.DoesNotExist:
+        logger.info(f"No files found to delete for report: {sac.report_id}")
+    except Exception as e:
+        logger.error(
+            f"Failed to delete files from fac-db and S3 for report: {sac.report_id}. Error: {e}"
+        )
+
+
+def delete_files_in_bulk(filenames, sac):
+    """Delete files from S3 in bulk."""
+    # This client uses the internal endpoint URL because we're making a request to S3 from within the app
+    s3_client = boto3_client(
+        service_name="s3",
+        region_name=settings.AWS_S3_PRIVATE_REGION_NAME,
+        aws_access_key_id=settings.AWS_PRIVATE_ACCESS_KEY_ID,
+        aws_secret_access_key=settings.AWS_PRIVATE_SECRET_ACCESS_KEY,
+        endpoint_url=settings.AWS_S3_PRIVATE_INTERNAL_ENDPOINT,
+        config=Config(signature_version="s3v4"),
+    )
+
+    try:
+        delete_objects = [{"Key": filename} for filename in filenames]
+
+        response = s3_client.delete_objects(
+            Bucket=settings.AWS_PRIVATE_STORAGE_BUCKET_NAME,
+            Delete={"Objects": delete_objects},
+        )
+
+        deleted_files = response.get("Deleted", [])
+        for deleted in deleted_files:
+            logger.info(
+                f"Successfully deleted {deleted['Key']} from S3 for report: {sac.report_id}"
+            )
+
+        errors = response.get("Errors", [])
+        if errors:
+            for error in errors:
+                logger.error(
+                    f"Failed to delete {error['Key']} from S3 for report: {sac.report_id}. Error: {error['Message']}"  # nosec B608
+                )
+
+    except ClientError as e:
+        logger.error(
+            f"Failed to delete files from S3 for report: {sac.report_id}. Error: {e}"
+        )

backend/dissemination/templates/summary.html

@@ -53,7 +53,8 @@ <h1 class="usa-legend usa-legend--large font-sans-2xl">Single audit summary</h1>
                             <a class="tooltip flex-align-self-center margin-bottom-2"
                                 href="https://support.fac.gov/hc/en-us/articles/29043839561869-I-just-submitted-my-audit-report-Why-can-t-I-download-my-report-from-Search-Is-there-an-error-with-my-submission"
                                 target="_blank"
-                                rel="noopener noreferrer">
+                                rel="noopener noreferrer"
+                                aria-label="SF-SACs are typically available for download a day after submission. Click to learn more">
                                 <svg class="usa-icon"
                                     aria-hidden="true"
                                     role="img">