[FDS-2491] Integration tests for Schematic API Test plan (#1512)

Integration tests for Schematic API Test plan

.github/workflows/test.yml

@@ -134,7 +134,16 @@ jobs:
           SERVICE_ACCOUNT_CREDS: ${{ secrets.SERVICE_ACCOUNT_CREDS }}
         run: >
           poetry run pytest --durations=0 --cov-append --cov-report=term --cov-report=html:htmlcov --cov-report=xml:coverage.xml --cov=schematic/
-          -m "not (rule_benchmark)" --reruns 4 -n 8 --ignore=tests/unit
+          -m "not (rule_benchmark or single_process_execution)" --reruns 4 -n 8 --ignore=tests/unit
+
+      - name: Run integration tests single process
+        if: ${{ contains(fromJSON('["3.10"]'), matrix.python-version) }}
+        env:
+          SYNAPSE_ACCESS_TOKEN: ${{ secrets.SYNAPSE_ACCESS_TOKEN }}
+          SERVICE_ACCOUNT_CREDS: ${{ secrets.SERVICE_ACCOUNT_CREDS }}
+        run: >
+          poetry run pytest --durations=0 --cov-append --cov-report=term --cov-report=html:htmlcov --cov-report=xml:coverage.xml --cov=schematic/
+          -m "single_process_execution" --reruns 4 --ignore=tests/unit
 
 
       - name: Upload pytest test results

env.example

@@ -12,4 +12,21 @@ SERVICE_ACCOUNT_CREDS='Provide service account creds'
 # TRACING_SERVICE_NAME=schematic-api
 # LOGGING_SERVICE_NAME=schematic-api
 ## Other examples: dev, staging, prod
-# DEPLOYMENT_ENVIRONMENT=local
+# DEPLOYMENT_ENVIRONMENT=local
+
+# Used during integration test run to determine if files will be output for manual
+# inspection. These tests cannot fully finish all validation via code. All of these
+# tests will be marked by pytest "manual_verification_required"
+# More information: https://sagebionetworks.jira.com/wiki/spaces/SCHEM/pages/3055779846/Schematic+API+test+plan 
+MANUAL_TEST_VERIFICATION=false
+
+# Used to determine if a local flask instance is created during integration testing. If
+# this is true schematic tests will use a schematic API server running outside of the
+# context of the integration test. The url used is defined below.
+USE_DEPLOYED_SCHEMATIC_API_SERVER=false
+
+# The URL used to execute integration tests for schematic API. Defaults to localhost.
+# dev: https://schematic-dev.api.sagebionetworks.org
+# staging: https://schematic-staging.api.sagebionetworks.org
+# prod: https://schematic.api.sagebionetworks.org
+SCHEMATIC_API_SERVER_URL=http://localhost:3001

pytest.ini

@@ -13,4 +13,7 @@ markers =
     table_operations: marks tests covering table operations that pass locally but fail on CI due to interactions with Synapse (skipped on GitHub CI)
     rule_benchmark: marks tests covering validation rule benchmarking
     synapse_credentials_needed: marks api tests that require synapse credentials to run
-    empty_token: marks api tests that send empty credentials in the request
+    empty_token: marks api tests that send empty credentials in the request
+    manual_verification_required: Tests that require manual verification to fully validate
+    local_or_remote_api: Tests that can be configured to run against a local or remote API
+    single_process_execution: Tests that should run without xdist due to tests being flakey

schematic/store/synapse.py

@@ -371,13 +371,15 @@ def query_fileview(
         self,
         columns: Optional[list] = None,
         where_clauses: Optional[list] = None,
+        force_requery: Optional[bool] = False,
     ) -> None:
         """
         Method to query the Synapse FileView and store the results in a pandas DataFrame. The results are stored in the storageFileviewTable attribute.
         Is called once during initialization of the SynapseStorage object and can be called again later to specify a specific, more limited scope for validation purposes.
         Args:
             columns (Optional[list], optional): List of columns to be selected from the table. Defaults behavior is to request all columns.
             where_clauses (Optional[list], optional): List of where clauses to be used to scope the query. Defaults to None.
+            force_requery (Optional[bool], optional): If True, forces a requery of the fileview. Defaults to False.
         """
         self._purge_synapse_cache()
 
@@ -394,8 +396,8 @@ def query_fileview(
         if previous_query_built:
             self.new_query_different = self.fileview_query != previous_query
 
-        # Only perform the query if it is different from the previous query
-        if self.new_query_different:
+        # Only perform the query if it is different from the previous query or we are forcing new results to be retrieved
+        if self.new_query_different or force_requery:
             try:
                 self.storageFileviewTable = self.syn.tableQuery(
                     query=self.fileview_query,
@@ -2845,7 +2847,7 @@ def getDatasetProject(self, datasetId: str) -> str:
         # re-query if no datasets found
         if dataset_row.empty:
             sleep(5)
-            self.query_fileview()
+            self.query_fileview(force_requery=True)
             # Subset main file view
             dataset_index = self.storageFileviewTable["id"] == datasetId
             dataset_row = self.storageFileviewTable[dataset_index]

tests/conftest.py

@@ -4,18 +4,25 @@
 import os
 import shutil
 import sys
+import tempfile
+from dataclasses import dataclass
 from typing import Callable, Generator, Set
 
+import flask
 import pytest
 from dotenv import load_dotenv
+from flask.testing import FlaskClient
 from opentelemetry import trace
+from synapseclient.client import Synapse
 
 from schematic.configuration.configuration import CONFIG, Configuration
 from schematic.models.metadata import MetadataModel
 from schematic.schemas.data_model_graph import DataModelGraph, DataModelGraphExplorer
 from schematic.schemas.data_model_parser import DataModelParser
 from schematic.store.synapse import SynapseStorage
 from schematic.utils.df_utils import load_df
+from schematic.utils.general import create_temp_folder
+from schematic_api.api import create_app
 from tests.utils import CleanupAction, CleanupItem
 
 tracer = trace.get_tracer("Schematic-Tests")
@@ -41,9 +48,26 @@ def dataset_id():
     yield "syn25614635"
 
 
+@pytest.fixture(scope="class")
+def flask_app() -> flask.Flask:
+    """Create a Flask app for testing."""
+    app = create_app()
+    return app
+
+
+@pytest.fixture(scope="class")
+def flask_client(flask_app: flask.Flask) -> Generator[FlaskClient, None, None]:
+    flask_app.config["SCHEMATIC_CONFIG"] = None
+
+    with flask_app.test_client() as client:
+        yield client
+
+
 # This class serves as a container for helper functions that can be
 # passed to individual tests using the `helpers` fixture. This approach
 # was required because fixture functions cannot take arguments.
+
+
 class Helpers:
     @staticmethod
     def get_data_path(path, *paths):
@@ -126,6 +150,60 @@ def synapse_store():
     yield SynapseStorage()
 
 
+@dataclass
+class ConfigurationForTesting:
+    """
+    Variables that are specific to testing. Specifically these are used to control
+    the flags used during manual verification of some integration test results.
+
+    Attributes:
+        manual_test_verification_enabled (bool): Whether manual verification is enabled.
+        manual_test_verification_path (str): The path to the directory where manual test
+            verification files are stored.
+        use_deployed_schematic_api_server (bool): Used to determine if a local flask
+            instance is created during integration testing. If this is true schematic
+            tests will use a schematic API server running outside of the context of the
+            integration test.
+        schematic_api_server_url (str): The URL of the schematic API server. Defaults to
+            http://localhost:3001.
+
+    """
+
+    manual_test_verification_enabled: bool
+    manual_test_verification_path: str
+    use_deployed_schematic_api_server: bool
+    schematic_api_server_url: str
+
+
+@pytest.fixture(scope="session")
+def testing_config(config: Configuration) -> ConfigurationForTesting:
+    """Configuration variables that are specific to testing."""
+    manual_test_verification_enabled = (
+        os.environ.get("MANUAL_TEST_VERIFICATION", "false").lower() == "true"
+    )
+    use_deployed_schematic_api_server = (
+        os.environ.get("USE_DEPLOYED_SCHEMATIC_API_SERVER", "false").lower() == "true"
+    )
+    schematic_api_server_url = os.environ.get(
+        "SCHEMATIC_API_SERVER_URL", "http://localhost:3001"
+    )
+
+    if manual_test_verification_enabled:
+        manual_test_verification_path = os.path.join(
+            config.manifest_folder, "manual_test_verification"
+        )
+        os.makedirs(manual_test_verification_path, exist_ok=True)
+    else:
+        manual_test_verification_path = ""
+
+    return ConfigurationForTesting(
+        manual_test_verification_enabled=manual_test_verification_enabled,
+        manual_test_verification_path=manual_test_verification_path,
+        use_deployed_schematic_api_server=use_deployed_schematic_api_server,
+        schematic_api_server_url=schematic_api_server_url,
+    )
+
+
 # These fixtures make copies of existing test manifests.
 # These copies can the be altered by a given test, and the copy will eb destroyed at the
 # end of the test
@@ -170,6 +248,23 @@ def syn_token(config: Configuration):
     return token
 
 
+@pytest.fixture(scope="class")
+def syn(syn_token) -> Synapse:
+    syn = Synapse()
+    syn.login(authToken=syn_token, silent=True)
+    return syn
+
+
+@pytest.fixture(scope="session")
+def download_location() -> Generator[str, None, None]:
+    download_location = create_temp_folder(path=tempfile.gettempdir())
+    yield download_location
+
+    # Cleanup after tests have used the temp folder
+    if os.path.exists(download_location):
+        shutil.rmtree(download_location)
+
+
 def metadata_model(helpers, data_model_labels):
     metadata_model = MetadataModel(
         inputMModelLocation=helpers.get_data_path("example.model.jsonld"),

tests/data/mock_manifests/TestManifestSubmission_test_submit_manifest_with_blacklisted_characters.csv

@@ -0,0 +1,2 @@
+Filename,Sample-ID,File-Format,Component,Genome Build,Genome FASTA,entityId
+Manifest Submission - Manual test - file-based manifest submission/test-annotation-key-table-column-name/sample A.txt,100,FASTQ,BulkRNA-seqAssay,GRCh38,,syn63607043

tests/data/mock_manifests/TestManifestSubmission_test_submit_manifest_with_hide_blanks.csv

@@ -0,0 +1,2 @@
+Filename,Sample ID,File Format,Component,Genome Build,Genome FASTA,entityId
+Manifest Submission - Manual test - file-based manifest submission/hide-blanks-true/sample A.txt,1,FASTQ,BulkRNA-seqAssay,,,syn63606862

tests/data/mock_manifests/mock_example_biospecimen_manifest.csv

@@ -0,0 +1,2 @@
+Sample ID,Patient ID,Tissue Status,Component
+1,1,Healthy,Biospecimen

tests/data/mock_manifests/mock_example_bulkrnaseq_manifest.csv

@@ -0,0 +1,2 @@
+Filename,Sample ID,File Format,Component,Genome Build,Genome FASTA,entityId
+Manifest Submission - Manual test - file-based manifest submission/test-dataset/sample A.txt,1,FASTQ,BulkRNA-seqAssay,,,syn63561932