feat: Add Import of FSE's

* Add new importer.py to handle imports
* Add new dialog with states to go through the import process
* Use a newly spawned thread to process the import async

Author: dhaselhan

backend/lcfs/db/dependencies.py

@@ -2,7 +2,6 @@
 from typing import AsyncGenerator
 
 from fastapi import Request
-from redis import asyncio as aioredis
 from sqlalchemy import text
 from sqlalchemy.engine import make_url
 from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine
@@ -30,7 +29,9 @@ async def set_user_context(session: AsyncSession, username: str):
         raise e
 
 
-async def get_async_db_session(request: Request) -> AsyncGenerator[AsyncSession, None]:
+async def get_async_db_session(
+    request: Request,
+) -> AsyncGenerator[AsyncSession, None]:
     """
     Create and get database session.
     :yield: database session.

backend/lcfs/services/clamav/client.py

@@ -48,5 +48,4 @@ def scan_file(self, file):
                     result=result,
                 )
                 raise VirusScanException(f"Virus detected: {result}")
-
         return result

backend/lcfs/tests/final_supply_equipment/test_final_supply_equipment_importer.py

@@ -0,0 +1,143 @@
+import pytest
+import asyncio
+from unittest.mock import MagicMock, AsyncMock, patch
+import concurrent.futures
+
+from lcfs.web.api.final_supply_equipment.importer import FinalSupplyEquipmentImporter
+from lcfs.web.api.final_supply_equipment.repo import FinalSupplyEquipmentRepository
+from lcfs.web.api.final_supply_equipment.services import FinalSupplyEquipmentServices
+from lcfs.web.api.compliance_report.services import ComplianceReportServices
+from lcfs.services.clamav.client import ClamAVService
+from redis.asyncio import Redis
+
+
+@pytest.fixture
+def mock_repo() -> FinalSupplyEquipmentRepository:
+    repo = MagicMock(spec=FinalSupplyEquipmentRepository)
+    return repo
+
+
+@pytest.fixture
+def mock_fse_service() -> FinalSupplyEquipmentServices:
+    service = MagicMock(spec=FinalSupplyEquipmentServices)
+    return service
+
+
+@pytest.fixture
+def mock_compliance_service() -> ComplianceReportServices:
+    service = MagicMock(spec=ComplianceReportServices)
+    return service
+
+
+@pytest.fixture
+def mock_clamav() -> ClamAVService:
+    clamav = MagicMock(spec=ClamAVService)
+    return clamav
+
+
+@pytest.fixture
+def mock_redis() -> Redis:
+    redis_client = MagicMock(spec=Redis)
+    redis_client.set = AsyncMock()
+    redis_client.get = AsyncMock()
+    return redis_client
+
+
+@pytest.fixture
+def mock_executor():
+    executor = concurrent.futures.ThreadPoolExecutor(max_workers=1)
+    yield executor
+    executor.shutdown(wait=False)
+
+
+@pytest.fixture
+def importer_instance(
+    mock_repo,
+    mock_fse_service,
+    mock_compliance_service,
+    mock_clamav,
+    mock_redis,
+    mock_executor,
+):
+    """
+    Creates a FinalSupplyEquipmentImporter with mocked dependencies.
+    """
+    return FinalSupplyEquipmentImporter(
+        repo=mock_repo,
+        fse_service=mock_fse_service,
+        compliance_report_services=mock_compliance_service,
+        clamav_service=mock_clamav,
+        redis_client=mock_redis,
+        executor=mock_executor,
+    )
+
+
+@pytest.mark.anyio
+async def test_import_data_success(importer_instance, mock_redis):
+    file_mock = MagicMock()
+    file_mock.filename = "test.xlsx"
+    file_mock.read = AsyncMock(return_value=b"fake-excel-contents")
+
+    user_mock = MagicMock()
+    user_mock.organization.organization_code = "TEST-ORG"
+
+    with patch(
+        "lcfs.web.api.final_supply_equipment.importer.import_async",
+        new=AsyncMock(return_value=None),
+    ) as mock_import_task:
+
+        job_id = await importer_instance.import_data(
+            compliance_report_id=123, user=user_mock, file=file_mock, overwrite=False
+        )
+
+        assert isinstance(job_id, str)
+        assert len(job_id) > 0
+
+        # Check Redis progress was initialized
+        mock_redis.set.assert_called()
+        # Check our background task was scheduled
+        mock_import_task.assert_called()
+
+
+@pytest.mark.anyio
+async def test_import_data_with_clamav(importer_instance, mock_clamav, mock_redis):
+    with patch("lcfs.settings.settings.clamav_enabled", True):
+        file_mock = MagicMock()
+        file_mock.filename = "test.xlsx"
+        file_mock.read = AsyncMock(return_value=b"excel-data")
+        user_mock = MagicMock()
+
+        with patch(
+            "lcfs.web.api.final_supply_equipment.importer.import_async", new=AsyncMock()
+        ) as mock_import_task:
+            job_id = await importer_instance.import_data(
+                compliance_report_id=999, user=user_mock, file=file_mock, overwrite=True
+            )
+
+            assert job_id
+            mock_import_task.assert_called()
+
+
+@pytest.mark.anyio
+async def test_get_status_no_job_found(importer_instance, mock_redis):
+    """
+    Tests that get_status returns a default response if redis has no record for job_id.
+    """
+    mock_redis.get = AsyncMock(return_value=None)
+
+    result = await importer_instance.get_status("non-existent-id")
+
+    assert result["progress"] == 0
+    assert "No job found" in result["status"]
+
+
+@pytest.mark.anyio
+async def test_get_status_invalid_json(importer_instance, mock_redis):
+    """
+    Tests that get_status handles invalid JSON from redis gracefully.
+    """
+    mock_redis.get = AsyncMock(return_value=b"not-valid-json")
+
+    result = await importer_instance.get_status("corrupt-id")
+    assert result["progress"] == 0
+    assert "Invalid status data found." in result["status"]

backend/lcfs/tests/final_supply_equipment/test_final_supply_equipment_services.py

@@ -50,7 +50,6 @@ def service(mock_request, mock_repo, mock_comp_report_repo):
     Instantiate the service class with mocked dependencies.
     """
     return FinalSupplyEquipmentServices(
-        request=mock_request,
         repo=mock_repo,
         compliance_report_repo=mock_comp_report_repo,
     )
@@ -90,7 +89,7 @@ async def test_get_fse_options_success(service, mock_repo, mock_request):
         ["OrgA", "OrgB"],
     )
 
-    result = await service.get_fse_options()
+    result = await service.get_fse_options(mock_request.user)
     assert "intended_use_types" in result
     assert "levels_of_equipment" in result
     assert "intended_user_types" in result
@@ -101,14 +100,14 @@ async def test_get_fse_options_success(service, mock_repo, mock_request):
 
 
 @pytest.mark.anyio
-async def test_get_fse_options_exception(service, mock_repo):
+async def test_get_fse_options_exception(service, mock_repo, mock_request):
     """
     Test get_fse_options raises HTTP 400 if repo call fails.
     """
     mock_repo.get_fse_options.side_effect = Exception("Repo Error")
 
     with pytest.raises(HTTPException, match="Error retrieving FSE options") as exc:
-        await service.get_fse_options()
+        await service.get_fse_options(mock_request.user)
 
     assert exc.value.status_code == 400
 
@@ -235,6 +234,7 @@ async def test_update_final_supply_equipment_not_found(
 async def test_create_final_supply_equipment_success(
     service,
     mock_repo,
+    mock_request,
     valid_final_supply_equipment_schema,
     valid_final_supply_equipment_create_schema,
 ):
@@ -248,7 +248,7 @@ async def test_create_final_supply_equipment_success(
     mock_repo.increment_seq_by_org_and_postal_code.return_value = None
 
     new_fse = await service.create_final_supply_equipment(
-        valid_final_supply_equipment_create_schema
+        valid_final_supply_equipment_create_schema, mock_request.user
     )
     assert new_fse is not None
     mock_repo.create_final_supply_equipment.assert_awaited_once()
@@ -272,24 +272,26 @@ async def test_generate_registration_number_success(service, mock_repo, mock_req
     Test generating a registration number with a valid postal code.
     """
     mock_repo.get_current_seq_by_org_and_postal_code.return_value = 1
-    reg_num = await service.generate_registration_number("A1A 1A1")
+    reg_num = await service.generate_registration_number(mock_request.user, "A1A 1A1")
     # Format => ORGCODE-A1A1A1-002 (for next_number = 2)
     assert reg_num.startswith("TESTORG-A1A1A1-")
     seq_str = reg_num.split("-")[-1]
     assert seq_str == "002"  # Because current seq was 1, next is 2
 
 
 @pytest.mark.anyio
-async def test_generate_registration_number_invalid_postal(service):
+async def test_generate_registration_number_invalid_postal(service, mock_request):
     """
     Test invalid postal code raises HTTP 400.
     """
     with pytest.raises(ValueError, match="Invalid Canadian postal code format"):
-        await service.generate_registration_number("12345")
+        await service.generate_registration_number(mock_request.user, "12345")
 
 
 @pytest.mark.anyio
-async def test_generate_registration_number_exceeds_limit(service, mock_repo):
+async def test_generate_registration_number_exceeds_limit(
+    service, mock_repo, mock_request
+):
     """
     Test exceeding maximum registration numbers raises ValueError.
     """
@@ -299,7 +301,7 @@ async def test_generate_registration_number_exceeds_limit(service, mock_repo):
         ValueError,
         match="Exceeded maximum registration numbers for the given postal code",
     ):
-        await service.generate_registration_number("A1A 1A1")
+        await service.generate_registration_number(mock_request.user, "A1A 1A1")
 
 
 @pytest.mark.anyio

backend/lcfs/utils/constants.py

@@ -87,3 +87,5 @@ class FILE_MEDIA_TYPE(Enum):
 }
 
 default_ci = {"Gasoline": 93.67, "Diesel": 100.21, "Jet fuel": 88.83}
+
+POSTAL_REGEX = r"^[A-Za-z]\d[A-Za-z] \d[A-Za-z]\d$"