From f619a593adb87c181c3878fcd071e47378cc22b6 Mon Sep 17 00:00:00 2001 From: Nargis Sultani Date: Fri, 8 Dec 2023 01:02:43 -0500 Subject: [PATCH 01/11] Updated institution model, modified unit tests, and added new tests --- src/entities/models/__init__.py | 16 ++ src/entities/models/dao.py | 46 +++++- src/entities/models/dto.py | 60 +++++++ src/entities/repos/institutions_repo.py | 21 ++- src/routers/institutions.py | 16 ++ tests/api/conftest.py | 16 ++ tests/api/routers/test_institutions_api.py | 112 +++++++++++++- .../entities/repos/test_institutions_repo.py | 146 +++++++++++++++++- 8 files changed, 427 insertions(+), 6 deletions(-) diff --git a/src/entities/models/__init__.py b/src/entities/models/__init__.py index 3bd37af..ace01c3 100644 --- a/src/entities/models/__init__.py +++ b/src/entities/models/__init__.py @@ -11,6 +11,14 @@ "DeniedDomainDto", "UserProfile", "AuthenticatedUser", + "FederalRegulatorDao", + "HMDAInstitutionTypeDao", + "SBLInstitutionTypeDao", + "AddressStateDao", + "FederalRegulatorDto", + "HMDAInstitutionTypeDto", + "SBLInstitutionTypeDto", + "AddressStateDto", ] from .dao import ( @@ -18,6 +26,10 @@ FinancialInstitutionDao, FinancialInstitutionDomainDao, DeniedDomainDao, + FederalRegulatorDao, + HMDAInstitutionTypeDao, + SBLInstitutionTypeDao, + AddressStateDao, ) from .dto import ( FinancialInstitutionDto, @@ -28,4 +40,8 @@ DeniedDomainDto, UserProfile, AuthenticatedUser, + FederalRegulatorDto, + HMDAInstitutionTypeDto, + SBLInstitutionTypeDto, + AddressStateDto, ) diff --git a/src/entities/models/dao.py b/src/entities/models/dao.py index 3599188..1f0172c 100644 --- a/src/entities/models/dao.py +++ b/src/entities/models/dao.py @@ -1,6 +1,6 @@ from datetime import datetime from typing import List -from sqlalchemy import ForeignKey, func +from sqlalchemy import ForeignKey, func, String from sqlalchemy.orm import Mapped, mapped_column, relationship from sqlalchemy.orm import DeclarativeBase from sqlalchemy.ext.asyncio import AsyncAttrs @@ -21,6 +21,26 @@ class FinancialInstitutionDao(AuditMixin, Base): domains: Mapped[List["FinancialInstitutionDomainDao"]] = relationship( "FinancialInstitutionDomainDao", back_populates="fi" ) + tax_id: Mapped[str] = mapped_column(String(9), unique=True) + rssd_id: Mapped[int] = mapped_column(unique=True) + primary_federal_regulator_id: Mapped[str] = mapped_column(ForeignKey("federal_regulator.id")) + primary_federal_regulator: Mapped["FederalRegulatorDao"] = relationship(lazy="selectin") + hmda_institution_type_id: Mapped[str] = mapped_column(ForeignKey("hmda_institution_type.id")) + hmda_institution_type: Mapped["HMDAInstitutionTypeDao"] = relationship(lazy="selectin") + sbl_institution_type_id: Mapped[str] = mapped_column(ForeignKey("sbl_institution_type.id")) + sbl_institution_type: Mapped["SBLInstitutionTypeDao"] = relationship(lazy="selectin") + hq_address_street_1: Mapped[str] = mapped_column(nullable=False) + hq_address_street_2: Mapped[str] + hq_address_city: Mapped[str] + hq_address_state_code: Mapped[str] = mapped_column(ForeignKey("address_state.code")) + hq_address_state: Mapped["AddressStateDao"] = relationship(lazy="selectin") + hq_address_zip: Mapped[str] = mapped_column(String(5), nullable=False) + parent_lei: Mapped[str] = mapped_column(String(20)) + parent_legal_name: Mapped[str] + parent_rssd_id: Mapped[int] + top_holder_lei: Mapped[str] = mapped_column(String(20)) + top_holder_legal_name: Mapped[str] + top_holder_rssd_id: Mapped[int] class FinancialInstitutionDomainDao(AuditMixin, Base): @@ -33,3 +53,27 @@ class FinancialInstitutionDomainDao(AuditMixin, Base): class DeniedDomainDao(AuditMixin, Base): __tablename__ = "denied_domains" domain: Mapped[str] = mapped_column(index=True, primary_key=True) + + +class FederalRegulatorDao(AuditMixin, Base): + __tablename__ = "federal_regulator" + id: Mapped[str] = mapped_column(String(4), index=True, primary_key=True, unique=True) + name: Mapped[str] = mapped_column(unique=True, nullable=False) + + +class HMDAInstitutionTypeDao(AuditMixin, Base): + __tablename__ = "hmda_institution_type" + id: Mapped[str] = mapped_column(index=True, primary_key=True, unique=True) + name: Mapped[str] = mapped_column(unique=True) + + +class SBLInstitutionTypeDao(AuditMixin, Base): + __tablename__ = "sbl_institution_type" + id: Mapped[str] = mapped_column(index=True, primary_key=True, unique=True) + name: Mapped[str] = mapped_column(unique=True, nullable=False) + + +class AddressStateDao(AuditMixin, Base): + __tablename__ = "address_state" + code: Mapped[str] = mapped_column(String(2), index=True, primary_key=True, unique=True) + name: Mapped[str] = mapped_column(unique=True, nullable=False) diff --git a/src/entities/models/dto.py b/src/entities/models/dto.py index a09062b..9cea2a2 100644 --- a/src/entities/models/dto.py +++ b/src/entities/models/dto.py @@ -24,6 +24,22 @@ class FinancialInstitutionBase(BaseModel): class FinancialInstitutionDto(FinancialInstitutionBase): lei: str + tax_id: str + rssd_id: int + primary_federal_regulator_id: str + hmda_institution_type_id: str + sbl_institution_type_id: str + hq_address_street_1: str + hq_address_street_2: str + hq_address_city: str + hq_address_state_code: str + hq_address_zip: str + parent_lei: str + parent_legal_name: str + parent_rssd_id: int + top_holder_lei: str + top_holder_legal_name: str + top_holder_rssd_id: int class Config: from_attributes = True @@ -53,6 +69,50 @@ class FinanicialInstitutionAssociationDto(FinancialInstitutionDto): approved: bool +class FederalRegulatorBase(BaseModel): + id: str + + +class FederalRegulatorDto(FederalRegulatorBase): + name: str + + class Config: + from_attributes = True + + +class HMDAInstitutionTypeBase(BaseModel): + id: str + + +class HMDAInstitutionTypeDto(HMDAInstitutionTypeBase): + name: str + + class Config: + from_attributes = True + + +class SBLInstitutionTypeBase(BaseModel): + id: str + + +class SBLInstitutionTypeDto(SBLInstitutionTypeBase): + name: str + + class Config: + from_attributes = True + + +class AddressStateBase(BaseModel): + code: str + + +class AddressStateDto(AddressStateBase): + name: str + + class Config: + from_attributes = True + + class AuthenticatedUser(BaseUser, BaseModel): claims: Dict[str, Any] name: str diff --git a/src/entities/repos/institutions_repo.py b/src/entities/repos/institutions_repo.py index 4442b50..4597c72 100644 --- a/src/entities/repos/institutions_repo.py +++ b/src/entities/repos/institutions_repo.py @@ -51,7 +51,26 @@ async def upsert_institution(session: AsyncSession, fi: FinancialInstitutionDto) res = await session.execute(stmt) db_fi = res.scalar_one_or_none() if db_fi is None: - db_fi = FinancialInstitutionDao(lei=fi.lei, name=fi.name) + db_fi = FinancialInstitutionDao( + lei=fi.lei, + name=fi.name, + tax_id=fi.tax_id, + rssd_id=fi.rssd_id, + primary_federal_regulator_id=fi.primary_federal_regulator_id, + hmda_institution_type_id=fi.hmda_institution_type_id, + sbl_institution_type_id=fi.sbl_institution_type_id, + hq_address_street_1=fi.hq_address_street_1, + hq_address_street_2=fi.hq_address_street_2, + hq_address_city=fi.hq_address_city, + hq_address_state_code=fi.hq_address_state_code, + hq_address_zip=fi.hq_address_zip, + parent_lei=fi.parent_lei, + parent_legal_name=fi.parent_legal_name, + parent_rssd_id=fi.parent_rssd_id, + top_holder_lei=fi.top_holder_lei, + top_holder_legal_name=fi.top_holder_legal_name, + top_holder_rssd_id=fi.top_holder_rssd_id, + ) session.add(db_fi) else: db_fi.name = fi.name diff --git a/src/routers/institutions.py b/src/routers/institutions.py index 78b5d5b..371c2ee 100644 --- a/src/routers/institutions.py +++ b/src/routers/institutions.py @@ -58,6 +58,22 @@ async def get_associated_institutions(request: Request): FinanicialInstitutionAssociationDto( name=institution.name, lei=institution.lei, + tax_id=institution.tax_id, + rssd_id=institution.rssd_id, + primary_federal_regulator_id=institution.primary_federal_regulator_id, + hmda_institution_type_id=institution.hmda_institution_type_id, + sbl_institution_type_id=institution.sbl_institution_type_id, + hq_address_street_1=institution.hq_address_street_1, + hq_address_street_2=institution.hq_address_street_2, + hq_address_city=institution.hq_address_city, + hq_address_state_code=institution.hq_address_state_code, + hq_address_zip=institution.hq_address_zip, + parent_lei=institution.parent_lei, + parent_legal_name=institution.parent_legal_name, + parent_rssd_id=institution.parent_rssd_id, + top_holder_lei=institution.top_holder_lei, + top_holder_legal_name=institution.top_holder_legal_name, + top_holder_rssd_id=institution.top_holder_rssd_id, approved=email_domain in [inst_domain.domain for inst_domain in institution.domains], ) for institution in associated_institutions diff --git a/tests/api/conftest.py b/tests/api/conftest.py index 451ba6b..73577f4 100644 --- a/tests/api/conftest.py +++ b/tests/api/conftest.py @@ -55,6 +55,22 @@ def get_institutions_mock(mocker: MockerFixture) -> Mock: name="Test Bank 123", lei="TESTBANK123", domains=[FinancialInstitutionDomainDao(domain="test.bank", lei="TESTBANK123")], + tax_id="123456789", + rssd_id=1234, + primary_federal_regulator_id="FRI1", + hmda_institution_type_id="HIT1", + sbl_institution_type_id="SIT1", + hq_address_street_1="Test Address Street 1", + hq_address_street_2="", + hq_address_city="Test City 1", + hq_address_state_code="GA", + hq_address_zip="00000", + parent_lei="PARENTTESTBANK123", + parent_legal_name="PARENT TEST BANK 123", + parent_rssd_id=12345, + top_holder_lei="TOPHOLDERLEI123", + top_holder_legal_name="TOP HOLDER LEI 123", + top_holder_rssd_id=123456, ) ] return mock diff --git a/tests/api/routers/test_institutions_api.py b/tests/api/routers/test_institutions_api.py index dd6d706..cd6bc13 100644 --- a/tests/api/routers/test_institutions_api.py +++ b/tests/api/routers/test_institutions_api.py @@ -33,11 +33,49 @@ def test_create_institution_authed(self, mocker: MockerFixture, app_fixture: Fas name="testName", lei="testLei", domains=[FinancialInstitutionDomainDao(domain="test.bank", lei="TESTBANK123")], + tax_id="123456789", + rssd_id=1234, + primary_federal_regulator_id="FRI2", + hmda_institution_type_id="HIT2", + sbl_institution_type_id="SIT2", + hq_address_street_1="Test Address Street 1", + hq_address_street_2="", + hq_address_city="Test City 1", + hq_address_state_code="VA", + hq_address_zip="00000", + parent_lei="PARENTTESTBANK123", + parent_legal_name="PARENT TEST BANK 123", + parent_rssd_id=12345, + top_holder_lei="TOPHOLDERLEI123", + top_holder_legal_name="TOP HOLDER LEI 123", + top_holder_rssd_id=123456, ) upsert_group_mock = mocker.patch("oauth2.oauth2_admin.OAuth2Admin.upsert_group") upsert_group_mock.return_value = "leiGroup" client = TestClient(app_fixture) - res = client.post("/v1/institutions/", json={"name": "testName", "lei": "testLei"}) + res = client.post( + "/v1/institutions/", + json={ + "name": "testName", + "lei": "testLei", + "tax_id": "123456789", + "rssd_id": 12344, + "primary_federal_regulator_id": "FRI2", + "hmda_institution_type_id": "HIT2", + "sbl_institution_type_id": "SIT2", + "hq_address_street_1": "Test Address Street 1", + "hq_address_street_2": "", + "hq_address_city": "Test City 1", + "hq_address_state_code": "VA", + "hq_address_zip": "00000", + "parent_lei": "PARENTTESTBANK123", + "parent_legal_name": "PARENT TEST BANK 123", + "parent_rssd_id": 12345, + "top_holder_lei": "TOPHOLDERLEI123", + "top_holder_legal_name": "TOP HOLDER LEI 123", + "top_holder_rssd_id": 123456, + }, + ) assert res.status_code == 200 assert res.json()[1].get("name") == "testName" @@ -53,7 +91,29 @@ def test_create_institution_authed_no_permission(self, app_fixture: FastAPI, aut AuthenticatedUser.from_claim(claims), ) client = TestClient(app_fixture) - res = client.post("/v1/institutions/", json={"name": "testName", "lei": "testLei"}) + res = client.post( + "/v1/institutions/", + json={ + "name": "testName", + "lei": "testLei", + "tax_id": "123456789", + "rssd_id": 12344, + "primary_federal_regulator_id": "FIR2", + "hmda_institution_type_id": "HIT2", + "sbl_institution_type_id": "SIT2", + "hq_address_street_1": "Test Address Street 1", + "hq_address_street_2": "", + "hq_address_city": "Test City 1", + "hq_address_state_code": "VA", + "hq_address_zip": "00000", + "parent_lei": "PARENTTESTBANK123", + "parent_legal_name": "PARENT TEST BANK 123", + "parent_rssd_id": 12345, + "top_holder_lei": "TOPHOLDERLEI123", + "top_holder_legal_name": "TOP HOLDER LEI 123", + "top_holder_rssd_id": 123456, + }, + ) assert res.status_code == 403 def test_get_institution_unauthed(self, app_fixture: FastAPI, unauthed_user_mock: Mock): @@ -68,6 +128,22 @@ def test_get_institution_authed(self, mocker: MockerFixture, app_fixture: FastAP name="Test Bank 123", lei="TESTBANK123", domains=[FinancialInstitutionDomainDao(domain="test.bank", lei="TESTBANK123")], + tax_id="123456789", + rssd_id=1234, + primary_federal_regulator_id="FRI1", + hmda_institution_type_id="HIT1", + sbl_institution_type_id="SIT1", + hq_address_street_1="Test Address Street 1", + hq_address_street_2="", + hq_address_city="Test City 1", + hq_address_state_code="GA", + hq_address_zip="00000", + parent_lei="PARENTTESTBANK123", + parent_legal_name="PARENT TEST BANK 123", + parent_rssd_id=12345, + top_holder_lei="TOPHOLDERLEI123", + top_holder_legal_name="TOP HOLDER LEI 123", + top_holder_rssd_id=123456, ) client = TestClient(app_fixture) lei_path = "testLeiPath" @@ -145,11 +221,43 @@ def test_get_associated_institutions( name="Test Bank 123", lei="TESTBANK123", domains=[FinancialInstitutionDomainDao(domain="test123.bank", lei="TESTBANK123")], + tax_id="123456789", + rssd_id=1234, + primary_federal_regulator_id="FRI1", + hmda_institution_type_id="HIT1", + sbl_institution_type_id="SIT1", + hq_address_street_1="Test Address Street 1", + hq_address_street_2="", + hq_address_city="Test City 1", + hq_address_state_code="GA", + hq_address_zip="00000", + parent_lei="PARENTTESTBANK123", + parent_legal_name="PARENT TEST BANK 123", + parent_rssd_id=12345, + top_holder_lei="TOPHOLDERLEI123", + top_holder_legal_name="TOP HOLDER LEI 123", + top_holder_rssd_id=123456, ), FinancialInstitutionDao( name="Test Bank 234", lei="TESTBANK234", domains=[FinancialInstitutionDomainDao(domain="test234.bank", lei="TESTBANK234")], + tax_id="123456879", + rssd_id=6879, + primary_federal_regulator_id="FRI1", + hmda_institution_type_id="HIT1", + sbl_institution_type_id="SIT1", + hq_address_street_1="Test Address Street 2", + hq_address_street_2="", + hq_address_city="Test City 2", + hq_address_state_code="GA", + hq_address_zip="00000", + parent_lei="PARENTTESTBANK123", + parent_legal_name="PARENT TEST BANK 123", + parent_rssd_id=14523, + top_holder_lei="TOPHOLDERLEI123", + top_holder_legal_name="TOP HOLDER LEI 123", + top_holder_rssd_id=341256, ), ] claims = { diff --git a/tests/entities/repos/test_institutions_repo.py b/tests/entities/repos/test_institutions_repo.py index 6af55cd..f2095d8 100644 --- a/tests/entities/repos/test_institutions_repo.py +++ b/tests/entities/repos/test_institutions_repo.py @@ -6,7 +6,13 @@ FinancialInstitutionDomainDao, FinancialInsitutionDomainCreate, ) -from entities.models import DeniedDomainDao +from entities.models import ( + DeniedDomainDao, + AddressStateDao, + FederalRegulatorDao, + HMDAInstitutionTypeDao, + SBLInstitutionTypeDao, +) from entities.repos import institutions_repo as repo @@ -16,23 +22,108 @@ async def setup( self, transaction_session: AsyncSession, ): + state_ga, state_ca, state_fl = ( + AddressStateDao(code="GA", name="Georgia"), + AddressStateDao(code="CA", name="California"), + AddressStateDao(code="FL", name="Florida"), + ) + fr_dao_fri1, fr_dao_fri2, fr_dao_fri3 = ( + FederalRegulatorDao(id="FRI1", name="Test Federal Regulator ID 1"), + FederalRegulatorDao(id="FRI2", name="Test Federal Regulator ID 2"), + FederalRegulatorDao(id="FRI3", name="Test Federal Regulator ID 3"), + ) + hmda_it_dao_hit1, hmda_it_dao_hit2, hmda_it_dao_hit3 = ( + HMDAInstitutionTypeDao(id="HIT1", name="Test HMDA Instituion ID 1"), + HMDAInstitutionTypeDao(id="HIT2", name="Test HMDA Instituion ID 2"), + HMDAInstitutionTypeDao(id="HIT3", name="Test HMDA Instituion ID 3"), + ) + sbl_it_dao_sit1, sbl_it_dao_sit2, sbl_it_dao_sit3 = ( + SBLInstitutionTypeDao(id="SIT1", name="Test SBL Instituion ID 1"), + SBLInstitutionTypeDao(id="SIT2", name="Test SBL Instituion ID 2"), + SBLInstitutionTypeDao(id="SIT3", name="Test SBL Instituion ID 3"), + ) fi_dao_123, fi_dao_456, fi_dao_sub_456 = ( FinancialInstitutionDao( name="Test Bank 123", lei="TESTBANK123", domains=[FinancialInstitutionDomainDao(domain="test.bank.1", lei="TESTBANK123")], + tax_id="123456789", + rssd_id=1234, + primary_federal_regulator_id="FRI1", + hmda_institution_type_id="HIT1", + sbl_institution_type_id="SIT1", + hq_address_street_1="Test Address Street 1", + hq_address_street_2="", + hq_address_city="Test City 1", + hq_address_state_code="GA", + hq_address_zip="00000", + parent_lei="PARENTTESTBANK123", + parent_legal_name="PARENT TEST BANK 123", + parent_rssd_id=12345, + top_holder_lei="TOPHOLDERLEI123", + top_holder_legal_name="TOP HOLDER LEI 123", + top_holder_rssd_id=123456, ), FinancialInstitutionDao( name="Test Bank 456", lei="TESTBANK456", domains=[FinancialInstitutionDomainDao(domain="test.bank.2", lei="TESTBANK456")], + tax_id="987654321", + rssd_id=4321, + primary_federal_regulator_id="FRI2", + hmda_institution_type_id="HIT2", + sbl_institution_type_id="SIT2", + hq_address_street_1="Test Address Street 2", + hq_address_street_2="", + hq_address_city="Test City 2", + hq_address_state_code="CA", + hq_address_zip="11111", + parent_lei="PARENTTESTBANK456", + parent_legal_name="PARENT TEST BANK 456", + parent_rssd_id=54321, + top_holder_lei="TOPHOLDERLEI456", + top_holder_legal_name="TOP HOLDER LEI 456", + top_holder_rssd_id=654321, ), FinancialInstitutionDao( name="Test Sub Bank 456", lei="TESTSUBBANK456", domains=[FinancialInstitutionDomainDao(domain="sub.test.bank.2", lei="TESTSUBBANK456")], + tax_id="765432198", + rssd_id=2134, + primary_federal_regulator_id="FRI3", + hmda_institution_type_id="HIT3", + sbl_institution_type_id="SIT3", + hq_address_street_1="Test Address Street 3", + hq_address_street_2="", + hq_address_city="Test City 3", + hq_address_state_code="FL", + hq_address_zip="11111", + parent_lei="PARENTTESTBANK456", + parent_legal_name="PARENT TEST SUB BANK 456", + parent_rssd_id=21435, + top_holder_lei="TOPHOLDERLEI456", + top_holder_legal_name="TOP HOLDER LEI SUB BANK 456", + top_holder_rssd_id=321654, ), ) + + transaction_session.add(state_ga) + transaction_session.add(state_ca) + transaction_session.add(state_fl) + + transaction_session.add(fr_dao_fri1) + transaction_session.add(fr_dao_fri2) + transaction_session.add(fr_dao_fri3) + + transaction_session.add(hmda_it_dao_hit1) + transaction_session.add(hmda_it_dao_hit2) + transaction_session.add(hmda_it_dao_hit3) + + transaction_session.add(sbl_it_dao_sit1) + transaction_session.add(sbl_it_dao_sit2) + transaction_session.add(sbl_it_dao_sit3) + transaction_session.add(fi_dao_123) transaction_session.add(fi_dao_456) transaction_session.add(fi_dao_sub_456) @@ -72,7 +163,26 @@ async def test_get_institutions_by_lei_list_item_not_existing(self, query_sessio async def test_add_institution(self, transaction_session: AsyncSession): await repo.upsert_institution( transaction_session, - FinancialInstitutionDao(name="New Bank 123", lei="NEWBANK123"), + FinancialInstitutionDao( + name="New Bank 123", + lei="NEWBANK123", + tax_id="654321987", + rssd_id=6543, + primary_federal_regulator_id="FRI3", + hmda_institution_type_id="HIT3", + sbl_institution_type_id="SIT3", + hq_address_street_1="Test Address Street 3", + hq_address_street_2="", + hq_address_city="Test City 3", + hq_address_state_code="FL", + hq_address_zip="22222", + parent_lei="PARENTNEWBANK123", + parent_legal_name="PARENT NEW BANK 123", + parent_rssd_id=76543, + top_holder_lei="TOPHOLDERNEWBANKLEI123", + top_holder_legal_name="TOP HOLDER NEW BANK LEI 123", + top_holder_rssd_id=876543, + ), ) res = await repo.get_institutions(transaction_session) assert len(res) == 4 @@ -101,3 +211,35 @@ async def test_domain_allowed(self, transaction_session: AsyncSession): await transaction_session.commit() assert await repo.is_domain_allowed(transaction_session, "yahoo.com") is False assert await repo.is_domain_allowed(transaction_session, "gmail.com") is True + + async def test_institutions_mapped_to_state_dao_valid(self, query_session: AsyncSession): + res = await repo.get_institutions(query_session, leis=["TESTBANK123"]) + assert res[0].hq_address_state.name == "Georgia" + + async def test_institutions_mapped_to_state_dao_invalid(self, query_session: AsyncSession): + res = await repo.get_institutions(query_session, leis=["TESTBANK456"]) + assert res[0].hq_address_state.name != "Georgia" + + async def test_institutions_mapped_to_federal_regulator_dao_valid(self, query_session: AsyncSession): + res = await repo.get_institutions(query_session, leis=["TESTBANK456"]) + assert res[0].primary_federal_regulator.name != "Test Federal Regulator ID 1" + + async def test_institutions_mapped_to_federal_regulator_dao_invalid(self, query_session: AsyncSession): + res = await repo.get_institutions(query_session, leis=["TESTBANK123"]) + assert res[0].primary_federal_regulator.name == "Test Federal Regulator ID 1" + + async def test_institutions_mapped_to_hmda_it_dao_valid(self, query_session: AsyncSession): + res = await repo.get_institutions(query_session, leis=["TESTBANK123"]) + assert res[0].hmda_institution_type.name == "Test HMDA Instituion ID 1" + + async def test_institutions_mapped_to_hmda_it_dao_invalid(self, query_session: AsyncSession): + res = await repo.get_institutions(query_session, leis=["TESTBANK456"]) + assert res[0].hmda_institution_type.name != "Test HMDA Instituion ID 1" + + async def test_institutions_mapped_to_sbl_it_dao_valid(self, query_session: AsyncSession): + res = await repo.get_institutions(query_session, leis=["TESTBANK123"]) + assert res[0].sbl_institution_type.name == "Test SBL Instituion ID 1" + + async def test_institutions_mapped_to_sbl_it_dao_invalid(self, query_session: AsyncSession): + res = await repo.get_institutions(query_session, leis=["TESTBANK456"]) + assert res[0].sbl_institution_type.name != "Test SBL Instituion ID 1" From 2957ad65aff394f664204a79ba4ee9c3368a4bd6 Mon Sep 17 00:00:00 2001 From: Nargis Sultani Date: Fri, 8 Dec 2023 04:47:53 -0500 Subject: [PATCH 02/11] Added Alembic files and changes --- db_revisions/env.py | 6 +- ...050_update_financial_institutions_table.py | 119 ++++++++++++++++++ ...1f6c33f20a2e_create_address_state_table.py | 37 ++++++ ...612bf1c9_create_federal_regulator_table.py | 36 ++++++ ...cd2d4_create_sbl_institution_type_table.py | 37 ++++++ ...275b_create_hmda_institution_type_table.py | 37 ++++++ tests/migrations/test_migrations.py | 4 + 7 files changed, 275 insertions(+), 1 deletion(-) create mode 100644 db_revisions/versions/045aa502e050_update_financial_institutions_table.py create mode 100644 db_revisions/versions/1f6c33f20a2e_create_address_state_table.py create mode 100644 db_revisions/versions/549c612bf1c9_create_federal_regulator_table.py create mode 100644 db_revisions/versions/56ef0b5cd2d4_create_sbl_institution_type_table.py create mode 100644 db_revisions/versions/8b1ba6a3275b_create_hmda_institution_type_table.py diff --git a/db_revisions/env.py b/db_revisions/env.py index 811e720..92ef7b1 100644 --- a/db_revisions/env.py +++ b/db_revisions/env.py @@ -65,6 +65,7 @@ def run_migrations_offline() -> None: target_metadata=target_metadata, literal_binds=True, dialect_opts={"paramstyle": "named"}, + render_as_batch=True, ) with context.begin_transaction(): @@ -90,7 +91,10 @@ def run_migrations_online() -> None: with connectable.connect() as connection: context.configure( - connection=connection, target_metadata=target_metadata, version_table_schema=target_metadata.schema + connection=connection, + target_metadata=target_metadata, + version_table_schema=target_metadata.schema, + render_as_batch=True, ) with context.begin_transaction(): diff --git a/db_revisions/versions/045aa502e050_update_financial_institutions_table.py b/db_revisions/versions/045aa502e050_update_financial_institutions_table.py new file mode 100644 index 0000000..8e47a8f --- /dev/null +++ b/db_revisions/versions/045aa502e050_update_financial_institutions_table.py @@ -0,0 +1,119 @@ +"""Update Financial Institutions Table + +Revision ID: 045aa502e050 +Revises: "549c612bf1c9" +Create Date: 2023-11-29 11:55:10.328766 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +from db_revisions.utils import table_exists + + +# revision identifiers, used by Alembic. +revision: str = "045aa502e050" +down_revision: Union[str, None] = "549c612bf1c9" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + if table_exists("financial_institutions"): + op.add_column("financial_institutions", sa.Column("tax_id", sa.String(length=9), nullable=True)) + op.add_column("financial_institutions", sa.Column("rssd_id", sa.Integer(), nullable=True)) + op.create_unique_constraint("unique_association", "financial_institutions", ["tax_id", "rssd_id"]) + op.add_column( + "financial_institutions", sa.Column("primary_federal_regulator_id", sa.String(length=4), nullable=True) + ) + op.add_column("financial_institutions", sa.Column("hmda_institution_type_id", sa.String(), nullable=True)) + op.add_column("financial_institutions", sa.Column("sbl_institution_type_id", sa.String(), nullable=True)) + op.add_column("financial_institutions", sa.Column("hq_address_street_1", sa.String(), nullable=False)) + op.add_column("financial_institutions", sa.Column("hq_address_street_2", sa.String(), nullable=True)) + op.add_column("financial_institutions", sa.Column("hq_address_city", sa.String(), nullable=True)) + op.add_column("financial_institutions", sa.Column("hq_address_state", sa.String(length=2), nullable=True)) + op.add_column("financial_institutions", sa.Column("hq_address_zip", sa.String(length=5), nullable=False)) + op.add_column("financial_institutions", sa.Column("parent_lei", sa.String(length=20), nullable=True)) + op.add_column("financial_institutions", sa.Column("parent_legal_name", sa.String(), nullable=True)) + op.add_column("financial_institutions", sa.Column("parent_rssd_id", sa.Integer(), nullable=True)) + op.add_column("financial_institutions", sa.Column("top_holder_lei", sa.String(length=20), nullable=True)) + op.add_column("financial_institutions", sa.Column("top_holder_legal_name", sa.String(), nullable=True)) + op.add_column("financial_institutions", sa.Column("top_holder_rssd_id", sa.Integer(), nullable=True)) + op.create_index( + op.f("ix_financial_institutions_hmda_institution_type_id"), + "financial_institutions", + ["hmda_institution_type_id"], + unique=False, + ) + op.create_index( + op.f("ix_financial_institutions_hq_address_state"), + "financial_institutions", + ["hq_address_state"], + unique=False, + ) + op.create_index( + op.f("ix_financial_institutions_primary_federal_regulator_id"), + "financial_institutions", + ["primary_federal_regulator_id"], + unique=False, + ) + op.create_index( + op.f("ix_financial_institutions_sbl_institution_type_id"), + "financial_institutions", + ["sbl_institution_type_id"], + unique=False, + ) + with op.batch_alter_table("financial_institutions") as batch_op: + batch_op.create_foreign_key( + "fk_federal_regulator_financial_institutions", + "federal_regulator", + ["primary_federal_regulator_id"], + ["id"], + ) + batch_op.create_foreign_key( + "fk_address_state_financial_institutions", "address_state", ["hq_address_state"], ["code"] + ) + batch_op.create_foreign_key( + "fk_hmda_institution_type_financial_institutions", + "hmda_institution_type", + ["hmda_institution_type_id"], + ["id"], + ) + batch_op.create_foreign_key( + "fk_sbl_institution_type_financial_institutions", + "sbl_institution_type", + ["sbl_institution_type_id"], + ["id"], + ) + + +def downgrade() -> None: + op.drop_constraint( + constraint_name="fk_federal_regulator_financial_institutions", table_name="financial_institutions" + ) + op.drop_constraint(constraint_name="fk_address_state_financial_institutions", table_name="financial_institutions") + op.drop_constraint( + constraint_name="fk_hmda_institution_type_financial_institutions", table_name="financial_institutions" + ) + op.drop_constraint( + constraint_name="fk_sbl_institution_type_financial_institutions", table_name="financial_institutions" + ) + op.drop_column("financial_institutions", "top_holder_rssd_id") + op.drop_column("financial_institutions", "top_holder_legal_name") + op.drop_column("financial_institutions", "top_holder_lei") + op.drop_column("financial_institutions", "parent_rssd_id") + op.drop_column("financial_institutions", "parent_legal_name") + op.drop_column("financial_institutions", "parent_lei") + op.drop_column("financial_institutions", "hq_address_zip") + op.drop_column("financial_institutions", "hq_address_state") + op.drop_column("financial_institutions", "hq_address_city") + op.drop_column("financial_institutions", "hq_address_street_2") + op.drop_column("financial_institutions", "hq_address_street_1") + op.drop_column("financial_institutions", "sbl_institution_type_id") + op.drop_column("financial_institutions", "hmda_institution_type_id") + op.drop_column("financial_institutions", "primary_federal_regulator_id") + op.drop_column("financial_institutions", "rssd_id") + op.drop_column("financial_institutions", "tax_id") diff --git a/db_revisions/versions/1f6c33f20a2e_create_address_state_table.py b/db_revisions/versions/1f6c33f20a2e_create_address_state_table.py new file mode 100644 index 0000000..60c65ac --- /dev/null +++ b/db_revisions/versions/1f6c33f20a2e_create_address_state_table.py @@ -0,0 +1,37 @@ +"""Create Address State Table + +Revision ID: 1f6c33f20a2e +Revises: "20e0d51d8be9" +Create Date: 2023-11-29 12:03:41.737864 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + +from db_revisions.utils import table_exists + + +# revision identifiers, used by Alembic. +revision: str = "1f6c33f20a2e" +down_revision: Union[str, None] = "20e0d51d8be9" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + if not table_exists("address_state"): + op.create_table( + "address_state", + sa.Column("code", sa.String(length=2), nullable=False), + sa.Column("name", sa.String(), nullable=False), + sa.Column("event_time", sa.DateTime(), server_default=sa.func.now(), nullable=False), + sa.PrimaryKeyConstraint("code"), + sa.UniqueConstraint("name"), + ) + op.create_index(op.f("ix_address_state_code"), "address_state", ["code"], unique=False) + + +def downgrade() -> None: + op.drop_table("address_state") diff --git a/db_revisions/versions/549c612bf1c9_create_federal_regulator_table.py b/db_revisions/versions/549c612bf1c9_create_federal_regulator_table.py new file mode 100644 index 0000000..f37f893 --- /dev/null +++ b/db_revisions/versions/549c612bf1c9_create_federal_regulator_table.py @@ -0,0 +1,36 @@ +"""Create Federal Regulator Table + +Revision ID: 549c612bf1c9 +Revises: "8b1ba6a3275b" +Create Date: 2023-11-29 12:09:20.012400 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + +from db_revisions.utils import table_exists + + +# revision identifiers, used by Alembic. +revision: str = "549c612bf1c9" +down_revision: Union[str, None] = "8b1ba6a3275b" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + if not table_exists("federal_regulator"): + op.create_table( + "federal_regulator", + sa.Column("id", sa.String(length=4), nullable=False), + sa.Column("name", sa.String(), nullable=False), + sa.Column("event_time", sa.DateTime(), server_default=sa.func.now(), nullable=False), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("name"), + ) + + +def downgrade() -> None: + op.drop_table("federal_regulator") diff --git a/db_revisions/versions/56ef0b5cd2d4_create_sbl_institution_type_table.py b/db_revisions/versions/56ef0b5cd2d4_create_sbl_institution_type_table.py new file mode 100644 index 0000000..aa69bc3 --- /dev/null +++ b/db_revisions/versions/56ef0b5cd2d4_create_sbl_institution_type_table.py @@ -0,0 +1,37 @@ +"""Create SBL Institution Type Table + +Revision ID: 56ef0b5cd2d4 +Revises: "1f6c33f20a2e" +Create Date: 2023-11-29 12:20:05.593826 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + +from db_revisions.utils import table_exists + + +# revision identifiers, used by Alembic. +revision: str = "56ef0b5cd2d4" +down_revision: Union[str, None] = "1f6c33f20a2e" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + if not table_exists("sbl_institution_type"): + op.create_table( + "sbl_institution_type", + sa.Column("id", sa.String(), nullable=False), + sa.Column("name", sa.String(), nullable=False), + sa.Column("event_time", sa.DateTime(), server_default=sa.func.now(), nullable=False), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("name"), + ) + op.create_index(op.f("ix_sbl_institution_type_id"), "sbl_institution_type", ["id"], unique=False) + + +def downgrade() -> None: + op.drop_table("sbl_institution_type") diff --git a/db_revisions/versions/8b1ba6a3275b_create_hmda_institution_type_table.py b/db_revisions/versions/8b1ba6a3275b_create_hmda_institution_type_table.py new file mode 100644 index 0000000..91beed5 --- /dev/null +++ b/db_revisions/versions/8b1ba6a3275b_create_hmda_institution_type_table.py @@ -0,0 +1,37 @@ +"""Create HMDA Institution Type Table + +Revision ID: 8b1ba6a3275b +Revises: "56ef0b5cd2d4" +Create Date: 2023-11-29 12:14:16.694281 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + +from db_revisions.utils import table_exists + + +# revision identifiers, used by Alembic. +revision: str = "8b1ba6a3275b" +down_revision: Union[str, None] = "56ef0b5cd2d4" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + if not table_exists("hmda_institution_type"): + op.create_table( + "hmda_institution_type", + sa.Column("id", sa.String(), nullable=False), + sa.Column("name", sa.String(), nullable=False), + sa.Column("event_time", sa.DateTime(), server_default=sa.func.now(), nullable=False), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("name"), + ) + op.create_index(op.f("ix_hmda_institution_type_id"), "hmda_institution_type", ["id"], unique=False) + + +def downgrade() -> None: + op.drop_table("hmda_institution_type") diff --git a/tests/migrations/test_migrations.py b/tests/migrations/test_migrations.py index 33dda1a..378e73b 100644 --- a/tests/migrations/test_migrations.py +++ b/tests/migrations/test_migrations.py @@ -18,3 +18,7 @@ def test_tables_exist_after_migration(alembic_runner: MigrationContext, alembic_ assert "denied_domains" in tables assert "financial_institutions" in tables assert "financial_institution_domains" in tables + assert "address_state" in tables + assert "federal_regulator" in tables + assert "hmda_institution_type" in tables + assert "sbl_institution_type" in tables From 8ab4c6e529cfe3fd5a44dbe7f4a5e7620f0a4d44 Mon Sep 17 00:00:00 2001 From: Nargis Sultani Date: Fri, 8 Dec 2023 10:40:07 -0500 Subject: [PATCH 03/11] Removed the table_exists condition from updating financial institution alembic script --- ...050_update_financial_institutions_table.py | 125 +++++++++--------- 1 file changed, 60 insertions(+), 65 deletions(-) diff --git a/db_revisions/versions/045aa502e050_update_financial_institutions_table.py b/db_revisions/versions/045aa502e050_update_financial_institutions_table.py index 8e47a8f..3f83674 100644 --- a/db_revisions/versions/045aa502e050_update_financial_institutions_table.py +++ b/db_revisions/versions/045aa502e050_update_financial_institutions_table.py @@ -10,10 +10,6 @@ from alembic import op import sqlalchemy as sa - -from db_revisions.utils import table_exists - - # revision identifiers, used by Alembic. revision: str = "045aa502e050" down_revision: Union[str, None] = "549c612bf1c9" @@ -22,72 +18,71 @@ def upgrade() -> None: - if table_exists("financial_institutions"): - op.add_column("financial_institutions", sa.Column("tax_id", sa.String(length=9), nullable=True)) - op.add_column("financial_institutions", sa.Column("rssd_id", sa.Integer(), nullable=True)) - op.create_unique_constraint("unique_association", "financial_institutions", ["tax_id", "rssd_id"]) - op.add_column( - "financial_institutions", sa.Column("primary_federal_regulator_id", sa.String(length=4), nullable=True) - ) - op.add_column("financial_institutions", sa.Column("hmda_institution_type_id", sa.String(), nullable=True)) - op.add_column("financial_institutions", sa.Column("sbl_institution_type_id", sa.String(), nullable=True)) - op.add_column("financial_institutions", sa.Column("hq_address_street_1", sa.String(), nullable=False)) - op.add_column("financial_institutions", sa.Column("hq_address_street_2", sa.String(), nullable=True)) - op.add_column("financial_institutions", sa.Column("hq_address_city", sa.String(), nullable=True)) - op.add_column("financial_institutions", sa.Column("hq_address_state", sa.String(length=2), nullable=True)) - op.add_column("financial_institutions", sa.Column("hq_address_zip", sa.String(length=5), nullable=False)) - op.add_column("financial_institutions", sa.Column("parent_lei", sa.String(length=20), nullable=True)) - op.add_column("financial_institutions", sa.Column("parent_legal_name", sa.String(), nullable=True)) - op.add_column("financial_institutions", sa.Column("parent_rssd_id", sa.Integer(), nullable=True)) - op.add_column("financial_institutions", sa.Column("top_holder_lei", sa.String(length=20), nullable=True)) - op.add_column("financial_institutions", sa.Column("top_holder_legal_name", sa.String(), nullable=True)) - op.add_column("financial_institutions", sa.Column("top_holder_rssd_id", sa.Integer(), nullable=True)) - op.create_index( - op.f("ix_financial_institutions_hmda_institution_type_id"), - "financial_institutions", - ["hmda_institution_type_id"], - unique=False, + op.add_column("financial_institutions", sa.Column("tax_id", sa.String(length=9), nullable=True)) + op.add_column("financial_institutions", sa.Column("rssd_id", sa.Integer(), nullable=True)) + op.create_unique_constraint("unique_association", "financial_institutions", ["tax_id", "rssd_id"]) + op.add_column( + "financial_institutions", sa.Column("primary_federal_regulator_id", sa.String(length=4), nullable=True) + ) + op.add_column("financial_institutions", sa.Column("hmda_institution_type_id", sa.String(), nullable=True)) + op.add_column("financial_institutions", sa.Column("sbl_institution_type_id", sa.String(), nullable=True)) + op.add_column("financial_institutions", sa.Column("hq_address_street_1", sa.String(), nullable=False)) + op.add_column("financial_institutions", sa.Column("hq_address_street_2", sa.String(), nullable=True)) + op.add_column("financial_institutions", sa.Column("hq_address_city", sa.String(), nullable=True)) + op.add_column("financial_institutions", sa.Column("hq_address_state", sa.String(length=2), nullable=True)) + op.add_column("financial_institutions", sa.Column("hq_address_zip", sa.String(length=5), nullable=False)) + op.add_column("financial_institutions", sa.Column("parent_lei", sa.String(length=20), nullable=True)) + op.add_column("financial_institutions", sa.Column("parent_legal_name", sa.String(), nullable=True)) + op.add_column("financial_institutions", sa.Column("parent_rssd_id", sa.Integer(), nullable=True)) + op.add_column("financial_institutions", sa.Column("top_holder_lei", sa.String(length=20), nullable=True)) + op.add_column("financial_institutions", sa.Column("top_holder_legal_name", sa.String(), nullable=True)) + op.add_column("financial_institutions", sa.Column("top_holder_rssd_id", sa.Integer(), nullable=True)) + op.create_index( + op.f("ix_financial_institutions_hmda_institution_type_id"), + "financial_institutions", + ["hmda_institution_type_id"], + unique=False, + ) + op.create_index( + op.f("ix_financial_institutions_hq_address_state"), + "financial_institutions", + ["hq_address_state"], + unique=False, + ) + op.create_index( + op.f("ix_financial_institutions_primary_federal_regulator_id"), + "financial_institutions", + ["primary_federal_regulator_id"], + unique=False, + ) + op.create_index( + op.f("ix_financial_institutions_sbl_institution_type_id"), + "financial_institutions", + ["sbl_institution_type_id"], + unique=False, + ) + with op.batch_alter_table("financial_institutions") as batch_op: + batch_op.create_foreign_key( + "fk_federal_regulator_financial_institutions", + "federal_regulator", + ["primary_federal_regulator_id"], + ["id"], ) - op.create_index( - op.f("ix_financial_institutions_hq_address_state"), - "financial_institutions", - ["hq_address_state"], - unique=False, + batch_op.create_foreign_key( + "fk_address_state_financial_institutions", "address_state", ["hq_address_state"], ["code"] ) - op.create_index( - op.f("ix_financial_institutions_primary_federal_regulator_id"), - "financial_institutions", - ["primary_federal_regulator_id"], - unique=False, + batch_op.create_foreign_key( + "fk_hmda_institution_type_financial_institutions", + "hmda_institution_type", + ["hmda_institution_type_id"], + ["id"], ) - op.create_index( - op.f("ix_financial_institutions_sbl_institution_type_id"), - "financial_institutions", + batch_op.create_foreign_key( + "fk_sbl_institution_type_financial_institutions", + "sbl_institution_type", ["sbl_institution_type_id"], - unique=False, + ["id"], ) - with op.batch_alter_table("financial_institutions") as batch_op: - batch_op.create_foreign_key( - "fk_federal_regulator_financial_institutions", - "federal_regulator", - ["primary_federal_regulator_id"], - ["id"], - ) - batch_op.create_foreign_key( - "fk_address_state_financial_institutions", "address_state", ["hq_address_state"], ["code"] - ) - batch_op.create_foreign_key( - "fk_hmda_institution_type_financial_institutions", - "hmda_institution_type", - ["hmda_institution_type_id"], - ["id"], - ) - batch_op.create_foreign_key( - "fk_sbl_institution_type_financial_institutions", - "sbl_institution_type", - ["sbl_institution_type_id"], - ["id"], - ) def downgrade() -> None: From 9bce1b045116a4521f4d812333c0dd9e8a3ce96a Mon Sep 17 00:00:00 2001 From: Nargis Sultani Date: Fri, 8 Dec 2023 14:24:33 -0500 Subject: [PATCH 04/11] Fixed alembic issues --- ...050_update_financial_institutions_table.py | 81 +++++++++---------- .../entities/repos/test_institutions_repo.py | 16 ++-- tests/migrations/test_migrations.py | 2 +- 3 files changed, 47 insertions(+), 52 deletions(-) diff --git a/db_revisions/versions/045aa502e050_update_financial_institutions_table.py b/db_revisions/versions/045aa502e050_update_financial_institutions_table.py index 3f83674..2d5a3c9 100644 --- a/db_revisions/versions/045aa502e050_update_financial_institutions_table.py +++ b/db_revisions/versions/045aa502e050_update_financial_institutions_table.py @@ -18,50 +18,45 @@ def upgrade() -> None: - op.add_column("financial_institutions", sa.Column("tax_id", sa.String(length=9), nullable=True)) - op.add_column("financial_institutions", sa.Column("rssd_id", sa.Integer(), nullable=True)) - op.create_unique_constraint("unique_association", "financial_institutions", ["tax_id", "rssd_id"]) - op.add_column( - "financial_institutions", sa.Column("primary_federal_regulator_id", sa.String(length=4), nullable=True) - ) - op.add_column("financial_institutions", sa.Column("hmda_institution_type_id", sa.String(), nullable=True)) - op.add_column("financial_institutions", sa.Column("sbl_institution_type_id", sa.String(), nullable=True)) - op.add_column("financial_institutions", sa.Column("hq_address_street_1", sa.String(), nullable=False)) - op.add_column("financial_institutions", sa.Column("hq_address_street_2", sa.String(), nullable=True)) - op.add_column("financial_institutions", sa.Column("hq_address_city", sa.String(), nullable=True)) - op.add_column("financial_institutions", sa.Column("hq_address_state", sa.String(length=2), nullable=True)) - op.add_column("financial_institutions", sa.Column("hq_address_zip", sa.String(length=5), nullable=False)) - op.add_column("financial_institutions", sa.Column("parent_lei", sa.String(length=20), nullable=True)) - op.add_column("financial_institutions", sa.Column("parent_legal_name", sa.String(), nullable=True)) - op.add_column("financial_institutions", sa.Column("parent_rssd_id", sa.Integer(), nullable=True)) - op.add_column("financial_institutions", sa.Column("top_holder_lei", sa.String(length=20), nullable=True)) - op.add_column("financial_institutions", sa.Column("top_holder_legal_name", sa.String(), nullable=True)) - op.add_column("financial_institutions", sa.Column("top_holder_rssd_id", sa.Integer(), nullable=True)) - op.create_index( - op.f("ix_financial_institutions_hmda_institution_type_id"), - "financial_institutions", - ["hmda_institution_type_id"], - unique=False, - ) - op.create_index( - op.f("ix_financial_institutions_hq_address_state"), - "financial_institutions", - ["hq_address_state"], - unique=False, - ) - op.create_index( - op.f("ix_financial_institutions_primary_federal_regulator_id"), - "financial_institutions", - ["primary_federal_regulator_id"], - unique=False, - ) - op.create_index( - op.f("ix_financial_institutions_sbl_institution_type_id"), - "financial_institutions", - ["sbl_institution_type_id"], - unique=False, - ) with op.batch_alter_table("financial_institutions") as batch_op: + batch_op.add_column(sa.Column("tax_id", sa.String(length=9), nullable=True)) + batch_op.add_column(sa.Column("rssd_id", sa.Integer(), nullable=True)) + batch_op.add_column(sa.Column("primary_federal_regulator_id", sa.String(length=4), nullable=True)) + batch_op.add_column(sa.Column("hmda_institution_type_id", sa.String(), nullable=True)) + batch_op.add_column(sa.Column("sbl_institution_type_id", sa.String(), nullable=True)) + batch_op.add_column(sa.Column("hq_address_street_1", sa.String(), nullable=False)) + batch_op.add_column(sa.Column("hq_address_street_2", sa.String(), nullable=True)) + batch_op.add_column(sa.Column("hq_address_city", sa.String(), nullable=True)) + batch_op.add_column(sa.Column("hq_address_state", sa.String(length=2), nullable=True)) + batch_op.add_column(sa.Column("hq_address_zip", sa.String(length=5), nullable=False)) + batch_op.add_column(sa.Column("parent_lei", sa.String(length=20), nullable=True)) + batch_op.add_column(sa.Column("parent_legal_name", sa.String(), nullable=True)) + batch_op.add_column(sa.Column("parent_rssd_id", sa.Integer(), nullable=True)) + batch_op.add_column(sa.Column("top_holder_lei", sa.String(length=20), nullable=True)) + batch_op.add_column(sa.Column("top_holder_legal_name", sa.String(), nullable=True)) + batch_op.add_column(sa.Column("top_holder_rssd_id", sa.Integer(), nullable=True)) + batch_op.create_index( + batch_op.f("ix_financial_institutions_hmda_institution_type_id"), + ["hmda_institution_type_id"], + unique=False, + ) + batch_op.create_index( + batch_op.f("ix_financial_institutions_hq_address_state"), + ["hq_address_state"], + unique=False, + ) + batch_op.create_index( + batch_op.f("ix_financial_institutions_primary_federal_regulator_id"), + ["primary_federal_regulator_id"], + unique=False, + ) + batch_op.create_index( + batch_op.f("ix_financial_institutions_sbl_institution_type_id"), + ["sbl_institution_type_id"], + unique=False, + ) + batch_op.create_unique_constraint("unique_financial_institutions_tax_id", ["tax_id"]) + batch_op.create_unique_constraint("unique_financial_institutions_rssd_id", ["rssd_id"]) batch_op.create_foreign_key( "fk_federal_regulator_financial_institutions", "federal_regulator", diff --git a/tests/entities/repos/test_institutions_repo.py b/tests/entities/repos/test_institutions_repo.py index f2095d8..63d93aa 100644 --- a/tests/entities/repos/test_institutions_repo.py +++ b/tests/entities/repos/test_institutions_repo.py @@ -212,34 +212,34 @@ async def test_domain_allowed(self, transaction_session: AsyncSession): assert await repo.is_domain_allowed(transaction_session, "yahoo.com") is False assert await repo.is_domain_allowed(transaction_session, "gmail.com") is True - async def test_institutions_mapped_to_state_dao_valid(self, query_session: AsyncSession): + async def test_institution_mapped_to_state_valid(self, query_session: AsyncSession): res = await repo.get_institutions(query_session, leis=["TESTBANK123"]) assert res[0].hq_address_state.name == "Georgia" - async def test_institutions_mapped_to_state_dao_invalid(self, query_session: AsyncSession): + async def test_institution_mapped_to_state_invalid(self, query_session: AsyncSession): res = await repo.get_institutions(query_session, leis=["TESTBANK456"]) assert res[0].hq_address_state.name != "Georgia" - async def test_institutions_mapped_to_federal_regulator_dao_valid(self, query_session: AsyncSession): + async def test_institution_mapped_to_federal_regulator_valid(self, query_session: AsyncSession): res = await repo.get_institutions(query_session, leis=["TESTBANK456"]) assert res[0].primary_federal_regulator.name != "Test Federal Regulator ID 1" - async def test_institutions_mapped_to_federal_regulator_dao_invalid(self, query_session: AsyncSession): + async def test_institution_mapped_to_federal_regulator_invalid(self, query_session: AsyncSession): res = await repo.get_institutions(query_session, leis=["TESTBANK123"]) assert res[0].primary_federal_regulator.name == "Test Federal Regulator ID 1" - async def test_institutions_mapped_to_hmda_it_dao_valid(self, query_session: AsyncSession): + async def test_institution_mapped_to_hmda_it_valid(self, query_session: AsyncSession): res = await repo.get_institutions(query_session, leis=["TESTBANK123"]) assert res[0].hmda_institution_type.name == "Test HMDA Instituion ID 1" - async def test_institutions_mapped_to_hmda_it_dao_invalid(self, query_session: AsyncSession): + async def test_institution_mapped_to_hmda_it_invalid(self, query_session: AsyncSession): res = await repo.get_institutions(query_session, leis=["TESTBANK456"]) assert res[0].hmda_institution_type.name != "Test HMDA Instituion ID 1" - async def test_institutions_mapped_to_sbl_it_dao_valid(self, query_session: AsyncSession): + async def test_institution_mapped_to_sbl_it_valid(self, query_session: AsyncSession): res = await repo.get_institutions(query_session, leis=["TESTBANK123"]) assert res[0].sbl_institution_type.name == "Test SBL Instituion ID 1" - async def test_institutions_mapped_to_sbl_it_dao_invalid(self, query_session: AsyncSession): + async def test_institution_mapped_to_sbl_it_invalid(self, query_session: AsyncSession): res = await repo.get_institutions(query_session, leis=["TESTBANK456"]) assert res[0].sbl_institution_type.name != "Test SBL Instituion ID 1" diff --git a/tests/migrations/test_migrations.py b/tests/migrations/test_migrations.py index 378e73b..2a27197 100644 --- a/tests/migrations/test_migrations.py +++ b/tests/migrations/test_migrations.py @@ -11,7 +11,7 @@ def test_tables_exist_after_migration(alembic_runner: MigrationContext, alembic_engine: Engine): - alembic_runner.migrate_up_to("20e0d51d8be9") + alembic_runner.migrate_up_to("045aa502e050") inspector = sqlalchemy.inspect(alembic_engine) tables = inspector.get_table_names() From 609863dfe5e8b13a33b7dbd44b10f5e8b01c1e15 Mon Sep 17 00:00:00 2001 From: Nargis Sultani Date: Sat, 9 Dec 2023 03:32:54 -0500 Subject: [PATCH 05/11] changed address_state_code name in db_revision to match that of in the models --- ...045aa502e050_update_financial_institutions_table.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/db_revisions/versions/045aa502e050_update_financial_institutions_table.py b/db_revisions/versions/045aa502e050_update_financial_institutions_table.py index 2d5a3c9..7932387 100644 --- a/db_revisions/versions/045aa502e050_update_financial_institutions_table.py +++ b/db_revisions/versions/045aa502e050_update_financial_institutions_table.py @@ -27,7 +27,7 @@ def upgrade() -> None: batch_op.add_column(sa.Column("hq_address_street_1", sa.String(), nullable=False)) batch_op.add_column(sa.Column("hq_address_street_2", sa.String(), nullable=True)) batch_op.add_column(sa.Column("hq_address_city", sa.String(), nullable=True)) - batch_op.add_column(sa.Column("hq_address_state", sa.String(length=2), nullable=True)) + batch_op.add_column(sa.Column("hq_address_state_code", sa.String(length=2), nullable=True)) batch_op.add_column(sa.Column("hq_address_zip", sa.String(length=5), nullable=False)) batch_op.add_column(sa.Column("parent_lei", sa.String(length=20), nullable=True)) batch_op.add_column(sa.Column("parent_legal_name", sa.String(), nullable=True)) @@ -41,8 +41,8 @@ def upgrade() -> None: unique=False, ) batch_op.create_index( - batch_op.f("ix_financial_institutions_hq_address_state"), - ["hq_address_state"], + batch_op.f("ix_financial_institutions_hq_address_state_code"), + ["hq_address_state_code"], unique=False, ) batch_op.create_index( @@ -64,7 +64,7 @@ def upgrade() -> None: ["id"], ) batch_op.create_foreign_key( - "fk_address_state_financial_institutions", "address_state", ["hq_address_state"], ["code"] + "fk_address_state_code_financial_institutions", "address_state", ["hq_address_state_code"], ["code"] ) batch_op.create_foreign_key( "fk_hmda_institution_type_financial_institutions", @@ -98,7 +98,7 @@ def downgrade() -> None: op.drop_column("financial_institutions", "parent_legal_name") op.drop_column("financial_institutions", "parent_lei") op.drop_column("financial_institutions", "hq_address_zip") - op.drop_column("financial_institutions", "hq_address_state") + op.drop_column("financial_institutions", "hq_address_state_code") op.drop_column("financial_institutions", "hq_address_city") op.drop_column("financial_institutions", "hq_address_street_2") op.drop_column("financial_institutions", "hq_address_street_1") From b7af90fe1e4c8d0c7d79e52395633596d601c75b Mon Sep 17 00:00:00 2001 From: Nargis Sultani Date: Thu, 14 Dec 2023 05:31:25 -0500 Subject: [PATCH 06/11] Ticket 61: Feeding lookup tables with alembic --- db_revisions/feed/address_state.csv | 58 +++++++++++++++++++ db_revisions/feed/federal_regulator.csv | 14 +++++ db_revisions/feed/hmda_institution_type.csv | 19 ++++++ db_revisions/feed/sbl_institution_type.csv | 14 +++++ db_revisions/utils.py | 13 +++++ ...a742d97ad9_feed_federal_regulator_table.py | 28 +++++++++ .../7b6ff51002b5_feed_address_state_table.py | 28 +++++++++ ...1b1e109_feed_sbl_institution_type_table.py | 27 +++++++++ ...1aa6df_feed_hmda_institution_type_table.py | 28 +++++++++ tests/migrations/test_migrations.py | 36 ++++++++++++ 10 files changed, 265 insertions(+) create mode 100644 db_revisions/feed/address_state.csv create mode 100644 db_revisions/feed/federal_regulator.csv create mode 100644 db_revisions/feed/hmda_institution_type.csv create mode 100644 db_revisions/feed/sbl_institution_type.csv create mode 100644 db_revisions/versions/26a742d97ad9_feed_federal_regulator_table.py create mode 100644 db_revisions/versions/7b6ff51002b5_feed_address_state_table.py create mode 100644 db_revisions/versions/a41281b1e109_feed_sbl_institution_type_table.py create mode 100644 db_revisions/versions/f4ff7d1aa6df_feed_hmda_institution_type_table.py diff --git a/db_revisions/feed/address_state.csv b/db_revisions/feed/address_state.csv new file mode 100644 index 0000000..8cf1d0a --- /dev/null +++ b/db_revisions/feed/address_state.csv @@ -0,0 +1,58 @@ +code|name +AL|Alabama +AK|Alaska +AZ|Arizona +AR|Arkansas +CA|California +CO|Colorado +CT|Connecticut +DE|Delaware +FL|Florida +GA|Georgia +HI|Hawaii +ID|Idaho +IL|Illinois +IN|Indiana +IA|Iowa +KS|Kansas +KY|Kentucky +LA|Louisiana +ME|Maine +MD|Maryland +MA|Massachusetts +MI|Michigan +MN|Minnesota +MS|Mississippi +MO|Missouri +MT|Montana +NE|Nebraska +NV|Nevada +NH|New Hampshire +NJ|New Jersey +NM|New Mexico +NY|New York +NC|North Carolina +ND|North Dakota +OH|Ohio +OK|Oklahoma +OR|Oregon +PA|Pennsylvania +RI|Rhode Island +SC|South Carolina +SD|South Dakota +TN|Tennessee +TX|Texas +UT|Utah +VT|Vermont +VA|Virginia +WA|Washington +WV|West Virginia +WI|Wisconsin +WY|Wyoming +DC|District of Columbia +AS|American Samoa +GU|Guam +MP|Northern Mariana Islands +PR|Puerto Rico +UM|United States Minor Outlying Islands +VI|Virgin Islands, U.S. \ No newline at end of file diff --git a/db_revisions/feed/federal_regulator.csv b/db_revisions/feed/federal_regulator.csv new file mode 100644 index 0000000..43a0273 --- /dev/null +++ b/db_revisions/feed/federal_regulator.csv @@ -0,0 +1,14 @@ +id|name +FCA|Farm Credit Administration +FDIC|Federal Deposit Insurance Corporation +FHFA|Federal Housing Finance Agency +FRS|Federal Reserve System +NCUA|National Credit Union Administration +OCC|Office of the Comptroller of the Currency +OTS|Office of Thrift Supervision (only valid until July 21, 2011) + + + + + + diff --git a/db_revisions/feed/hmda_institution_type.csv b/db_revisions/feed/hmda_institution_type.csv new file mode 100644 index 0000000..49d71d7 --- /dev/null +++ b/db_revisions/feed/hmda_institution_type.csv @@ -0,0 +1,19 @@ +id|name +1|National Bank (OCC supervised) +2|State Member Bank (FRS Supervised): +3|State non-member bank (FDIC supervised) +4|State Chartered Thrift (FDIC supervised) +5|Federal Chartered Thrift (OCC supervised) +6|Credit Union (NCUA supervised) +7|Federal Branch or Agency of Foreign Banking Organization (FBO) +8|Branch or Agency of FBO (FRS supervised) +9|MBS of national Bank (OCC supervised) +10|MBS of state member bank (FRS supervised) +11|MBS of state non-member bank (FDIC supervised) +12|MBS of Bank Holding Company (BHC) (FRS supervised) +13|MBS of credit union (NCUA supervised) +14|independent MBS, no depository affiliation +15|MBS of Savings and Loan Holding Co +16|MBS of state chartered Thrift +17|MBS of federally chartered thrift (OCC supervised) +18|Affiliate of depository institution. MBS is in the same ownership org as a depository. \ No newline at end of file diff --git a/db_revisions/feed/sbl_institution_type.csv b/db_revisions/feed/sbl_institution_type.csv new file mode 100644 index 0000000..5d150ae --- /dev/null +++ b/db_revisions/feed/sbl_institution_type.csv @@ -0,0 +1,14 @@ +id|name +1|Bank or savings association. +2|Minority depository institution. +3|Credit union. +4|Nondepository institution. +5|Community development financial institution (CDFI). +6|Other nonprofit financial institution. +7|Farm Credit System institution. +8|Government lender. +9|Commercial finance company. +10|Equipment finance company. +11|Industrial loan company. +12|Online lender. +13|Other \ No newline at end of file diff --git a/db_revisions/utils.py b/db_revisions/utils.py index d84fa39..2119987 100644 --- a/db_revisions/utils.py +++ b/db_revisions/utils.py @@ -1,6 +1,8 @@ from alembic import op from sqlalchemy import engine_from_config import sqlalchemy +from csv import DictReader +import os def table_exists(table_name): @@ -11,3 +13,14 @@ def table_exists(table_name): inspector = sqlalchemy.inspect(engine) tables = inspector.get_table_names() return table_name in tables + + +def get_feed_data_from_file(table_name): + file_dir = os.path.dirname(os.path.realpath(__file__)) + data_file_path = f"{file_dir}/feed/%s.csv" % table_name + data_file = open(data_file_path, "r") + reader = DictReader(data_file, delimiter="|") + output_list = list() + for dictionary in reader: + output_list.append(dictionary) + return output_list diff --git a/db_revisions/versions/26a742d97ad9_feed_federal_regulator_table.py b/db_revisions/versions/26a742d97ad9_feed_federal_regulator_table.py new file mode 100644 index 0000000..31d69cf --- /dev/null +++ b/db_revisions/versions/26a742d97ad9_feed_federal_regulator_table.py @@ -0,0 +1,28 @@ +"""Feed Federal Regulator table + +Revision ID: 26a742d97ad9 +Revises: 7b6ff51002b5 +Create Date: 2023-12-14 01:23:17.872728 + +""" +from typing import Sequence, Union +from alembic import op +from db_revisions.utils import get_feed_data_from_file +from entities.models import FederalRegulatorDao + + +# revision identifiers, used by Alembic. +revision: str = "26a742d97ad9" +down_revision: Union[str, None] = "7b6ff51002b5" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + data = get_feed_data_from_file(FederalRegulatorDao.__tablename__) + + op.bulk_insert(FederalRegulatorDao.__table__, data) + + +def downgrade() -> None: + op.execute(FederalRegulatorDao.__table__.delete()) diff --git a/db_revisions/versions/7b6ff51002b5_feed_address_state_table.py b/db_revisions/versions/7b6ff51002b5_feed_address_state_table.py new file mode 100644 index 0000000..35fe95a --- /dev/null +++ b/db_revisions/versions/7b6ff51002b5_feed_address_state_table.py @@ -0,0 +1,28 @@ +"""Feed Address State table + +Revision ID: 7b6ff51002b5 +Revises: 045aa502e050 +Create Date: 2023-12-14 01:21:48.325752 + +""" +from typing import Sequence, Union +from alembic import op +from db_revisions.utils import get_feed_data_from_file +from entities.models import AddressStateDao + + +# revision identifiers, used by Alembic. +revision: str = "7b6ff51002b5" +down_revision: Union[str, None] = "045aa502e050" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + data = get_feed_data_from_file(AddressStateDao.__tablename__) + + op.bulk_insert(AddressStateDao.__table__, data) + + +def downgrade() -> None: + op.execute(AddressStateDao.__table__.delete()) diff --git a/db_revisions/versions/a41281b1e109_feed_sbl_institution_type_table.py b/db_revisions/versions/a41281b1e109_feed_sbl_institution_type_table.py new file mode 100644 index 0000000..b79001b --- /dev/null +++ b/db_revisions/versions/a41281b1e109_feed_sbl_institution_type_table.py @@ -0,0 +1,27 @@ +"""Feed SBL Institution Type table + +Revision ID: a41281b1e109 +Revises: f4ff7d1aa6df +Create Date: 2023-12-14 01:24:00.120073 + +""" +from typing import Sequence, Union +from alembic import op +from db_revisions.utils import get_feed_data_from_file +from entities.models import SBLInstitutionTypeDao + +# revision identifiers, used by Alembic. +revision: str = "a41281b1e109" +down_revision: Union[str, None] = "f4ff7d1aa6df" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + data = get_feed_data_from_file(SBLInstitutionTypeDao.__tablename__) + + op.bulk_insert(SBLInstitutionTypeDao.__table__, data) + + +def downgrade() -> None: + op.execute(SBLInstitutionTypeDao.__table__.delete()) diff --git a/db_revisions/versions/f4ff7d1aa6df_feed_hmda_institution_type_table.py b/db_revisions/versions/f4ff7d1aa6df_feed_hmda_institution_type_table.py new file mode 100644 index 0000000..297f5d8 --- /dev/null +++ b/db_revisions/versions/f4ff7d1aa6df_feed_hmda_institution_type_table.py @@ -0,0 +1,28 @@ +"""Feed HMDA Institution Type table + +Revision ID: f4ff7d1aa6df +Revises: 26a742d97ad9 +Create Date: 2023-12-14 01:23:47.017878 + +""" +from typing import Sequence, Union +from alembic import op +from db_revisions.utils import get_feed_data_from_file +from entities.models import HMDAInstitutionTypeDao + + +# revision identifiers, used by Alembic. +revision: str = "f4ff7d1aa6df" +down_revision: Union[str, None] = "26a742d97ad9" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + data = get_feed_data_from_file(HMDAInstitutionTypeDao.__tablename__) + + op.bulk_insert(HMDAInstitutionTypeDao.__table__, data) + + +def downgrade() -> None: + op.execute(HMDAInstitutionTypeDao.__table__.delete()) diff --git a/tests/migrations/test_migrations.py b/tests/migrations/test_migrations.py index 2a27197..e705a03 100644 --- a/tests/migrations/test_migrations.py +++ b/tests/migrations/test_migrations.py @@ -5,6 +5,7 @@ ) import sqlalchemy +from sqlalchemy import text from sqlalchemy.engine import Engine from pytest_alembic import MigrationContext @@ -22,3 +23,38 @@ def test_tables_exist_after_migration(alembic_runner: MigrationContext, alembic_ assert "federal_regulator" in tables assert "hmda_institution_type" in tables assert "sbl_institution_type" in tables + + +def test_gnarly_migration_xyz123(alembic_runner: MigrationContext, alembic_engine: Engine): + # Migrate up to, but not including this new migration + alembic_runner.migrate_up_before("045aa502e050") + + # Test address_state feed + alembic_runner.insert_into("address_state", dict(code="TS", name="Test State")) + alembic_runner.migrate_up_one() + with alembic_engine.connect() as conn: + address_state_rows = conn.execute(text("SELECT code from address_state")).fetchall() + assert address_state_rows == [("TS",)] + + # Test federal_regulator feed + alembic_runner.insert_into("federal_regulator", dict(id="TFCA", name="Test Farm Credit Administration")) + alembic_runner.migrate_up_one() + with alembic_engine.connect() as conn: + address_state_rows = conn.execute(text("SELECT id from federal_regulator")).fetchall() + assert address_state_rows == [("TFCA",)] + + # Test hmda_institution_type feed + alembic_runner.insert_into( + "hmda_institution_type", dict(id="T8", name="Test Branch or Agency of FBO (FRS supervised)") + ) + alembic_runner.migrate_up_one() + with alembic_engine.connect() as conn: + address_state_rows = conn.execute(text("SELECT id from hmda_institution_type")).fetchall() + assert address_state_rows == [("T8",)] + + # Test sbl_institution_type feed + alembic_runner.insert_into("sbl_institution_type", dict(id="T12", name="Test Online lender")) + alembic_runner.migrate_up_one() + with alembic_engine.connect() as conn: + address_state_rows = conn.execute(text("SELECT id from sbl_institution_type")).fetchall() + assert address_state_rows == [("T12",)] From a59f6a60d8a075545217bf8a3a2d73377ad94f78 Mon Sep 17 00:00:00 2001 From: Nargis Sultani Date: Thu, 14 Dec 2023 13:21:49 -0500 Subject: [PATCH 07/11] Modified test cases --- db_revisions/feed/federal_regulator.csv | 8 +-- tests/migrations/test_migrations.py | 66 +++++++++++++++++++------ 2 files changed, 51 insertions(+), 23 deletions(-) diff --git a/db_revisions/feed/federal_regulator.csv b/db_revisions/feed/federal_regulator.csv index 43a0273..a7cdab5 100644 --- a/db_revisions/feed/federal_regulator.csv +++ b/db_revisions/feed/federal_regulator.csv @@ -5,10 +5,4 @@ FHFA|Federal Housing Finance Agency FRS|Federal Reserve System NCUA|National Credit Union Administration OCC|Office of the Comptroller of the Currency -OTS|Office of Thrift Supervision (only valid until July 21, 2011) - - - - - - +OTS|Office of Thrift Supervision (only valid until July 21, 2011) \ No newline at end of file diff --git a/tests/migrations/test_migrations.py b/tests/migrations/test_migrations.py index c021527..8c9bcec 100644 --- a/tests/migrations/test_migrations.py +++ b/tests/migrations/test_migrations.py @@ -7,6 +7,9 @@ import sqlalchemy from sqlalchemy import text from sqlalchemy.engine import Engine +import os +import csv +from entities.models import AddressStateDao, FederalRegulatorDao, HMDAInstitutionTypeDao, SBLInstitutionTypeDao from pytest_alembic import MigrationContext @@ -25,36 +28,67 @@ def test_tables_exist_after_migration(alembic_runner: MigrationContext, alembic_ assert "sbl_institution_type" in tables -def test_data_feed_after_migration_045aa502e050(alembic_runner: MigrationContext, alembic_engine: Engine): +def data_feed_migration_helper(table_name): + file_dir = os.path.dirname(os.path.realpath(__file__)) + data_file_path = f"{file_dir}/../../db_revisions/feed/%s.csv" % table_name + with open(data_file_path) as f: + reader = csv.reader(f, delimiter="|") + next(reader) + output_list = list(tuple(line) for line in reader) + return output_list + + +def test_address_state_data_feed(alembic_runner: MigrationContext, alembic_engine: Engine): # Migrate up to, but not including this new migration - alembic_runner.migrate_up_before("045aa502e050") + alembic_runner.migrate_up_before("7b6ff51002b5") # Test address_state feed - alembic_runner.insert_into("address_state", dict(code="TS", name="Test State")) + address_state_tablename = AddressStateDao.__tablename__ alembic_runner.migrate_up_one() with alembic_engine.connect() as conn: - address_state_rows = conn.execute(text("SELECT code from address_state")).fetchall() - assert address_state_rows == [("TS",)] + address_state_rows = conn.execute(text("SELECT code, name from %s" % address_state_tablename)).fetchall() + address_state_expected = data_feed_migration_helper(address_state_tablename) + assert address_state_rows == address_state_expected + + +def test_federal_regulator_data_feed(alembic_runner: MigrationContext, alembic_engine: Engine): + # Migrate up to, but not including this new migration + alembic_runner.migrate_up_before("26a742d97ad9") # Test federal_regulator feed - alembic_runner.insert_into("federal_regulator", dict(id="TFCA", name="Test Farm Credit Administration")) + federal_regulator_tablename = FederalRegulatorDao.__tablename__ alembic_runner.migrate_up_one() with alembic_engine.connect() as conn: - address_state_rows = conn.execute(text("SELECT id from federal_regulator")).fetchall() - assert address_state_rows == [("TFCA",)] + federal_regulator_rows = conn.execute(text("SELECT id, name from %s" % federal_regulator_tablename)).fetchall() + federal_regulator_expected = data_feed_migration_helper(federal_regulator_tablename) + assert federal_regulator_rows == federal_regulator_expected + + +def test_hmda_institution_type_data_feed(alembic_runner: MigrationContext, alembic_engine: Engine): + # Migrate up to, but not including this new migration + alembic_runner.migrate_up_before("f4ff7d1aa6df") # Test hmda_institution_type feed - alembic_runner.insert_into( - "hmda_institution_type", dict(id="T8", name="Test Branch or Agency of FBO (FRS supervised)") - ) + hmda_institution_type_tablename = HMDAInstitutionTypeDao.__tablename__ alembic_runner.migrate_up_one() with alembic_engine.connect() as conn: - address_state_rows = conn.execute(text("SELECT id from hmda_institution_type")).fetchall() - assert address_state_rows == [("T8",)] + hmda_institution_type_rows = conn.execute( + text("SELECT id, name from %s" % hmda_institution_type_tablename) + ).fetchall() + hmda_institution_type_expected = data_feed_migration_helper(hmda_institution_type_tablename) + assert hmda_institution_type_rows == hmda_institution_type_expected + + +def test_sbl_institution_type_data_feed(alembic_runner: MigrationContext, alembic_engine: Engine): + # Migrate up to, but not including this new migration + alembic_runner.migrate_up_before("a41281b1e109") # Test sbl_institution_type feed - alembic_runner.insert_into("sbl_institution_type", dict(id="T12", name="Test Online lender")) + sbl_institution_type_tablename = SBLInstitutionTypeDao.__tablename__ alembic_runner.migrate_up_one() with alembic_engine.connect() as conn: - address_state_rows = conn.execute(text("SELECT id from sbl_institution_type")).fetchall() - assert address_state_rows == [("T12",)] + sbl_institution_type_rows = conn.execute( + text("SELECT id, name from %s" % sbl_institution_type_tablename) + ).fetchall() + sbl_institution_type_expected = data_feed_migration_helper(sbl_institution_type_tablename) + assert sbl_institution_type_rows == sbl_institution_type_expected From b39dd24fac6b6895c9cf43e4f576bb927cd5af93 Mon Sep 17 00:00:00 2001 From: Nargis Sultani Date: Thu, 14 Dec 2023 16:27:40 -0500 Subject: [PATCH 08/11] Address the comment and moved data-feed-related tests to a new file (test_lookup_tables_data_feed) --- .../test_lookup_tables_data_feed.py | 70 +++++++++++++++++++ tests/migrations/test_migrations.py | 70 ------------------- 2 files changed, 70 insertions(+), 70 deletions(-) create mode 100644 tests/migrations/test_lookup_tables_data_feed.py diff --git a/tests/migrations/test_lookup_tables_data_feed.py b/tests/migrations/test_lookup_tables_data_feed.py new file mode 100644 index 0000000..42e9349 --- /dev/null +++ b/tests/migrations/test_lookup_tables_data_feed.py @@ -0,0 +1,70 @@ +import os +import csv +from sqlalchemy import select +from sqlalchemy.engine import Engine +from pytest_alembic import MigrationContext +from entities.models import AddressStateDao, FederalRegulatorDao, HMDAInstitutionTypeDao, SBLInstitutionTypeDao + + +def data_feed_helper(table_name): + file_dir = os.path.dirname(os.path.realpath(__file__)) + data_file_path = f"{file_dir}/../../db_revisions/feed/%s.csv" % table_name + with open(data_file_path) as f: + reader = csv.reader(f, delimiter="|") + next(reader) + output_list = list(tuple(line) for line in reader) + return output_list + + +def test_address_state_data_feed(alembic_runner: MigrationContext, alembic_engine: Engine): + # Migrate up to, but not including this new migration + alembic_runner.migrate_up_before("7b6ff51002b5") + + # Test address_state feed + alembic_runner.migrate_up_one() + + with alembic_engine.connect() as conn: + # address_state_rows = conn.execute(text("SELECT code, name from %s" % address_state_tablename)).fetchall() + address_state_rows = conn.execute(select(AddressStateDao.code, AddressStateDao.name)).fetchall() + address_state_expected = data_feed_helper(AddressStateDao.__tablename__) + assert address_state_rows == address_state_expected + + +def test_federal_regulator_data_feed(alembic_runner: MigrationContext, alembic_engine: Engine): + # Migrate up to, but not including this new migration + alembic_runner.migrate_up_before("26a742d97ad9") + + # Test federal_regulator feed + alembic_runner.migrate_up_one() + with alembic_engine.connect() as conn: + federal_regulator_rows = conn.execute(select(FederalRegulatorDao.id, FederalRegulatorDao.name)).fetchall() + federal_regulator_expected = data_feed_helper(FederalRegulatorDao.__tablename__) + assert federal_regulator_rows == federal_regulator_expected + + +def test_hmda_institution_type_data_feed(alembic_runner: MigrationContext, alembic_engine: Engine): + # Migrate up to, but not including this new migration + alembic_runner.migrate_up_before("f4ff7d1aa6df") + + # Test hmda_institution_type feed + alembic_runner.migrate_up_one() + with alembic_engine.connect() as conn: + hmda_institution_type_rows = conn.execute( + select(HMDAInstitutionTypeDao.id, HMDAInstitutionTypeDao.name) + ).fetchall() + hmda_institution_type_expected = data_feed_helper(HMDAInstitutionTypeDao.__tablename__) + assert hmda_institution_type_rows == hmda_institution_type_expected + + +def test_sbl_institution_type_data_feed(alembic_runner: MigrationContext, alembic_engine: Engine): + # Migrate up to, but not including this new migration + alembic_runner.migrate_up_before("a41281b1e109") + + # Test sbl_institution_type feed + alembic_runner.migrate_up_one() + with alembic_engine.connect() as conn: + sbl_institution_type_rows = conn.execute( + select(SBLInstitutionTypeDao.id, SBLInstitutionTypeDao.name) + ).fetchall() + sbl_institution_type_expected = data_feed_helper(SBLInstitutionTypeDao.__tablename__) + assert sbl_institution_type_rows == sbl_institution_type_expected diff --git a/tests/migrations/test_migrations.py b/tests/migrations/test_migrations.py index 8c9bcec..2a27197 100644 --- a/tests/migrations/test_migrations.py +++ b/tests/migrations/test_migrations.py @@ -5,11 +5,7 @@ ) import sqlalchemy -from sqlalchemy import text from sqlalchemy.engine import Engine -import os -import csv -from entities.models import AddressStateDao, FederalRegulatorDao, HMDAInstitutionTypeDao, SBLInstitutionTypeDao from pytest_alembic import MigrationContext @@ -26,69 +22,3 @@ def test_tables_exist_after_migration(alembic_runner: MigrationContext, alembic_ assert "federal_regulator" in tables assert "hmda_institution_type" in tables assert "sbl_institution_type" in tables - - -def data_feed_migration_helper(table_name): - file_dir = os.path.dirname(os.path.realpath(__file__)) - data_file_path = f"{file_dir}/../../db_revisions/feed/%s.csv" % table_name - with open(data_file_path) as f: - reader = csv.reader(f, delimiter="|") - next(reader) - output_list = list(tuple(line) for line in reader) - return output_list - - -def test_address_state_data_feed(alembic_runner: MigrationContext, alembic_engine: Engine): - # Migrate up to, but not including this new migration - alembic_runner.migrate_up_before("7b6ff51002b5") - - # Test address_state feed - address_state_tablename = AddressStateDao.__tablename__ - alembic_runner.migrate_up_one() - with alembic_engine.connect() as conn: - address_state_rows = conn.execute(text("SELECT code, name from %s" % address_state_tablename)).fetchall() - address_state_expected = data_feed_migration_helper(address_state_tablename) - assert address_state_rows == address_state_expected - - -def test_federal_regulator_data_feed(alembic_runner: MigrationContext, alembic_engine: Engine): - # Migrate up to, but not including this new migration - alembic_runner.migrate_up_before("26a742d97ad9") - - # Test federal_regulator feed - federal_regulator_tablename = FederalRegulatorDao.__tablename__ - alembic_runner.migrate_up_one() - with alembic_engine.connect() as conn: - federal_regulator_rows = conn.execute(text("SELECT id, name from %s" % federal_regulator_tablename)).fetchall() - federal_regulator_expected = data_feed_migration_helper(federal_regulator_tablename) - assert federal_regulator_rows == federal_regulator_expected - - -def test_hmda_institution_type_data_feed(alembic_runner: MigrationContext, alembic_engine: Engine): - # Migrate up to, but not including this new migration - alembic_runner.migrate_up_before("f4ff7d1aa6df") - - # Test hmda_institution_type feed - hmda_institution_type_tablename = HMDAInstitutionTypeDao.__tablename__ - alembic_runner.migrate_up_one() - with alembic_engine.connect() as conn: - hmda_institution_type_rows = conn.execute( - text("SELECT id, name from %s" % hmda_institution_type_tablename) - ).fetchall() - hmda_institution_type_expected = data_feed_migration_helper(hmda_institution_type_tablename) - assert hmda_institution_type_rows == hmda_institution_type_expected - - -def test_sbl_institution_type_data_feed(alembic_runner: MigrationContext, alembic_engine: Engine): - # Migrate up to, but not including this new migration - alembic_runner.migrate_up_before("a41281b1e109") - - # Test sbl_institution_type feed - sbl_institution_type_tablename = SBLInstitutionTypeDao.__tablename__ - alembic_runner.migrate_up_one() - with alembic_engine.connect() as conn: - sbl_institution_type_rows = conn.execute( - text("SELECT id, name from %s" % sbl_institution_type_tablename) - ).fetchall() - sbl_institution_type_expected = data_feed_migration_helper(sbl_institution_type_tablename) - assert sbl_institution_type_rows == sbl_institution_type_expected From 4424a46b3a1a23883e5c5e40ad7eb627f2c81b42 Mon Sep 17 00:00:00 2001 From: Nargis Sultani Date: Thu, 14 Dec 2023 16:30:15 -0500 Subject: [PATCH 09/11] removed commented code --- tests/migrations/test_lookup_tables_data_feed.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/migrations/test_lookup_tables_data_feed.py b/tests/migrations/test_lookup_tables_data_feed.py index 42e9349..9d983a4 100644 --- a/tests/migrations/test_lookup_tables_data_feed.py +++ b/tests/migrations/test_lookup_tables_data_feed.py @@ -24,7 +24,6 @@ def test_address_state_data_feed(alembic_runner: MigrationContext, alembic_engin alembic_runner.migrate_up_one() with alembic_engine.connect() as conn: - # address_state_rows = conn.execute(text("SELECT code, name from %s" % address_state_tablename)).fetchall() address_state_rows = conn.execute(select(AddressStateDao.code, AddressStateDao.name)).fetchall() address_state_expected = data_feed_helper(AddressStateDao.__tablename__) assert address_state_rows == address_state_expected From 35d8c4c4cce6c4e5c28321702d09100591642df2 Mon Sep 17 00:00:00 2001 From: Nargis Sultani Date: Fri, 15 Dec 2023 13:40:48 -0500 Subject: [PATCH 10/11] harcoded table names in the revision script and test cases instead of coupling them with dao --- ...a742d97ad9_feed_federal_regulator_table.py | 2 +- .../7b6ff51002b5_feed_address_state_table.py | 2 +- ...1b1e109_feed_sbl_institution_type_table.py | 2 +- ...1aa6df_feed_hmda_institution_type_table.py | 2 +- .../test_lookup_tables_data_feed.py | 23 +++++++++++-------- 5 files changed, 17 insertions(+), 14 deletions(-) diff --git a/db_revisions/versions/26a742d97ad9_feed_federal_regulator_table.py b/db_revisions/versions/26a742d97ad9_feed_federal_regulator_table.py index 31d69cf..5302b0d 100644 --- a/db_revisions/versions/26a742d97ad9_feed_federal_regulator_table.py +++ b/db_revisions/versions/26a742d97ad9_feed_federal_regulator_table.py @@ -19,7 +19,7 @@ def upgrade() -> None: - data = get_feed_data_from_file(FederalRegulatorDao.__tablename__) + data = get_feed_data_from_file("federal_regulator") op.bulk_insert(FederalRegulatorDao.__table__, data) diff --git a/db_revisions/versions/7b6ff51002b5_feed_address_state_table.py b/db_revisions/versions/7b6ff51002b5_feed_address_state_table.py index 35fe95a..5a1ed40 100644 --- a/db_revisions/versions/7b6ff51002b5_feed_address_state_table.py +++ b/db_revisions/versions/7b6ff51002b5_feed_address_state_table.py @@ -19,7 +19,7 @@ def upgrade() -> None: - data = get_feed_data_from_file(AddressStateDao.__tablename__) + data = get_feed_data_from_file("address_state") op.bulk_insert(AddressStateDao.__table__, data) diff --git a/db_revisions/versions/a41281b1e109_feed_sbl_institution_type_table.py b/db_revisions/versions/a41281b1e109_feed_sbl_institution_type_table.py index b79001b..058087f 100644 --- a/db_revisions/versions/a41281b1e109_feed_sbl_institution_type_table.py +++ b/db_revisions/versions/a41281b1e109_feed_sbl_institution_type_table.py @@ -18,7 +18,7 @@ def upgrade() -> None: - data = get_feed_data_from_file(SBLInstitutionTypeDao.__tablename__) + data = get_feed_data_from_file("sbl_institution_type") op.bulk_insert(SBLInstitutionTypeDao.__table__, data) diff --git a/db_revisions/versions/f4ff7d1aa6df_feed_hmda_institution_type_table.py b/db_revisions/versions/f4ff7d1aa6df_feed_hmda_institution_type_table.py index 297f5d8..1502c9e 100644 --- a/db_revisions/versions/f4ff7d1aa6df_feed_hmda_institution_type_table.py +++ b/db_revisions/versions/f4ff7d1aa6df_feed_hmda_institution_type_table.py @@ -19,7 +19,7 @@ def upgrade() -> None: - data = get_feed_data_from_file(HMDAInstitutionTypeDao.__tablename__) + data = get_feed_data_from_file("hmda_institution_type") op.bulk_insert(HMDAInstitutionTypeDao.__table__, data) diff --git a/tests/migrations/test_lookup_tables_data_feed.py b/tests/migrations/test_lookup_tables_data_feed.py index 9d983a4..6181871 100644 --- a/tests/migrations/test_lookup_tables_data_feed.py +++ b/tests/migrations/test_lookup_tables_data_feed.py @@ -1,6 +1,6 @@ import os import csv -from sqlalchemy import select +from sqlalchemy import text from sqlalchemy.engine import Engine from pytest_alembic import MigrationContext from entities.models import AddressStateDao, FederalRegulatorDao, HMDAInstitutionTypeDao, SBLInstitutionTypeDao @@ -21,11 +21,11 @@ def test_address_state_data_feed(alembic_runner: MigrationContext, alembic_engin alembic_runner.migrate_up_before("7b6ff51002b5") # Test address_state feed + address_state_tablename = "address_state" alembic_runner.migrate_up_one() - with alembic_engine.connect() as conn: - address_state_rows = conn.execute(select(AddressStateDao.code, AddressStateDao.name)).fetchall() - address_state_expected = data_feed_helper(AddressStateDao.__tablename__) + address_state_rows = conn.execute(text("SELECT code, name from %s" % address_state_tablename)).fetchall() + address_state_expected = data_feed_helper(address_state_tablename) assert address_state_rows == address_state_expected @@ -34,10 +34,11 @@ def test_federal_regulator_data_feed(alembic_runner: MigrationContext, alembic_e alembic_runner.migrate_up_before("26a742d97ad9") # Test federal_regulator feed + federal_regulator_tablename = "federal_regulator" alembic_runner.migrate_up_one() with alembic_engine.connect() as conn: - federal_regulator_rows = conn.execute(select(FederalRegulatorDao.id, FederalRegulatorDao.name)).fetchall() - federal_regulator_expected = data_feed_helper(FederalRegulatorDao.__tablename__) + federal_regulator_rows = conn.execute(text("SELECT id, name from %s" % federal_regulator_tablename)).fetchall() + federal_regulator_expected = data_feed_helper(federal_regulator_tablename) assert federal_regulator_rows == federal_regulator_expected @@ -46,12 +47,13 @@ def test_hmda_institution_type_data_feed(alembic_runner: MigrationContext, alemb alembic_runner.migrate_up_before("f4ff7d1aa6df") # Test hmda_institution_type feed + hmda_institution_type_tablename = "hmda_institution_type" alembic_runner.migrate_up_one() with alembic_engine.connect() as conn: hmda_institution_type_rows = conn.execute( - select(HMDAInstitutionTypeDao.id, HMDAInstitutionTypeDao.name) + text("SELECT id, name from %s" % hmda_institution_type_tablename) ).fetchall() - hmda_institution_type_expected = data_feed_helper(HMDAInstitutionTypeDao.__tablename__) + hmda_institution_type_expected = data_feed_helper(hmda_institution_type_tablename) assert hmda_institution_type_rows == hmda_institution_type_expected @@ -60,10 +62,11 @@ def test_sbl_institution_type_data_feed(alembic_runner: MigrationContext, alembi alembic_runner.migrate_up_before("a41281b1e109") # Test sbl_institution_type feed + sbl_institution_type_tablename = "sbl_institution_type" alembic_runner.migrate_up_one() with alembic_engine.connect() as conn: sbl_institution_type_rows = conn.execute( - select(SBLInstitutionTypeDao.id, SBLInstitutionTypeDao.name) + text("SELECT id, name from %s" % sbl_institution_type_tablename) ).fetchall() - sbl_institution_type_expected = data_feed_helper(SBLInstitutionTypeDao.__tablename__) + sbl_institution_type_expected = data_feed_helper(sbl_institution_type_tablename) assert sbl_institution_type_rows == sbl_institution_type_expected From bea2615332c28bc2f0c1f6c93fb085da43d47c72 Mon Sep 17 00:00:00 2001 From: Nargis Sultani Date: Fri, 15 Dec 2023 13:51:21 -0500 Subject: [PATCH 11/11] removed unused imports --- tests/migrations/test_lookup_tables_data_feed.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/migrations/test_lookup_tables_data_feed.py b/tests/migrations/test_lookup_tables_data_feed.py index 6181871..ed74161 100644 --- a/tests/migrations/test_lookup_tables_data_feed.py +++ b/tests/migrations/test_lookup_tables_data_feed.py @@ -3,7 +3,6 @@ from sqlalchemy import text from sqlalchemy.engine import Engine from pytest_alembic import MigrationContext -from entities.models import AddressStateDao, FederalRegulatorDao, HMDAInstitutionTypeDao, SBLInstitutionTypeDao def data_feed_helper(table_name):