Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Moved the feed data for the lookup tables from csv file into src/conf… #67

Merged
merged 6 commits into from
Jan 4, 2024
Merged

Moved the feed data for the lookup tables from csv file into src/conf… #67

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
b570fd6
Moved the feed data for the lookup tables from csv file into src/conf…
Dec 21, 2023
f96e709
Address the comment by moving the seed data from config.py to db_revi…
Dec 21, 2023
f584799
Modified comments, changed feed to seed
Dec 21, 2023
21052bb
Addressed the comments
Jan 2, 2024
4fd0d11
addressed tge comment
Jan 2, 2024
8ee8223
Addressed the comments
Jan 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 0 additions & 58 deletions db_revisions/feed/address_state.csv
View file
Open in desktop

This file was deleted.

8 changes: 0 additions & 8 deletions db_revisions/feed/federal_regulator.csv
View file
Open in desktop

This file was deleted.

19 changes: 0 additions & 19 deletions db_revisions/feed/hmda_institution_type.csv
View file
Open in desktop

This file was deleted.

14 changes: 0 additions & 14 deletions db_revisions/feed/sbl_institution_type.csv
View file
Open in desktop

This file was deleted.

13 changes: 0 additions & 13 deletions db_revisions/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
from alembic import op
from sqlalchemy import engine_from_config
import sqlalchemy
from csv import DictReader
import os


def table_exists(table_name):
Expand All @@ -13,14 +11,3 @@ def table_exists(table_name):
inspector = sqlalchemy.inspect(engine)
tables = inspector.get_table_names()
return table_name in tables


def get_feed_data_from_file(table_name):
file_dir = os.path.dirname(os.path.realpath(__file__))
data_file_path = f"{file_dir}/feed/%s.csv" % table_name
data_file = open(data_file_path, "r")
reader = DictReader(data_file, delimiter="|")
output_list = list()
for dictionary in reader:
output_list.append(dictionary)
return output_list
4 changes: 3 additions & 1 deletion db_revisions/versions/045aa502e050_update_financial_institutions_table.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,9 @@ def downgrade() -> None:
op.drop_constraint(
constraint_name="fk_federal_regulator_financial_institutions", table_name="financial_institutions"
)
op.drop_constraint(constraint_name="fk_address_state_financial_institutions", table_name="financial_institutions")
op.drop_constraint(
constraint_name="fk_address_state_code_financial_institutions", table_name="financial_institutions"
)
op.drop_constraint(
constraint_name="fk_hmda_institution_type_financial_institutions", table_name="financial_institutions"
)
Expand Down
6 changes: 2 additions & 4 deletions db_revisions/versions/26a742d97ad9_feed_federal_regulator_table.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@
"""
from typing import Sequence, Union
from alembic import op
from db_revisions.utils import get_feed_data_from_file
from entities.models import FederalRegulatorDao
from config import federal_regulator_feed


# revision identifiers, used by Alembic.
Expand All @@ -19,9 +19,7 @@


def upgrade() -> None:
data = get_feed_data_from_file("federal_regulator")

op.bulk_insert(FederalRegulatorDao.__table__, data)
op.bulk_insert(FederalRegulatorDao.__table__, federal_regulator_feed)


def downgrade() -> None:
Expand Down
6 changes: 2 additions & 4 deletions db_revisions/versions/7b6ff51002b5_feed_address_state_table.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@
"""
from typing import Sequence, Union
from alembic import op
from db_revisions.utils import get_feed_data_from_file
from entities.models import AddressStateDao
from config import address_state_feed


# revision identifiers, used by Alembic.
Expand All @@ -19,9 +19,7 @@


def upgrade() -> None:
data = get_feed_data_from_file("address_state")

op.bulk_insert(AddressStateDao.__table__, data)
op.bulk_insert(AddressStateDao.__table__, address_state_feed)


def downgrade() -> None:
Expand Down
6 changes: 2 additions & 4 deletions db_revisions/versions/a41281b1e109_feed_sbl_institution_type_table.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@
"""
from typing import Sequence, Union
from alembic import op
from db_revisions.utils import get_feed_data_from_file
from entities.models import SBLInstitutionTypeDao
from config import sbl_institution_type_feed

# revision identifiers, used by Alembic.
revision: str = "a41281b1e109"
Expand All @@ -18,9 +18,7 @@


def upgrade() -> None:
data = get_feed_data_from_file("sbl_institution_type")

op.bulk_insert(SBLInstitutionTypeDao.__table__, data)
op.bulk_insert(SBLInstitutionTypeDao.__table__, sbl_institution_type_feed)


def downgrade() -> None:
Expand Down
6 changes: 2 additions & 4 deletions db_revisions/versions/f4ff7d1aa6df_feed_hmda_institution_type_table.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@
"""
from typing import Sequence, Union
from alembic import op
from db_revisions.utils import get_feed_data_from_file
from entities.models import HMDAInstitutionTypeDao
from config import hmda_institution_type_feed


# revision identifiers, used by Alembic.
Expand All @@ -19,9 +19,7 @@


def upgrade() -> None:
data = get_feed_data_from_file("hmda_institution_type")

op.bulk_insert(HMDAInstitutionTypeDao.__table__, data)
op.bulk_insert(HMDAInstitutionTypeDao.__table__, hmda_institution_type_feed)


def downgrade() -> None:
Expand Down
119 changes: 119 additions & 0 deletions src/config.py
View file
Open in desktop
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't know if this was discussed already in the other PRs or with Le, but should the individual feed lists go into their corresponding migration scripts to keep the seed data with the creation of the table the data is going into? The config class seems to be specific to env configurations we pull from either env vars or .env files.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I can move it inside its individual scripts. That's the way I initially had it, but then I needed to access the same data for testing. So, I wanted to have them just in one place.

I can go ahead and move it inside the individual scripts.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Gotcha, thought it might be that because of the comment in there. It's great that the tests check that the expected seed data is in tables so hopefully the tests can access the lists in the migrations (which is a little annoying to import each especially since the module names are horrific) or at the worst copy the list to the test testing the table.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should still have them inside the individual scripts for data integrity, considering the small data set. For testing purposes, you can test just a small sample of the data; or create test resources.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@lchen-2101 Got it, yeah makes more sense.

Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,125 @@
from pydantic.types import SecretStr
from pydantic_settings import BaseSettings, SettingsConfigDict

"""
START Feed data for the lookup tables:
address_state
federal_regulator
hmda_institution_type
sbl_institution_type
These are accessed in db_revisions/versions/* scripts and in test/migrations/test_lookup_tables_data_feed .
"""

address_state_feed = [
{"code": "AL", "name": "Alabama"},
{"code": "AK", "name": "Alaska"},
{"code": "AZ", "name": "Arizona"},
{"code": "AR", "name": "Arkansas"},
{"code": "CA", "name": "California"},
{"code": "CO", "name": "Colorado"},
{"code": "CT", "name": "Connecticut"},
{"code": "DE", "name": "Delaware"},
{"code": "FL", "name": "Florida"},
{"code": "GA", "name": "Georgia"},
{"code": "HI", "name": "Hawaii"},
{"code": "ID", "name": "Idaho"},
{"code": "IL", "name": "Illinois"},
{"code": "IN", "name": "Indiana"},
{"code": "IA", "name": "Iowa"},
{"code": "KS", "name": "Kansas"},
{"code": "KY", "name": "Kentucky"},
{"code": "LA", "name": "Louisiana"},
{"code": "ME", "name": "Maine"},
{"code": "MD", "name": "Maryland"},
{"code": "MA", "name": "Massachusetts"},
{"code": "MI", "name": "Michigan"},
{"code": "MN", "name": "Minnesota"},
{"code": "MS", "name": "Mississippi"},
{"code": "MO", "name": "Missouri"},
{"code": "MT", "name": "Montana"},
{"code": "NE", "name": "Nebraska"},
{"code": "NV", "name": "Nevada"},
{"code": "NH", "name": "New Hampshire"},
{"code": "NJ", "name": "New Jersey"},
{"code": "NM", "name": "New Mexico"},
{"code": "NY", "name": "New York"},
{"code": "NC", "name": "North Carolina"},
{"code": "ND", "name": "North Dakota"},
{"code": "OH", "name": "Ohio"},
{"code": "OK", "name": "Oklahoma"},
{"code": "OR", "name": "Oregon"},
{"code": "PA", "name": "Pennsylvania"},
{"code": "RI", "name": "Rhode Island"},
{"code": "SC", "name": "South Carolina"},
{"code": "SD", "name": "South Dakota"},
{"code": "TN", "name": "Tennessee"},
{"code": "TX", "name": "Texas"},
{"code": "UT", "name": "Utah"},
{"code": "VT", "name": "Vermont"},
{"code": "VA", "name": "Virginia"},
{"code": "WA", "name": "Washington"},
{"code": "WV", "name": "West Virginia"},
{"code": "WI", "name": "Wisconsin"},
{"code": "WY", "name": "Wyoming"},
{"code": "DC", "name": "District of Columbia"},
{"code": "AS", "name": "American Samoa"},
{"code": "GU", "name": "Guam"},
{"code": "MP", "name": "Northern Mariana Islands"},
{"code": "PR", "name": "Puerto Rico"},
{"code": "UM", "name": "United States Minor Outlying Islands"},
{"code": "VI", "name": "Virgin Islands, U.S."},
]

federal_regulator_feed = [
{"id": "FCA", "name": "Farm Credit Administration"},
{"id": "FDIC", "name": "Federal Deposit Insurance Corporation"},
{"id": "FHFA", "name": "Federal Housing Finance Agency"},
{"id": "FRS", "name": "Federal Reserve System"},
{"id": "NCUA", "name": "National Credit Union Administration"},
{"id": "OCC", "name": "Office of the Comptroller of the Currency"},
{"id": "OTS", "name": "Office of Thrift Supervision (only valid until July 21, 2011)"},
]

sbl_institution_type_feed = [
{"id": "1", "name": "Bank or savings association."},
{"id": "2", "name": "Minority depository institution."},
{"id": "3", "name": "Credit union."},
{"id": "4", "name": "Nondepository institution."},
{"id": "5", "name": "Community development financial institution (CDFI)."},
{"id": "6", "name": "Other nonprofit financial institution."},
{"id": "7", "name": "Farm Credit System institution."},
{"id": "8", "name": "Government lender."},
{"id": "9", "name": "Commercial finance company."},
{"id": "10", "name": "Equipment finance company."},
{"id": "11", "name": "Industrial loan company."},
{"id": "12", "name": "Online lender."},
{"id": "13", "name": "Other"},
]

hmda_institution_type_feed = [
{"id": "1", "name": "National Bank (OCC supervised)"},
{"id": "2", "name": "State Member Bank (FRS Supervised)"},
{"id": "3", "name": "State non-member bank (FDIC supervised)"},
{"id": "4", "name": "State Chartered Thrift (FDIC supervised)"},
{"id": "5", "name": "Federal Chartered Thrift (OCC supervised)"},
{"id": "6", "name": "Credit Union (NCUA supervised)"},
{"id": "7", "name": "Federal Branch or Agency of Foreign Banking Organization (FBO)"},
{"id": "8", "name": "Branch or Agency of FBO (FRS supervised)"},
{"id": "9", "name": "MBS of national Bank (OCC supervised)"},
{"id": "10", "name": "MBS of state member bank (FRS supervised)"},
{"id": "11", "name": "MBS of state non-member bank (FDIC supervised)"},
{"id": "12", "name": "MBS of Bank Holding Company (BHC) (FRS supervised)"},
{"id": "13", "name": "MBS of credit union (NCUA supervised)"},
{"id": "14", "name": "independent MBS, no depository affiliation"},
{"id": "15", "name": "MBS of Savings and Loan Holding Co"},
{"id": "16", "name": "MBS of state chartered Thrift"},
{"id": "17", "name": "MBS of federally chartered thrift (OCC supervised)"},
{"id": "18", "name": "Affiliate of depository institution. MBS is in the same ownership org as a depository."},
]

"""
END Feed data for the lookup tables:
"""
JWT_OPTS_PREFIX = "jwt_opts_"

env_files_to_load = [".env"]
Expand Down
Loading
Loading