Skip to content

Commit

Permalink
prepare for qwat tests and remove subset db code
Browse files Browse the repository at this point in the history
  • Loading branch information
@olivierdalang
olivierdalang committed Apr 23, 2021
1 parent f667243 commit 9446f92
Show file tree
Hide file tree
Showing 5 changed files with 66 additions and 123 deletions.
6 changes: 1 addition & 5 deletions qgepqwat2ili/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ def main(args):

parser_setupdb = subparsers.add_parser("setupdb", help="setup test db")
parser_setupdb.set_defaults(parser="setupdb")
parser_setupdb.add_argument("type", choices=["empty", "full", "subset"], help="type")
parser_setupdb.add_argument("type", choices=["empty", "full"], help="type")

args = parser.parse_args(args)

Expand Down Expand Up @@ -125,10 +125,6 @@ def main(args):
utils.templates.generate_template("qwat", "wasser", BaseQwat, BaseWasser, QWATMAPPING)

elif args.parser == "setupdb":
if args.type == "subset":
raise Exception(
"subset is currently disabled as quite slow, uncomment corresponding lines utils/various.py"
)
utils.various.setup_test_db(args.type)

else:
Expand Down
Empty file added qgepqwat2ili/tests/__init__.py
View file
Empty file.
21 changes: 8 additions & 13 deletions qgepqwat2ili/tests.py → qgepqwat2ili/tests/test_qgep.py
View file
Original file line number Diff line number Diff line change
@@ -1,19 +1,14 @@
"""
python -m unittest interlis.tests
"""

import decimal
import logging
import os
import sys
import tempfile
import unittest

from qgepqwat2ili import main, utils
from qgepqwat2ili.qgep.model_qgep import get_qgep_model
from sqlalchemy.orm import Session

from . import main, utils
from .qgep.model_qgep import get_qgep_model

# Display logging in unittest output
logger = logging.getLogger()
logger.setLevel(logging.WARNING)
Expand All @@ -31,7 +26,7 @@ def test_case_a_import_wincan_xtf(self):
"""

# Validate the incomming XTF
path = os.path.join(os.path.dirname(__file__), "data", "test_data", "case_a_import_from_wincan.xtf")
path = os.path.join(os.path.dirname(__file__), "..", "data", "test_data", "case_a_import_from_wincan.xtf")
utils.ili2db.validate_xtf_data(path)

# Prepare db (we import in a full schema)
Expand Down Expand Up @@ -85,7 +80,7 @@ def test_case_b_export_complete_qgep_to_xtf(self):
# B. export the whole QGEP model to interlis
"""

# Prepare subset db
# Prepare db
main(["setupdb", "full"])

path = os.path.join(tempfile.mkdtemp(), "export.xtf")
Expand All @@ -103,7 +98,7 @@ def test_case_c_import_complete_xtf_to_qgep(self):

# Incomming XTF
# THIS INPUT FILE IS INVALID !
path = os.path.join(os.path.dirname(__file__), "data", "test_data", "case_c_import_all.xtf")
path = os.path.join(os.path.dirname(__file__), "..", "data", "test_data", "case_c_import_all.xtf")

# Prepare subset db (we import in an empty schema)
main(["setupdb", "empty"])
Expand Down Expand Up @@ -131,7 +126,7 @@ def test_case_d_export_subset(self):
# D. export a subset
"""

# Prepare subset db
# Prepare db
main(["setupdb", "full"])

path = os.path.join(tempfile.mkdtemp(), "export.xtf")
Expand Down Expand Up @@ -165,10 +160,10 @@ def test_regression_001_self_referencing_organisation(self):
"""

path = os.path.join(
os.path.dirname(__file__), "data", "test_data", "regression_001_self_referencing_organisation.xtf"
os.path.dirname(__file__), "..", "data", "test_data", "regression_001_self_referencing_organisation.xtf"
)

# Prepare subset db (we import in an empty schema)
# Prepare db (we import in an empty schema)
main(["setupdb", "empty"])

QGEP = get_qgep_model()
Expand Down
Empty file added qgepqwat2ili/tests/test_qwat.py
View file
Empty file.
162 changes: 57 additions & 105 deletions qgepqwat2ili/utils/various.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,11 +34,11 @@ def exec_(command, check=True):

def setup_test_db(template="full"):
"""
As initializing demo data can be a bit slow (esp. if we want only a subset),
As initializing demo data can be a bit slow,
we do these steps in a Docker container then commit it to an image, so we can
start a clean database relatively quickly.
We prepare three template databases : full, subset, empty
We prepare two template databases : full, empty
And then copy those to config.PGDATABASE when needed to initialize an fresh db
for testing.
"""
Expand All @@ -57,111 +57,63 @@ def dexec_(cmd, check=True):
f"docker run -d --rm -v qgepqwat_db:/var/lib/postgresql/data -p 5432:5432 --name qgepqwat -e POSTGRES_PASSWORD={pgconf['password'] or 'postgres'} -e POSTGRES_DB={pgconf['dbname'] or 'qgep_prod'} postgis/postgis"
)

# Wait for PG
while exec_("docker exec qgepqwat pg_isready", check=False) != 0:
logger.info("Postgres not ready... we wait...")
time.sleep(1)
# Wait for PG
while exec_("docker exec qgepqwat pg_isready", check=False) != 0:
logger.info("Postgres not ready... we wait...")
time.sleep(1)

db_didnt_exist = (
dexec_("createdb -U postgres tpl_empty", check=False) == 0
or dexec_("createdb -U postgres tpl_subset", check=False) == 0
or dexec_("createdb -U postgres tpl_full", check=False) == 0
db_didnt_exist = (
dexec_("createdb -U postgres tpl_empty", check=False) == 0
or dexec_("createdb -U postgres tpl_full", check=False) == 0
)
if db_didnt_exist:
logger.info("Test templates don't exist, we create them")

dexec_("dropdb -U postgres tpl_empty")
dexec_("dropdb -U postgres tpl_full")

dexec_("apt-get update")
dexec_("apt-get install -y wget")

# Getting data
dexec_(
"wget https://github.com/QGEP/datamodel/releases/download/1.5.4/qgep_1.5.4_structure_and_demo_data.backup"
)
dexec_(
"wget https://github.com/QGEP/datamodel/releases/download/1.5.4/qgep_1.5.4_structure_with_value_lists.sql"
)
dexec_(
"wget https://github.com/qwat/qwat-data-model/releases/download/1.3.5/qwat_v1.3.5_data_and_structure_sample.backup"
)
dexec_("wget https://github.com/qwat/qwat-data-model/releases/download/1.3.5/qwat_v1.3.5_structure_only.sql")
dexec_(
"wget https://github.com/qwat/qwat-data-model/releases/download/1.3.5/qwat_v1.3.5_value_list_data_only.sql"
)
if db_didnt_exist:
logger.info("Test templates don't exist, we create them")

dexec_("dropdb -U postgres tpl_empty --if-exists")
dexec_("dropdb -U postgres tpl_subset --if-exists")
dexec_("dropdb -U postgres tpl_full --if-exists")

dexec_("apt-get update")
dexec_("apt-get install -y wget")

# Getting data
dexec_(
"wget https://github.com/QGEP/datamodel/releases/download/1.5.4/qgep_1.5.4_structure_and_demo_data.backup"
)
dexec_(
"wget https://github.com/QGEP/datamodel/releases/download/1.5.4/qgep_1.5.4_structure_with_value_lists.sql"
)
dexec_(
"wget https://github.com/qwat/qwat-data-model/releases/download/1.3.5/qwat_v1.3.5_data_and_structure_sample.backup"
)
dexec_(
"wget https://github.com/qwat/qwat-data-model/releases/download/1.3.5/qwat_v1.3.5_structure_only.sql"
)
dexec_(
"wget https://github.com/qwat/qwat-data-model/releases/download/1.3.5/qwat_v1.3.5_value_list_data_only.sql"
)

# Creating the template DB with empty structure
dexec_("psql -f qgep_1.5.4_structure_with_value_lists.sql qgep_prod postgres")
dexec_("psql -f qwat_v1.3.5_structure_only.sql qgep_prod postgres")
dexec_("psql -f qwat_v1.3.5_value_list_data_only.sql qgep_prod postgres")
dexec_("createdb -U postgres --template=qgep_prod tpl_empty")

# Creating the template DB with full data
dexec_(
'psql -U postgres -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE pid<>pg_backend_pid();"'
)
dexec_("dropdb -U postgres qgep_prod --if-exists")
dexec_("createdb -U postgres qgep_prod")
dexec_(
"pg_restore -U postgres --dbname qgep_prod --verbose --no-privileges --exit-on-error qgep_1.5.4_structure_and_demo_data.backup"
)
dexec_(
"pg_restore -U postgres --dbname qgep_prod --verbose --no-privileges --exit-on-error qwat_v1.3.5_data_and_structure_sample.backup"
)
dexec_("createdb -U postgres --template=qgep_prod tpl_full")

# Hotfix invalid data
delta_path = os.path.join(
os.path.dirname(__file__), "..", "data", "test_data", "qgep_demodata_hotfix.sql"
)
exec_(f"docker cp {delta_path} qgepqwat:/tpl_full_hotfix.sql")
dexec_("psql -U postgres -d tpl_full -f /tpl_full_hotfix.sql")

# Creating the template DB with subset data
# THIS IS QUITE SLOW, WE DISABLE IT FOR NOW
# connection = psycopg2.connect(
# f"host={pgconf['host']} port={pgconf['port']} dbname={pgconf['dbname']} user={pgconf['user']} password={pgconf['password']}"
# )
# connection.set_session(autocommit=True)
# cursor = connection.cursor()
# for schema in ["qgep_od", "qwat_od"]:
# # For each table that has a geometry column, we keep only features that are in a specific extent.
# cursor.execute("SELECT qgep_sys.drop_symbology_triggers();")
# cursor.execute(
# f"""SELECT c.table_name, c.column_name
# FROM information_schema.columns c
# JOIN information_schema.tables t ON t.table_schema = c.table_schema AND t.table_name = c.table_name AND t.table_type = 'BASE TABLE'
# JOIN pg_type typ ON c.udt_name = typ.typname
# WHERE c.table_schema = '{schema}' AND typname = 'geometry';"""
# )
# for row in cursor.fetchall():
# table, column = row
# logger.info(f"KEEPING ONLY A SUBSET OF {schema}.{table} (this can take a while)...")
# logger.info(f"DELETE FROM {schema}.{table} WHERE ({column} && ST_MakeEnvelope(2750260.6, 1264318.8, 2750335.0,1264367.7, 2056)) = FALSE;")
# try:
# cursor.execute(
# f"DELETE FROM {schema}.{table} WHERE ({column} && ST_MakeEnvelope(2750260.6, 1264318.8, 2750335.0,1264367.7, 2056)) = FALSE;"
# )
# except psycopg2.errors.ForeignKeyViolation as e:
# logger.warning(f"Exception {e} !! we still continue...")
# pass

# if schema == 'qgep_od':
# # We delete orphaned wastewater_structure
# cursor.execute("DELETE FROM qgep_od.wastewater_structure WHERE fk_main_wastewater_node IS NULL;")

# cursor.execute("SELECT qgep_sys.create_symbology_triggers();")
# cursor.close()
# dexec_(f'psql -U postgres -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE pid<>pg_backend_pid();"')
# dexec_(f"createdb -U postgres --template=qgep_prod tpl_subset")

# # add our QWAT migrations
# exec_(r'docker cp C:\Users\Olivier\Code\QWAT\data-model\update\delta\delta_1.3.6_add_vl_for_SIA_export.sql qgepqwatbuilder:/delta_1.3.6_add_vl_for_SIA_export.sql')
# dexec_(f'psql -U postgres -d qgep_prod -f /delta_1.3.6_add_vl_for_SIA_export.sql')

# Creating the template DB with empty structure
dexec_("psql -f qgep_1.5.4_structure_with_value_lists.sql qgep_prod postgres")
dexec_("psql -f qwat_v1.3.5_structure_only.sql qgep_prod postgres")
dexec_("psql -f qwat_v1.3.5_value_list_data_only.sql qgep_prod postgres")
dexec_("createdb -U postgres --template=qgep_prod tpl_empty")

# Creating the template DB with full data
dexec_(
'psql -U postgres -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE pid<>pg_backend_pid();"'
)
dexec_("dropdb -U postgres qgep_prod --if-exists")
dexec_("createdb -U postgres qgep_prod")
dexec_(
"pg_restore -U postgres --dbname qgep_prod --verbose --no-privileges --exit-on-error qgep_1.5.4_structure_and_demo_data.backup"
)
dexec_(
"pg_restore -U postgres --dbname qgep_prod --verbose --no-privileges --exit-on-error qwat_v1.3.5_data_and_structure_sample.backup"
)
dexec_("createdb -U postgres --template=qgep_prod tpl_full")

# Hotfix invalid data
delta_path = os.path.join(os.path.dirname(__file__), "..", "data", "test_data", "qgep_demodata_hotfix.sql")
exec_(f"docker cp {delta_path} qgepqwat:/tpl_full_hotfix.sql")
dexec_("psql -U postgres -d tpl_full -f /tpl_full_hotfix.sql")

dexec_(
f'psql -U postgres -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE pid<>pg_backend_pid();"'
Expand Down

0 comments on commit 9446f92

Please sign in to comment.