diff --git a/CHANGES.rst b/CHANGES.rst index 6ad7583..cbd6312 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -5,7 +5,11 @@ Changelog of threedi-schema 0.222.3 (unreleased) -------------------- -- Nothing changed yet. +- Implement changes for schema version 300 concerning inflow +- Replace v2_surface and v2_impervious_surface (and maps) with surface and dry_weather_flow tables +- Redistribute data from v2_surface or v2_impervious_surface, depending on simulation_template_settings.use_0d_inflow, over suface and dry_weather_flow tables +- Populate surface_parameters and dry_weather_flow_distribution tables with default data +- A full overview can be obtained from the migration code (`threedi_schema/migrations/versions/0223_upgrade_db_inflow.py`) 0.222.2 (2024-06-13) @@ -26,7 +30,7 @@ Changelog of threedi-schema - Implement changes for schema version 300 concerning simulation settings - Reduce all settings tables to a single row. Multiple settings per schematisation are no longer allowed. -- A full overview can most easily be obtained from the migration; to summarize: +- A full overview can most easily be obtained from the migration code (`threedi_schema/migrations/versions/0222_upgrade_db_settings.py`); to summarize: - Rename settings tables from "v2_foo" to "foo" - Rename several columns in settings tables - Move settings to context specific tables instead of a single generic table diff --git a/MANIFEST.in b/MANIFEST.in index 389693c..d4e1e74 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,6 +1,8 @@ include LICENSE # Include docs in the root. include *.rst +# Include migration data +include threedi_schema/migrations/versions/data/* # Exclude byte-compiled code recursive-exclude .venv * recursive-exclude * __pycache__ diff --git a/pytest.ini b/pytest.ini index 566791b..7463624 100644 --- a/pytest.ini +++ b/pytest.ini @@ -3,4 +3,5 @@ markers = migrations: all migration tests migration_222: migration to schema 222 + migration_223: migration to schema 223 diff --git a/threedi_schema/__init__.py b/threedi_schema/__init__.py index 5331563..77842f4 100644 --- a/threedi_schema/__init__.py +++ b/threedi_schema/__init__.py @@ -2,5 +2,5 @@ from .domain import constants, custom_types, models # NOQA # fmt: off -__version__ = '0.222.3.dev0' +__version__ = '0.223.dev0' # fmt: on diff --git a/threedi_schema/domain/models.py b/threedi_schema/domain/models.py index 109bdb4..9544370 100644 --- a/threedi_schema/domain/models.py +++ b/threedi_schema/domain/models.py @@ -163,7 +163,7 @@ def max_infiltration_capacity_file(self): class SurfaceParameter(Base): - __tablename__ = "v2_surface_parameters" + __tablename__ = "surface_parameters" id = Column(Integer, primary_key=True) outflow_delay = Column(Float, nullable=False) surface_layer_thickness = Column(Float, nullable=False) @@ -172,33 +172,63 @@ class SurfaceParameter(Base): min_infiltration_capacity = Column(Float, nullable=False) infiltration_decay_constant = Column(Float, nullable=False) infiltration_recovery_constant = Column(Float, nullable=False) - - surface = relationship( - "Surface", - back_populates="surface_parameters", - ) + tags = Column(Text) + description = Column(Text) class Surface(Base): - __tablename__ = "v2_surface" + __tablename__ = "surface" id = Column(Integer, primary_key=True) - display_name = Column(String(255)) code = Column(String(100)) - zoom_category = Column(IntegerEnum(constants.ZoomCategories)) - nr_of_inhabitants = Column(Float) - dry_weather_flow = Column(Float) - function = Column(String(64)) + display_name = Column(String(255)) area = Column(Float) surface_parameters_id = Column( Integer, ForeignKey(SurfaceParameter.__tablename__ + ".id"), nullable=False ) - the_geom = Column( - Geometry("GEOMETRY"), + geom = Column( + Geometry("POLYGON"), nullable=True, ) - surface_parameters = relationship( - SurfaceParameter, foreign_keys=surface_parameters_id, back_populates="surface" + tags = Column(Text) + + +class DryWeatherFlow(Base): + __tablename__ = "dry_weather_flow" + id = Column(Integer, primary_key=True) + multiplier = Column(Float) + dry_weather_flow_distribution_id = Column(Text) + daily_total = Column(Float) + interpolate = Column(Boolean) + display_name = Column(String(255)) + code = Column(String(100)) + geom = Column( + Geometry("POLYGON"), + nullable=False, + ) + tags = Column(Text) + + +class DryWeatherFlowMap(Base): + __tablename__ = "dry_weather_flow_map" + id = Column(Integer, primary_key=True) + connection_node_id = Column(Integer) + dry_weather_flow_id = Column(Integer) + display_name = Column(String(255)) + code = Column(String(100)) + geom = Column( + Geometry("LINESTRING"), + nullable=False, ) + percentage = Column(Float) + tags = Column(Text) + + +class DryWeatherFlowDistribution(Base): + __tablename__ = "dry_weather_flow_distribution" + id = Column(Integer, primary_key=True) + description = Column(Text) + tags = Column(Text) + distribution = Column(Text) class GroundWater(Base): @@ -289,9 +319,6 @@ class ConnectionNode(Base): boundary_conditions = relationship( "BoundaryCondition1D", back_populates="connection_node" ) - impervious_surface_maps = relationship( - "ImperviousSurfaceMap", back_populates="connection_node" - ) laterals1d = relationship("Lateral1d", back_populates="connection_node") @@ -497,13 +524,17 @@ class BoundaryCondition1D(Base): class SurfaceMap(Base): - __tablename__ = "v2_surface_map" + __tablename__ = "surface_map" id = Column(Integer, primary_key=True) surface_id = Column(Integer, nullable=False) connection_node_id = Column( Integer, ForeignKey(ConnectionNode.__tablename__ + ".id"), nullable=False ) percentage = Column(Float) + geom = Column(Geometry("LINESTRING"), nullable=False) + tags = Column(Text) + code = Column(String(100)) + display_name = Column(String(255)) class Channel(Base): @@ -772,47 +803,6 @@ class Obstacle(Base): the_geom = Column(Geometry("LINESTRING"), nullable=False) -class ImperviousSurface(Base): - __tablename__ = "v2_impervious_surface" - id = Column(Integer, primary_key=True) - code = Column(String(100)) - display_name = Column(String(255)) - surface_inclination = Column( - VarcharEnum(constants.SurfaceInclinationType), nullable=False - ) - surface_class = Column(VarcharEnum(constants.SurfaceClass), nullable=False) - surface_sub_class = Column(String(128)) - zoom_category = Column(IntegerEnum(constants.ZoomCategories)) - nr_of_inhabitants = Column(Float) - area = Column(Float) - dry_weather_flow = Column(Float) - the_geom = Column( - Geometry("GEOMETRY"), - nullable=True, - ) - impervious_surface_maps = relationship( - "ImperviousSurfaceMap", back_populates="impervious_surface" - ) - - -class ImperviousSurfaceMap(Base): - __tablename__ = "v2_impervious_surface_map" - id = Column(Integer, primary_key=True) - percentage = Column(Float, nullable=False) - impervious_surface_id = Column( - Integer, ForeignKey(ImperviousSurface.__tablename__ + ".id"), nullable=False - ) - impervious_surface = relationship( - ImperviousSurface, back_populates="impervious_surface_maps" - ) - connection_node_id = Column( - Integer, ForeignKey(ConnectionNode.__tablename__ + ".id"), nullable=False - ) - connection_node = relationship( - ConnectionNode, back_populates="impervious_surface_maps" - ) - - class PotentialBreach(Base): __tablename__ = "v2_potential_breach" id = Column(Integer, primary_key=True) @@ -843,6 +833,12 @@ class ExchangeLine(Base): exchange_level = Column(Float) +class Tags(Base): + __tablename__ = "tags" + id = Column(Integer, primary_key=True) + description = Column(Text) + + DECLARED_MODELS = [ AggregationSettings, BoundaryCondition1D, @@ -862,14 +858,15 @@ class ExchangeLine(Base): CrossSectionLocation, Culvert, DemAverageArea, + DryWeatherFlow, + DryWeatherFlowMap, + DryWeatherFlowDistribution, ExchangeLine, Floodfill, ModelSettings, GridRefinement, GridRefinementArea, GroundWater, - ImperviousSurface, - ImperviousSurfaceMap, Interflow, Lateral1d, Lateral2D, diff --git a/threedi_schema/migrations/versions/0223_upgrade_db_inflow.py b/threedi_schema/migrations/versions/0223_upgrade_db_inflow.py new file mode 100644 index 0000000..864842f --- /dev/null +++ b/threedi_schema/migrations/versions/0223_upgrade_db_inflow.py @@ -0,0 +1,438 @@ +"""Upgrade settings in schema + +Revision ID: 0223 +Revises: +Create Date: 2024-05-27 10:35 + +""" +import json +from pathlib import Path +from typing import Dict, List, Tuple + +import sqlalchemy as sa +from alembic import op +from geoalchemy2 import load_spatialite +from sqlalchemy import Boolean, Column, Float, Integer, String, Text +from sqlalchemy.event import listen +from sqlalchemy.orm import declarative_base + +from threedi_schema.domain.custom_types import Geometry + +# revision identifiers, used by Alembic. +revision = "0223" +down_revision = "0222" +branch_labels = None +depends_on = None + +Base = declarative_base() + +data_dir = Path(__file__).parent / "data" + +# (source table, destination table) +RENAME_TABLES = [ + ("v2_surface_parameters", "surface_parameters"), +] + +ADD_COLUMNS = [ + ("surface_parameters", Column("description", Text)), + ("surface_parameters", Column("tags", Text)), +] + +ADD_TABLES = { + "surface": [ + Column("area", Float), + Column("surface_parameters_id", Integer, default=1), + Column("tags", Text), + Column("code", String(100)), + Column("display_name", String(255)), + ], + "dry_weather_flow": [ + Column("multiplier", Float), + Column("dry_weather_flow_distribution_id", Integer, default=1), + Column("daily_total", Float), + Column("interpolate", Boolean, default=False), + Column("tags", Text), + Column("code", String(100)), + Column("display_name", String(255)), + ], + "surface_map": [ + Column("connection_node_id", Integer), + Column("surface_id", Integer), + Column("percentage", Float), + Column("tags", Text), + Column("code", String(100)), + Column("display_name", String(255)), + ], + "dry_weather_flow_map": [ + Column("connection_node_id", Integer), + Column("dry_weather_flow_id", Integer), + Column("percentage", Float), + Column("tags", Text), + Column("code", String(100)), + Column("display_name", String(255)), + ], + "dry_weather_flow_distribution": [ + Column("description", Text), + Column("tags", Text), + Column("distribution", Text) + ], + "tags": [ + Column("description", Text) + ] +} + +# Geom columns need to be added using geoalchemy, so therefore that's a seperate task +NEW_GEOM_COLUMNS = { + ("surface", Column("geom", Geometry("POLYGON"), nullable=False)), + ("dry_weather_flow", Column("geom", Geometry("POLYGON"), nullable=False)), + ("surface_map", Column("geom", Geometry("LINESTRING"), nullable=False)), + ("dry_weather_flow_map", Column("geom", Geometry("LINESTRING"), nullable=False)) +} + +REMOVE_TABLES = [ + "v2_impervious_surface", + "v2_impervious_surface_map", + "v2_surface", + "v2_surface_map" +] + + +def rename_tables(table_sets: List[Tuple[str, str]]): + # no checks for existence are done, this will fail if a source table doesn't exist + for src_name, dst_name in table_sets: + op.rename_table(src_name, dst_name) + + +def create_new_tables(new_tables: Dict[str, sa.Column]): + # no checks for existence are done, this will fail if any table already exists + for table_name, columns in new_tables.items(): + op.create_table(table_name, sa.Column("id", sa.Integer(), primary_key=True), + *columns) + + +def add_columns_to_tables(table_columns: List[Tuple[str, Column]]): + # no checks for existence are done, this will fail if any column already exists + for dst_table, col in table_columns: + if isinstance(col.type, Geometry): + add_geometry_column(dst_table, col) + else: + with op.batch_alter_table(dst_table) as batch_op: + batch_op.add_column(col) + + +def add_geometry_column(table: str, geocol: Column): + # Adding geometry columns via alembic doesn't work + # https://postgis.net/docs/AddGeometryColumn.html + geotype = geocol.type + query = ( + f"SELECT AddGeometryColumn('{table}', '{geocol.name}', {geotype.srid}, '{geotype.geometry_type}', 'XY', 0);") + op.execute(sa.text(query)) + + +def remove_tables(tables: List[str]): + for table in tables: + op.drop_table(table) + + +def copy_values_to_new_table(src_table: str, src_columns: List[str], dst_table: str, dst_columns: List[str]): + query = f'INSERT INTO {dst_table} ({", ".join(dst_columns)}) SELECT {", ".join(src_columns)} FROM {src_table}' + op.execute(sa.text(query)) + + +def copy_v2_data_to_surface(src_table: str): + src_columns = ["id", "code", "display_name", "sur_geom", "area"] + dst_columns = ["id", "code", "display_name", "geom", "area"] + if src_table == "v2_surface": + src_columns += ["surface_parameters_id"] + dst_columns += ["surface_parameters_id"] + copy_values_to_new_table(src_table, src_columns, "surface", dst_columns) + op.execute(sa.text("DELETE FROM surface WHERE area = 0 OR area IS NULL;")) + + +def copy_v2_data_to_dry_weather_flow(src_table: str): + src_columns = ["id", "code", "display_name", "dwf_geom", "nr_of_inhabitants", "dry_weather_flow"] + dst_columns = ["id", "code", "display_name", "geom", "multiplier", "daily_total"] + copy_values_to_new_table(src_table, src_columns, "dry_weather_flow", dst_columns) + op.execute(sa.text("DELETE FROM dry_weather_flow " + "WHERE multiplier = 0 OR daily_total = 0 OR multiplier IS NULL OR daily_total IS NULL;")) + + +def remove_orphans_from_map(basename: str): + query = f"DELETE FROM {basename}_map WHERE {basename}_id NOT IN (SELECT id FROM {basename});" + op.execute(sa.text(query)) + + +def copy_v2_data_to_dry_weather_flow_map(src_table: str): + src_columns = ["connection_node_id", "percentage", src_table.strip('v2_').replace('_map', '_id')] + dst_columns = ["connection_node_id", "percentage", "dry_weather_flow_id"] + copy_values_to_new_table(src_table, src_columns, "dry_weather_flow_map", dst_columns) + + +def copy_v2_data_to_surface_map(src_table: str): + src_columns = ["connection_node_id", "percentage", src_table.strip('v2_').replace('_map', '_id')] + dst_columns = ["connection_node_id", "percentage", "surface_id"] + copy_values_to_new_table(src_table, src_columns, "surface_map", dst_columns) + + +def add_map_geometries(src_table: str): + # Add geometries to a map table that connects the connection node and the surface / dry_weather_flow + query = f""" + UPDATE {src_table}_map + SET geom = ( + SELECT MakeLine(c.the_geom, ClosestPoint(s.geom, c.the_geom)) + FROM v2_connection_nodes c + JOIN {src_table}_map m ON c.id = m.connection_node_id + JOIN {src_table} s ON s.id = m.{src_table}_id); + """ + op.execute(sa.text(query)) + + +def get_global_srid(): + conn = op.get_bind() + use_0d_inflow = conn.execute(sa.text("SELECT use_0d_inflow FROM simulation_template_settings LIMIT 1")).fetchone() + if use_0d_inflow is not None: + srid = conn.execute(sa.text("SELECT epsg_code FROM model_settings LIMIT 1")).fetchone() + if (srid is not None) and (srid[0] is not None): + return srid[0] + return 28992 + + +def get_area_str(geom_str) -> str: + # Get SQLite statement to compute area for a given geometry + return f'ST_Area(ST_Transform({geom_str},{get_global_srid()}))' + + +def copy_polygons(src_table: str, tmp_geom: str): + # Copy existing polygons in src_table to a new column (tmp_geom): + # - directly copy polygons + # - copy the first item of all multipolygons + # - add new rows for each extra polygon inside a multipolygon + conn = op.get_bind() + # Copy polygons directly + op.execute(sa.text(f"UPDATE {src_table} SET {tmp_geom} = the_geom WHERE GeometryType(the_geom) = 'POLYGON';")) + # Copy first polygon of each multipolygon and correct the area + op.execute(sa.text(f""" + UPDATE {src_table} + SET {tmp_geom} = ST_GeometryN(the_geom,1), area = {get_area_str('ST_GeometryN(the_geom,1)')} + WHERE GeometryType(the_geom) = 'MULTIPOLYGON' + AND GeometryType(ST_GeometryN(the_geom,1)) = 'POLYGON'; + """)) + # Copy the remaining polygons for multipolygons with more than one polygon + # select column names that we will copy directly + col_names = [col_info[1] for col_info in + conn.execute(sa.text(f"PRAGMA table_info({src_table})")).fetchall()] + col_str = ', '.join(list(set(col_names) - {'id', 'the_geom', 'tmp_geom', 'area'})) + conn = op.get_bind() + rows = conn.execute(sa.text(f""" + SELECT id, {col_str}, NumGeometries(the_geom) FROM {src_table} + WHERE GeometryType(the_geom) = 'MULTIPOLYGON' + AND GeometryType(ST_GeometryN(the_geom,1)) = 'POLYGON' + AND NumGeometries(the_geom) > 1;""")).fetchall() + id_next = conn.execute(sa.text(f"SELECT MAX(id) FROM {src_table}")).fetchall()[0][0] + surf_id = f"{src_table.strip('v2_')}_id" + for row in rows: + id = row[0] + nof_polygons = row[-1] + # Retrieve data from map table + conn_node_id, percentage = conn.execute(sa.text(f""" + SELECT connection_node_id, percentage FROM {src_table}_map + WHERE {surf_id} = {id}""")).fetchall()[0] + for i in range(2, nof_polygons + 1): + id_next += 1 + # Copy polygon to new row + op.execute(sa.text(f""" + INSERT INTO {src_table} (id, the_geom, {tmp_geom}, area, {col_str}) + SELECT {id_next}, the_geom, ST_GeometryN(the_geom, {i}), + {get_area_str(f'ST_GeometryN(the_geom,{i})')}, {col_str} + FROM {src_table} WHERE id = {id} LIMIT 1 + """)) + # Add new row to the map + op.execute(sa.text(f""" + INSERT INTO {src_table}_map ({surf_id}, connection_node_id, percentage) + VALUES ({id_next}, {conn_node_id}, {percentage}) + """)) + + +def create_buffer_polygons(src_table: str, tmp_geom: str): + # create circular polygon of area 1 around the connection node + surf_id = f"{src_table.strip('v2_')}_id" + op.execute(sa.text(f""" + UPDATE {src_table} + SET {tmp_geom} = ( + SELECT ST_Buffer(v2_connection_nodes.the_geom, 1) + FROM v2_connection_nodes + JOIN {src_table}_map + ON v2_connection_nodes.id = {src_table}_map.connection_node_id + WHERE {src_table}.id = {src_table}_map.{surf_id} + ) + WHERE {tmp_geom} IS NULL + AND id IN ( + SELECT {src_table}_map.{surf_id} + FROM v2_connection_nodes + JOIN {src_table}_map + ON v2_connection_nodes.id = {src_table}_map.connection_node_id + ); + """)) + + +def create_square_polygons(src_table: str, tmp_geom: str): + # create square polygon with area area around the connection node + side_expr = f'sqrt({src_table}.area)' + surf_id = f"{src_table.strip('v2_')}_id" + # When no geometry is defined, a square with area matching the area column + # with the center at the connection node is added + srid = get_global_srid() + query_str = f""" + WITH center AS ( + SELECT {src_table}.id AS item_id, + ST_Centroid(ST_Collect( + ST_Transform(v2_connection_nodes.the_geom, {srid}))) AS geom + FROM {src_table}_map + JOIN v2_connection_nodes ON {src_table}_map.connection_node_id = v2_connection_nodes.id + JOIN {src_table} ON {src_table}_map.{surf_id} = {src_table}.id + WHERE {src_table}_map.{surf_id} = {src_table}.id + GROUP BY {src_table}.id + ), + side_length AS ( + SELECT {side_expr} AS side + ) + UPDATE {src_table} + SET {tmp_geom} = ( + SELECT ST_Transform( + SetSRID( + ST_GeomFromText('POLYGON((' || + (ST_X(center.geom) - side_length.side / 2) || ' ' || (ST_Y(center.geom) - side_length.side / 2) || ',' || + (ST_X(center.geom) + side_length.side / 2) || ' ' || (ST_Y(center.geom) - side_length.side / 2) || ',' || + (ST_X(center.geom) + side_length.side / 2) || ' ' || (ST_Y(center.geom) + side_length.side / 2) || ',' || + (ST_X(center.geom) - side_length.side / 2) || ' ' || (ST_Y(center.geom) + side_length.side / 2) || ',' || + (ST_X(center.geom) - side_length.side / 2) || ' ' || (ST_Y(center.geom) - side_length.side / 2) || + '))'), + {srid}), + 4326 + ) AS transformed_geom + FROM center, side_length + WHERE center.item_id = {src_table}.id + ) + WHERE {tmp_geom} IS NULL; + """ + op.execute(sa.text(query_str)) + + +def fix_src_geometry(src_table: str, tmp_geom: str, create_polygons): + conn = op.get_bind() + # create columns to store the derived geometries to + op.execute(sa.text(f"SELECT AddGeometryColumn('{src_table}', '{tmp_geom}', 4326, 'POLYGON', 'XY', 0);")) + # Copy existing polygons + copy_polygons(src_table, tmp_geom) + # Check if any existing geometries where not copied + not_copied = conn.execute(sa.text(f'SELECT id FROM {src_table} ' + f'WHERE {tmp_geom} IS NULL ' + f'AND the_geom IS NOT NULL')).fetchall() + if len(not_copied) > 0: + raise BaseException(f'Found {len(not_copied)} geometries in {src_table} that could not' + f'be converted to a POLYGON geometry: {not_copied}') + # Create polygons for rows where no geometry was defined + create_polygons(src_table, tmp_geom) + + +def populate_surface_and_dry_weather_flow(): + conn = op.get_bind() + use_0d_inflow = conn.execute(sa.text("SELECT use_0d_inflow FROM simulation_template_settings LIMIT 1")).fetchone() + if (use_0d_inflow is None) or (len(use_0d_inflow) == 0) or (use_0d_inflow[0] not in [1, 2]): + return + use_0d_inflow = use_0d_inflow[0] + # Use use_0d_inflow setting to determine wether to copy any data and if so from what table + src_table = "v2_impervious_surface" if use_0d_inflow == 1 else "v2_surface" + # Remove rows with insufficient data + op.execute(sa.text(f"DELETE FROM {src_table} WHERE area = 0 " + "AND (nr_of_inhabitants = 0 OR dry_weather_flow = 0);")) + # Create geometries for non-specified ones + # Add geometries for surfaces and dwf by adding extra columns + # This has to be done in advance because NULL geometries cannot be copied + # And this had to be done seperately because the geometries for surfaces and + # DWF are not by definition the same + fix_src_geometry(src_table, 'sur_geom', create_square_polygons) + fix_src_geometry(src_table, 'dwf_geom', create_buffer_polygons) + # Copy data to new tables + copy_v2_data_to_surface(src_table) + copy_v2_data_to_dry_weather_flow(src_table) + copy_v2_data_to_surface_map(f"{src_table}_map") + copy_v2_data_to_dry_weather_flow_map(f"{src_table}_map") + # Remove rows in maps that refer to non-existing objects + remove_orphans_from_map(basename="surface") + remove_orphans_from_map(basename="dry_weather_flow") + # Create geometries in new maps + add_map_geometries("surface") + add_map_geometries("dry_weather_flow") + # Set surface parameter id + if use_0d_inflow == 1: + set_surface_parameters_id() + # Populate tables with default values + populate_dry_weather_flow_distribution() + populate_surface_parameters() + + +def set_surface_parameters_id(): + # Make sure not to call this on an empty database + with open(data_dir.joinpath('0223_surface_parameters_map.json'), 'r') as f: + parameter_map = json.load(f) + conn = op.get_bind() + surface_class, surface_inclination = conn.execute( + sa.text("SELECT surface_class, surface_inclination FROM v2_impervious_surface")).fetchone() + parameter_id = parameter_map[f'{surface_class} - {surface_inclination}'] + op.execute(f'UPDATE surface SET surface_parameters_id = {parameter_id}') + + +def populate_surface_parameters(): + # Make sure not to call this on an empty database + with open(data_dir.joinpath('0223_surface_parameters_contents.json'), 'r') as f: + data_to_insert = json.load(f) + keys_str = "(" + ",".join(data_to_insert[0].keys()) + ")" + for row in data_to_insert: + val_str = "(" + ",".join([repr(item) for item in row.values()]) + ")" + sql_query = f"INSERT INTO surface_parameters {keys_str} VALUES {val_str}" + op.execute(sa.text(sql_query)) + + +def populate_dry_weather_flow_distribution(): + with open(data_dir.joinpath('0223_dry_weather_flow_distribution.csv'), 'r') as f: + distr = f.read().strip() + description = "Kennisbank Stichting Rioned - https://www.riool.net/huishoudelijk-afvalwater" + sql_query = f"INSERT INTO dry_weather_flow_distribution (description, distribution) VALUES ('{description}', '{distr}')" + op.execute(sa.text(sql_query)) + + +def fix_geometry_columns(): + GEO_COL_INFO = [ + ('dry_weather_flow', 'geom', 'POLYGON'), + ('dry_weather_flow_map', 'geom', 'LINESTRING'), + ('surface', 'geom', 'POLYGON'), + ('surface_map', 'geom', 'LINESTRING'), + ] + for table, column, geotype in GEO_COL_INFO: + with op.batch_alter_table(table) as batch_op: + batch_op.alter_column(column_name=column, nullable=False) + migration_query = f"SELECT RecoverGeometryColumn('{table}', '{column}', {4326}, '{geotype}', 'XY')" + op.execute(sa.text(migration_query)) + + +def upgrade(): + connection = op.get_bind() + listen(connection.engine, "connect", load_spatialite) + # create new tables and rename existing tables + create_new_tables(ADD_TABLES) + rename_tables(RENAME_TABLES) + # add new columns to existing tables + add_columns_to_tables(ADD_COLUMNS) + add_columns_to_tables(NEW_GEOM_COLUMNS) + # migrate values from old tables to new tables + populate_surface_and_dry_weather_flow() + # recover geometry columns + fix_geometry_columns() + # remove old tables + remove_tables(REMOVE_TABLES) + + +def downgrade(): + # Not implemented on purpose + raise NotImplementedError("Downgrade back from 0.3xx is not supported") diff --git a/threedi_schema/migrations/versions/data/0223_dry_weather_flow_distribution.csv b/threedi_schema/migrations/versions/data/0223_dry_weather_flow_distribution.csv new file mode 100644 index 0000000..9de9804 --- /dev/null +++ b/threedi_schema/migrations/versions/data/0223_dry_weather_flow_distribution.csv @@ -0,0 +1 @@ +3,1.5,1,1,0.5,0.5,2.5,8,7.5,6,5.5,5,4.5,4,4,3.5,3.5,4,5.5,8,7,5.5,4.5,4 \ No newline at end of file diff --git a/threedi_schema/migrations/versions/data/0223_surface_parameters_contents.json b/threedi_schema/migrations/versions/data/0223_surface_parameters_contents.json new file mode 100644 index 0000000..29e1c3b --- /dev/null +++ b/threedi_schema/migrations/versions/data/0223_surface_parameters_contents.json @@ -0,0 +1,167 @@ +[ + { + "id": 101, + "description": "gesloten verharding, hellend", + "outflow_delay": 0.5, + "surface_layer_thickness": 0.0, + "infiltration": 0, + "max_infiltration_capacity": 0, + "min_infiltration_capacity": 0.0, + "infiltration_decay_constant": 0, + "infiltration_recovery_constant": 0.0 + }, + { + "id": 102, + "description": "gesloten verharding, vlak", + "outflow_delay": 0.2, + "surface_layer_thickness": 0.5, + "infiltration": 0, + "max_infiltration_capacity": 0, + "min_infiltration_capacity": 0.0, + "infiltration_decay_constant": 0, + "infiltration_recovery_constant": 0.0 + }, + { + "id": 103, + "description": "gesloten verharding, vlak uitgestrekt", + "outflow_delay": 0.1, + "surface_layer_thickness": 1.0, + "infiltration": 0, + "max_infiltration_capacity": 0, + "min_infiltration_capacity": 0.0, + "infiltration_decay_constant": 0, + "infiltration_recovery_constant": 0.0 + }, + { + "id": 104, + "description": "open verharding, hellend", + "outflow_delay": 0.5, + "surface_layer_thickness": 0.0, + "infiltration": 1, + "max_infiltration_capacity": 2, + "min_infiltration_capacity": 0.5, + "infiltration_decay_constant": 3, + "infiltration_recovery_constant": 0.1 + }, + { + "id": 105, + "description": "open verharding, vlak", + "outflow_delay": 0.2, + "surface_layer_thickness": 0.5, + "infiltration": 1, + "max_infiltration_capacity": 2, + "min_infiltration_capacity": 0.5, + "infiltration_decay_constant": 3, + "infiltration_recovery_constant": 0.1 + }, + { + "id": 106, + "description": "open verharding, vlak uitgestrekt", + "outflow_delay": 0.1, + "surface_layer_thickness": 1.0, + "infiltration": 1, + "max_infiltration_capacity": 2, + "min_infiltration_capacity": 0.5, + "infiltration_decay_constant": 3, + "infiltration_recovery_constant": 0.1 + }, + { + "id": 107, + "description": "dak, hellend", + "outflow_delay": 0.5, + "surface_layer_thickness": 0.0, + "infiltration": 0, + "max_infiltration_capacity": 0, + "min_infiltration_capacity": 0.0, + "infiltration_decay_constant": 0, + "infiltration_recovery_constant": 0.0 + }, + { + "id": 108, + "description": "dak, vlak", + "outflow_delay": 0.2, + "surface_layer_thickness": 2.0, + "infiltration": 0, + "max_infiltration_capacity": 0, + "min_infiltration_capacity": 0.0, + "infiltration_decay_constant": 0, + "infiltration_recovery_constant": 0.0 + }, + { + "id": 109, + "description": "dak, vlak uitgestrekt", + "outflow_delay": 0.1, + "surface_layer_thickness": 4.0, + "infiltration": 0, + "max_infiltration_capacity": 0, + "min_infiltration_capacity": 0.0, + "infiltration_decay_constant": 0, + "infiltration_recovery_constant": 0.0 + }, + { + "id": 110, + "description": "onverhard, hellend", + "outflow_delay": 0.5, + "surface_layer_thickness": 2.0, + "infiltration": 1, + "max_infiltration_capacity": 5, + "min_infiltration_capacity": 1.0, + "infiltration_decay_constant": 3, + "infiltration_recovery_constant": 0.1 + }, + { + "id": 111, + "description": "onverhard, vlak", + "outflow_delay": 0.2, + "surface_layer_thickness": 4.0, + "infiltration": 1, + "max_infiltration_capacity": 5, + "min_infiltration_capacity": 1.0, + "infiltration_decay_constant": 3, + "infiltration_recovery_constant": 0.1 + }, + { + "id": 112, + "description": "onverhard, vlak uitgestrekt", + "outflow_delay": 0.1, + "surface_layer_thickness": 6.0, + "infiltration": 1, + "max_infiltration_capacity": 5, + "min_infiltration_capacity": 1.0, + "infiltration_decay_constant": 3, + "infiltration_recovery_constant": 0.1 + }, + { + "id": 113, + "description": "half verhard, hellend", + "outflow_delay": 0.5, + "surface_layer_thickness": 2.0, + "infiltration": 1, + "max_infiltration_capacity": 5, + "min_infiltration_capacity": 1.0, + "infiltration_decay_constant": 3, + "infiltration_recovery_constant": 0.1 + }, + { + "id": 114, + "description": "half verhard, vlak", + "outflow_delay": 0.2, + "surface_layer_thickness": 4.0, + "infiltration": 1, + "max_infiltration_capacity": 5, + "min_infiltration_capacity": 1.0, + "infiltration_decay_constant": 3, + "infiltration_recovery_constant": 0.1 + }, + { + "id": 115, + "description": "half verhard, vlak uitgestrekt", + "outflow_delay": 0.1, + "surface_layer_thickness": 6.0, + "infiltration": 1, + "max_infiltration_capacity": 5, + "min_infiltration_capacity": 1.0, + "infiltration_decay_constant": 3, + "infiltration_recovery_constant": 0.1 + } +] \ No newline at end of file diff --git a/threedi_schema/migrations/versions/data/0223_surface_parameters_map.json b/threedi_schema/migrations/versions/data/0223_surface_parameters_map.json new file mode 100644 index 0000000..2dba02f --- /dev/null +++ b/threedi_schema/migrations/versions/data/0223_surface_parameters_map.json @@ -0,0 +1,17 @@ +{ + "gesloten verharding - hellend": 101, + "gesloten verharding - vlak": 102, + "gesloten verharding - uitgestrekt": 103, + "open verharding - hellend": 104, + "open verharding - vlak": 105, + "open verharding - uitgestrekt": 106, + "pand - hellend": 107, + "pand - vlak": 108, + "pand - uitgestrekt": 109, + "onverhard - hellend": 110, + "onverhard - vlak": 111, + "onverhard - uitgestrekt": 112, + "half verhard - hellend": 113, + "half verhard - vlak": 114, + "half verhard - uitgestrekt": 115 +} \ No newline at end of file diff --git a/threedi_schema/tests/conftest.py b/threedi_schema/tests/conftest.py index ed4185f..cb9ae0d 100644 --- a/threedi_schema/tests/conftest.py +++ b/threedi_schema/tests/conftest.py @@ -36,9 +36,8 @@ def empty_sqlite_v4(tmp_path): @pytest.fixture def south_latest_sqlite(tmp_path): """An empty SQLite that is in its latest South migration state""" - # TODO: replace with geopackage - tmp_sqlite = tmp_path / "south_latest.gpkg" - shutil.copyfile(data_dir / "south_latest.gpkg", tmp_sqlite) + tmp_sqlite = tmp_path / "south_latest.sqlite" + shutil.copyfile(data_dir / "south_latest.sqlite", tmp_sqlite) return ThreediDatabase(tmp_sqlite) diff --git a/threedi_schema/tests/data/south_latest.gpkg b/threedi_schema/tests/data/south_latest.gpkg deleted file mode 100644 index f56438e..0000000 Binary files a/threedi_schema/tests/data/south_latest.gpkg and /dev/null differ diff --git a/threedi_schema/tests/data/staging-test-0d1d2d-simple-infiltration.sqlite b/threedi_schema/tests/data/staging-test-0d1d2d-simple-infiltration.sqlite new file mode 100644 index 0000000..39271ff --- /dev/null +++ b/threedi_schema/tests/data/staging-test-0d1d2d-simple-infiltration.sqlite @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c886d84ce1eae66119ff9b223b215fdddf76c3e8be32baff66e15198243794da +size 8105984 diff --git a/threedi_schema/tests/data/staging-test-0d1d2d-simple-infiltration_surface.sqlite b/threedi_schema/tests/data/staging-test-0d1d2d-simple-infiltration_surface.sqlite new file mode 100644 index 0000000..df03947 --- /dev/null +++ b/threedi_schema/tests/data/staging-test-0d1d2d-simple-infiltration_surface.sqlite @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d7bbdac56fd93576eddd972a1ff14770ab2d071311b30f66b862120d10c0336 +size 8105984 diff --git a/threedi_schema/tests/data/v2_bergermeer_221.gpkg b/threedi_schema/tests/data/v2_bergermeer_221.gpkg deleted file mode 100644 index 843c260..0000000 Binary files a/threedi_schema/tests/data/v2_bergermeer_221.gpkg and /dev/null differ diff --git a/threedi_schema/tests/data/v2_bergermeer_221.sqlite b/threedi_schema/tests/data/v2_bergermeer_221.sqlite new file mode 100644 index 0000000..d34ed9c --- /dev/null +++ b/threedi_schema/tests/data/v2_bergermeer_221.sqlite @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c3912c24f0ac81f29463a669108446d2c37b59806107a503dcce04b228ea480 +size 6354944 diff --git a/threedi_schema/tests/test_migration.py b/threedi_schema/tests/test_migration.py index 6604f7b..192068e 100644 --- a/threedi_schema/tests/test_migration.py +++ b/threedi_schema/tests/test_migration.py @@ -4,6 +4,7 @@ from pathlib import Path import pytest +from geoalchemy2 import Geometry from sqlalchemy import inspect from threedi_schema import ModelSchema, ThreediDatabase @@ -21,7 +22,7 @@ @pytest.fixture(scope="session") def sqlite_path(): - return data_dir.joinpath("v2_bergermeer_221.gpkg") + return data_dir.joinpath("v2_bergermeer_221.sqlite") @pytest.fixture(scope="session") def schema_upgraded(tmp_path_factory, sqlite_path): @@ -51,13 +52,19 @@ def get_cursor_for_schema(schema): def get_columns_from_schema(schema, table_name): inspector = inspect(schema.db.get_engine()) columns = inspector.get_columns(table_name) - return {c['name']: (str(c['type']), c['nullable']) for c in columns} + return {column['name']: (str(column['type']).lower(), column['nullable']) for column in columns + if not 'geom' in column['name']} def get_columns_from_sqlite(cursor, table_name): cursor.execute(f"PRAGMA table_info({table_name})") - columns = cursor.fetchall() - return {c[1]: (c[2], not c[3]) for c in columns} + col_map = {} + for c in cursor.fetchall(): + if 'geom' in c[1]: + continue + type_str = c[2].lower() if c[2] != 'bool' else 'boolean' + col_map[c[1]] = (type_str, not c[3]) + return col_map def get_values_from_sqlite(cursor, table_name, column_name): @@ -65,14 +72,54 @@ def get_values_from_sqlite(cursor, table_name, column_name): return cursor.fetchall() +@pytest.mark.parametrize("sqlite_file", + ["v2_bergermeer_221.sqlite", + "staging-test-0d1d2d-simple-infiltration.sqlite", + "staging-test-0d1d2d-simple-infiltration_surface.sqlite"]) +def test_upgrade_success(sqlite_file, tmp_path_factory): + tmp_sqlite = tmp_path_factory.mktemp("custom_dir").joinpath(sqlite_file) + shutil.copy(data_dir.joinpath(sqlite_file), tmp_sqlite) + schema = ModelSchema(ThreediDatabase(tmp_sqlite)) + # Test if running upgrade doesn't run into any exceptions + try: + schema.upgrade(backup=False) + except Exception: + pytest.fail(f"Failed to upgrade {sqlite_file}") + + + +class TestMigration223: + pytestmark = pytest.mark.migration_223 + removed_tables = set(['v2_surface', 'v2_surface_parameters', 'v2_surface_map', + 'v2_impervious_surface', 'v2_impervious_surface_map']) + added_tables = set(['surface', 'surface_map', 'surface_parameters', 'tags', + 'dry_weather_flow', 'dry_weather_flow_map','dry_weather_flow_distribution']) + + def test_tables(self, schema_ref, schema_upgraded): + # Test whether the added tables are present + # and whether the removed tables are not present* + tables_new = set(get_sql_tables(get_cursor_for_schema(schema_upgraded))) + assert self.added_tables.issubset(tables_new) + assert self.removed_tables.isdisjoint(tables_new) + + def test_columns_added_tables(self, schema_upgraded): + # Note that only the added tables are touched. + # So this check covers both added and removed columns. + cursor = get_cursor_for_schema(schema_upgraded) + for table in self.added_tables: + cols_sqlite = get_columns_from_sqlite(cursor, table) + cols_schema = get_columns_from_schema(schema_upgraded, table) + assert cols_sqlite == cols_schema + + class TestMigration222: pytestmark = pytest.mark.migration_222 with open(data_dir.joinpath('migration_222.csv'), 'r') as file: # src_table, src_column, dst_table, dst_column migration_map = [[row[0], row[1], row[2], row[3]] for row in csv.reader(file)] - removed_tables = list(set([row[0] for row in migration_map])) - added_tables = list(set([row[2] for row in migration_map])) + removed_tables = set([row[0] for row in migration_map]) + added_tables = set([row[2] for row in migration_map]) bool_settings_id = [ ("use_groundwater_storage", "groundwater_settings_id", "groundwater"), ("use_interflow", "interflow_settings_id", "interflow"), @@ -92,12 +139,11 @@ class TestMigration222: "interception"] def test_tables(self, schema_ref, schema_upgraded): - # Test whether renaming removed the correct columns, - # and whether adding/renaming added the correct columns. - tables_ref = set(get_sql_tables(get_cursor_for_schema(schema_ref))) + # Test whether the added tables are present + # and whether the removed tables are not present* tables_new = set(get_sql_tables(get_cursor_for_schema(schema_upgraded))) - assert sorted(self.removed_tables) == sorted(list(tables_ref - tables_new)) - assert sorted(self.added_tables) == sorted(list(tables_new - tables_ref)) + assert self.added_tables.issubset(tables_new) + assert self.removed_tables.isdisjoint(tables_new) def test_columns_added_tables(self, schema_upgraded): # Note that only the added tables are touched. diff --git a/threedi_schema/tests/test_spatalite_versions.py b/threedi_schema/tests/test_spatalite_versions.py index 131c43f..4d7ad4b 100644 --- a/threedi_schema/tests/test_spatalite_versions.py +++ b/threedi_schema/tests/test_spatalite_versions.py @@ -45,25 +45,26 @@ def test_copy_invalid_geometry(empty_sqlite_v3, empty_sqlite_v4): """Copying an invalid geometry (ST_IsValid evaluates to False) is possible""" db_from = empty_sqlite_v3 db_to = empty_sqlite_v4 - - obj = models.Surface( + # Note MP: this only works when this object is not involved in a migratin + # This may cause issues with future database upgrades + obj = models.GridRefinementArea( id=3, code="test", display_name="test", the_geom="SRID=4326;POLYGON((0 0, 10 10, 0 10, 10 0, 0 0))", - surface_parameters_id=1, + refinement_level=1, ) with db_from.session_scope() as session: session.add(obj) session.commit() - copy_model(db_from, db_to, models.Surface) + copy_model(db_from, db_to, models.GridRefinementArea) with db_to.session_scope() as session: records = list( session.query( - models.Surface.id, - func.ST_AsText(models.Surface.the_geom), + models.GridRefinementArea.id, + func.ST_AsText(models.GridRefinementArea.the_geom), ) )