diff --git a/.ghsci_version b/.ghsci_version index c966188e..f12d1f24 100644 --- a/.ghsci_version +++ b/.ghsci_version @@ -1 +1 @@ -4.4.7 +4.4.8 diff --git a/.test-compose.yml b/.test-compose.yml index 4517408f..f5d1103e 100644 --- a/.test-compose.yml +++ b/.test-compose.yml @@ -1,7 +1,7 @@ version: "3" services: ghsci: - image: globalhealthyliveablecities/global-indicators:v4.4.5 + image: globalhealthyliveablecities/global-indicators:v4.4.8 container_name: ghsci shm_size: 2g stdin_open: true # docker run -i diff --git a/docker-compose.yml b/docker-compose.yml index 89fa8c74..781ca4b8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ version: "3" services: ghsci: - image: globalhealthyliveablecities/global-indicators:v4.4.5 + image: globalhealthyliveablecities/global-indicators:v4.4.8 container_name: ghsci shm_size: 2g stdin_open: true # docker run -i diff --git a/docker/Dockerfile b/docker/Dockerfile index dcb070c9..ff92cc2d 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -54,7 +54,8 @@ RUN mkdir /env && \ ENV PATH="/env/bin/:$PATH" ENV PROJ_LIB="/env/share/proj" -RUN pip install -e git+https://github.com/carlhiggs/gtfs-lite.git#egg=gtfs-lite && \ +RUN pip install -e git+https://github.com/wklumpen/gtfs-lite.git#egg=gtfs-lite && \ + pip install -e git+https://github.com/anerv/BikeDNA_BIG#egg=src && \ echo 'alias configure="python configure.py"' >> ~/.bashrc && \ echo 'alias analysis="python analysis.py"' >> ~/.bashrc && \ echo 'alias generate="python generate.py"' >> ~/.bashrc && \ diff --git a/docker/environment.yml b/docker/environment.yml index 7ec2d9be..6e4f2252 100644 --- a/docker/environment.yml +++ b/docker/environment.yml @@ -7,9 +7,10 @@ dependencies: - fpdf2=2.7.* - sqlalchemy=1.4.* - nicegui>=1.2.24 - - cryptography=41.0.* + - cryptography>=41.0.2 - requests=2.31.* - tornado>=6.3.2 + - starlette>=0.27.0 - pyrosm # for building network graphs from OpenStreetMap .pbf files - momepy # for building network graphs from external network files - openpyxl diff --git a/docker/requirements.txt b/docker/requirements.txt index 68d84ca9..191ca0aa 100644 --- a/docker/requirements.txt +++ b/docker/requirements.txt @@ -1,10 +1,11 @@ affine==2.4.0 aiofiles==23.1.0 -anyio==3.7.0 +annotated-types==0.5.0 +anyio==3.7.1 argon2-cffi==21.3.0 argon2-cffi-bindings==21.2.0 asttokens==2.2.1 -async-lru==2.0.2 +async-lru==2.0.3 attrs==23.1.0 Babel==2.12.1 backcall==0.2.0 @@ -15,20 +16,20 @@ bleach==6.0.0 branca==0.6.0 Brotli==1.0.9 Cartopy==0.21.1 -certifi==2023.5.7 +certifi==2023.7.22 cffi==1.15.1 -charset-normalizer==3.1.0 -click==8.1.3 +charset-normalizer==3.2.0 +click==8.1.6 click-plugins==1.1.1 cligj==0.7.2 colorama==0.4.6 comm==0.1.3 contextily==1.3.0 contourpy==1.1.0 -cryptography==41.0.1 +cryptography==41.0.2 cycler==0.11.0 cykhash==2.0.1 -Cython==0.29.35 +Cython==3.0.0 dataclasses==0.8 debugpy==1.6.7 decorator==5.1.1 @@ -37,51 +38,57 @@ entrypoints==0.4 et-xmlfile==1.1.0 exceptiongroup==1.1.2 executing==1.2.0 -fastapi==0.92.0 +fastapi==0.100.0 fastapi-socketio==0.0.10 -fastjsonschema==2.17.1 +fastjsonschema==2.18.0 Fiona==1.9.4 flit_core==3.9.0 folium==0.14.0 -fonttools==4.40.0 +fonttools==4.41.1 fpdf2==2.7.4 GDAL==3.7.0 -GeoAlchemy2==0.13.3 +GeoAlchemy2==0.14.0 geographiclib==1.52 geopandas==0.13.2 geopy==2.3.0 greenlet==2.0.2 gtfs-lite==0.2.0 h11==0.14.0 +h2==4.1.0 +hpack==4.0.0 +httpcore==0.17.3 +httpx==0.24.1 +hyperframe==6.0.1 idna==3.4 -importlib-metadata==6.7.0 -importlib-resources==5.12.0 +importlib-metadata==6.8.0 +importlib-resources==6.0.0 ipykernel==6.24.0 ipython==8.14.0 -ipywidgets==8.0.6 +ipywidgets==8.0.7 itsdangerous==2.1.2 jedi==0.18.2 Jinja2==3.1.2 joblib==1.3.0 json5==0.9.14 -jsonschema==4.17.3 +jsonschema==4.18.4 +jsonschema-specifications==2023.7.1 jupyter_client==8.3.0 jupyter_core==5.3.1 jupyter-events==0.6.3 jupyter-lsp==2.2.0 jupyter_server==2.7.0 jupyter_server_terminals==0.4.4 -jupyterlab==4.0.2 +jupyterlab==4.0.3 jupyterlab-pygments==0.2.2 jupyterlab_server==2.23.0 -jupyterlab-widgets==3.0.7 +jupyterlab-widgets==3.0.8 kiwisolver==1.4.4 libpysal==4.7.0 -lxml==4.9.2 +lxml==4.9.3 mapclassify==2.5.0 markdown2==2.4.9 MarkupSafe==2.1.3 -matplotlib==3.7.1 +matplotlib==3.7.2 matplotlib-inline==0.1.6 mercantile==1.2.1 mistune==3.0.0 @@ -89,17 +96,18 @@ momepy==0.0.0 munch==4.0.0 munkres==1.1.4 nbclient==0.8.0 -nbconvert==7.6.0 -nbformat==5.9.0 +nbconvert==7.7.2 +nbformat==5.9.1 nest-asyncio==1.5.6 +netifaces==0.11.0 networkx==3.1 nicegui==0.1.0 notebook_shim==0.2.3 numexpr==2.8.4 -numpy==1.25.0 +numpy==1.25.1 openpyxl==3.1.2 orjson==3.8.14 -osmnx==1.5.0 +osmnx==1.5.1 overrides==7.3.1 OWSLib==0.29.2 packaging==23.1 @@ -110,33 +118,32 @@ parso==0.8.3 pexpect==4.8.0 pickleshare==0.7.5 Pillow==10.0.0 -pip==23.1.2 +pip==23.2.1 pkgutil_resolve_name==1.3.10 -platformdirs==3.8.0 -plotly==5.13.1 +platformdirs==3.9.1 pooch==1.7.0 -prometheus-client==0.17.0 -prompt-toolkit==3.0.38 +prometheus-client==0.17.1 +prompt-toolkit==3.0.39 pscript==0.7.7 psutil==5.9.5 -psycopg2==2.9.3 +psycopg2==2.9.6 ptyprocess==0.7.0 pure-eval==0.2.2 py-cpuinfo==9.0.0 pycparser==2.21 -pydantic==1.10.10 +pydantic==2.0.3 +pydantic_core==2.3.0 pygeometa==0.15.0 pygeos==0.14 Pygments==2.15.1 -pyparsing==3.1.0 +pyparsing==3.0.9 pyproj==3.6.0 pyrobuf==0.9.3 pyrosm==0.6.1 -pyrsistent==0.19.3 pyshp==2.3.1 PySocks==1.7.1 python-dateutil==2.8.2 -python-engineio==4.4.1 +python-engineio==4.5.1 python-json-logger==2.0.7 python-multipart==0.0.6 python-rapidjson==1.10 @@ -145,9 +152,11 @@ pytz==2023.3 PyYAML==6.0 pyzmq==25.1.0 rasterio==1.3.8 +referencing==0.30.0 requests==2.31.0 rfc3339-validator==0.1.4 rfc3986-validator==0.1.1 +rpds-py==0.9.2 Rtree==1.0.1 scikit-learn==1.3.0 scipy==1.11.1 @@ -158,13 +167,13 @@ six==1.16.0 sniffio==1.3.0 snuggs==1.4.7 soupsieve==2.3.2.post1 -SQLAlchemy==1.4.48 +SQLAlchemy==1.4.49 +src==0.0.0 stack-data==0.6.2 -starlette==0.25.0 +starlette==0.27.0 tables==3.8.0 -tenacity==8.2.2 terminado==0.17.1 -threadpoolctl==3.1.0 +threadpoolctl==3.2.0 tinycss2==1.2.1 tomli==2.0.1 tornado==6.3.2 @@ -174,15 +183,15 @@ typing_extensions==4.7.1 typing-utils==0.1.0 tzdata==2023.3 unicodedata2==15.0.0 -urllib3==2.0.3 -uvicorn==0.20.0 +urllib3==2.0.4 +uvicorn==0.23.1 vbuild==0.8.1 watchfiles==0.18.1 wcwidth==0.2.6 webencodings==0.5.1 websocket-client==1.6.1 websockets==11.0.3 -wheel==0.40.0 -widgetsnbextension==4.0.7 -xyzservices==2023.5.0 -zipp==3.15.0 +wheel==0.41.0 +widgetsnbextension==4.0.8 +xyzservices==2023.7.0 +zipp==3.16.2 diff --git a/process/compare.py b/process/compare.py index 17e9c772..344877fe 100644 --- a/process/compare.py +++ b/process/compare.py @@ -49,12 +49,40 @@ def compare(r, comparison_codename): dfs[file] = pd.read_csv(files[file]) else: sys.exit( - f"""Compare a reference city to a comparison city, and save the comparison as a CSV file.\n\nThe summary results file ({files[file]}) could not be located.\n\nPlease try again by entering codenames from the list of configured cities {region_names} that have been fully analysed with resources generated:\npython 4_compare.py \n\nAlternatively, enter the shortcut command:\ncompare """, + f"""Compare a reference city to a comparison city, and save the comparison as a CSV file.\n\nThe summary results file ({files[file]}) could not be located.\n\nPlease try again by entering codenames from the list of configured cities {get_region_names()} that have been fully analysed with resources generated:\npython 4_compare.py \n\nAlternatively, enter the shortcut command:\ncompare """, ) + # ordered set of columns shared between dataframes + shared_columns = [ + x + for x in dfs[codename].columns + if x in dfs[comparison_codename].columns + ] + # store unshared columns from each dataframe + unshared_columns = { + codename: [ + x + for x in dfs[codename].columns + if x not in dfs[comparison_codename].columns + ], + comparison_codename: [ + x + for x in dfs[comparison_codename].columns + if x not in dfs[codename].columns + ], + } + print(f'\nColumns shared across both datasets: {shared_columns}') + for name in unshared_columns: + print(f'\nColumns unique to {name}: {unshared_columns[name]}') # print(pd.concat(dfs).transpose()) comparison = ( - dfs[codename] - .compare(dfs[comparison_codename], align_axis=0) + dfs[codename][shared_columns] + .compare( + dfs[comparison_codename][shared_columns], + align_axis=0, + keep_shape=True, + keep_equal=True, + result_names=(codename, comparison_codename), + ) .droplevel(0) .transpose() ) @@ -63,8 +91,6 @@ def compare(r, comparison_codename): f'The results contained in the generated summaries for {codename} and {comparison_codename} are identical.', ) else: - comparison.columns = [codename, comparison_codename] - # print(f'\n{comparison}') comparison.to_csv( f"{r.config['region_dir']}/compare_{r.codename}_{comparison_codename}_{date_hhmm}.csv", ) diff --git a/process/gui.py b/process/gui.py index 23ab64cd..d764cb39 100644 --- a/process/gui.py +++ b/process/gui.py @@ -1,29 +1,16 @@ #!/usr/bin/env python3 """GHSCI graphical user interface; run and access at https://localhost:8080.""" -# import asyncio + import os.path -# import geopandas as gpd import pandas as pd - -# import yaml -# from analysis import analysis -# from compare import compare from configure import configuration from nicegui import Client, app, ui from subprocesses import ghsci - -# from subprocesses._utils import plot_choropleth_map from subprocesses.leaflet import leaflet - -# from generate import generate -# from geoalchemy2 import Geometry, WKTElement from subprocesses.local_file_picker import local_file_picker -# import platform -# import shlex - class Region: """Minimal class to define a region.""" @@ -528,16 +515,6 @@ def format_policy_checklist(xlsx) -> dict: return df -async def save_policy_report() -> str: - """Generate policy report.""" - from policy_report import PDF_Policy_Report - - report = PDF_Policy_Report( - 'data/policy_review/Urban policy checklist_1000 Cities Challenge_version 1.0.0 - test.xlsx', - ) - return await report.generate_policy_report() - - @ui.refreshable def studyregion_ui() -> None: ui.html(region.config['header_name']).style( @@ -598,11 +575,17 @@ async def load_policy_checklist() -> None: 'wrap-cells=true table-style="{vertical-align: text-top}"', ) as table: with table.add_slot('top-left'): - ui.button('Generate PDF').props( - 'icon=download_for_offline outline', - ).classes('shadow-lg').on( - 'click', - PDF_Policy_Report(xlsx[0]).generate_policy_report, + ui.button( + 'Generate PDF', + on_click=lambda: ( + ui.notify( + PDF_Policy_Report( + xlsx[0], + ).generate_policy_report(), + ) + ), + ).props('icon=download_for_offline outline').classes( + 'shadow-lg', ).tooltip( f"Save an indexed PDF of the policy checklist to {xlsx[0].replace('.xlsx','.pdf')}. Please wait a few moments for this to be generated after clicking.", ).style( diff --git a/process/subprocesses/_04_create_population_grid.py b/process/subprocesses/_04_create_population_grid.py index 4da8fa10..80266a2e 100644 --- a/process/subprocesses/_04_create_population_grid.py +++ b/process/subprocesses/_04_create_population_grid.py @@ -5,209 +5,50 @@ population raster dataset tiles. """ -import os -import subprocess as sp import sys import time # Set up project and region parameters for GHSCIC analyses import ghsci import pandas as pd -from osgeo import gdal from script_running_log import script_running_log -from sqlalchemy import create_engine, inspect, text +from sqlalchemy import text -# disable noisy GDAL logging -# gdal.SetConfigOption('CPL_LOG', 'NUL') # Windows -gdal.SetConfigOption('CPL_LOG', '/dev/null') # Linux/MacOS - -def reproject_raster(inpath, outpath, new_crs): - import rasterio - from rasterio.warp import ( - Resampling, - calculate_default_transform, - reproject, - ) - - dst_crs = new_crs # CRS for web meractor - with rasterio.open(inpath) as src: - transform, width, height = calculate_default_transform( - src.crs, dst_crs, src.width, src.height, *src.bounds, +def population_to_db(r): + if r.config['population']['data_type'].startswith('raster'): + # raster source + r.raster_to_db( + raster='population', + config=r.config['population'], + field='pop_est', + reference_grid=True, ) - kwargs = src.meta.copy() - kwargs.update( - { - 'crs': dst_crs, - 'transform': transform, - 'width': width, - 'height': height, - }, - ) - with rasterio.open(outpath, 'w', **kwargs) as dst: - for i in range(1, src.count + 1): - reproject( - source=rasterio.band(src, i), - destination=rasterio.band(dst, i), - src_transform=src.transform, - src_crs=src.crs, - dst_transform=transform, - dst_crs=dst_crs, - resampling=Resampling.nearest, - ) - - -def extract_population_from_raster(r): - print('Extracting population raster data...') - db = r.config['db'] - db_host = r.config['db_host'] - db_pwd = r.config['db_pwd'] - population_stub = ( - f'{r.config["region_dir"]}/{r.config["population_grid"]}_{r.codename}' - ) - # construct virtual raster table - vrt = f'{r.config["population"]["data_dir"]}/{r.config["population_grid"]}_{r.config["population"]["crs_srid"]}.vrt' - population_raster_clipped = ( - f'{population_stub}_{r.config["population"]["crs_srid"]}.tif' - ) - population_raster_projected = ( - f'{population_stub}_{r.config["crs"]["srid"]}.tif' - ) - print('Raster population dataset...', end='', flush=True) - if not os.path.isfile(vrt): - tif_folder = f'{r.config["population"]["data_dir"]}' - tif_files = [ - os.path.join(tif_folder, file) - for file in os.listdir(tif_folder) - if os.path.splitext(file)[-1] == '.tif' - ] - gdal.BuildVRT(vrt, tif_files) - print(f' has now been indexed ({vrt}).') else: - print(f' has already been indexed ({vrt}).') - print('\nPopulation data clipped to region...', end='', flush=True) - if not os.path.isfile(population_raster_clipped): - # extract study region boundary in projection of tiles - clipping_query = ( - f'SELECT geom FROM {r.config["buffered_urban_study_region"]}' + # vector source + r.ogr_to_db( + source=r.config['population']['data_dir'], + layer=r.config['population_grid'], + promote_to_multi=True, + source_crs=f"{r.config['population']['crs_srid']}", ) - clipping = r.get_gdf(text(clipping_query), geom_col='geom').to_crs( - r.config['population']['crs_srid'], - ) - # get clipping boundary values in required order for gdal translate - bbox = list( - clipping.bounds[['minx', 'maxy', 'maxx', 'miny']].values[0], - ) - # bbox = list(clipping.bounds.values[0]) - gdal.Translate(population_raster_clipped, vrt, projWin=bbox) - print(f' has now been created ({population_raster_clipped}).') - else: - print(f' has already been created ({population_raster_clipped}).') - print('\nPopulation data projected for region...', end='', flush=True) - if not os.path.isfile(population_raster_projected): - # reproject and save the re-projected clipped raster - # (see config file for reprojection function) - reproject_raster( - inpath=population_raster_clipped, - outpath=population_raster_projected, - new_crs=r.config['crs']['srid'], - ) - print(f' has now been created ({population_raster_projected}).') - else: - print(f' has already been created ({population_raster_projected}).') - if r.config['population_grid'] not in r.tables: - print( - f'\nImport population grid {r.config["population_grid"]} to database... ', - end='', - flush=True, - ) - # import raster to postgis and vectorise, as per http://www.brianmcgill.org/postgis_zonal.pdf - command = ( - f'raster2pgsql -d -s {r.config["crs"]["srid"]} -I -Y ' - f"-N {r.config['population']['raster_nodata']} " - f'-t 1x1 {population_raster_projected} {r.config["population_grid"]} ' - f'| PGPASSWORD={db_pwd} psql -U postgres -h {db_host} -d {db} ' - '>> /dev/null' - ) - sp.call(command, shell=True) - print('Done.') - else: - print(f'{r.config["population_grid"]} has been imported to database.') - - -def raster_sql_processing(r): - queries = [ - f"""ALTER TABLE {r.config["population_grid"]} DROP COLUMN rid;""", - f"""DELETE FROM {r.config["population_grid"]} WHERE (ST_SummaryStats(rast)).sum IS NULL;""", - f"""ALTER TABLE {r.config["population_grid"]} ADD grid_id bigserial;""", - f"""ALTER TABLE {r.config["population_grid"]} ADD COLUMN IF NOT EXISTS pop_est int;""", - f"""ALTER TABLE {r.config["population_grid"]} ADD COLUMN IF NOT EXISTS geom geometry;""", - f"""UPDATE {r.config["population_grid"]} SET geom = ST_ConvexHull(rast);""", - f"""CREATE INDEX {r.config["population_grid"]}_ix ON {r.config["population_grid"]} (grid_id);""", - f"""CREATE INDEX {r.config["population_grid"]}_gix ON {r.config["population_grid"]} USING GIST(geom);""", - f"""UPDATE {r.config["population_grid"]} SET pop_est = (ST_SummaryStats(rast)).sum;""", - f"""ALTER TABLE {r.config["population_grid"]} DROP COLUMN rast;""", - ] - for sql in queries: - with r.engine.begin() as connection: - connection.execute(text(sql)) - - -def extract_population_from_vector(r): - print('Extracting population vector data...') - db = r.config['db'] - db_host = r.config['db_host'] - db_port = r.config['db_port'] - db_user = r.config['db_user'] - db_pwd = r.config['db_pwd'] - population_data = r.config['population']['data_dir'] - if '.gpkg:' in population_data: - gpkg = population_data.split(':') - population_data = gpkg[0] - query = gpkg[1] - else: - query = '' - command = ( - ' ogr2ogr -overwrite -progress -f "PostgreSQL" ' - f' PG:"host={db_host} port={db_port} dbname={db}' - f' user={db_user} password={db_pwd}" ' - f' "{population_data}" ' - f' -lco geometry_name="geom" -lco precision=NO ' - f' -t_srs {r.config["crs_srid"]} -nln "{r.config["population_grid"]}" ' - f' -nlt PROMOTE_TO_MULTI' - f' {query}' - ) - print(command) - failure = sp.call(command, shell=True) - if failure == 1: - sys.exit( - f"Error reading in population data '{population_data}' (check format)", - ) - # except Exception as e: - # raise Exception(f'Error reading in boundary data (check format): {e}') + queries = [ + f"""ALTER TABLE {r.config["population_grid"]} ADD grid_id bigserial;""", + f"""ALTER TABLE {r.config["population_grid"]} RENAME {r.config["population"]["vector_population_data_field"]} TO pop_est;""", + f"""CREATE INDEX {r.config["population_grid"]}_ix ON {r.config["population_grid"]} (grid_id);""", + ] + for sql in queries: + with r.engine.begin() as connection: + connection.execute(text(sql)) print('Done.') -def vector_sql_processing(r): - queries = [ - f"""ALTER TABLE {r.config["population_grid"]} ADD grid_id bigserial;""", - f"""ALTER TABLE {r.config["population_grid"]} RENAME {r.config["population"]["vector_population_data_field"]} TO pop_est;""", - ] - for sql in queries: - with r.engine.begin() as connection: - connection.execute(text(sql)) - - def derive_population_grid_variables(r): print( 'Derive population grid variables and summaries... ', end='', flush=True, ) - if r.config['population']['data_type'].startswith('vector'): - vector_sql_processing(r) - else: - raster_sql_processing(r) queries = [ f"""ALTER TABLE {r.config["population_grid"]} ADD COLUMN IF NOT EXISTS area_sqkm float;""", f"""ALTER TABLE {r.config["population_grid"]} ADD COLUMN IF NOT EXISTS pop_per_sqkm float;""", @@ -292,11 +133,9 @@ def create_population_grid(codename): if r.config['population_grid'] in tables: print('Population grid already exists in database.') else: - # population raster set up - if r.config['population']['data_type'].startswith('vector'): - extract_population_from_vector(r) - else: - extract_population_from_raster(r) + # import data + population_to_db(r) + # derive variables derive_population_grid_variables(r) pop = r.get_df(r.config['population_grid']) pd.options.display.float_format = ( diff --git a/process/subprocesses/_08_destination_summary.py b/process/subprocesses/_08_destination_summary.py index 00f4e642..684fab80 100644 --- a/process/subprocesses/_08_destination_summary.py +++ b/process/subprocesses/_08_destination_summary.py @@ -42,7 +42,7 @@ def destination_summary(codename): a.area_sqkm, a.pop_per_sqkm, t.count/a.area_sqkm AS dest_per_sqkm, - t.count/a.area_sqkm/(pop_est/10000) AS dest_per_sqkm_per_10kpop + t.count/a.area_sqkm/(pop_est/10000.0) AS dest_per_sqkm_per_10kpop FROM urban_study_region a, (SELECT d.dest_name_full, COUNT(d.*) count diff --git a/process/subprocesses/_utils.py b/process/subprocesses/_utils.py index 2deac5bb..e426dee5 100644 --- a/process/subprocesses/_utils.py +++ b/process/subprocesses/_utils.py @@ -22,7 +22,6 @@ from mpl_toolkits.axes_grid1 import make_axes_locatable from mpl_toolkits.axes_grid1.anchored_artists import AnchoredSizeBar from mpl_toolkits.axes_grid1.inset_locator import inset_axes -from subprocesses.batlow import batlow_map as cmap # 'pretty' text wrapping as per https://stackoverflow.com/questions/37572837/how-can-i-make-python-3s-print-fit-the-size-of-the-command-prompt @@ -266,6 +265,8 @@ def check_and_update_config_reporting_parameters(config): def generate_report_for_language( r, language, indicators, policies, ): + from subprocesses.batlow import batlow_map as cmap + """Generate report for a processed city in a given language.""" font = get_and_setup_font(language, r.config) # set up policies @@ -1570,6 +1571,7 @@ def study_region_map( import cartopy.io.img_tiles as cimgt import cartopy.io.ogc_clients as ogcc from shapely.geometry import box + from subprocesses.batlow import batlow_map as cmap file_name = re.sub(r'\W+', '_', file_name) filepath = f'{region_config["region_dir"]}/figures/{file_name}.png' @@ -1788,3 +1790,38 @@ def buffered_box(total_bounds, distance): buffer_distance = [x * distance for x in mod] new_bounds = [total_bounds[x] + buffer_distance[x] for x in range(0, 4)] return new_bounds + + +def reproject_raster(inpath, outpath, new_crs): + import rasterio + from rasterio.warp import ( + Resampling, + calculate_default_transform, + reproject, + ) + + dst_crs = new_crs # CRS for web meractor + with rasterio.open(inpath) as src: + transform, width, height = calculate_default_transform( + src.crs, dst_crs, src.width, src.height, *src.bounds, + ) + kwargs = src.meta.copy() + kwargs.update( + { + 'crs': dst_crs, + 'transform': transform, + 'width': width, + 'height': height, + }, + ) + with rasterio.open(outpath, 'w', **kwargs) as dst: + for i in range(1, src.count + 1): + reproject( + source=rasterio.band(src, i), + destination=rasterio.band(dst, i), + src_transform=src.transform, + src_crs=src.crs, + dst_transform=transform, + dst_crs=dst_crs, + resampling=Resampling.nearest, + ) diff --git a/process/subprocesses/ghsci.py b/process/subprocesses/ghsci.py index 74bc42de..8e042166 100644 --- a/process/subprocesses/ghsci.py +++ b/process/subprocesses/ghsci.py @@ -129,13 +129,23 @@ def compare(self, comparison): comparison = compare_resources(self, comparison) return comparison - def drop(self): + def drop(self, table=''): """Attempt to drop database results for this study region.""" - from _drop_study_region_database import ( - drop_study_region_database as drop_resources, - ) + if table == '': + from _drop_study_region_database import ( + drop_study_region_database as drop_resources, + ) - drop_resources(self) + drop_resources(self) + else: + with self.engine.begin() as connection: + try: + print('Dropping table {table}...}') + connection.execute( + text(f"""DROP TABLE IF EXISTS {table};"""), + ) + except Exception as e: + print(f'Error: {e}') def _create_database(self): """Create database for this study region.""" @@ -387,7 +397,13 @@ def ogr_to_db( """Read spatial data with ogr2ogr and save to Postgis database.""" import subprocess as sp - name = self.config['name'] + if source.count(':') == 1: + # appears to be using optional query syntax as could be used for a geopackage + parts = source.split(':') + source = parts[0] + query = parts[1] + del parts + crs_srid = self.config['crs_srid'] db = self.config['db'] db_host = self.config['db_host'] @@ -414,6 +430,120 @@ def ogr_to_db( else: return failure + def raster_to_db( + self, + raster: str, + config: dict, + field: str, + to_vector: bool = True, + reference_grid=False, + ): + """Read raster data save to Postgis database, optionally adding and indexing a unique grid_id variable for use as a reference grid for analysis.""" + import subprocess as sp + + from _utils import reproject_raster + from osgeo import gdal + + # disable noisy GDAL logging + # gdal.SetConfigOption('CPL_LOG', 'NUL') # Windows + gdal.SetConfigOption('CPL_LOG', '/dev/null') # Linux/MacOS + """Extract data from raster tiles and import to database.""" + print('Extracting raster data...') + raster_grid = self.config['population_grid'] + raster_stub = ( + f'{self.config["region_dir"]}/{raster_grid}_{self.codename}' + ) + # construct virtual raster table + vrt = f'{config["data_dir"]}/{raster_grid}_{config["crs_srid"]}.vrt' + raster_clipped = f'{raster_stub}_{config["crs_srid"]}.tif' + raster_projected = f'{raster_stub}_{self.config["crs"]["srid"]}.tif' + print(f'{raster} dataset...', end='', flush=True) + if not os.path.isfile(vrt): + tif_folder = f'{config["data_dir"]}' + tif_files = [ + os.path.join(tif_folder, file) + for file in os.listdir(tif_folder) + if os.path.splitext(file)[-1] == '.tif' + ] + gdal.BuildVRT(vrt, tif_files) + print(f'{raster} has now been indexed ({vrt}).') + else: + print(f'{raster} has already been indexed ({vrt}).') + print(f'\n{raster} data clipped to region...', end='', flush=True) + if not os.path.isfile(raster_clipped): + # extract study region boundary in projection of tiles + clipping_query = f'SELECT geom FROM {self.config["buffered_urban_study_region"]}' + clipping = self.get_gdf( + text(clipping_query), geom_col='geom', + ).to_crs(config['crs_srid']) + # get clipping boundary values in required order for gdal translate + bbox = list( + clipping.bounds[['minx', 'maxy', 'maxx', 'miny']].values[0], + ) + gdal.Translate(raster_clipped, vrt, projWin=bbox) + print(f'{raster} has now been created ({raster_clipped}).') + else: + print(f'{raster} has already been created ({raster_clipped}).') + print(f'\n{raster} projected for region...', end='', flush=True) + if not os.path.isfile(raster_projected): + # reproject and save the re-projected clipped raster + reproject_raster( + inpath=raster_clipped, + outpath=raster_projected, + new_crs=self.config['crs']['srid'], + ) + print(f' has now been created ({raster_projected}).') + else: + print(f' has already been created ({raster_projected}).') + if raster_grid not in self.tables: + print( + f'\nImport grid {raster_grid} to database... ', + end='', + flush=True, + ) + # import raster to postgis and vectorise, as per http://www.brianmcgill.org/postgis_zonal.pdf + if to_vector: + command = ( + f'raster2pgsql -d -s {self.config["crs"]["srid"]} -I -Y ' + f"-N {config['raster_nodata']} " + f'-t 1x1 {raster_projected} {raster_grid} ' + f'| PGPASSWORD={self.config["db_pwd"]} psql -U postgres -h {self.config["db_host"]} -d {self.config["db"]} ' + '>> /dev/null' + ) + sp.call(command, shell=True) + # remove empty cells + with self.engine.begin() as connection: + connection.execute( + text( + f"""DELETE FROM {raster_grid} WHERE (ST_SummaryStats(rast)).sum IS NULL;""", + ), + ) + # if reference grid add and index grid id + if reference_grid: + queries = [ + f"""ALTER TABLE {raster_grid} DROP COLUMN rid;""", + f"""ALTER TABLE {raster_grid} ADD grid_id bigserial;""", + f"""CREATE INDEX {raster_grid}_ix ON {raster_grid} (grid_id);""", + ] + for sql in queries: + with self.engine.begin() as connection: + connection.execute(text(sql)) + # add geometry column and calculate statistic + queries = [ + f"""ALTER TABLE {raster_grid} ADD COLUMN IF NOT EXISTS geom geometry;""", + f"""UPDATE {raster_grid} SET geom = ST_ConvexHull(rast);""", + f"""CREATE INDEX {raster_grid}_gix ON {raster_grid} USING GIST(geom);""", + f"""ALTER TABLE {raster_grid} ADD COLUMN IF NOT EXISTS {field} int;""", + f"""UPDATE {raster_grid} SET {field} = (ST_SummaryStats(rast)).sum;""", + f"""ALTER TABLE {raster_grid} DROP COLUMN rast;""", + ] + for sql in queries: + with self.engine.begin() as connection: + connection.execute(text(sql)) + print('Done.') + else: + print(f'{raster_grid} has been imported to database.') + def choropleth( self, field: str, @@ -423,6 +553,7 @@ def choropleth( save=True, attribution: str = 'Global Healthy and Sustainable City Indicators Collaboration', ): + """Plot a choropleth map of a specified field in a specified layer, with a custom title and attribution, with optional saving to an html file.""" from _utils import plot_choropleth_map tables = self.get_tables() diff --git a/process/subprocesses/leaflet.py b/process/subprocesses/leaflet.py index 21534e3e..2cd15ef5 100644 --- a/process/subprocesses/leaflet.py +++ b/process/subprocesses/leaflet.py @@ -3,17 +3,13 @@ from typing import Tuple from nicegui import ui -from nicegui.dependencies import register_component -from nicegui.element import Element -register_component('leaflet', __file__, 'leaflet.js') - -class leaflet(Element): +class leaflet(ui.element, component='leaflet.js'): """Leaflet helper script for GHSCI map, drawing on NiceGUI example.""" def __init__(self) -> None: - super().__init__('leaflet') + super().__init__() ui.add_head_html( """ None: paths = list(self.path.glob('*')) if not self.show_hidden_files: @@ -83,8 +98,8 @@ def update_grid(self) -> None: ) self.grid.update() - async def handle_double_click(self, msg: Dict) -> None: - self.path = Path(msg['args']['data']['path']) + def handle_double_click(self, e: events.GenericEventArguments) -> None: + self.path = Path(e.args['data']['path']) if self.path.is_dir(): self.update_grid() else: diff --git a/process/subprocesses/policy_report.py b/process/subprocesses/policy_report.py index bafc3d6d..b0ca1c29 100644 --- a/process/subprocesses/policy_report.py +++ b/process/subprocesses/policy_report.py @@ -503,6 +503,7 @@ def generate_policy_report(self): ) self.format_criteria(collection_details) self.format_policy_checklist(self.checklist) - report_file = f'{self.file.replace(".xlsx","")}.pdf' + report_file = f'{self.file.replace(".xlsx",".pdf")}' self.output(report_file) + print(f'Report saved to {report_file}') return report_file