Merge pull request #352 from global-healthy-liveable-cities/enhancements

Enhancements

.ghsci_version

@@ -1 +1 @@
-4.4.7
+4.4.8

.test-compose.yml

@@ -1,7 +1,7 @@
 version: "3"
 services:
   ghsci:
-    image: globalhealthyliveablecities/global-indicators:v4.4.5
+    image: globalhealthyliveablecities/global-indicators:v4.4.8
     container_name: ghsci
     shm_size: 2g
     stdin_open: true # docker run -i

docker-compose.yml

@@ -1,7 +1,7 @@
 version: "3"
 services:
   ghsci:
-    image: globalhealthyliveablecities/global-indicators:v4.4.5
+    image: globalhealthyliveablecities/global-indicators:v4.4.8
     container_name: ghsci
     shm_size: 2g
     stdin_open: true # docker run -i

docker/Dockerfile

@@ -54,7 +54,8 @@ RUN mkdir /env && \
 ENV PATH="/env/bin/:$PATH"
 ENV PROJ_LIB="/env/share/proj"
 
-RUN pip install -e git+https://github.com/carlhiggs/gtfs-lite.git#egg=gtfs-lite && \
+RUN pip install -e git+https://github.com/wklumpen/gtfs-lite.git#egg=gtfs-lite && \
+    pip install -e git+https://github.com/anerv/BikeDNA_BIG#egg=src && \
     echo 'alias configure="python configure.py"' >> ~/.bashrc && \
     echo 'alias analysis="python analysis.py"' >> ~/.bashrc && \
     echo 'alias generate="python generate.py"' >> ~/.bashrc && \

docker/environment.yml

@@ -7,9 +7,10 @@ dependencies:
   - fpdf2=2.7.*
   - sqlalchemy=1.4.*
   - nicegui>=1.2.24
-  - cryptography=41.0.*
+  - cryptography>=41.0.2
   - requests=2.31.*
   - tornado>=6.3.2
+  - starlette>=0.27.0
   - pyrosm # for building network graphs from OpenStreetMap .pbf files
   - momepy # for building network graphs from external network files
   - openpyxl

docker/requirements.txt

@@ -1,10 +1,11 @@
 affine==2.4.0
 aiofiles==23.1.0
-anyio==3.7.0
+annotated-types==0.5.0
+anyio==3.7.1
 argon2-cffi==21.3.0
 argon2-cffi-bindings==21.2.0
 asttokens==2.2.1
-async-lru==2.0.2
+async-lru==2.0.3
 attrs==23.1.0
 Babel==2.12.1
 backcall==0.2.0
@@ -15,20 +16,20 @@ bleach==6.0.0
 branca==0.6.0
 Brotli==1.0.9
 Cartopy==0.21.1
-certifi==2023.5.7
+certifi==2023.7.22
 cffi==1.15.1
-charset-normalizer==3.1.0
-click==8.1.3
+charset-normalizer==3.2.0
+click==8.1.6
 click-plugins==1.1.1
 cligj==0.7.2
 colorama==0.4.6
 comm==0.1.3
 contextily==1.3.0
 contourpy==1.1.0
-cryptography==41.0.1
+cryptography==41.0.2
 cycler==0.11.0
 cykhash==2.0.1
-Cython==0.29.35
+Cython==3.0.0
 dataclasses==0.8
 debugpy==1.6.7
 decorator==5.1.1
@@ -37,69 +38,76 @@ entrypoints==0.4
 et-xmlfile==1.1.0
 exceptiongroup==1.1.2
 executing==1.2.0
-fastapi==0.92.0
+fastapi==0.100.0
 fastapi-socketio==0.0.10
-fastjsonschema==2.17.1
+fastjsonschema==2.18.0
 Fiona==1.9.4
 flit_core==3.9.0
 folium==0.14.0
-fonttools==4.40.0
+fonttools==4.41.1
 fpdf2==2.7.4
 GDAL==3.7.0
-GeoAlchemy2==0.13.3
+GeoAlchemy2==0.14.0
 geographiclib==1.52
 geopandas==0.13.2
 geopy==2.3.0
 greenlet==2.0.2
 gtfs-lite==0.2.0
 h11==0.14.0
+h2==4.1.0
+hpack==4.0.0
+httpcore==0.17.3
+httpx==0.24.1
+hyperframe==6.0.1
 idna==3.4
-importlib-metadata==6.7.0
-importlib-resources==5.12.0
+importlib-metadata==6.8.0
+importlib-resources==6.0.0
 ipykernel==6.24.0
 ipython==8.14.0
-ipywidgets==8.0.6
+ipywidgets==8.0.7
 itsdangerous==2.1.2
 jedi==0.18.2
 Jinja2==3.1.2
 joblib==1.3.0
 json5==0.9.14
-jsonschema==4.17.3
+jsonschema==4.18.4
+jsonschema-specifications==2023.7.1
 jupyter_client==8.3.0
 jupyter_core==5.3.1
 jupyter-events==0.6.3
 jupyter-lsp==2.2.0
 jupyter_server==2.7.0
 jupyter_server_terminals==0.4.4
-jupyterlab==4.0.2
+jupyterlab==4.0.3
 jupyterlab-pygments==0.2.2
 jupyterlab_server==2.23.0
-jupyterlab-widgets==3.0.7
+jupyterlab-widgets==3.0.8
 kiwisolver==1.4.4
 libpysal==4.7.0
-lxml==4.9.2
+lxml==4.9.3
 mapclassify==2.5.0
 markdown2==2.4.9
 MarkupSafe==2.1.3
-matplotlib==3.7.1
+matplotlib==3.7.2
 matplotlib-inline==0.1.6
 mercantile==1.2.1
 mistune==3.0.0
 momepy==0.0.0
 munch==4.0.0
 munkres==1.1.4
 nbclient==0.8.0
-nbconvert==7.6.0
-nbformat==5.9.0
+nbconvert==7.7.2
+nbformat==5.9.1
 nest-asyncio==1.5.6
+netifaces==0.11.0
 networkx==3.1
 nicegui==0.1.0
 notebook_shim==0.2.3
 numexpr==2.8.4
-numpy==1.25.0
+numpy==1.25.1
 openpyxl==3.1.2
 orjson==3.8.14
-osmnx==1.5.0
+osmnx==1.5.1
 overrides==7.3.1
 OWSLib==0.29.2
 packaging==23.1
@@ -110,33 +118,32 @@ parso==0.8.3
 pexpect==4.8.0
 pickleshare==0.7.5
 Pillow==10.0.0
-pip==23.1.2
+pip==23.2.1
 pkgutil_resolve_name==1.3.10
-platformdirs==3.8.0
-plotly==5.13.1
+platformdirs==3.9.1
 pooch==1.7.0
-prometheus-client==0.17.0
-prompt-toolkit==3.0.38
+prometheus-client==0.17.1
+prompt-toolkit==3.0.39
 pscript==0.7.7
 psutil==5.9.5
-psycopg2==2.9.3
+psycopg2==2.9.6
 ptyprocess==0.7.0
 pure-eval==0.2.2
 py-cpuinfo==9.0.0
 pycparser==2.21
-pydantic==1.10.10
+pydantic==2.0.3
+pydantic_core==2.3.0
 pygeometa==0.15.0
 pygeos==0.14
 Pygments==2.15.1
-pyparsing==3.1.0
+pyparsing==3.0.9
 pyproj==3.6.0
 pyrobuf==0.9.3
 pyrosm==0.6.1
-pyrsistent==0.19.3
 pyshp==2.3.1
 PySocks==1.7.1
 python-dateutil==2.8.2
-python-engineio==4.4.1
+python-engineio==4.5.1
 python-json-logger==2.0.7
 python-multipart==0.0.6
 python-rapidjson==1.10
@@ -145,9 +152,11 @@ pytz==2023.3
 PyYAML==6.0
 pyzmq==25.1.0
 rasterio==1.3.8
+referencing==0.30.0
 requests==2.31.0
 rfc3339-validator==0.1.4
 rfc3986-validator==0.1.1
+rpds-py==0.9.2
 Rtree==1.0.1
 scikit-learn==1.3.0
 scipy==1.11.1
@@ -158,13 +167,13 @@ six==1.16.0
 sniffio==1.3.0
 snuggs==1.4.7
 soupsieve==2.3.2.post1
-SQLAlchemy==1.4.48
+SQLAlchemy==1.4.49
+src==0.0.0
 stack-data==0.6.2
-starlette==0.25.0
+starlette==0.27.0
 tables==3.8.0
-tenacity==8.2.2
 terminado==0.17.1
-threadpoolctl==3.1.0
+threadpoolctl==3.2.0
 tinycss2==1.2.1
 tomli==2.0.1
 tornado==6.3.2
@@ -174,15 +183,15 @@ typing_extensions==4.7.1
 typing-utils==0.1.0
 tzdata==2023.3
 unicodedata2==15.0.0
-urllib3==2.0.3
-uvicorn==0.20.0
+urllib3==2.0.4
+uvicorn==0.23.1
 vbuild==0.8.1
 watchfiles==0.18.1
 wcwidth==0.2.6
 webencodings==0.5.1
 websocket-client==1.6.1
 websockets==11.0.3
-wheel==0.40.0
-widgetsnbextension==4.0.7
-xyzservices==2023.5.0
-zipp==3.15.0
+wheel==0.41.0
+widgetsnbextension==4.0.8
+xyzservices==2023.7.0
+zipp==3.16.2

process/compare.py

@@ -49,12 +49,40 @@ def compare(r, comparison_codename):
             dfs[file] = pd.read_csv(files[file])
         else:
             sys.exit(
-                f"""Compare a reference city to a comparison city, and save the comparison as a CSV file.\n\nThe summary results file ({files[file]}) could not be located.\n\nPlease try again by entering codenames from the list of configured cities {region_names} that have been fully analysed with resources generated:\npython 4_compare.py <reference> <comparison>\n\nAlternatively, enter the shortcut command:\ncompare <reference> <comparison>""",
+                f"""Compare a reference city to a comparison city, and save the comparison as a CSV file.\n\nThe summary results file ({files[file]}) could not be located.\n\nPlease try again by entering codenames from the list of configured cities {get_region_names()} that have been fully analysed with resources generated:\npython 4_compare.py <reference> <comparison>\n\nAlternatively, enter the shortcut command:\ncompare <reference> <comparison>""",
             )
+    # ordered set of columns shared between dataframes
+    shared_columns = [
+        x
+        for x in dfs[codename].columns
+        if x in dfs[comparison_codename].columns
+    ]
+    # store unshared columns from each dataframe
+    unshared_columns = {
+        codename: [
+            x
+            for x in dfs[codename].columns
+            if x not in dfs[comparison_codename].columns
+        ],
+        comparison_codename: [
+            x
+            for x in dfs[comparison_codename].columns
+            if x not in dfs[codename].columns
+        ],
+    }
+    print(f'\nColumns shared across both datasets: {shared_columns}')
+    for name in unshared_columns:
+        print(f'\nColumns unique to {name}: {unshared_columns[name]}')
     # print(pd.concat(dfs).transpose())
     comparison = (
-        dfs[codename]
-        .compare(dfs[comparison_codename], align_axis=0)
+        dfs[codename][shared_columns]
+        .compare(
+            dfs[comparison_codename][shared_columns],
+            align_axis=0,
+            keep_shape=True,
+            keep_equal=True,
+            result_names=(codename, comparison_codename),
+        )
         .droplevel(0)
         .transpose()
     )
@@ -63,8 +91,6 @@ def compare(r, comparison_codename):
             f'The results contained in the generated summaries for {codename} and {comparison_codename} are identical.',
         )
     else:
-        comparison.columns = [codename, comparison_codename]
-        # print(f'\n{comparison}')
         comparison.to_csv(
             f"{r.config['region_dir']}/compare_{r.codename}_{comparison_codename}_{date_hhmm}.csv",
         )

process/gui.py

@@ -1,29 +1,16 @@
 #!/usr/bin/env python3
 """GHSCI graphical user interface; run and access at https://localhost:8080."""
 
-# import asyncio
+
 import os.path
 
-# import geopandas as gpd
 import pandas as pd
-
-# import yaml
-# from analysis import analysis
-# from compare import compare
 from configure import configuration
 from nicegui import Client, app, ui
 from subprocesses import ghsci
-
-# from subprocesses._utils import plot_choropleth_map
 from subprocesses.leaflet import leaflet
-
-# from generate import generate
-# from geoalchemy2 import Geometry, WKTElement
 from subprocesses.local_file_picker import local_file_picker
 
-# import platform
-# import shlex
-
 
 class Region:
     """Minimal class to define a region."""
@@ -528,16 +515,6 @@ def format_policy_checklist(xlsx) -> dict:
     return df
 
 
-async def save_policy_report() -> str:
-    """Generate policy report."""
-    from policy_report import PDF_Policy_Report
-
-    report = PDF_Policy_Report(
-        'data/policy_review/Urban policy checklist_1000 Cities Challenge_version 1.0.0 - test.xlsx',
-    )
-    return await report.generate_policy_report()
-
-
 @ui.refreshable
 def studyregion_ui() -> None:
     ui.html(region.config['header_name']).style(
@@ -598,11 +575,17 @@ async def load_policy_checklist() -> None:
                 'wrap-cells=true table-style="{vertical-align: text-top}"',
             ) as table:
                 with table.add_slot('top-left'):
-                    ui.button('Generate PDF').props(
-                        'icon=download_for_offline outline',
-                    ).classes('shadow-lg').on(
-                        'click',
-                        PDF_Policy_Report(xlsx[0]).generate_policy_report,
+                    ui.button(
+                        'Generate PDF',
+                        on_click=lambda: (
+                            ui.notify(
+                                PDF_Policy_Report(
+                                    xlsx[0],
+                                ).generate_policy_report(),
+                            )
+                        ),
+                    ).props('icon=download_for_offline outline').classes(
+                        'shadow-lg',
                     ).tooltip(
                         f"Save an indexed PDF of the policy checklist to {xlsx[0].replace('.xlsx','.pdf')}.  Please wait a few moments for this to be generated after clicking.",
                     ).style(