diff --git a/process/1_create_project_configuration_files.py b/process/1_create_project_configuration_files.py index 22bdecf2..c305c2c3 100644 --- a/process/1_create_project_configuration_files.py +++ b/process/1_create_project_configuration_files.py @@ -4,11 +4,56 @@ Copies configuration templates to configuration folder for custom modification. """ -import os.path +import os import shutil +import sys source_folder = './configuration/templates' dest_folder = './configuration' +region_names = [ + x.split('.yml')[0] + for x in os.listdir('./configuration/regions') + if x.endswith('.yml') +] + +region_template = """ +name: Full study region name, e.g. Las Palmas de Gran Canaria +year: Target year for analysis, e.g. 2023 +country: Fully country name, e.g. España +country_code: Two character country code (ISO3166 Alpha-2 code), e.g. ES +continent: Continent name, e.g. Europe +crs: + name: name of the coordinate reference system (CRS), e.g. REGCAN95 / LAEA Europe + standard: acronym of the standard catalogue defining this CRS, eg. EPSG + srid: spatial reference identifier (SRID) integer that identifies this CRS according to the specified standard, e.g. 5635 (see https://spatialreference.org/, or search for what is commonly used in your city or country; e.g. a national CRS like those listed at https://en.wikipedia.org/wiki/List_of_national_coordinate_reference_systems ) + utm: UTM grid if EPSG code is not known (used to manually derive EPSG code= 326** is for Northern Hemisphere, 327** is for Southern Hemisphere), e.g. 28N (see https://mangomap.com/robertyoung/maps/69585/what-utm-zone-am-i-in- ) +study_region_boundary: + data: Region boundary data (path relative to project directory, or GHS:variable='value'), e.g. "region_boundaries/Example/Las Palmas de Gran Canaria - Centro Nacional de Información Geográfica - WGS84 - EPSG4326.geojson" + source: The name of the provider of this data, e.g. Centro Nacional de Información Geográfica + publication_date: Publication date for study region area data source, or date of currency, e.g. 2019-02-01 + url: URL for the source dataset, or its provider, e.g. https://datos.gob.es/en/catalogo/e00125901-spaignllm + licence: Licence for the data, e.g. CC-BY-4.0 + licence_url: URL for the data licence, e.g. https://www.ign.es/resources/licencia/Condiciones_licenciaUso_IGN.pdf + ghsl_urban_intersection: Whether the provided study region boundary will be further restricted to an urban area defined by its intersection with a linked urban region dataset (see urban_region), e.g. true +population: the population record in datasets.yml to be used for this region, e.g. example_las_palmas_pop_2022 +OpenStreetMap: the OpenStreetMap record defined in datasets.yml to be used for this region, e.g. las_palmas_20230221 +network: + osmnx_retain_all: If false, only retain main connected network when retrieving OSM roads, e.g. false + buffered_region: Recommended to set to 'true'. If false, use the study region boundary without buffering for network extraction (this may appropriate for true islands, but could be problematic for anywhere else where the network and associated amenities may be accessible beyond the edge of the study region boundary). + polygon_iteration: Iterate over and combine polygons (this may be appropriate for a series of islands), e.g. false + connection_threshold: Minimum distance to retain (a useful parameter for islands, like Hong Kong, but for most purposes you can leave this blank) + intersection_tolerance: Tolerance in metres for cleaning intersections. This is an important methodological choice, and the chosen parameter should be robust to a variety of network topologies in the city being studied. See https://github.com/gboeing/osmnx-examples/blob/main/notebooks/04-simplify-graph-consolidate-nodes.ipynb. For example: 12 +urban_region: the urban region dataset defined in datasets.yml to be optionally used for this region (such as data from the Global Human Settlements Layer Urban Centres Database, GHSL-UCDB), e.g. GHS-Example-Las-Palmas +urban_query: GHS or other linkage of covariate data (GHS:variable='value', or path:variable='value' for a dataset with equivalently named fields defined in project parameters for air_pollution_covariates), e.g. GHS:UC_NM_MN=='Las Palmas de Gran Canaria' and CTR_MN_NM=='Spain' +country_gdp: + classification: Country GDP classification, e.g. lower-middle + reference: Citation for the GDP classification, e.g. The World Bank. 2020. World Bank country and lending groups. https://datahelpdesk.worldbank.org/knowledgebase/articles/906519-world-bank-country-and-lending-groups +covariate_data: additional covariates to be linked (currently this is designed to direct the process to retrieve additional city statistics from the GHSL UCDB entry for the city, where available), e.g. urban_query +custom_destinations: Details of custom destinations to use (e.g. as done for Maiduguri, Nigeria), in addition to those from OSM (optional, as required; else, leave blank) file name (located in study region folder), category plain name field, category full name field, Y coordinate, X coordinate, EPSG number, attribution +gtfs_feeds: Details for city specific parent folder (within GTFS data dir defined in config.yml) and feed-specific unzipped sub-folders containing GTFS feed data. For each feed, the data provider, year, start and end dates to use, and mapping of modes are specified (only required if feed does not conform to the GTFS specification https://developers.google.com/transit/gtfs/reference). For cities with no GTFS feeds identified, this may be left blank. Otherwise, copy the example in the example_ES_Las_Palmas_2023.yml file or other example files where multiple GTFS feeds for different modes have been defined to get started and customise this for your city. +policy_review: Optional path to results of policy indicator review for inclusion in generated reports +notes: Notes for this region +""" print( 'Creating project configuration files, if not already existing in the configuration folder...', @@ -22,39 +67,45 @@ else: shutil.copyfile(path_file, path_file.replace('templates/', '')) print(f'\t- created {file}') +except Exception as e: + raise Exception(f'An error occurred: {e}') +if len(sys.argv) >= 2: + codename = sys.argv[1] + if os.path.exists(f'./configuration/regions/{codename}.yml'): + print( + f"\nConfiguration file for the specified study region codename '{codename}' already exists: \nconfiguration/regions/{codename}.yml.\n\nPlease open and edit this file in a text editor following the provided example directions in order to complete configuration for your study region. A completed example study region configuration can be viewed in the file 'configuration/regions/example_ES_Las_Palmas_2023.yml'.\n\nTo view additional guidance on configuration, run this script again without a codename. Once configuration has been completed, to proceed to analysis for this city, enter:\npython 2_analyse_region.py {codename}\n", + ) + else: + with open(f'./configuration/regions/{codename}.yml', 'w') as f: + f.write(region_template) + print( + f"\nNew region configuration file has been initialised using the codename, '{codename}', in the folder:\nconfiguration/regions/{codename}.yml\n\nPlease open and edit this file in a text editor following the provided example directions in order to complete configuration for your study region. A completed example study region configuration can be viewed in the file 'configuration/regions/example_ES_Las_Palmas_2023.yml'.\n\nTo view additional guidance on configuration, run this script again without a codename.\n", + ) +else: print( - """ -All required configuration files are present in the configuration folder, and may be customised as required: + f""" +Before commencing analysis, your study regions will need to be configured. -config.yml: - Configuration of overall project +Study regions are configured using .yml files located within the `configuration/regions` sub-folder. An example configuration for Las Palmas de Gran Canaria (for which data supporting analysis is included) has been provided in the file `process/configuration/regions/example_ES_Las_Palmas_2023.yml`, and further additional example regions have also been provided. The name of the file, for example `example_ES_Las_Palmas_2023`, acts a codename for the city when used in processing and avoids issues with ambiguity when analysing multiple cities across different regions and time points: 'example' clarifies that this is an example, 'ES' clarifies that this is a Spanish city, 'Las_Palmas' is a common short way of writing the city's name, and the analysis uses data chosen to target 2023. Using a naming convention like this is recommended when creating new region configuration files (e.g. ES_Barcelona_2023.yml, or AU_Melbourne_2023.yml). -regions.yml - Configuration of study regions of interest +The study region configuration files have a file extension .yml (YAML files), which means they are text files that can be edited in a text editor to define region specific details, including which datasets used - eg cities from a particular region could share common excerpts of population and OpenStreetMap, potentially). -datasets.yml - Configuration of datasets, which may be referenced by study regions of interest +Additional configuration may be required to define datasets referenced for regions, configuration of language for reporting using the following files: -indicators.yml - Configuration of indicators +datasets.yml: Configuration of datasets, which may be referenced by study regions of interest -osm_destination_definitions.csv - Configuration of key-value pairs used to identify features of interest using OpenStreetMap data +_report_configuration.xlsx: (required to generate reports) Use to configure generation of images and reports generated for processed cities; in particular, the translation of prose and executive summaries for different languages. This can be edited in a spreadsheet program like Microsoft Excel. -osm_open_space.yml - Configuration of queries used to derive a dataset of areas of open space using OpenStreetMap data +config.yml: (optional) Configuration of overall project, including your time zone for logging start and end times of analyses -resources.json - A file which may be used to log details of users processing environments (not currently implemented in code, but may be manually edited by users for their records). +Optional configuration of other parameters is also possible. Please visit our tool's website for further guidance on how to use this tool: +https://global-healthy-liveable-cities.github.io/ -policies.yml - Configuration of policies for reporting in generated reports +Currently configured study regions : {', '.join(region_names)} -_report_configuration.xlsx - Use to configure generation of report PDFs for processed cities; in particular, the translation of prose and executive summaries for different languages +To initialise a new study region configuration file, you can run this script again providing your choice of codename, for example: +python 1_create_project_configuration_files.py example_ES_Las_Palmas_2023 """, ) -except Exception as e: - raise Exception(f'An error occurred: {e}') diff --git a/process/2_analyse_region.py b/process/2_analyse_region.py index 39fe7c2f..74f5dda3 100644 --- a/process/2_analyse_region.py +++ b/process/2_analyse_region.py @@ -12,15 +12,17 @@ # Load study region configuration from subprocesses._project_setup import ( + analysis_timezone, authors, codename, config_path, date_hhmm, folder_path, name, + region_config, region_dir, region_names, - regions, + time, version, ) from tqdm.auto import tqdm @@ -43,7 +45,7 @@ current_parameters = { 'date': date_hhmm, 'project': project_configuration, - codename: regions[codename], + codename: region_config, } if os.path.isfile(f'{region_dir}/_parameters.yml'): @@ -87,6 +89,11 @@ f'A dated copy of project and region parameters has been saved as {region_dir}/_parameters.yml.\n\n', ) +print( + f'Analysis time zone: {analysis_timezone} (to set time zone for where you are, edit config.yml)', +) +start_analysis = time.time() +print(f"Analysis start: {time.strftime('%Y-%m-%d_%H%M')}") study_region_setup = { '_00_create_database.py': 'Create database', '_01_create_study_region.py': 'Create study region', @@ -103,7 +110,12 @@ '_12_neighbourhood_analysis.py': 'Analyse neighbourhoods', '_13_aggregation.py': 'Aggregate region summary analyses', } -pbar = tqdm(study_region_setup, position=0, leave=True) +pbar = tqdm( + study_region_setup, + position=0, + leave=True, + bar_format='{desc:35} {percentage:3.0f}%|{bar:30}| ({n_fmt}/{total_fmt})', +) append_to_log_file = open( f'{region_dir}/__{name}__{codename}_processing_log.txt', 'a', ) @@ -122,6 +134,7 @@ f'\n\nProcessing {step} failed: {e}\n\n Please review the processing log file for this study region for more information on what caused this error and how to resolve it. The file __{name}__{codename}_processing_log.txt is located in the output directory and may be opened for viewing in a text editor.', ) finally: + duration = (time.time() - start_analysis) / 60 print( - '\n\nOnce the setup of study region resources has been successfully completed, we encourage you to inspect the region-specific resources located in the output directory (e.g. text log file, geopackage output, csv files, PDF report and image files).', + f'{time.strftime("%Y-%m-%d_%H%M")} (analysis duration of approximately {duration:.1f} minutes)\n\nOnce the setup of study region resources has been successfully completed, we encourage you to inspect the region-specific resources located in the output directory (e.g. text log file, geopackage output, csv files, PDF report and image files).', ) diff --git a/process/3_generate_reports.py b/process/3_generate_resources.py similarity index 90% rename from process/3_generate_reports.py rename to process/3_generate_resources.py index e51819e6..187c1219 100644 --- a/process/3_generate_reports.py +++ b/process/3_generate_resources.py @@ -14,7 +14,8 @@ folder_path, indicators, policies, - regions, + region_config, + region_names, ) @@ -29,13 +30,13 @@ class config: configuration = './configuration/_report_configuration.xlsx' -if config.city not in regions: +if config.city not in region_names: sys.exit( - f'Specified city ({config.city}) does not appear to be in the list of configured cities ({list(regions.keys())})', + f'Specified city ({config.city}) does not appear to be in the list of configured cities ({region_names})', ) config.folder_path = folder_path -config.region = regions[config.city] +config.region = region_config if not os.path.exists(config.region['region_dir']): sys.exit( f"\n\nProcessed resource folder for this city couldn't be located:" diff --git a/process/configuration/regions/additional example regions/AT_Graz_2020.yml b/process/configuration/regions/additional example regions/AT_Graz_2020.yml new file mode 100644 index 00000000..15f7e7c3 --- /dev/null +++ b/process/configuration/regions/additional example regions/AT_Graz_2020.yml @@ -0,0 +1,43 @@ +name: Graz +year: 2020 +country: Austria +country_code: AT +continent: Europe +crs: + name: + standard: EPSG + srid: 32633 + utm: 33N +study_region_boundary: + data: region_boundaries/boundaries.gpkg:Graz + source: + publication_date: + url: + licence: + licence_url: + ghsl_urban_intersection: true +population: ghsl_r2019a_2015 +OpenStreetMap: global_indicators_25_cities +network: + osmnx_retain_all: false + buffered_region: true + polygon_iteration: + connection_threshold: + intersection_tolerance: 12 +urban_region: GHS-URBAN +urban_query: GHS:UC_NM_MN=='Graz' and CTR_MN_NM=='Austria' +country_gdp: + classification: High-income + reference: The World Bank. 2020. World Bank country and lending groups. https://datahelpdesk.worldbank.org/knowledgebase/articles/906519-world-bank-country-and-lending-groups +covariate_data: urban_query +custom_destinations: + file: + dest_name: + dest_name_full: + lat: + lon: + epsg: + attribution: +gtfs_feeds: +policy_review: Graz_GHSCIC_policy_graded_2020.xlsx +notes: diff --git a/process/configuration/regions/additional example regions/AU_Adelaide_2020.yml b/process/configuration/regions/additional example regions/AU_Adelaide_2020.yml new file mode 100644 index 00000000..de8534df --- /dev/null +++ b/process/configuration/regions/additional example regions/AU_Adelaide_2020.yml @@ -0,0 +1,50 @@ +name: Adelaide +year: 2020 +country: Australia +country_code: AU +continent: Australasia +crs: + name: + standard: EPSG + srid: 7845 + utm: +study_region_boundary: + data: region_boundaries/boundaries.gpkg:Adelaide + source: + publication_date: + url: + licence: + licence_url: + ghsl_urban_intersection: true +population: ghsl_r2019a_2015 +OpenStreetMap: global_indicators_25_cities +network: + osmnx_retain_all: false + buffered_region: true + polygon_iteration: + connection_threshold: + intersection_tolerance: 12 +urban_region: GHS-URBAN +urban_query: GHS:UC_NM_MN=='Adelaide' and CTR_MN_NM=='Australia' +country_gdp: + classification: High-income + reference: The World Bank. 2020. World Bank country and lending groups. https://datahelpdesk.worldbank.org/knowledgebase/articles/906519-world-bank-country-and-lending-groups +covariate_data: urban_query +custom_destinations: + file: + dest_name: + dest_name_full: + lat: + lon: + epsg: + attribution: +gtfs_feeds: + folder: gtfs_au_adelaide + gtfs_au_sa_adelaidemetro_20191004: + gtfs_provider: AdelaideMetro + gtfs_year: 2019 + start_date_mmdd: 20191008 + end_date_mmdd: 20191205 + modes: +policy_review: Adelaide_GHSCIC_policy_graded_2020.xlsx +notes: diff --git a/process/configuration/regions/additional example regions/AU_Melbourne_2020.yml b/process/configuration/regions/additional example regions/AU_Melbourne_2020.yml new file mode 100644 index 00000000..e1d0c9f2 --- /dev/null +++ b/process/configuration/regions/additional example regions/AU_Melbourne_2020.yml @@ -0,0 +1,50 @@ +name: Melbourne +year: 2020 +country: Australia +country_code: AU +continent: Australasia +crs: + name: + standard: EPSG + srid: 7845 + utm: +study_region_boundary: + data: region_boundaries/boundaries.gpkg:Melbourne + source: + publication_date: + url: + licence: + licence_url: + ghsl_urban_intersection: true +population: ghsl_r2019a_2015 +OpenStreetMap: global_indicators_25_cities +network: + osmnx_retain_all: false + buffered_region: true + polygon_iteration: + connection_threshold: + intersection_tolerance: 12 +urban_region: GHS-URBAN +urban_query: GHS:UC_NM_MN=='Melbourne' and CTR_MN_NM=='Australia' +country_gdp: + classification: High-income + reference: The World Bank. 2020. World Bank country and lending groups. https://datahelpdesk.worldbank.org/knowledgebase/articles/906519-world-bank-country-and-lending-groups +covariate_data: urban_query +custom_destinations: + file: + dest_name: + dest_name_full: + lat: + lon: + epsg: + attribution: +gtfs_feeds: + folder: gtfs_au_melbourne + gtfs_au_vic_ptv_20191004: + gtfs_provider: PublicTransportVictoria + gtfs_year: 2019 + start_date_mmdd: 20191008 + end_date_mmdd: 20191205 + modes: +policy_review: Melbourne_GHSCIC_policy_graded_2020.xlsx +notes: diff --git a/process/configuration/regions/additional example regions/AU_Sydney_2020.yml b/process/configuration/regions/additional example regions/AU_Sydney_2020.yml new file mode 100644 index 00000000..dc598b39 --- /dev/null +++ b/process/configuration/regions/additional example regions/AU_Sydney_2020.yml @@ -0,0 +1,92 @@ +name: Sydney +year: 2020 +country: Australia +country_code: AU +continent: Australasia +crs: + name: + standard: EPSG + srid: 7845 + utm: +study_region_boundary: + data: region_boundaries/boundaries.gpkg:Sydney + source: + publication_date: + url: + licence: + licence_url: + ghsl_urban_intersection: true +population: ghsl_r2019a_2015 +OpenStreetMap: global_indicators_25_cities +network: + osmnx_retain_all: false + buffered_region: true + polygon_iteration: + connection_threshold: + intersection_tolerance: 12 +urban_region: GHS-URBAN +urban_query: GHS:UC_NM_MN=='Sydney' and CTR_MN_NM=='Australia' +country_gdp: + classification: High-income + reference: The World Bank. 2020. World Bank country and lending groups. https://datahelpdesk.worldbank.org/knowledgebase/articles/906519-world-bank-country-and-lending-groups +covariate_data: urban_query +custom_destinations: + file: + dest_name: + dest_name_full: + lat: + lon: + epsg: + attribution: +gtfs_feeds: + folder: gtfs_au_sydney + gtfs_au_nsw_tfnsw_complete_20190619: + gtfs_provider: NSW + gtfs_year: 2019 + start_date_mmdd: 20191008 + end_date_mmdd: 20191205 + modes: + Tram: + route_types: + - 0 + agency_id: + Metro: + route_types: + - 401 + agency_id: + Rail: + route_types: + - 401 + agency_id: + Bus: + route_types: + - 700 + - 712 + - 714 + agency_id: + Ferry: + route_types: + - 4 + agency_id: + Cable tram: + route_types: + - 5 + agency_id: + Aerial lift: + route_types: + - 6 + agency_id: + Funicular: + route_types: + - 7 + agency_id: + Trolleybus: + route_types: + - 11 + agency_id: + Monorail: + route_types: + - 12 + agency_id: +policy_review: Sydney_GHSCIC_policy_graded_2020.xlsx +notes: diff --git a/process/configuration/regions/additional example regions/BE_Ghent_2020.yml b/process/configuration/regions/additional example regions/BE_Ghent_2020.yml new file mode 100644 index 00000000..62173423 --- /dev/null +++ b/process/configuration/regions/additional example regions/BE_Ghent_2020.yml @@ -0,0 +1,50 @@ +name: Ghent +year: 2020 +country: Belgium +country_code: BE +continent: Europe +crs: + name: + standard: EPSG + srid: 32631 + utm: 31N +study_region_boundary: + data: region_boundaries/boundaries.gpkg:Ghent + source: + publication_date: + url: + licence: + licence_url: + ghsl_urban_intersection: true +population: ghsl_r2019a_2015 +OpenStreetMap: global_indicators_25_cities +network: + osmnx_retain_all: false + buffered_region: true + polygon_iteration: + connection_threshold: + intersection_tolerance: 12 +urban_region: GHS-URBAN +urban_query: GHS:UC_NM_MN=='Ghent' and CTR_MN_NM=='Belgium' +country_gdp: + classification: High-income + reference: The World Bank. 2020. World Bank country and lending groups. https://datahelpdesk.worldbank.org/knowledgebase/articles/906519-world-bank-country-and-lending-groups +covariate_data: urban_query +custom_destinations: + file: + dest_name: + dest_name_full: + lat: + lon: + epsg: + attribution: +gtfs_feeds: + folder: Belgium + de_lijn-gtfs_20230117: + gtfs_provider: De Lijn + gtfs_year: 2023 + start_date_mmdd: 20230117 + end_date_mmdd: 20230130 + modes: +policy_review: Ghent_GHSCIC_policy_graded_2020.xlsx +notes: diff --git a/process/configuration/regions/additional example regions/BE_Ghent_2022.yml b/process/configuration/regions/additional example regions/BE_Ghent_2022.yml new file mode 100644 index 00000000..766ab83b --- /dev/null +++ b/process/configuration/regions/additional example regions/BE_Ghent_2022.yml @@ -0,0 +1,43 @@ +name: Ghent +year: 2022 +country: Belgium +country_code: BE +continent: Europe +crs: + name: + standard: EPSG + srid: 3812 + utm: 31N +study_region_boundary: + data: region_boundaries/Belgium/adminvector_3812.gpkg:municipality -where fid==277 + source: National Geographic Institute + publication_date: + url: https://ac.ngi.be/remoteclient-open/GeoBePartners-open/NGI-IGN/AdminVector/fb1e2993-2020-428c-9188-eb5f75e284b9_geopackage+sqlite3_3812.zip + licence: no limitations to public access + licence_url: http://inspire.ec.europa.eu/metadata-codelist/LimitationsOnPublicAccess/noLimitations + ghsl_urban_intersection: false +population: ghsl_r2022a +OpenStreetMap: belgium_20221023 +network: + osmnx_retain_all: false + buffered_region: true + polygon_iteration: + connection_threshold: + intersection_tolerance: 12 +urban_region: GHS-URBAN +urban_query: GHS:UC_NM_MN=='Ghent' and CTR_MN_NM=='Belgium' +country_gdp: + classification: High-income + reference: The World Bank. 2020. World Bank country and lending groups. https://datahelpdesk.worldbank.org/knowledgebase/articles/906519-world-bank-country-and-lending-groups +covariate_data: urban_query +custom_destinations: + file: + dest_name: + dest_name_full: + lat: + lon: + epsg: + attribution: +gtfs_feeds: +policy_review: process/data/policy_review/Ghent_GHSCIC_policy_graded_2020.xlsx +notes: Delfien van Dyck recommended that the official boundary would be most useful, not using the GHSL UCDB intersection. Downloaded dataset is a derived boundary compilation by Belgium's National Geographic Institute of administrative boundaries sourced from The General Administration of Patrimonial Documentation of the FPS Finance (the federal authorities named as the authentic source of Belgian administrative limits, according to geo.be). diff --git a/process/configuration/regions/additional example regions/BR_Sao_Paulo_2020.yml b/process/configuration/regions/additional example regions/BR_Sao_Paulo_2020.yml new file mode 100644 index 00000000..0356f2c8 --- /dev/null +++ b/process/configuration/regions/additional example regions/BR_Sao_Paulo_2020.yml @@ -0,0 +1,50 @@ +name: São Paulo +year: 2020 +country: Brazil +country_code: BR +continent: America, South +crs: + name: + standard: EPSG + srid: 32723 + utm: 23S +study_region_boundary: + data: region_boundaries/boundaries.gpkg:Sao Paulo + source: + publication_date: + url: + licence: + licence_url: + ghsl_urban_intersection: true +population: ghsl_r2019a_2015 +OpenStreetMap: global_indicators_25_cities +network: + osmnx_retain_all: false + buffered_region: true + polygon_iteration: + connection_threshold: + intersection_tolerance: 12 +urban_region: GHS-URBAN +urban_query: "GHS:UC_NM_MN=='S\xE3o Paulo' and CTR_MN_NM=='Brazil'" +country_gdp: + classification: Upper-middle-income + reference: The World Bank. 2020. World Bank country and lending groups. https://datahelpdesk.worldbank.org/knowledgebase/articles/906519-world-bank-country-and-lending-groups +covariate_data: urban_query +custom_destinations: + file: + dest_name: + dest_name_full: + lat: + lon: + epsg: + attribution: +gtfs_feeds: + folder: gtfs_brazil_sao_paulo + gtfs_brazil_sao_paulo_SPTrans_20080101: + gtfs_provider: SPTrans + gtfs_year: 2019 + start_date_mmdd: 20191008 + end_date_mmdd: 20191205 + modes: +policy_review: Sao Paulo_GHSCIC_policy_graded_2020.xlsx +notes: diff --git a/process/configuration/regions/additional example regions/CH_Bern_2020.yml b/process/configuration/regions/additional example regions/CH_Bern_2020.yml new file mode 100644 index 00000000..715a7dde --- /dev/null +++ b/process/configuration/regions/additional example regions/CH_Bern_2020.yml @@ -0,0 +1,95 @@ +name: Bern +year: 2020 +country: Switzerland +country_code: CH +continent: Europe +crs: + name: + standard: EPSG + srid: 32633 + utm: 33N +study_region_boundary: + data: region_boundaries/boundaries.gpkg:Bern + source: + publication_date: + url: + licence: + licence_url: + ghsl_urban_intersection: true +population: ghsl_r2019a_2015 +OpenStreetMap: global_indicators_25_cities +network: + osmnx_retain_all: false + buffered_region: true + polygon_iteration: + connection_threshold: + intersection_tolerance: 12 +urban_region: GHS-URBAN +urban_query: GHS:UC_NM_MN=='Bern' and CTR_MN_NM=='Switzerland' +country_gdp: + classification: High-income + reference: The World Bank. 2020. World Bank country and lending groups. https://datahelpdesk.worldbank.org/knowledgebase/articles/906519-world-bank-country-and-lending-groups +covariate_data: urban_query +custom_destinations: + file: + dest_name: + dest_name_full: + lat: + lon: + epsg: + attribution: +gtfs_feeds: + folder: gtfs_swiss_bern + gtfs_swiss_bern_SCF_20181209: + gtfs_provider: opentransportdata.swiss + gtfs_year: 2019 + start_date_mmdd: 20190405 + end_date_mmdd: 20190605 + modes: + Tram: + route_types: + - 900 + agency_id: + Metro: + route_types: + - 100 + - 400 + - 401 + agency_id: + Rail: + route_types: + - 102 + - 103 + - 106 + - 1700 + agency_id: + Bus: + route_types: + - 700 + agency_id: + Ferry: + route_types: + - 1000 + agency_id: + Cable tram: + route_types: + - 5 + agency_id: + Aerial lift: + route_types: + - 1300 + agency_id: + Funicular: + route_types: + - 1400 + agency_id: + Trolleybus: + route_types: + - 11 + agency_id: + Monorail: + route_types: + - 12 + agency_id: +policy_review: Bern_GHSCIC_policy_graded_2020.xlsx +notes: diff --git a/process/configuration/regions/additional example regions/CL_Santiago_2022.yml b/process/configuration/regions/additional example regions/CL_Santiago_2022.yml new file mode 100644 index 00000000..b8c05116 --- /dev/null +++ b/process/configuration/regions/additional example regions/CL_Santiago_2022.yml @@ -0,0 +1,51 @@ +name: Santiago +year: 2022 +country: Chile +country_code: CL +continent: South America +crs: + name: SIRGAS-Chile 2002 / UTM zone 19S + standard: EPSG + srid: 5361 + utm: 19S +study_region_boundary: + data: region_boundaries/Chile/Chile - Metropolitana de Santiago - 2020 - EPSG5361.geojson + source: "La Subsecretar\xEDa de Desarrollo Regional y Administrativo (Subdere)" + publication_date: + url: https://datos.gob.cl/dataset/15883 (https://www.ide.cl/descargas/capas/subdere/DivisionPoliticoAdministrativa2020.zip) + licence: Creative Commons Non-Commercial (Any) + licence_url: + ghsl_urban_intersection: true +population: ghsl_r2022a_Chile_partial_2020 +OpenStreetMap: chile_20221212 +network: + osmnx_retain_all: false + buffered_region: true + polygon_iteration: + connection_threshold: + intersection_tolerance: 12 +urban_region: GHS-URBAN +urban_query: GHS:UC_NM_MN=='Santiago' and CTR_MN_NM=='Chile' +country_gdp: + classification: + reference: +covariate_data: urban_query +custom_destinations: + file: + dest_name: + dest_name_full: + lat: + lon: + epsg: + attribution: +gtfs_feeds: + folder: gtfs_chile_santiago + gtfs-v76-po20221105: + gtfs_provider: "Subsecretar\xEDa de Transporte - la Red Metropolitana de Movilidad (RED)" + gtfs_year: 2022 + gtfs_url: https://datos.gob.cl/dataset/33245 + start_date_mmdd: 20221105 + end_date_mmdd: 20221205 + modes: +policy_review: +notes: diff --git a/process/configuration/regions/additional example regions/CZ_Olomouc_2020.yml b/process/configuration/regions/additional example regions/CZ_Olomouc_2020.yml new file mode 100644 index 00000000..df7fc01b --- /dev/null +++ b/process/configuration/regions/additional example regions/CZ_Olomouc_2020.yml @@ -0,0 +1,43 @@ +name: Olomouc +year: 2020 +country: Czech Republic +country_code: CZ +continent: Europe +crs: + name: + standard: EPSG + srid: 32633 + utm: 33N +study_region_boundary: + data: region_boundaries/boundaries.gpkg:Olomouc + source: + publication_date: + url: + licence: + licence_url: + ghsl_urban_intersection: true +population: ghsl_r2019a_2015 +OpenStreetMap: global_indicators_25_cities +network: + osmnx_retain_all: false + buffered_region: true + polygon_iteration: + connection_threshold: + intersection_tolerance: 12 +urban_region: GHS-URBAN +urban_query: GHS:UC_NM_MN=='Olomouc' and CTR_MN_NM=='Czech Republic' +country_gdp: + classification: High-income + reference: The World Bank. 2020. World Bank country and lending groups. https://datahelpdesk.worldbank.org/knowledgebase/articles/906519-world-bank-country-and-lending-groups +covariate_data: urban_query +custom_destinations: + file: + dest_name: + dest_name_full: + lat: + lon: + epsg: + attribution: +gtfs_feeds: +policy_review: Olomouc_GHSCIC_policy_graded_2020.xlsx +notes: diff --git a/process/configuration/regions/additional example regions/DE_Cologne_2020.yml b/process/configuration/regions/additional example regions/DE_Cologne_2020.yml new file mode 100644 index 00000000..cb730fd3 --- /dev/null +++ b/process/configuration/regions/additional example regions/DE_Cologne_2020.yml @@ -0,0 +1,50 @@ +name: Cologne +year: 2020 +country: Germany +country_code: DE +continent: Europe +crs: + name: + standard: EPSG + srid: 32631 + utm: 31N +study_region_boundary: + data: region_boundaries/boundaries.gpkg:Cologne + source: + publication_date: + url: + licence: + licence_url: + ghsl_urban_intersection: true +population: ghsl_r2019a_2015 +OpenStreetMap: global_indicators_25_cities +network: + osmnx_retain_all: false + buffered_region: true + polygon_iteration: + connection_threshold: + intersection_tolerance: 12 +urban_region: GHS-URBAN +urban_query: GHS:UC_NM_MN=='Cologne' and CTR_MN_NM=='Germany' +country_gdp: + classification: High-income + reference: The World Bank. 2020. World Bank country and lending groups. https://datahelpdesk.worldbank.org/knowledgebase/articles/906519-world-bank-country-and-lending-groups +covariate_data: urban_query +custom_destinations: + file: + dest_name: + dest_name_full: + lat: + lon: + epsg: + attribution: +gtfs_feeds: + folder: gtfs_germany_cologne + gtfs_germany_cologne_VR_20171210: + gtfs_provider: VRS + gtfs_year: 2018 + start_date_mmdd: 20180405 + end_date_mmdd: 20180605 + modes: + policy_review: Cologne_GHSCIC_policy_graded_2020.xlsx + notes: diff --git a/process/configuration/regions/additional example regions/DK_Odense_2020.yml b/process/configuration/regions/additional example regions/DK_Odense_2020.yml new file mode 100644 index 00000000..7f556099 --- /dev/null +++ b/process/configuration/regions/additional example regions/DK_Odense_2020.yml @@ -0,0 +1,50 @@ +name: Odense +year: 2020 +country: Denmark +country_code: DK +continent: Europe +crs: + name: + standard: EPSG + srid: 32632 + utm: 32N +study_region_boundary: + data: region_boundaries/boundaries.gpkg:Odense + source: + publication_date: + url: + licence: + licence_url: + ghsl_urban_intersection: true +population: ghsl_r2019a_2015 +OpenStreetMap: global_indicators_25_cities +network: + osmnx_retain_all: false + buffered_region: true + polygon_iteration: + connection_threshold: + intersection_tolerance: 12 +urban_region: GHS-URBAN +urban_query: GHS:UC_NM_MN=='Odense' and CTR_MN_NM=='Denmark' +country_gdp: + classification: High-income + reference: The World Bank. 2020. World Bank country and lending groups. https://datahelpdesk.worldbank.org/knowledgebase/articles/906519-world-bank-country-and-lending-groups +covariate_data: urban_query +custom_destinations: + file: + dest_name: + dest_name_full: + lat: + lon: + epsg: + attribution: +gtfs_feeds: + folder: gtfs_denmark_odense + gtfs_denmark_odense_Rejseplanen_20190314: + gtfs_provider: https://www.rejseplanen.dk/ (via OpenMobilityData) + gtfs_year: 2019 + start_date_mmdd: 20190405 + end_date_mmdd: 20190605 + modes: +policy_review: Odense_GHSCIC_policy_graded_2020.xlsx +notes: diff --git a/process/configuration/regions/additional example regions/ES_Barcelona_2020.yml b/process/configuration/regions/additional example regions/ES_Barcelona_2020.yml new file mode 100644 index 00000000..ffea72ed --- /dev/null +++ b/process/configuration/regions/additional example regions/ES_Barcelona_2020.yml @@ -0,0 +1,62 @@ +name: Barcelona +year: 2020 +country: Spain +country_code: ES +continent: Europe +crs: + name: + standard: EPSG + srid: 25831 + utm: +study_region_boundary: + data: region_boundaries/boundaries.gpkg:Barcelona + source: + publication_date: + url: + licence: + licence_url: + ghsl_urban_intersection: true +population: ghsl_r2019a_2015 +OpenStreetMap: global_indicators_25_cities +network: + osmnx_retain_all: false + buffered_region: true + polygon_iteration: + connection_threshold: + intersection_tolerance: 12 +urban_region: GHS-URBAN +urban_query: GHS:UC_NM_MN=='Barcelona' and CTR_MN_NM=='Spain' +country_gdp: + classification: High-income + reference: The World Bank. 2020. World Bank country and lending groups. https://datahelpdesk.worldbank.org/knowledgebase/articles/906519-world-bank-country-and-lending-groups +covariate_data: urban_query +custom_destinations: + file: + dest_name: + dest_name_full: + lat: + lon: + epsg: + attribution: +gtfs_feeds: + folder: gtfs_spain_barcelona + gtfs_spain_barcelona_AMB_20190404: + gtfs_provider: AMB + gtfs_year: 2019 + start_date_mmdd: 20190405 + end_date_mmdd: 20190605 + modes: + gtfs_spain_barcelona_TMB_20190402: + gtfs_provider: TMB + gtfs_year: 2019 + start_date_mmdd: 20190405 + end_date_mmdd: 20190605 + modes: + gtfs_spain_barcelona_Trambaix_20190303: + gtfs_provider: TMB + gtfs_year: 2019 + start_date_mmdd: 20190405 + end_date_mmdd: 20190605 + modes: +policy_review: Barcelona_GHSCIC_policy_graded_2020.xlsx +notes: diff --git a/process/configuration/regions/additional example regions/ES_Valencia_2020.yml b/process/configuration/regions/additional example regions/ES_Valencia_2020.yml new file mode 100644 index 00000000..56904b13 --- /dev/null +++ b/process/configuration/regions/additional example regions/ES_Valencia_2020.yml @@ -0,0 +1,56 @@ +name: Valencia +year: 2020 +country: Spain +country_code: ES +continent: Europe +crs: + name: + standard: EPSG + srid: 25830 + utm: 30N +study_region_boundary: + data: region_boundaries/boundaries.gpkg:Valencia + source: + publication_date: + url: + licence: + licence_url: + ghsl_urban_intersection: true +population: ghsl_r2019a_2015 +OpenStreetMap: global_indicators_25_cities +network: + osmnx_retain_all: false + buffered_region: true + polygon_iteration: + connection_threshold: + intersection_tolerance: 12 +urban_region: GHS-URBAN +urban_query: GHS:UC_NM_MN=='Valencia' and CTR_MN_NM=='Spain' +country_gdp: + classification: High-income + reference: The World Bank. 2020. World Bank country and lending groups. https://datahelpdesk.worldbank.org/knowledgebase/articles/906519-world-bank-country-and-lending-groups +covariate_data: urban_query +custom_destinations: + file: + dest_name: + dest_name_full: + lat: + lon: + epsg: + attribution: +gtfs_feeds: + folder: gtfs_spain_valencia + gtfs_spain_valencia_MetroValencia_20190403: + gtfs_provider: metroValencia + gtfs_year: 2019 + start_date_mmdd: 20190405 + end_date_mmdd: 20190605 + modes: + gtfs_spain_valencia_EMT_20190403: + gtfs_provider: EMT + gtfs_year: 2019 + start_date_mmdd: 20190405 + end_date_mmdd: 20190605 + modes: +policy_review: Valencia_GHSCIC_policy_graded_2020.xlsx +notes: diff --git a/process/configuration/regions/additional example regions/ES_Valencia_2022.yml b/process/configuration/regions/additional example regions/ES_Valencia_2022.yml new file mode 100644 index 00000000..6e6c9907 --- /dev/null +++ b/process/configuration/regions/additional example regions/ES_Valencia_2022.yml @@ -0,0 +1,56 @@ +name: Valencia +year: 2022 +country: Spain +country_code: ES +continent: Europe +crs: + name: + standard: EPSG + srid: 25830 + utm: 30N +study_region_boundary: + data: region_boundaries/boundaries.gpkg:Valencia + source: + publication_date: + url: + licence: + licence_url: + ghsl_urban_intersection: true +population: ghsl_r2022a +OpenStreetMap: valencia_20221015 +network: + osmnx_retain_all: false + buffered_region: true + polygon_iteration: + connection_threshold: + intersection_tolerance: 12 +urban_region: GHS-URBAN +urban_query: GHS:UC_NM_MN=='Valencia' and CTR_MN_NM=='Spain' +country_gdp: + classification: High-income + reference: The World Bank. 2020. World Bank country and lending groups. https://datahelpdesk.worldbank.org/knowledgebase/articles/906519-world-bank-country-and-lending-groups +covariate_data: urban_query +custom_destinations: + file: + dest_name: + dest_name_full: + lat: + lon: + crs: + attribution: +gtfs_feeds: + folder: gtfs_spain_valencia + gtfs_spain_valencia_MetroValencia_20190403: + gtfs_provider: metroValencia + gtfs_year: 2019 + start_date_mmdd: 20190405 + end_date_mmdd: 20190605 + modes: + gtfs_spain_valencia_EMT_20190403: + gtfs_provider: EMT + gtfs_year: 2019 + start_date_mmdd: 20190405 + end_date_mmdd: 20190605 + modes: +policy_review: process/data/policy_review/Valencia_GHSCIC_policy_graded_2020.xlsx +notes: diff --git a/process/configuration/regions/additional example regions/ES_Vic_2020.yml b/process/configuration/regions/additional example regions/ES_Vic_2020.yml new file mode 100644 index 00000000..bea33ac6 --- /dev/null +++ b/process/configuration/regions/additional example regions/ES_Vic_2020.yml @@ -0,0 +1,43 @@ +name: Vic +year: 2020 +country: Spain +country_code: ES +continent: Europe +crs: + name: + standard: EPSG + srid: 25831 + utm: +study_region_boundary: + data: region_boundaries/boundaries.gpkg:Vic + source: + publication_date: + url: + licence: + licence_url: + ghsl_urban_intersection: false +population: ghsl_r2019a_2015 +OpenStreetMap: global_indicators_25_cities +network: + osmnx_retain_all: false + buffered_region: true + polygon_iteration: + connection_threshold: + intersection_tolerance: 12 +urban_region: GHS-URBAN +urban_query: +country_gdp: + classification: High-income + reference: The World Bank. 2020. World Bank country and lending groups. https://datahelpdesk.worldbank.org/knowledgebase/articles/906519-world-bank-country-and-lending-groups +covariate_data: urban_query +custom_destinations: + file: + dest_name: + dest_name_full: + lat: + lon: + epsg: + attribution: +gtfs_feeds: +policy_review: process/data/policy_review/Vic_GHSCIC_policy_graded_2020.xlsx +notes: Vic appears to be well represented for its size and population with OSM destinations, however unlike all other cities it does not intersect an 'urban area' identified by the Global Human Settlements dataset. In this regard, it should be noted that Vic is exceptional compared with other cities, which were all identified as being largely 'urban'. diff --git a/process/configuration/regions/additional example regions/GB_Belfast_2020.yml b/process/configuration/regions/additional example regions/GB_Belfast_2020.yml new file mode 100644 index 00000000..9b00ba93 --- /dev/null +++ b/process/configuration/regions/additional example regions/GB_Belfast_2020.yml @@ -0,0 +1,50 @@ +name: Belfast +year: 2020 +country: United Kingdom +country_code: GB +continent: Europe +crs: + name: + standard: EPSG + srid: 29902 + utm: +study_region_boundary: + data: region_boundaries/boundaries.gpkg:Belfast + source: + publication_date: + url: + licence: + licence_url: + ghsl_urban_intersection: true +population: ghsl_r2019a_2015 +OpenStreetMap: global_indicators_25_cities +network: + osmnx_retain_all: false + buffered_region: true + polygon_iteration: + connection_threshold: + intersection_tolerance: 12 +urban_region: GHS-URBAN +urban_query: GHS:UC_NM_MN=='Belfast' and CTR_MN_NM=='United Kingdom' +country_gdp: + classification: High-income + reference: The World Bank. 2020. World Bank country and lending groups. https://datahelpdesk.worldbank.org/knowledgebase/articles/906519-world-bank-country-and-lending-groups +covariate_data: urban_query +custom_destinations: + file: + dest_name: + dest_name_full: + lat: + lon: + epsg: + attribution: +gtfs_feeds: + folder: gtfs_uk_belfast + data nicva org/metrogtfs: + gtfs_provider: Translink (via ODI Belfast / OpenDataNI) + gtfs_year: 2017 + start_date_mmdd: 20170405 + end_date_mmdd: 20170605 + modes: +policy_review: Belfast_GHSCIC_policy_graded_2020.xlsx +notes: diff --git a/process/configuration/regions/additional example regions/GB_Manchester_2022.yml b/process/configuration/regions/additional example regions/GB_Manchester_2022.yml new file mode 100644 index 00000000..168cad2d --- /dev/null +++ b/process/configuration/regions/additional example regions/GB_Manchester_2022.yml @@ -0,0 +1,43 @@ +name: Manchester +year: 2022 +country: England +country_code: GB +continent: Europe +crs: + name: + standard: EPSG + srid: 32630 + utm: 30N +study_region_boundary: + data: region_boundaries/Manchester/Greater_Manchester_Districts.gpkg:Greater_Manchester + source: + publication_date: + url: + licence: + licence_url: + ghsl_urban_intersection: true +population: ghsl_r2022a +OpenStreetMap: greater_manchester_20221015 +network: + osmnx_retain_all: false + buffered_region: true + polygon_iteration: + connection_threshold: + intersection_tolerance: 12 +urban_region: GHS-URBAN +urban_query: GHS:UC_NM_MN=='Manchester' and CTR_MN_NM=='United Kingdom' +country_gdp: + classification: High-income + reference: The World Bank. 2020. World Bank country and lending groups. https://datahelpdesk.worldbank.org/knowledgebase/articles/906519-world-bank-country-and-lending-groups +covariate_data: urban_query +custom_destinations: + file: + dest_name: + dest_name_full: + lat: + lon: + crs: + attribution: +gtfs_feeds: +policy_review: +notes: diff --git a/process/configuration/regions/additional example regions/HK_Hong_Kong_2020.yml b/process/configuration/regions/additional example regions/HK_Hong_Kong_2020.yml new file mode 100644 index 00000000..3438a240 --- /dev/null +++ b/process/configuration/regions/additional example regions/HK_Hong_Kong_2020.yml @@ -0,0 +1,50 @@ +name: Hong Kong +year: 2020 +country: China (SAR) +country_code: HK +continent: Asia +crs: + name: + standard: EPSG + srid: 32650 + utm: 50N +study_region_boundary: + data: region_boundaries/boundaries.gpkg:Hong Kong + source: + publication_date: + url: + licence: + licence_url: + ghsl_urban_intersection: true +population: ghsl_r2019a_2015 +OpenStreetMap: global_indicators_25_cities +network: + osmnx_retain_all: true + buffered_region: false + polygon_iteration: true + connection_threshold: 200 + intersection_tolerance: 12 +urban_region: GHS-URBAN +urban_query: GHS:UC_NM_MN=='Hong Kong' and CTR_MN_NM=='China' +country_gdp: + classification: High-income + reference: The World Bank. 2020. World Bank country and lending groups. https://datahelpdesk.worldbank.org/knowledgebase/articles/906519-world-bank-country-and-lending-groups +covariate_data: urban_query +custom_destinations: + file: + dest_name: + dest_name_full: + lat: + lon: + epsg: + attribution: +gtfs_feeds: + folder: gtfs_china_hongkong + gtfs_china_hongkong_hk_2019: + gtfs_provider: data.gov.hk + gtfs_year: 2019 + start_date_mmdd: 20190405 + end_date_mmdd: 20190605 + modes: +policy_review: Hong Kong_GHSCIC_policy_graded_2020.xlsx +notes: diff --git a/process/configuration/regions/additional example regions/IN_Chennai_2020.yml b/process/configuration/regions/additional example regions/IN_Chennai_2020.yml new file mode 100644 index 00000000..fdebf1e1 --- /dev/null +++ b/process/configuration/regions/additional example regions/IN_Chennai_2020.yml @@ -0,0 +1,56 @@ +name: Chennai +year: 2020 +country: India +country_code: IN +continent: Asia +crs: + name: + standard: EPSG + srid: 32644 + utm: 44N +study_region_boundary: + data: region_boundaries/boundaries.gpkg:Chennai + source: + publication_date: + url: + licence: + licence_url: + ghsl_urban_intersection: true +population: ghsl_r2019a_2015 +OpenStreetMap: global_indicators_25_cities +network: + osmnx_retain_all: false + buffered_region: true + polygon_iteration: + connection_threshold: + intersection_tolerance: 12 +urban_region: GHS-URBAN +urban_query: GHS:UC_NM_MN=='Chennai' and CTR_MN_NM=='India' +country_gdp: + classification: Lower-middle-income + reference: The World Bank. 2020. World Bank country and lending groups. https://datahelpdesk.worldbank.org/knowledgebase/articles/906519-world-bank-country-and-lending-groups +covariate_data: urban_query +custom_destinations: + file: + dest_name: + dest_name_full: + lat: + lon: + epsg: + attribution: +gtfs_feeds: + folder: gtfs_india_chennai + metropolitan-transport-corporation_20101218_1110: + gtfs_provider: Metropolitan Transport Corporation (via GTFS Exchange) + gtfs_year: 2010 + start_date_mmdd: 20100405 + end_date_mmdd: 20100605 + modes: + chennai-rail-gtfs-master/fixtures: + gtfs_provider: https://github.com/justjkk/chennai-rail-gtfs + gtfs_year: 2016 + start_date_mmdd: 20161008 + end_date_mmdd: 20161205 + modes: +policy_review: Chennai_GHSCIC_policy_graded_2020.xlsx +notes: diff --git a/process/configuration/regions/additional example regions/MX_Mexico_City_2020.yml b/process/configuration/regions/additional example regions/MX_Mexico_City_2020.yml new file mode 100644 index 00000000..5db4e353 --- /dev/null +++ b/process/configuration/regions/additional example regions/MX_Mexico_City_2020.yml @@ -0,0 +1,50 @@ +name: Mexico City +year: 2020 +country: Mexico +country_code: MX +continent: America, North +crs: + name: + standard: EPSG + srid: 32614 + utm: 14N +study_region_boundary: + data: region_boundaries/boundaries.gpkg:Mexico City + source: + publication_date: + url: + licence: + licence_url: + ghsl_urban_intersection: true +population: ghsl_r2019a_2015 +OpenStreetMap: global_indicators_25_cities +network: + osmnx_retain_all: false + buffered_region: true + polygon_iteration: + connection_threshold: + intersection_tolerance: 12 +urban_region: GHS-URBAN +urban_query: GHS:UC_NM_MN=='Mexico City' and CTR_MN_NM=='Mexico' +country_gdp: + classification: Upper-middle-income + reference: The World Bank. 2020. World Bank country and lending groups. https://datahelpdesk.worldbank.org/knowledgebase/articles/906519-world-bank-country-and-lending-groups +covariate_data: urban_query +custom_destinations: + file: + dest_name: + dest_name_full: + lat: + lon: + epsg: + attribution: +gtfs_feeds: + folder: gtfs_mexico_mexico_city + gtfs_mexico_mexico_city_fdg_20180101: + gtfs_provider: FederalDistrictGovernment + gtfs_year: 2019 + start_date_mmdd: 20190405 + end_date_mmdd: 20190605 + modes: +policy_review: Mexico City_GHSCIC_policy_graded_2020.xlsx +notes: diff --git a/process/configuration/regions/additional example regions/NG_Maiduguri_2020.yml b/process/configuration/regions/additional example regions/NG_Maiduguri_2020.yml new file mode 100644 index 00000000..8c94e114 --- /dev/null +++ b/process/configuration/regions/additional example regions/NG_Maiduguri_2020.yml @@ -0,0 +1,43 @@ +name: Maiduguri +year: 2020 +country: Nigeria +country_code: NG +continent: Africa +crs: + name: + standard: EPSG + srid: 32633 + utm: 31N +study_region_boundary: + data: GHS:UC_NM_MN='Maiduguri' + source: + publication_date: + url: + licence: + licence_url: + ghsl_urban_intersection: true +population: ghsl_r2019a_2015 +OpenStreetMap: global_indicators_25_cities +network: + osmnx_retain_all: false + buffered_region: true + polygon_iteration: + connection_threshold: + intersection_tolerance: 12 +urban_region: GHS-URBAN +urban_query: GHS:UC_NM_MN=='Maiduguri' and CTR_MN_NM=='Nigeria' +country_gdp: + classification: Lower-middle-income + reference: The World Bank. 2020. World Bank country and lending groups. https://datahelpdesk.worldbank.org/knowledgebase/articles/906519-world-bank-country-and-lending-groups +covariate_data: urban_query +custom_destinations: + file: Maiduguri_shops_convenience_complete_2020-07-13_categorised_final.csv + dest_name: dest_name + dest_name_full: dest_name_full + lat: Latitude + lon: Longitude + epsg: 4326 + attribution: Dr Garba Sambo (University of Maiduguri) and Assoc. Prof. Adewale Oyeyemi (University of Maiduguri) +gtfs_feeds: +policy_review: Maiduguri_GHSCIC_policy_graded_2020.xlsx +notes: diff --git a/process/configuration/regions/additional example regions/NZ_Auckland_2020.yml b/process/configuration/regions/additional example regions/NZ_Auckland_2020.yml new file mode 100644 index 00000000..17343115 --- /dev/null +++ b/process/configuration/regions/additional example regions/NZ_Auckland_2020.yml @@ -0,0 +1,50 @@ +name: Auckland +year: 2020 +country: New Zealand +country_code: NZ +continent: Australasia +crs: + name: + standard: EPSG + srid: 2193 + utm: +study_region_boundary: + data: region_boundaries/boundaries.gpkg:Auckland + source: + publication_date: + url: + licence: + licence_url: + ghsl_urban_intersection: true +population: ghsl_r2019a_2015 +OpenStreetMap: global_indicators_25_cities +network: + osmnx_retain_all: false + buffered_region: true + polygon_iteration: + connection_threshold: + intersection_tolerance: 12 +urban_region: GHS-URBAN +urban_query: GHS:UC_NM_MN=='Auckland' and CTR_MN_NM=='New Zealand' +country_gdp: + classification: High-income + reference: The World Bank. 2020. World Bank country and lending groups. https://datahelpdesk.worldbank.org/knowledgebase/articles/906519-world-bank-country-and-lending-groups +covariate_data: urban_query +custom_destinations: + file: + dest_name: + dest_name_full: + lat: + lon: + epsg: + attribution: +gtfs_feeds: + folder: gtfs_newzealand_auckland + gtfs_newzealand_auckland_AucklandTransport_20190928: + gtfs_provider: AucklandTransport + gtfs_year: 2019 + start_date_mmdd: 20191008 + end_date_mmdd: 20191205 + modes: + policy_review: Auckland_GHSCIC_policy_graded_2020.xlsx + notes: diff --git a/process/configuration/regions/additional example regions/PT_Lisbon_2020.yml b/process/configuration/regions/additional example regions/PT_Lisbon_2020.yml new file mode 100644 index 00000000..7a6d83a9 --- /dev/null +++ b/process/configuration/regions/additional example regions/PT_Lisbon_2020.yml @@ -0,0 +1,86 @@ +name: Lisbon +year: 2020 +country: Portugal +country_code: PT +continent: Europe +crs: + name: + standard: EPSG + srid: 3763 + utm: +study_region_boundary: + data: region_boundaries/boundaries.gpkg:Lisbon + source: + publication_date: + url: + licence: + licence_url: + ghsl_urban_intersection: true +population: ghsl_r2019a_2015 +OpenStreetMap: global_indicators_25_cities +network: + osmnx_retain_all: false + buffered_region: true + polygon_iteration: + connection_threshold: + intersection_tolerance: 12 +urban_region: GHS-URBAN +urban_query: GHS:UC_NM_MN=='Lisbon' and CTR_MN_NM=='Portugal' +country_gdp: + classification: High-income + reference: The World Bank. 2020. World Bank country and lending groups. https://datahelpdesk.worldbank.org/knowledgebase/articles/906519-world-bank-country-and-lending-groups +covariate_data: urban_query +custom_destinations: + file: + dest_name: + dest_name_full: + lat: + lon: + epsg: + attribution: +gtfs_feeds: + folder: gtfs_portugal_lisbon + carris/gtfs_portugal_lisbon_carris_20111015: + gtfs_provider: carris + gtfs_year: 2019 + start_date_mmdd: 20190405 + end_date_mmdd: 20190605 + modes: + metro-de-lisboa/gtfs_portugal_lisbon_metro-de-lisboa_20111015: + gtfs_provider: metro-de-lisboa + gtfs_year: 2019 + start_date_mmdd: 20190405 + end_date_mmdd: 20190605 + modes: + Fertagus/gtfs_portugal_lisbon_Fertagus_20111015: + gtfs_provider: Fertagus + gtfs_year: 2019 + start_date_mmdd: 20190405 + end_date_mmdd: 20190605 + modes: + MTS/gtfs_portugal_lisbon_MTS_20150105: + gtfs_provider: MTS + gtfs_year: 2019 + start_date_mmdd: 20190405 + end_date_mmdd: 20190605 + modes: + Soflusa/gtfs_portugal_lisbon_Soflusa_20111015: + gtfs_provider: Soflusa + gtfs_year: 2019 + start_date_mmdd: 20190405 + end_date_mmdd: 20190605 + modes: + transtejo/gtfs_portugal_lisbon_transtejo_20111015: + gtfs_provider: transtejo + gtfs_year: 2019 + start_date_mmdd: 20190405 + end_date_mmdd: 20190605 + modes: + CP/gtfs_portugal_lisbon_CP_20111015: + gtfs_provider: CP + gtfs_year: 2019 + start_date_mmdd: 20190405 + end_date_mmdd: 20190605 + modes: +policy_review: Lisbon_GHSCIC_policy_graded_2020.xlsx +notes: diff --git a/process/configuration/regions/additional example regions/TH_Bangkok_2020.yml b/process/configuration/regions/additional example regions/TH_Bangkok_2020.yml new file mode 100644 index 00000000..1d3a018d --- /dev/null +++ b/process/configuration/regions/additional example regions/TH_Bangkok_2020.yml @@ -0,0 +1,50 @@ +name: Bangkok +year: 2020 +country: Thailand +country_code: TH +continent: Asia +crs: + name: + standard: EPSG + srid: 32647 + utm: 47N +study_region_boundary: + data: region_boundaries/boundaries.gpkg:Bangkok + source: + publication_date: + url: + licence: + licence_url: + ghsl_urban_intersection: true +population: ghsl_r2019a_2015 +OpenStreetMap: global_indicators_25_cities +network: + osmnx_retain_all: false + buffered_region: true + polygon_iteration: + connection_threshold: + intersection_tolerance: 12 +urban_region: GHS-URBAN +urban_query: GHS:UC_NM_MN=='Bangkok' and CTR_MN_NM=='Thailand' +country_gdp: + classification: Upper-middle-income + reference: The World Bank. 2020. World Bank country and lending groups. https://datahelpdesk.worldbank.org/knowledgebase/articles/906519-world-bank-country-and-lending-groups +covariate_data: urban_query +custom_destinations: + file: + dest_name: + dest_name_full: + lat: + lon: + epsg: + attribution: +gtfs_feeds: + folder: gtfs_thailand_bangkok + gtfs_thailand_bangkok_2021: + gtfs_provider: https://namtang.otp.go.th/ + gtfs_year: 2021 + start_date_mmdd: 20210405 + end_date_mmdd: 20210605 + modes: + policy_review: Bangkok_GHSCIC_policy_graded_2020.xlsx + notes: diff --git a/process/configuration/regions/additional example regions/US_Baltimore_2020.yml b/process/configuration/regions/additional example regions/US_Baltimore_2020.yml new file mode 100644 index 00000000..3bd8c05b --- /dev/null +++ b/process/configuration/regions/additional example regions/US_Baltimore_2020.yml @@ -0,0 +1,50 @@ +name: Baltimore +year: 2020 +country: United States +country_code: US +continent: America, North +crs: + name: + standard: EPSG + srid: 32618 + utm: 18N +study_region_boundary: + data: urban_query + source: + publication_date: + url: + licence: + licence_url: + ghsl_urban_intersection: true +population: ghsl_r2019a_2015 +OpenStreetMap: global_indicators_25_cities +network: + osmnx_retain_all: false + buffered_region: true + polygon_iteration: + connection_threshold: + intersection_tolerance: 12 +urban_region: GHS-URBAN +urban_query: GHS:UC_NM_MN=='Baltimore' and CTR_MN_NM=='United States' +country_gdp: + classification: High-income + reference: The World Bank. 2020. World Bank country and lending groups. https://datahelpdesk.worldbank.org/knowledgebase/articles/906519-world-bank-country-and-lending-groups +covariate_data: urban_query +custom_destinations: + file: + dest_name: + dest_name_full: + lat: + lon: + epsg: + attribution: +gtfs_feeds: + folder: gtfs_us_baltimore + gtfs_us_baltimore_MarylandMTA_20180101: + gtfs_provider: MarylandMTA + gtfs_year: 2019 + start_date_mmdd: 20190405 + end_date_mmdd: 20190605 + modes: +policy_review: Baltimore_GHSCIC_policy_graded_2020.xlsx +notes: Based on advice of collaborator following data review, use GHS boundary diff --git a/process/configuration/regions/additional example regions/US_Phoenix_2020.yml b/process/configuration/regions/additional example regions/US_Phoenix_2020.yml new file mode 100644 index 00000000..1d1f516c --- /dev/null +++ b/process/configuration/regions/additional example regions/US_Phoenix_2020.yml @@ -0,0 +1,50 @@ +name: Phoenix +year: 2020 +country: United States +country_code: US +continent: America, North +crs: + name: + standard: EPSG + srid: 32612 + utm: 12N +study_region_boundary: + data: region_boundaries/boundaries.gpkg:Phoenix + source: + publication_date: + url: + licence: + licence_url: + ghsl_urban_intersection: true +population: ghsl_r2019a_2015 +OpenStreetMap: global_indicators_25_cities +network: + osmnx_retain_all: false + buffered_region: true + polygon_iteration: + connection_threshold: + intersection_tolerance: 12 +urban_region: GHS-URBAN +urban_query: GHS:UC_NM_MN=='Phoenix' and CTR_MN_NM=='United States' +country_gdp: + classification: High-income + reference: The World Bank. 2020. World Bank country and lending groups. https://datahelpdesk.worldbank.org/knowledgebase/articles/906519-world-bank-country-and-lending-groups +covariate_data: urban_query +custom_destinations: + file: + dest_name: + dest_name_full: + lat: + lon: + epsg: + attribution: +gtfs_feeds: + folder: gtfs_us_phoenix + gtfs_us_phoenix_valleymetro_190403: + gtfs_provider: Valleymetro + gtfs_year: 2019 + start_date_mmdd: 20190405 + end_date_mmdd: 20190605 + modes: +policy_review: Phoenix_GHSCIC_policy_graded_2020.xlsx +notes: diff --git a/process/configuration/regions/additional example regions/US_Seattle_2020.yml b/process/configuration/regions/additional example regions/US_Seattle_2020.yml new file mode 100644 index 00000000..a1783128 --- /dev/null +++ b/process/configuration/regions/additional example regions/US_Seattle_2020.yml @@ -0,0 +1,50 @@ +name: Seattle +year: 2020 +country: United States +country_code: US +continent: America, North +crs: + name: + standard: EPSG + srid: 32610 + utm: 10N +study_region_boundary: + data: urban_query + source: + publication_date: + url: + licence: + licence_url: + ghsl_urban_intersection: true +population: ghsl_r2019a_2015 +OpenStreetMap: global_indicators_25_cities +network: + osmnx_retain_all: false + buffered_region: true + polygon_iteration: + connection_threshold: + intersection_tolerance: 12 +urban_region: GHS-URBAN +urban_query: GHS:UC_NM_MN=='Seattle' and CTR_MN_NM=='United States' +country_gdp: + classification: High-income + reference: The World Bank. 2020. World Bank country and lending groups. https://datahelpdesk.worldbank.org/knowledgebase/articles/906519-world-bank-country-and-lending-groups +covariate_data: urban_query +custom_destinations: + file: + dest_name: + dest_name_full: + lat: + lon: + epsg: + attribution: +gtfs_feeds: + folder: gtfs_us_seattle + gtfs_us_seattle_kingcountymetro_20190319: + gtfs_provider: KingCountyMetro + gtfs_year: 2019 + start_date_mmdd: 20190405 + end_date_mmdd: 20190605 + modes: +policy_review: Seattle_GHSCIC_policy_graded_2020.xlsx +notes: Based on advice of collaborator following data review, use GHS boundary diff --git a/process/configuration/regions/additional example regions/VN_Hanoi_2020.yml b/process/configuration/regions/additional example regions/VN_Hanoi_2020.yml new file mode 100644 index 00000000..78e60c62 --- /dev/null +++ b/process/configuration/regions/additional example regions/VN_Hanoi_2020.yml @@ -0,0 +1,63 @@ +name: Hanoi +year: 2020 +country: Vietnam +country_code: VN +continent: Asia +crs: + name: + standard: EPSG + srid: 32648 + utm: 48N +study_region_boundary: + data: region_boundaries/boundaries.gpkg:Hanoi + source: + publication_date: + url: + licence: + licence_url: + ghsl_urban_intersection: true +population: ghsl_r2019a_2015 +OpenStreetMap: global_indicators_25_cities +network: + osmnx_retain_all: false + buffered_region: true + polygon_iteration: + connection_threshold: + intersection_tolerance: 12 +urban_region: GHS-URBAN +urban_query: GHS:UC_NM_MN=='Hanoi' and CTR_MN_NM=='Vietnam' +country_gdp: + classification: Lower-middle-income + reference: The World Bank. 2020. World Bank country and lending groups. https://datahelpdesk.worldbank.org/knowledgebase/articles/906519-world-bank-country-and-lending-groups +covariate_data: urban_query +custom_destinations: + file: + dest_name: + dest_name_full: + lat: + lon: + epsg: + attribution: +gtfs_feeds: + folder: gtfs_vietnam_hanoi + dissolve: true + hanoi_gtfs_am: + gtfs_provider: World Bank + gtfs_year: 2018 + start_date_mmdd: 20180405 + end_date_mmdd: 20180605 + modes: + hanoi_gtfs_md: + gtfs_provider: World Bank + gtfs_year: 2018 + start_date_mmdd: 20180405 + end_date_mmdd: 20180605 + modes: + hanoi_gtfs_pm: + gtfs_provider: World Bank + gtfs_year: 2018 + start_date_mmdd: 20180405 + end_date_mmdd: 20180605 + modes: +policy_review: Hanoi_GHSCIC_policy_graded_2020.xlsx +notes: diff --git a/process/configuration/regions/example_ES_Las_Palmas_2023.yml b/process/configuration/regions/example_ES_Las_Palmas_2023.yml new file mode 100644 index 00000000..b67d234c --- /dev/null +++ b/process/configuration/regions/example_ES_Las_Palmas_2023.yml @@ -0,0 +1,57 @@ +# Example configuration file for Las Palmas de Gran Canaria, Spain. +# Archivo de configuración de ejemplo para Las Palmas de Gran Canaria, España +# Exemple de fitxer de configuració per a Las Palmas de Gran Canaria, Espanya + +name: Las Palmas de Gran Canaria +year: 2023 +country: Spain +country_code: ES +continent: Europe +crs: + name: REGCAN95 / LAEA Europe + standard: EPSG + srid: 5635 + utm: 28N +study_region_boundary: + data: "region_boundaries/Example/Las Palmas de Gran Canaria - Centro Nacional de Información Geográfica - WGS84 - EPSG4326.geojson" + source: Centro Nacional de Información Geográfica + publication_date: 2019-02-01 + url: https://datos.gob.es/en/catalogo/e00125901-spaignllm + licence: CC-BY-4.0 + licence_url: https://www.ign.es/resources/licencia/Condiciones_licenciaUso_IGN.pdf + ghsl_urban_intersection: false + citation: "Instituto Geográfico Nacional (2019). Base de datos de divisiones administrativas de España. https://datos.gob.es/en/catalogo/e00125901-spaignllm. + notes: manually extracted municipal boundary for Las Palmas de Gran Canaria in WGS84 from the downloaded zip file 'lineas_limite.zip' using QGIS to a geojson file for demonstration purposes." +population: example_las_palmas_pop_2022 +OpenStreetMap: las_palmas_20230221 +network: + osmnx_retain_all: false + buffered_region: true + polygon_iteration: false + connection_threshold: + intersection_tolerance: 12 +urban_region: GHS-Example-Las-Palmas +urban_query: GHS:UC_NM_MN=='Las Palmas de Gran Canaria' and CTR_MN_NM=='Spain' +country_gdp: + classification: High-income + reference: The World Bank. 2020. World Bank country and lending groups. https://datahelpdesk.worldbank.org/knowledgebase/articles/906519-world-bank-country-and-lending-groups +covariate_data: urban_query +custom_destinations: + file: + dest_name: + dest_name_full: + lat: + lon: + epsg: + attribution: +gtfs_feeds: + folder: Example + gtfs_es_las_palmas_de_gran_canaria_guaguas_20230222: + gtfs_provider: Guaguas + gtfs_year: 2023 + gtfs_url: http://www.guaguas.com/transit/google_transit.zip + start_date_mmdd: 20230405 + end_date_mmdd: 20230605 + modes: +policy_review: process/data/policy_review/_policy_review_template_v0_TO-BE-UPDATED.xlsx +notes: diff --git a/process/configuration/remap_yml_config.py b/process/configuration/remap_yml_config.py deleted file mode 100644 index 8f0034a7..00000000 --- a/process/configuration/remap_yml_config.py +++ /dev/null @@ -1,149 +0,0 @@ -""" -Convert older (pre 16 Feb 2023) config files to updated config format. - -Changes include: nested keys, clearer names (eg not_urban_intersection=False becomes 'study_region'>'ghsl_urban_intersection'=True). -""" - -import os -import pprint -import shutil -import sys -import time - -import yaml - -date = time.strftime('%Y-%m-%d') - -if os.path.exists(f'{os.getcwd()}/../global-indicators.sh'): - folder_path = os.path.abspath(f'{os.getcwd()}/../') - sys.path.append(os.path.abspath('./subprocesses')) -elif os.path.exists(f'{os.getcwd()}/../../global-indicators.sh'): - folder_path = os.path.abspath(f'{os.getcwd()}/../../') - sys.path.append(os.path.abspath('.')) -else: - folder_path = os.getcwd() - -config_path = f'{folder_path}/process/configuration' - -with open(f'{config_path}/regions.yml') as f: - regions = yaml.safe_load(f) - -# check to see if an old value is in the regions description -if 'network_not_using_buffered_region' in regions[list(regions.keys())[0]]: - proceed = input( - f"Region configuration template schema requires updating. Would you like a copy to be made of your old file to '{config_path}/_obsolete_regions_{date}.yml', and convert the original file 'regions.yml' to use the updated schema (nested keys, clearer names, rephrased double negatives)? This is required to run the current version of the software. Type 'y' and hit enter to proceed, or another letter and enter to exit: ", - ) - if proceed != 'y': - sys.exit() - shutil.copyfile( - f'{config_path}/regions.yml', - f'{config_path}/_obsolete_regions_{date}.yml', - ) - new = {} - for r in regions: - new[r] = { - 'name': regions[r]['full_locale'], - 'year': regions[r]['year'], - 'country': regions[r]['country'], - 'country_code': regions[r]['region'], - 'continent': regions[r]['continent'], - 'crs': { - 'name': regions[r]['crs_name'], - 'standard': regions[r]['crs_standard'], - 'srid': regions[r]['crs_srid'], - 'utm': regions[r]['utm'], - }, - 'study_region_boundary': { - 'data': regions[r]['area_data'], - 'source': regions[r]['area_data_source'], - 'publication_date': None, - 'url': regions[r]['area_data_source_url'], - 'licence': regions[r]['area_data_licence'], - 'licence_url': regions[r]['area_data_licence_url'], - 'ghsl_urban_intersection': regions[r]['not_urban_intersection'] - is False, - }, - 'population': regions[r]['population'], - 'OpenStreetMap': regions[r]['OpenStreetMap'], - 'network': { - 'osmnx_retain_all': regions[r]['osmnx_retain_all'], - 'buffered_region': regions[r][ - 'network_not_using_buffered_region' - ] - is None, - 'polygon_iteration': regions[r]['network_polygon_iteration'], - 'connection_threshold': regions[r][ - 'network_connection_threshold' - ], - 'intersection_tolerance': regions[r]['intersection_tolerance'], - }, - 'urban_region': 'GHS-URBAN', - 'urban_query': regions[r]['covariate_data'], - 'country_gdp': regions[r]['country_gdp'], - 'custom_destinations': regions[r]['custom_destinations'], - 'gtfs_feeds': regions[r]['gtfs_feeds'], - 'policy_review': None, - 'notes': regions[r]['note'], - } - if 'area_data_date' in regions[r]: - new[r]['study_region_boundary']['publication_date']: regions[r][ - 'area_data_date' - ] - if 'policy_review' in regions[r]: - new[r]['policy_review'] = regions[r]['policy_review'] - - if 'description' in new: - new['description']['crs'][ - 'utm' - ] = 'UTM grid if EPSG code is not known (used to manually derive EPSG code= 326** is for Northern Hemisphere, 327** is for Southern Hemisphere)' - new['description']['study_region_boundary'][ - 'publication_date' - ] = 'Publication date for study region area data source, or date of currency' - new['description']['network'][ - 'polygon_iteration' - ] = 'Iterate over and combine polygons (e.g. islands)' - new['description']['network'][ - 'connection_threshold' - ] = 'Minimum distance to retain' - new['description']['network'][ - 'intersection_tolerance' - ] = 'Tolerance in metres for cleaning intersections. This is an important methodological choice, and the chosen parameter should be robust to a variety of network topologies in the city being studied. See https://github.com/gboeing/osmnx-examples/blob/main/notebooks/04-simplify-graph-consolidate-nodes.ipynb.' - for key in new['description']: - if type(new['description'][key]) is str: - new['description'][key] = ( - new['description'][key] - .replace('\n', '') - .replace("'''", '') - ) - elif type(new['description'][key]) is dict: - for skey in new['description'][key]: - if type(new['description'][key][skey]) is str: - new['description'][key][skey] = ( - new['description'][key][skey] - .replace('\n', '') - .replace("'''", '') - ) - elif type(new['description'][key][skey]) is dict: - for sskey in new['description'][key][skey]: - if ( - type(new['description'][key][skey][sskey]) - is str - ): - new['description'][key][skey][sskey] = ( - new['description'][key][skey][sskey] - .replace('\n', '') - .replace("'''", '') - ) - with open(f'{config_path}/regions.yml', 'w') as f: - yaml.safe_dump( - new, - f, - default_style=None, - default_flow_style=False, - sort_keys=False, - width=float('inf'), - ) -else: - print( - "The current regions.yml file doesn't appear to contain the key 'network_not_using_buffered_region', at least not for the first record (which is usually 'description'). No conversion has been undertaken, as it does not seem to be required.", - ) diff --git a/process/configuration/templates/_report_configuration.xlsx b/process/configuration/templates/_report_configuration.xlsx index 99439c13..4503b58b 100644 Binary files a/process/configuration/templates/_report_configuration.xlsx and b/process/configuration/templates/_report_configuration.xlsx differ diff --git a/process/configuration/templates/regions.yml b/process/configuration/templates/regions.yml deleted file mode 100644 index cf6f6eda..00000000 --- a/process/configuration/templates/regions.yml +++ /dev/null @@ -1,1695 +0,0 @@ -description: - name: Full study region name - year: Target year for analysis - country: Fully country name - country_code: Two character country code (ISO3166 Alpha-2 code) - continent: Continent name - crs: - name: name of the coordinate reference system (CRS; eg WGS84) - standard: acronym of the standard catalogue defining this CRS (eg. EPSG) - srid: spatial reference identifier (SRID) integer that identifies this CRS (eg 4326) according to the standard crs_standard - utm: UTM grid if EPSG code is not known (used to manually derive EPSG code= 326** is for Northern Hemisphere, 327** is for Southern Hemisphere) - study_region_boundary: - data: Region boundary data (path relative to project directory, or GHS:variable='value') - source: The name of the provider of this data - publication_date: Publication date for study region area data source, or date of currency - url: URL for the source dataset, or its provider - licence: Licence for the data - licence_url: URL for the data licence - ghsl_urban_intersection: true - population: the population record in datasets.yml to be used for this region - OpenStreetMap: the OpenStreetMap record in datasets.yml to be used for this region - network: - osmnx_retain_all: If false, only retain main connected network when retrieving OSM roads - buffered_region: If false, use the study region boundary without buffering for network extraction (this may appropriate for true islands, but could be problematic for anywhere else where the network and associated amenities may be accessible beyond the edge of the study region boundary). - polygon_iteration: Iterate over and combine polygons (e.g. islands) - connection_threshold: Minimum distance to retain - intersection_tolerance: Tolerance in metres for cleaning intersections. This is an important methodological choice, and the chosen parameter should be robust to a variety of network topologies in the city being studied. See https://github.com/gboeing/osmnx-examples/blob/main/notebooks/04-simplify-graph-consolidate-nodes.ipynb. - urban_region: GHS-URBAN - urban_query: GHS or other linkage of covariate data (GHS:variable='value', or path:variable='value' for a dataset with equivalently named fields defined in project parameters for air_pollution_covariates) - country_gdp: - classification: Country GDP classification (e.g. lower-middle) - reference: Citation for the GDP classification - covariate_data: urban_query - custom_destinations: Details of custom destinations to use, in addition to those from OSM (optional, as required; else, leave blank) file name (located in study region folder), category plain name field, category full name field, Y coordinate, X coordinate, EPSG number, attribution - gtfs_feeds: Details for city specific parent folder (within GTFS data dir defined in config.yml) and feed-specific unzipped sub-folders containing GTFS feed data. For each feed, the data provider, year, start and end dates to use, and mapping of modes are specified (only required if feed does not conform to the GTFS specification https://developers.google.com/transit/gtfs/reference). For cities with no GTFS feeds identified, this may be left blank. - policy_review: Optional path to results of policy indicator review for inclusion in generated reports - notes: Notes for this region -example_las_palmas_2023: - name: Las Palmas de Gran Canaria - year: 2023 - country: Spain - country_code: ES - continent: Europe - crs: - name: REGCAN95 / LAEA Europe - standard: EPSG - srid: 5635 - utm: 28N - study_region_boundary: - data: "region_boundaries/Example/Las Palmas de Gran Canaria - Centro Nacional de Información Geográfica - WGS84 - EPSG4326.geojson" - source: Centro Nacional de Información Geográfica - publication_date: null - url: https://datos.gob.es/en/catalogo/e00125901-spaignllm - licence: CC-BY-4.0 - licence_url: https://www.ign.es/resources/licencia/Condiciones_licenciaUso_IGN.pdf - ghsl_urban_intersection: false - citation: "Instituto Geográfico Nacional (2019). Base de datos de divisiones administrativas de España. https://datos.gob.es/en/catalogo/e00125901-spaignllm. - notes: manually extracted municipal boundary for Las Palmas de Gran Canaria in WGS84 from the downloaded zip file 'lineas_limite.zip' using QGIS to a geojson file for demonstration purposes." - population: example_las_palmas_pop_2022 - OpenStreetMap: las_palmas_20230221 - network: - osmnx_retain_all: false - buffered_region: true - polygon_iteration: null - connection_threshold: null - intersection_tolerance: 12 - urban_region: GHS-Example-Las-Palmas - urban_query: GHS:UC_NM_MN=='Las Palmas de Gran Canaria' and CTR_MN_NM=='Spain' - country_gdp: - classification: High-income - reference: The World Bank. 2020. World Bank country and lending groups. https://datahelpdesk.worldbank.org/knowledgebase/articles/906519-world-bank-country-and-lending-groups - covariate_data: urban_query - custom_destinations: - file: null - dest_name: null - dest_name_full: null - lat: null - lon: null - epsg: null - attribution: null - gtfs_feeds: - folder: Example - gtfs_es_las_palmas_de_gran_canaria_guaguas_20230222: - gtfs_provider: Guaguas - gtfs_year: 2023 - gtfs_url: http://www.guaguas.com/transit/google_transit.zip - start_date_mmdd: 20230405 - end_date_mmdd: 20230605 - modes: null - policy_review: process/data/policy_review/_policy_review_template_v0_TO-BE-UPDATED.xlsx - notes: -santiago_2022: - name: Santiago - year: 2022 - country: Chile - country_code: CL - continent: South America - crs: - name: SIRGAS-Chile 2002 / UTM zone 19S - standard: EPSG - srid: 5361 - utm: 19S - study_region_boundary: - data: region_boundaries/Chile/Chile - Metropolitana de Santiago - 2020 - EPSG5361.geojson - source: "La Subsecretar\xEDa de Desarrollo Regional y Administrativo (Subdere)" - publication_date: null - url: https://datos.gob.cl/dataset/15883 (https://www.ide.cl/descargas/capas/subdere/DivisionPoliticoAdministrativa2020.zip) - licence: Creative Commons Non-Commercial (Any) - licence_url: null - ghsl_urban_intersection: true - population: ghsl_r2022a_Chile_partial_2020 - OpenStreetMap: chile_20221212 - network: - osmnx_retain_all: false - buffered_region: true - polygon_iteration: null - connection_threshold: null - intersection_tolerance: 12 - urban_region: GHS-URBAN - urban_query: GHS:UC_NM_MN=='Santiago' and CTR_MN_NM=='Chile' - country_gdp: - classification: null - reference: null - covariate_data: urban_query - custom_destinations: - file: null - dest_name: null - dest_name_full: null - lat: null - lon: null - epsg: null - attribution: null - gtfs_feeds: - folder: gtfs_chile_santiago - gtfs-v76-po20221105: - gtfs_provider: "Subsecretar\xEDa de Transporte - la Red Metropolitana de Movilidad (RED)" - gtfs_year: 2022 - gtfs_url: https://datos.gob.cl/dataset/33245 - start_date_mmdd: 20221105 - end_date_mmdd: 20221205 - modes: null - policy_review: null - notes: null -ghent_2022: - name: Ghent - year: 2022 - country: Belgium - country_code: BE - continent: Europe - crs: - name: null - standard: EPSG - srid: 3812 - utm: 31N - study_region_boundary: - data: region_boundaries/Belgium/adminvector_3812.gpkg:municipality -where fid==277 - source: National Geographic Institute - publication_date: null - url: https://ac.ngi.be/remoteclient-open/GeoBePartners-open/NGI-IGN/AdminVector/fb1e2993-2020-428c-9188-eb5f75e284b9_geopackage+sqlite3_3812.zip - licence: no limitations to public access - licence_url: http://inspire.ec.europa.eu/metadata-codelist/LimitationsOnPublicAccess/noLimitations - ghsl_urban_intersection: false - population: ghsl_r2022a - OpenStreetMap: belgium_20221023 - network: - osmnx_retain_all: false - buffered_region: true - polygon_iteration: null - connection_threshold: null - intersection_tolerance: 12 - urban_region: GHS-URBAN - urban_query: GHS:UC_NM_MN=='Ghent' and CTR_MN_NM=='Belgium' - country_gdp: - classification: High-income - reference: The World Bank. 2020. World Bank country and lending groups. https://datahelpdesk.worldbank.org/knowledgebase/articles/906519-world-bank-country-and-lending-groups - covariate_data: urban_query - custom_destinations: - file: null - dest_name: null - dest_name_full: null - lat: null - lon: null - epsg: null - attribution: null - gtfs_feeds: null - policy_review: process/data/policy_review/Ghent_GHSCIC_policy_graded_2020.xlsx - notes: Delfien van Dyck recommended that the official boundary would be most useful, not using the GHSL UCDB intersection. Downloaded dataset is a derived boundary compilation by Belgium's National Geographic Institute of administrative boundaries sourced from The General Administration of Patrimonial Documentation of the FPS Finance (the federal authorities named as the authentic source of Belgian administrative limits, according to geo.be). -valencia_2022: - name: Valencia - year: 2022 - country: Spain - country_code: ES - continent: Europe - crs: - name: null - standard: EPSG - srid: 25830 - utm: 30N - study_region_boundary: - data: region_boundaries/boundaries.gpkg:Valencia - source: null - publication_date: null - url: null - licence: null - licence_url: null - ghsl_urban_intersection: true - population: ghsl_r2022a - OpenStreetMap: valencia_20221015 - network: - osmnx_retain_all: false - buffered_region: true - polygon_iteration: null - connection_threshold: null - intersection_tolerance: 12 - urban_region: GHS-URBAN - urban_query: GHS:UC_NM_MN=='Valencia' and CTR_MN_NM=='Spain' - country_gdp: - classification: High-income - reference: The World Bank. 2020. World Bank country and lending groups. https://datahelpdesk.worldbank.org/knowledgebase/articles/906519-world-bank-country-and-lending-groups - covariate_data: urban_query - custom_destinations: - file: null - dest_name: null - dest_name_full: null - lat: null - lon: null - crs: null - attribution: null - gtfs_feeds: - folder: gtfs_spain_valencia - gtfs_spain_valencia_MetroValencia_20190403: - gtfs_provider: metroValencia - gtfs_year: 2019 - start_date_mmdd: 20190405 - end_date_mmdd: 20190605 - modes: null - gtfs_spain_valencia_EMT_20190403: - gtfs_provider: EMT - gtfs_year: 2019 - start_date_mmdd: 20190405 - end_date_mmdd: 20190605 - modes: null - policy_review: process/data/policy_review/Valencia_GHSCIC_policy_graded_2020.xlsx - notes: null -manchester_2022: - name: Manchester - year: 2022 - country: England - country_code: GB - continent: Europe - crs: - name: null - standard: EPSG - srid: 32630 - utm: 30N - study_region_boundary: - data: region_boundaries/Manchester/Greater_Manchester_Districts.gpkg:Greater_Manchester - source: null - publication_date: null - url: null - licence: null - licence_url: null - ghsl_urban_intersection: true - population: ghsl_r2022a - OpenStreetMap: greater_manchester_20221015 - network: - osmnx_retain_all: false - buffered_region: true - polygon_iteration: null - connection_threshold: null - intersection_tolerance: 12 - urban_region: GHS-URBAN - urban_query: GHS:UC_NM_MN=='Manchester' and CTR_MN_NM=='United Kingdom' - country_gdp: - classification: High-income - reference: The World Bank. 2020. World Bank country and lending groups. https://datahelpdesk.worldbank.org/knowledgebase/articles/906519-world-bank-country-and-lending-groups - covariate_data: urban_query - custom_destinations: - file: null - dest_name: null - dest_name_full: null - lat: null - lon: null - crs: null - attribution: null - gtfs_feeds: null - policy_review: null - notes: null -adelaide: - name: Adelaide - year: 2020 - country: Australia - country_code: AU - continent: Australasia - crs: - name: null - standard: EPSG - srid: 7845 - utm: null - study_region_boundary: - data: region_boundaries/boundaries.gpkg:Adelaide - source: null - publication_date: null - url: null - licence: null - licence_url: null - ghsl_urban_intersection: true - population: ghsl_r2019a_2015 - OpenStreetMap: global_indicators_25_cities - network: - osmnx_retain_all: false - buffered_region: true - polygon_iteration: null - connection_threshold: null - intersection_tolerance: 12 - urban_region: GHS-URBAN - urban_query: GHS:UC_NM_MN=='Adelaide' and CTR_MN_NM=='Australia' - country_gdp: - classification: High-income - reference: The World Bank. 2020. World Bank country and lending groups. https://datahelpdesk.worldbank.org/knowledgebase/articles/906519-world-bank-country-and-lending-groups - covariate_data: urban_query - custom_destinations: - file: null - dest_name: null - dest_name_full: null - lat: null - lon: null - epsg: null - attribution: null - gtfs_feeds: - folder: gtfs_au_adelaide - gtfs_au_sa_adelaidemetro_20191004: - gtfs_provider: AdelaideMetro - gtfs_year: 2019 - start_date_mmdd: 20191008 - end_date_mmdd: 20191205 - modes: null - policy_review: Adelaide_GHSCIC_policy_graded_2020.xlsx - notes: null -auckland: - name: Auckland - year: 2020 - country: New Zealand - country_code: NZ - continent: Australasia - crs: - name: null - standard: EPSG - srid: 2193 - utm: null - study_region_boundary: - data: region_boundaries/boundaries.gpkg:Auckland - source: null - publication_date: null - url: null - licence: null - licence_url: null - ghsl_urban_intersection: true - population: ghsl_r2019a_2015 - OpenStreetMap: global_indicators_25_cities - network: - osmnx_retain_all: false - buffered_region: true - polygon_iteration: null - connection_threshold: null - intersection_tolerance: 12 - urban_region: GHS-URBAN - urban_query: GHS:UC_NM_MN=='Auckland' and CTR_MN_NM=='New Zealand' - country_gdp: - classification: High-income - reference: The World Bank. 2020. World Bank country and lending groups. https://datahelpdesk.worldbank.org/knowledgebase/articles/906519-world-bank-country-and-lending-groups - covariate_data: urban_query - custom_destinations: - file: null - dest_name: null - dest_name_full: null - lat: null - lon: null - epsg: null - attribution: null - gtfs_feeds: - folder: gtfs_newzealand_auckland - gtfs_newzealand_auckland_AucklandTransport_20190928: - gtfs_provider: AucklandTransport - gtfs_year: 2019 - start_date_mmdd: 20191008 - end_date_mmdd: 20191205 - modes: null - policy_review: Auckland_GHSCIC_policy_graded_2020.xlsx - notes: null -baltimore: - name: Baltimore - year: 2020 - country: United States - country_code: US - continent: America, North - crs: - name: null - standard: EPSG - srid: 32618 - utm: 18N - study_region_boundary: - data: urban_query - source: null - publication_date: null - url: null - licence: null - licence_url: null - ghsl_urban_intersection: true - population: ghsl_r2019a_2015 - OpenStreetMap: global_indicators_25_cities - network: - osmnx_retain_all: false - buffered_region: true - polygon_iteration: null - connection_threshold: null - intersection_tolerance: 12 - urban_region: GHS-URBAN - urban_query: GHS:UC_NM_MN=='Baltimore' and CTR_MN_NM=='United States' - country_gdp: - classification: High-income - reference: The World Bank. 2020. World Bank country and lending groups. https://datahelpdesk.worldbank.org/knowledgebase/articles/906519-world-bank-country-and-lending-groups - covariate_data: urban_query - custom_destinations: - file: null - dest_name: null - dest_name_full: null - lat: null - lon: null - epsg: null - attribution: null - gtfs_feeds: - folder: gtfs_us_baltimore - gtfs_us_baltimore_MarylandMTA_20180101: - gtfs_provider: MarylandMTA - gtfs_year: 2019 - start_date_mmdd: 20190405 - end_date_mmdd: 20190605 - modes: null - policy_review: Baltimore_GHSCIC_policy_graded_2020.xlsx - notes: Based on advice of collaborator following data review, use GHS boundary -bangkok: - name: Bangkok - year: 2020 - country: Thailand - country_code: TH - continent: Asia - crs: - name: null - standard: EPSG - srid: 32647 - utm: 47N - study_region_boundary: - data: region_boundaries/boundaries.gpkg:Bangkok - source: null - publication_date: null - url: null - licence: null - licence_url: null - ghsl_urban_intersection: true - population: ghsl_r2019a_2015 - OpenStreetMap: global_indicators_25_cities - network: - osmnx_retain_all: false - buffered_region: true - polygon_iteration: null - connection_threshold: null - intersection_tolerance: 12 - urban_region: GHS-URBAN - urban_query: GHS:UC_NM_MN=='Bangkok' and CTR_MN_NM=='Thailand' - country_gdp: - classification: Upper-middle-income - reference: The World Bank. 2020. World Bank country and lending groups. https://datahelpdesk.worldbank.org/knowledgebase/articles/906519-world-bank-country-and-lending-groups - covariate_data: urban_query - custom_destinations: - file: null - dest_name: null - dest_name_full: null - lat: null - lon: null - epsg: null - attribution: null - gtfs_feeds: - folder: gtfs_thailand_bangkok - gtfs_thailand_bangkok_2021: - gtfs_provider: https://namtang.otp.go.th/ - gtfs_year: 2021 - start_date_mmdd: 20210405 - end_date_mmdd: 20210605 - modes: null - policy_review: Bangkok_GHSCIC_policy_graded_2020.xlsx - notes: null -barcelona: - name: Barcelona - year: 2020 - country: Spain - country_code: ES - continent: Europe - crs: - name: null - standard: EPSG - srid: 25831 - utm: null - study_region_boundary: - data: region_boundaries/boundaries.gpkg:Barcelona - source: null - publication_date: null - url: null - licence: null - licence_url: null - ghsl_urban_intersection: true - population: ghsl_r2019a_2015 - OpenStreetMap: global_indicators_25_cities - network: - osmnx_retain_all: false - buffered_region: true - polygon_iteration: null - connection_threshold: null - intersection_tolerance: 12 - urban_region: GHS-URBAN - urban_query: GHS:UC_NM_MN=='Barcelona' and CTR_MN_NM=='Spain' - country_gdp: - classification: High-income - reference: The World Bank. 2020. World Bank country and lending groups. https://datahelpdesk.worldbank.org/knowledgebase/articles/906519-world-bank-country-and-lending-groups - covariate_data: urban_query - custom_destinations: - file: null - dest_name: null - dest_name_full: null - lat: null - lon: null - epsg: null - attribution: null - gtfs_feeds: - folder: gtfs_spain_barcelona - gtfs_spain_barcelona_AMB_20190404: - gtfs_provider: AMB - gtfs_year: 2019 - start_date_mmdd: 20190405 - end_date_mmdd: 20190605 - modes: null - gtfs_spain_barcelona_TMB_20190402: - gtfs_provider: TMB - gtfs_year: 2019 - start_date_mmdd: 20190405 - end_date_mmdd: 20190605 - modes: null - gtfs_spain_barcelona_Trambaix_20190303: - gtfs_provider: TMB - gtfs_year: 2019 - start_date_mmdd: 20190405 - end_date_mmdd: 20190605 - modes: null - policy_review: Barcelona_GHSCIC_policy_graded_2020.xlsx - notes: null -belfast: - name: Belfast - year: 2020 - country: United Kingdom - country_code: GB - continent: Europe - crs: - name: null - standard: EPSG - srid: 29902 - utm: null - study_region_boundary: - data: region_boundaries/boundaries.gpkg:Belfast - source: null - publication_date: null - url: null - licence: null - licence_url: null - ghsl_urban_intersection: true - population: ghsl_r2019a_2015 - OpenStreetMap: global_indicators_25_cities - network: - osmnx_retain_all: false - buffered_region: true - polygon_iteration: null - connection_threshold: null - intersection_tolerance: 12 - urban_region: GHS-URBAN - urban_query: GHS:UC_NM_MN=='Belfast' and CTR_MN_NM=='United Kingdom' - country_gdp: - classification: High-income - reference: The World Bank. 2020. World Bank country and lending groups. https://datahelpdesk.worldbank.org/knowledgebase/articles/906519-world-bank-country-and-lending-groups - covariate_data: urban_query - custom_destinations: - file: null - dest_name: null - dest_name_full: null - lat: null - lon: null - epsg: null - attribution: null - gtfs_feeds: - folder: gtfs_uk_belfast - data nicva org/metrogtfs: - gtfs_provider: Translink (via ODI Belfast / OpenDataNI) - gtfs_year: 2017 - start_date_mmdd: 20170405 - end_date_mmdd: 20170605 - modes: null - policy_review: Belfast_GHSCIC_policy_graded_2020.xlsx - notes: null -bern: - name: Bern - year: 2020 - country: Switzerland - country_code: CH - continent: Europe - crs: - name: null - standard: EPSG - srid: 32633 - utm: 33N - study_region_boundary: - data: region_boundaries/boundaries.gpkg:Bern - source: null - publication_date: null - url: null - licence: null - licence_url: null - ghsl_urban_intersection: true - population: ghsl_r2019a_2015 - OpenStreetMap: global_indicators_25_cities - network: - osmnx_retain_all: false - buffered_region: true - polygon_iteration: null - connection_threshold: null - intersection_tolerance: 12 - urban_region: GHS-URBAN - urban_query: GHS:UC_NM_MN=='Bern' and CTR_MN_NM=='Switzerland' - country_gdp: - classification: High-income - reference: The World Bank. 2020. World Bank country and lending groups. https://datahelpdesk.worldbank.org/knowledgebase/articles/906519-world-bank-country-and-lending-groups - covariate_data: urban_query - custom_destinations: - file: null - dest_name: null - dest_name_full: null - lat: null - lon: null - epsg: null - attribution: null - gtfs_feeds: - folder: gtfs_swiss_bern - gtfs_swiss_bern_SCF_20181209: - gtfs_provider: opentransportdata.swiss - gtfs_year: 2019 - start_date_mmdd: 20190405 - end_date_mmdd: 20190605 - modes: - Tram: - route_types: - - 900 - agency_id: null - Metro: - route_types: - - 100 - - 400 - - 401 - agency_id: null - Rail: - route_types: - - 102 - - 103 - - 106 - - 1700 - agency_id: null - Bus: - route_types: - - 700 - agency_id: null - Ferry: - route_types: - - 1000 - agency_id: null - Cable tram: - route_types: - - 5 - agency_id: null - Aerial lift: - route_types: - - 1300 - agency_id: null - Funicular: - route_types: - - 1400 - agency_id: null - Trolleybus: - route_types: - - 11 - agency_id: null - Monorail: - route_types: - - 12 - agency_id: null - policy_review: Bern_GHSCIC_policy_graded_2020.xlsx - notes: null -chennai: - name: Chennai - year: 2020 - country: India - country_code: IN - continent: Asia - crs: - name: null - standard: EPSG - srid: 32644 - utm: 44N - study_region_boundary: - data: region_boundaries/boundaries.gpkg:Chennai - source: null - publication_date: null - url: null - licence: null - licence_url: null - ghsl_urban_intersection: true - population: ghsl_r2019a_2015 - OpenStreetMap: global_indicators_25_cities - network: - osmnx_retain_all: false - buffered_region: true - polygon_iteration: null - connection_threshold: null - intersection_tolerance: 12 - urban_region: GHS-URBAN - urban_query: GHS:UC_NM_MN=='Chennai' and CTR_MN_NM=='India' - country_gdp: - classification: Lower-middle-income - reference: The World Bank. 2020. World Bank country and lending groups. https://datahelpdesk.worldbank.org/knowledgebase/articles/906519-world-bank-country-and-lending-groups - covariate_data: urban_query - custom_destinations: - file: null - dest_name: null - dest_name_full: null - lat: null - lon: null - epsg: null - attribution: null - gtfs_feeds: - folder: gtfs_india_chennai - metropolitan-transport-corporation_20101218_1110: - gtfs_provider: Metropolitan Transport Corporation (via GTFS Exchange) - gtfs_year: 2010 - start_date_mmdd: 20100405 - end_date_mmdd: 20100605 - modes: null - chennai-rail-gtfs-master/fixtures: - gtfs_provider: https://github.com/justjkk/chennai-rail-gtfs - gtfs_year: 2016 - start_date_mmdd: 20161008 - end_date_mmdd: 20161205 - modes: null - policy_review: Chennai_GHSCIC_policy_graded_2020.xlsx - notes: null -cologne: - name: Cologne - year: 2020 - country: Germany - country_code: DE - continent: Europe - crs: - name: null - standard: EPSG - srid: 32631 - utm: 31N - study_region_boundary: - data: region_boundaries/boundaries.gpkg:Cologne - source: null - publication_date: null - url: null - licence: null - licence_url: null - ghsl_urban_intersection: true - population: ghsl_r2019a_2015 - OpenStreetMap: global_indicators_25_cities - network: - osmnx_retain_all: false - buffered_region: true - polygon_iteration: null - connection_threshold: null - intersection_tolerance: 12 - urban_region: GHS-URBAN - urban_query: GHS:UC_NM_MN=='Cologne' and CTR_MN_NM=='Germany' - country_gdp: - classification: High-income - reference: The World Bank. 2020. World Bank country and lending groups. https://datahelpdesk.worldbank.org/knowledgebase/articles/906519-world-bank-country-and-lending-groups - covariate_data: urban_query - custom_destinations: - file: null - dest_name: null - dest_name_full: null - lat: null - lon: null - epsg: null - attribution: null - gtfs_feeds: - folder: gtfs_germany_cologne - gtfs_germany_cologne_VR_20171210: - gtfs_provider: VRS - gtfs_year: 2018 - start_date_mmdd: 20180405 - end_date_mmdd: 20180605 - modes: null - policy_review: Cologne_GHSCIC_policy_graded_2020.xlsx - notes: null -ghent: - name: Ghent - year: 2020 - country: Belgium - country_code: BE - continent: Europe - crs: - name: null - standard: EPSG - srid: 32631 - utm: 31N - study_region_boundary: - data: region_boundaries/boundaries.gpkg:Ghent - source: null - publication_date: null - url: null - licence: null - licence_url: null - ghsl_urban_intersection: true - population: ghsl_r2019a_2015 - OpenStreetMap: global_indicators_25_cities - network: - osmnx_retain_all: false - buffered_region: true - polygon_iteration: null - connection_threshold: null - intersection_tolerance: 12 - urban_region: GHS-URBAN - urban_query: GHS:UC_NM_MN=='Ghent' and CTR_MN_NM=='Belgium' - country_gdp: - classification: High-income - reference: The World Bank. 2020. World Bank country and lending groups. https://datahelpdesk.worldbank.org/knowledgebase/articles/906519-world-bank-country-and-lending-groups - covariate_data: urban_query - custom_destinations: - file: null - dest_name: null - dest_name_full: null - lat: null - lon: null - epsg: null - attribution: null - gtfs_feeds: - folder: Belgium - de_lijn-gtfs_20230117: - gtfs_provider: De Lijn - gtfs_year: 2023 - start_date_mmdd: 20230117 - end_date_mmdd: 20230130 - modes: - policy_review: Ghent_GHSCIC_policy_graded_2020.xlsx - notes: null -graz: - name: Graz - year: 2020 - country: Austria - country_code: AT - continent: Europe - crs: - name: null - standard: EPSG - srid: 32633 - utm: 33N - study_region_boundary: - data: region_boundaries/boundaries.gpkg:Graz - source: null - publication_date: null - url: null - licence: null - licence_url: null - ghsl_urban_intersection: true - population: ghsl_r2019a_2015 - OpenStreetMap: global_indicators_25_cities - network: - osmnx_retain_all: false - buffered_region: true - polygon_iteration: null - connection_threshold: null - intersection_tolerance: 12 - urban_region: GHS-URBAN - urban_query: GHS:UC_NM_MN=='Graz' and CTR_MN_NM=='Austria' - country_gdp: - classification: High-income - reference: The World Bank. 2020. World Bank country and lending groups. https://datahelpdesk.worldbank.org/knowledgebase/articles/906519-world-bank-country-and-lending-groups - covariate_data: urban_query - custom_destinations: - file: null - dest_name: null - dest_name_full: null - lat: null - lon: null - epsg: null - attribution: null - gtfs_feeds: null - policy_review: Graz_GHSCIC_policy_graded_2020.xlsx - notes: null -hanoi: - name: Hanoi - year: 2020 - country: Vietnam - country_code: VN - continent: Asia - crs: - name: null - standard: EPSG - srid: 32648 - utm: 48N - study_region_boundary: - data: region_boundaries/boundaries.gpkg:Hanoi - source: null - publication_date: null - url: null - licence: null - licence_url: null - ghsl_urban_intersection: true - population: ghsl_r2019a_2015 - OpenStreetMap: global_indicators_25_cities - network: - osmnx_retain_all: false - buffered_region: true - polygon_iteration: null - connection_threshold: null - intersection_tolerance: 12 - urban_region: GHS-URBAN - urban_query: GHS:UC_NM_MN=='Hanoi' and CTR_MN_NM=='Vietnam' - country_gdp: - classification: Lower-middle-income - reference: The World Bank. 2020. World Bank country and lending groups. https://datahelpdesk.worldbank.org/knowledgebase/articles/906519-world-bank-country-and-lending-groups - covariate_data: urban_query - custom_destinations: - file: null - dest_name: null - dest_name_full: null - lat: null - lon: null - epsg: null - attribution: null - gtfs_feeds: - folder: gtfs_vietnam_hanoi - dissolve: true - hanoi_gtfs_am: - gtfs_provider: World Bank - gtfs_year: 2018 - start_date_mmdd: 20180405 - end_date_mmdd: 20180605 - modes: null - hanoi_gtfs_md: - gtfs_provider: World Bank - gtfs_year: 2018 - start_date_mmdd: 20180405 - end_date_mmdd: 20180605 - modes: null - hanoi_gtfs_pm: - gtfs_provider: World Bank - gtfs_year: 2018 - start_date_mmdd: 20180405 - end_date_mmdd: 20180605 - modes: null - policy_review: Hanoi_GHSCIC_policy_graded_2020.xlsx - notes: null -hong_kong: - name: Hong Kong - year: 2020 - country: China (SAR) - country_code: HK - continent: Asia - crs: - name: null - standard: EPSG - srid: 32650 - utm: 50N - study_region_boundary: - data: region_boundaries/boundaries.gpkg:Hong Kong - source: null - publication_date: null - url: null - licence: null - licence_url: null - ghsl_urban_intersection: true - population: ghsl_r2019a_2015 - OpenStreetMap: global_indicators_25_cities - network: - osmnx_retain_all: true - buffered_region: false - polygon_iteration: true - connection_threshold: 200 - intersection_tolerance: 12 - urban_region: GHS-URBAN - urban_query: GHS:UC_NM_MN=='Hong Kong' and CTR_MN_NM=='China' - country_gdp: - classification: High-income - reference: The World Bank. 2020. World Bank country and lending groups. https://datahelpdesk.worldbank.org/knowledgebase/articles/906519-world-bank-country-and-lending-groups - covariate_data: urban_query - custom_destinations: - file: null - dest_name: null - dest_name_full: null - lat: null - lon: null - epsg: null - attribution: null - gtfs_feeds: - folder: gtfs_china_hongkong - gtfs_china_hongkong_hk_2019: - gtfs_provider: data.gov.hk - gtfs_year: 2019 - start_date_mmdd: 20190405 - end_date_mmdd: 20190605 - modes: null - policy_review: Hong Kong_GHSCIC_policy_graded_2020.xlsx - notes: null -lisbon: - name: Lisbon - year: 2020 - country: Portugal - country_code: PT - continent: Europe - crs: - name: null - standard: EPSG - srid: 3763 - utm: null - study_region_boundary: - data: region_boundaries/boundaries.gpkg:Lisbon - source: null - publication_date: null - url: null - licence: null - licence_url: null - ghsl_urban_intersection: true - population: ghsl_r2019a_2015 - OpenStreetMap: global_indicators_25_cities - network: - osmnx_retain_all: false - buffered_region: true - polygon_iteration: null - connection_threshold: null - intersection_tolerance: 12 - urban_region: GHS-URBAN - urban_query: GHS:UC_NM_MN=='Lisbon' and CTR_MN_NM=='Portugal' - country_gdp: - classification: High-income - reference: The World Bank. 2020. World Bank country and lending groups. https://datahelpdesk.worldbank.org/knowledgebase/articles/906519-world-bank-country-and-lending-groups - covariate_data: urban_query - custom_destinations: - file: null - dest_name: null - dest_name_full: null - lat: null - lon: null - epsg: null - attribution: null - gtfs_feeds: - folder: gtfs_portugal_lisbon - carris/gtfs_portugal_lisbon_carris_20111015: - gtfs_provider: carris - gtfs_year: 2019 - start_date_mmdd: 20190405 - end_date_mmdd: 20190605 - modes: null - metro-de-lisboa/gtfs_portugal_lisbon_metro-de-lisboa_20111015: - gtfs_provider: metro-de-lisboa - gtfs_year: 2019 - start_date_mmdd: 20190405 - end_date_mmdd: 20190605 - modes: null - Fertagus/gtfs_portugal_lisbon_Fertagus_20111015: - gtfs_provider: Fertagus - gtfs_year: 2019 - start_date_mmdd: 20190405 - end_date_mmdd: 20190605 - modes: null - MTS/gtfs_portugal_lisbon_MTS_20150105: - gtfs_provider: MTS - gtfs_year: 2019 - start_date_mmdd: 20190405 - end_date_mmdd: 20190605 - modes: null - Soflusa/gtfs_portugal_lisbon_Soflusa_20111015: - gtfs_provider: Soflusa - gtfs_year: 2019 - start_date_mmdd: 20190405 - end_date_mmdd: 20190605 - modes: null - transtejo/gtfs_portugal_lisbon_transtejo_20111015: - gtfs_provider: transtejo - gtfs_year: 2019 - start_date_mmdd: 20190405 - end_date_mmdd: 20190605 - modes: null - CP/gtfs_portugal_lisbon_CP_20111015: - gtfs_provider: CP - gtfs_year: 2019 - start_date_mmdd: 20190405 - end_date_mmdd: 20190605 - modes: null - policy_review: Lisbon_GHSCIC_policy_graded_2020.xlsx - notes: null -maiduguri: - name: Maiduguri - year: 2020 - country: Nigeria - country_code: NG - continent: Africa - crs: - name: null - standard: EPSG - srid: 32633 - utm: 31N - study_region_boundary: - data: GHS:UC_NM_MN='Maiduguri' - source: null - publication_date: null - url: null - licence: null - licence_url: null - ghsl_urban_intersection: true - population: ghsl_r2019a_2015 - OpenStreetMap: global_indicators_25_cities - network: - osmnx_retain_all: false - buffered_region: true - polygon_iteration: null - connection_threshold: null - intersection_tolerance: 12 - urban_region: GHS-URBAN - urban_query: GHS:UC_NM_MN=='Maiduguri' and CTR_MN_NM=='Nigeria' - country_gdp: - classification: Lower-middle-income - reference: The World Bank. 2020. World Bank country and lending groups. https://datahelpdesk.worldbank.org/knowledgebase/articles/906519-world-bank-country-and-lending-groups - covariate_data: urban_query - custom_destinations: - file: Maiduguri_shops_convenience_complete_2020-07-13_categorised_final.csv - dest_name: dest_name - dest_name_full: dest_name_full - lat: Latitude - lon: Longitude - epsg: 4326 - attribution: Dr Garba Sambo (University of Maiduguri) and Assoc. Prof. Adewale Oyeyemi (University of Maiduguri) - gtfs_feeds: null - policy_review: Maiduguri_GHSCIC_policy_graded_2020.xlsx - notes: null -melbourne: - name: Melbourne - year: 2020 - country: Australia - country_code: AU - continent: Australasia - crs: - name: null - standard: EPSG - srid: 7845 - utm: null - study_region_boundary: - data: region_boundaries/boundaries.gpkg:Melbourne - source: null - publication_date: null - url: null - licence: null - licence_url: null - ghsl_urban_intersection: true - population: ghsl_r2019a_2015 - OpenStreetMap: global_indicators_25_cities - network: - osmnx_retain_all: false - buffered_region: true - polygon_iteration: null - connection_threshold: null - intersection_tolerance: 12 - urban_region: GHS-URBAN - urban_query: GHS:UC_NM_MN=='Melbourne' and CTR_MN_NM=='Australia' - country_gdp: - classification: High-income - reference: The World Bank. 2020. World Bank country and lending groups. https://datahelpdesk.worldbank.org/knowledgebase/articles/906519-world-bank-country-and-lending-groups - covariate_data: urban_query - custom_destinations: - file: null - dest_name: null - dest_name_full: null - lat: null - lon: null - epsg: null - attribution: null - gtfs_feeds: - folder: gtfs_au_melbourne - gtfs_au_vic_ptv_20191004: - gtfs_provider: PublicTransportVictoria - gtfs_year: 2019 - start_date_mmdd: 20191008 - end_date_mmdd: 20191205 - modes: null - policy_review: Melbourne_GHSCIC_policy_graded_2020.xlsx - notes: null -mexico_city: - name: Mexico City - year: 2020 - country: Mexico - country_code: MX - continent: America, North - crs: - name: null - standard: EPSG - srid: 32614 - utm: 14N - study_region_boundary: - data: region_boundaries/boundaries.gpkg:Mexico City - source: null - publication_date: null - url: null - licence: null - licence_url: null - ghsl_urban_intersection: true - population: ghsl_r2019a_2015 - OpenStreetMap: global_indicators_25_cities - network: - osmnx_retain_all: false - buffered_region: true - polygon_iteration: null - connection_threshold: null - intersection_tolerance: 12 - urban_region: GHS-URBAN - urban_query: GHS:UC_NM_MN=='Mexico City' and CTR_MN_NM=='Mexico' - country_gdp: - classification: Upper-middle-income - reference: The World Bank. 2020. World Bank country and lending groups. https://datahelpdesk.worldbank.org/knowledgebase/articles/906519-world-bank-country-and-lending-groups - covariate_data: urban_query - custom_destinations: - file: null - dest_name: null - dest_name_full: null - lat: null - lon: null - epsg: null - attribution: null - gtfs_feeds: - folder: gtfs_mexico_mexico_city - gtfs_mexico_mexico_city_fdg_20180101: - gtfs_provider: FederalDistrictGovernment - gtfs_year: 2019 - start_date_mmdd: 20190405 - end_date_mmdd: 20190605 - modes: null - policy_review: Mexico City_GHSCIC_policy_graded_2020.xlsx - notes: null -odense: - name: Odense - year: 2020 - country: Denmark - country_code: DK - continent: Europe - crs: - name: null - standard: EPSG - srid: 32632 - utm: 32N - study_region_boundary: - data: region_boundaries/boundaries.gpkg:Odense - source: null - publication_date: null - url: null - licence: null - licence_url: null - ghsl_urban_intersection: true - population: ghsl_r2019a_2015 - OpenStreetMap: global_indicators_25_cities - network: - osmnx_retain_all: false - buffered_region: true - polygon_iteration: null - connection_threshold: null - intersection_tolerance: 12 - urban_region: GHS-URBAN - urban_query: GHS:UC_NM_MN=='Odense' and CTR_MN_NM=='Denmark' - country_gdp: - classification: High-income - reference: The World Bank. 2020. World Bank country and lending groups. https://datahelpdesk.worldbank.org/knowledgebase/articles/906519-world-bank-country-and-lending-groups - covariate_data: urban_query - custom_destinations: - file: null - dest_name: null - dest_name_full: null - lat: null - lon: null - epsg: null - attribution: null - gtfs_feeds: - folder: gtfs_denmark_odense - gtfs_denmark_odense_Rejseplanen_20190314: - gtfs_provider: https://www.rejseplanen.dk/ (via OpenMobilityData) - gtfs_year: 2019 - start_date_mmdd: 20190405 - end_date_mmdd: 20190605 - modes: null - policy_review: Odense_GHSCIC_policy_graded_2020.xlsx - notes: null -olomouc: - name: Olomouc - year: 2020 - country: Czech Republic - country_code: CZ - continent: Europe - crs: - name: null - standard: EPSG - srid: 32633 - utm: 33N - study_region_boundary: - data: region_boundaries/boundaries.gpkg:Olomouc - source: null - publication_date: null - url: null - licence: null - licence_url: null - ghsl_urban_intersection: true - population: ghsl_r2019a_2015 - OpenStreetMap: global_indicators_25_cities - network: - osmnx_retain_all: false - buffered_region: true - polygon_iteration: null - connection_threshold: null - intersection_tolerance: 12 - urban_region: GHS-URBAN - urban_query: GHS:UC_NM_MN=='Olomouc' and CTR_MN_NM=='Czech Republic' - country_gdp: - classification: High-income - reference: The World Bank. 2020. World Bank country and lending groups. https://datahelpdesk.worldbank.org/knowledgebase/articles/906519-world-bank-country-and-lending-groups - covariate_data: urban_query - custom_destinations: - file: null - dest_name: null - dest_name_full: null - lat: null - lon: null - epsg: null - attribution: null - gtfs_feeds: null - policy_review: Olomouc_GHSCIC_policy_graded_2020.xlsx - notes: null -phoenix: - name: Phoenix - year: 2020 - country: United States - country_code: US - continent: America, North - crs: - name: null - standard: EPSG - srid: 32612 - utm: 12N - study_region_boundary: - data: region_boundaries/boundaries.gpkg:Phoenix - source: null - publication_date: null - url: null - licence: null - licence_url: null - ghsl_urban_intersection: true - population: ghsl_r2019a_2015 - OpenStreetMap: global_indicators_25_cities - network: - osmnx_retain_all: false - buffered_region: true - polygon_iteration: null - connection_threshold: null - intersection_tolerance: 12 - urban_region: GHS-URBAN - urban_query: GHS:UC_NM_MN=='Phoenix' and CTR_MN_NM=='United States' - country_gdp: - classification: High-income - reference: The World Bank. 2020. World Bank country and lending groups. https://datahelpdesk.worldbank.org/knowledgebase/articles/906519-world-bank-country-and-lending-groups - covariate_data: urban_query - custom_destinations: - file: null - dest_name: null - dest_name_full: null - lat: null - lon: null - epsg: null - attribution: null - gtfs_feeds: - folder: gtfs_us_phoenix - gtfs_us_phoenix_valleymetro_190403: - gtfs_provider: Valleymetro - gtfs_year: 2019 - start_date_mmdd: 20190405 - end_date_mmdd: 20190605 - modes: null - policy_review: Phoenix_GHSCIC_policy_graded_2020.xlsx - notes: null -sao_paulo: - name: Sao Paulo - year: 2020 - country: Brazil - country_code: BR - continent: America, South - crs: - name: null - standard: EPSG - srid: 32723 - utm: 23S - study_region_boundary: - data: region_boundaries/boundaries.gpkg:Sao Paulo - source: null - publication_date: null - url: null - licence: null - licence_url: null - ghsl_urban_intersection: true - population: ghsl_r2019a_2015 - OpenStreetMap: global_indicators_25_cities - network: - osmnx_retain_all: false - buffered_region: true - polygon_iteration: null - connection_threshold: null - intersection_tolerance: 12 - urban_region: GHS-URBAN - urban_query: "GHS:UC_NM_MN=='S\xE3o Paulo' and CTR_MN_NM=='Brazil'" - country_gdp: - classification: Upper-middle-income - reference: The World Bank. 2020. World Bank country and lending groups. https://datahelpdesk.worldbank.org/knowledgebase/articles/906519-world-bank-country-and-lending-groups - covariate_data: urban_query - custom_destinations: - file: null - dest_name: null - dest_name_full: null - lat: null - lon: null - epsg: null - attribution: null - gtfs_feeds: - folder: gtfs_brazil_sao_paulo - gtfs_brazil_sao_paulo_SPTrans_20080101: - gtfs_provider: SPTrans - gtfs_year: 2019 - start_date_mmdd: 20191008 - end_date_mmdd: 20191205 - modes: null - policy_review: Sao Paulo_GHSCIC_policy_graded_2020.xlsx - notes: null -seattle: - name: Seattle - year: 2020 - country: United States - country_code: US - continent: America, North - crs: - name: null - standard: EPSG - srid: 32610 - utm: 10N - study_region_boundary: - data: urban_query - source: null - publication_date: null - url: null - licence: null - licence_url: null - ghsl_urban_intersection: true - population: ghsl_r2019a_2015 - OpenStreetMap: global_indicators_25_cities - network: - osmnx_retain_all: false - buffered_region: true - polygon_iteration: null - connection_threshold: null - intersection_tolerance: 12 - urban_region: GHS-URBAN - urban_query: GHS:UC_NM_MN=='Seattle' and CTR_MN_NM=='United States' - country_gdp: - classification: High-income - reference: The World Bank. 2020. World Bank country and lending groups. https://datahelpdesk.worldbank.org/knowledgebase/articles/906519-world-bank-country-and-lending-groups - covariate_data: urban_query - custom_destinations: - file: null - dest_name: null - dest_name_full: null - lat: null - lon: null - epsg: null - attribution: null - gtfs_feeds: - folder: gtfs_us_seattle - gtfs_us_seattle_kingcountymetro_20190319: - gtfs_provider: KingCountyMetro - gtfs_year: 2019 - start_date_mmdd: 20190405 - end_date_mmdd: 20190605 - modes: null - policy_review: Seattle_GHSCIC_policy_graded_2020.xlsx - notes: Based on advice of collaborator following data review, use GHS boundary -sydney: - name: Sydney - year: 2020 - country: Australia - country_code: AU - continent: Australasia - crs: - name: null - standard: EPSG - srid: 7845 - utm: null - study_region_boundary: - data: region_boundaries/boundaries.gpkg:Sydney - source: null - publication_date: null - url: null - licence: null - licence_url: null - ghsl_urban_intersection: true - population: ghsl_r2019a_2015 - OpenStreetMap: global_indicators_25_cities - network: - osmnx_retain_all: false - buffered_region: true - polygon_iteration: null - connection_threshold: null - intersection_tolerance: 12 - urban_region: GHS-URBAN - urban_query: GHS:UC_NM_MN=='Sydney' and CTR_MN_NM=='Australia' - country_gdp: - classification: High-income - reference: The World Bank. 2020. World Bank country and lending groups. https://datahelpdesk.worldbank.org/knowledgebase/articles/906519-world-bank-country-and-lending-groups - covariate_data: urban_query - custom_destinations: - file: null - dest_name: null - dest_name_full: null - lat: null - lon: null - epsg: null - attribution: null - gtfs_feeds: - folder: gtfs_au_sydney - gtfs_au_nsw_tfnsw_complete_20190619: - gtfs_provider: NSW - gtfs_year: 2019 - start_date_mmdd: 20191008 - end_date_mmdd: 20191205 - modes: - Tram: - route_types: - - 0 - agency_id: null - Metro: - route_types: - - 401 - agency_id: null - Rail: - route_types: - - 401 - agency_id: null - Bus: - route_types: - - 700 - - 712 - - 714 - agency_id: null - Ferry: - route_types: - - 4 - agency_id: null - Cable tram: - route_types: - - 5 - agency_id: null - Aerial lift: - route_types: - - 6 - agency_id: null - Funicular: - route_types: - - 7 - agency_id: null - Trolleybus: - route_types: - - 11 - agency_id: null - Monorail: - route_types: - - 12 - agency_id: null - policy_review: Sydney_GHSCIC_policy_graded_2020.xlsx - notes: null -valencia: - name: Valencia - year: 2020 - country: Spain - country_code: ES - continent: Europe - crs: - name: null - standard: EPSG - srid: 25830 - utm: 30N - study_region_boundary: - data: region_boundaries/boundaries.gpkg:Valencia - source: null - publication_date: null - url: null - licence: null - licence_url: null - ghsl_urban_intersection: true - population: ghsl_r2019a_2015 - OpenStreetMap: global_indicators_25_cities - network: - osmnx_retain_all: false - buffered_region: true - polygon_iteration: null - connection_threshold: null - intersection_tolerance: 12 - urban_region: GHS-URBAN - urban_query: GHS:UC_NM_MN=='Valencia' and CTR_MN_NM=='Spain' - country_gdp: - classification: High-income - reference: The World Bank. 2020. World Bank country and lending groups. https://datahelpdesk.worldbank.org/knowledgebase/articles/906519-world-bank-country-and-lending-groups - covariate_data: urban_query - custom_destinations: - file: null - dest_name: null - dest_name_full: null - lat: null - lon: null - epsg: null - attribution: null - gtfs_feeds: - folder: gtfs_spain_valencia - gtfs_spain_valencia_MetroValencia_20190403: - gtfs_provider: metroValencia - gtfs_year: 2019 - start_date_mmdd: 20190405 - end_date_mmdd: 20190605 - modes: null - gtfs_spain_valencia_EMT_20190403: - gtfs_provider: EMT - gtfs_year: 2019 - start_date_mmdd: 20190405 - end_date_mmdd: 20190605 - modes: null - policy_review: Valencia_GHSCIC_policy_graded_2020.xlsx - notes: null -vic: - name: Vic - year: 2020 - country: Spain - country_code: ES - continent: Europe - crs: - name: null - standard: EPSG - srid: 25831 - utm: null - study_region_boundary: - data: region_boundaries/boundaries.gpkg:Vic - source: null - publication_date: null - url: null - licence: null - licence_url: null - ghsl_urban_intersection: false - population: ghsl_r2019a_2015 - OpenStreetMap: global_indicators_25_cities - network: - osmnx_retain_all: false - buffered_region: true - polygon_iteration: null - connection_threshold: null - intersection_tolerance: 12 - urban_region: GHS-URBAN - urban_query: null - country_gdp: - classification: High-income - reference: The World Bank. 2020. World Bank country and lending groups. https://datahelpdesk.worldbank.org/knowledgebase/articles/906519-world-bank-country-and-lending-groups - covariate_data: urban_query - custom_destinations: - file: null - dest_name: null - dest_name_full: null - lat: null - lon: null - epsg: null - attribution: null - gtfs_feeds: null - policy_review: process/data/policy_review/Vic_GHSCIC_policy_graded_2020.xlsx - notes: Vic appears to be well represented for its size and population with OSM destinations, however unlike all other cities it does not intersect an 'urban area' identified by the Global Human Settlements dataset. In this regard, it should be noted that Vic is exceptional compared with other cities, which were all identified as being largely 'urban'. diff --git a/process/configuration/templates/resources.json b/process/configuration/templates/resources.json deleted file mode 100644 index 87e1aa41..00000000 --- a/process/configuration/templates/resources.json +++ /dev/null @@ -1 +0,0 @@ -{"cpus": 0} diff --git a/process/data/policy_review/_policy_review_template_v0_TO-BE-UPDATED.xlsx b/process/data/policy_review/_policy_review_template_v0_TO-BE-UPDATED.xlsx index bd772fef..b89b1a5c 100644 Binary files a/process/data/policy_review/_policy_review_template_v0_TO-BE-UPDATED.xlsx and b/process/data/policy_review/_policy_review_template_v0_TO-BE-UPDATED.xlsx differ diff --git a/process/readme.md b/process/readme.md index 4a9886ae..08dd350d 100644 --- a/process/readme.md +++ b/process/readme.md @@ -4,15 +4,15 @@ The Global Healthy and Sustainable Cities Indicators Collaboration (GHSCIC) spat 1. Configuration 2. Region analysis -3. Generate reports +3. Generate resources As a result of running the process, a geopackage of spatial features for a specified and configured urban region is generated, including indicators for point locations, a small area grid (eg 100m), and overall city estimates. In addition CSV files containing indicators for small area grid cells and the overall city are also generated, omitting geometry. Optionally, PDF 'scorecard' reports summarising policy and spatial indicator results may be generated for dissemination. +Detailed usage notes are provided on the Global Healthy and Sustainable City Indicators tool [website]. + ## Software installation and set up -1. Download and unzip a software release: - - https://github.com/global-healthy-liveable-cities/global-indicators/releases/tag/v2.0.0 - - https://github.com/global-healthy-liveable-cities/global-indicators/releases/tag/v3.0.0 (pre-release) +1. Download and unzip the [latest software release](https://github.com/global-healthy-liveable-cities/global-indicators/releases) 2. Install and run [Docker Desktop](https://www.docker.com/) according to the guidelines for your operating system of choice 3. Run the software a command prompt at the project directory - on Windows in cmd.exe enter '.\global-indicators.bat' @@ -22,7 +22,11 @@ Those scripts get Docker to retrieve the computational environment and dependenc ## 1. Configuration and data sourcing -Before commencing analysis, your project and study regions will need to be configured. Configuration files which may be modified can first be initialised by running: +Before commencing analysis, your project and study regions will need to be configured. + +Regions are configured using .yml files located within the `configuration/regions` sub-folder. An example region configuration for Las Palmas de Gran Canaria (for which data supporting analysis is included) has been provided in the file `process/configuration/regions/example_ES_Las_Palmas_2023.yml`, and further additional example regions have been provided. The name of the file, for example `example_ES_Las_Palmas_2023`, acts a codename for the city when used in processing and avoids issues with ambiguity when analysing multiple cities across different regions and time points: 'example' clarifies that this is an example, 'ES' clarifies that this is a Spanish city, 'Las_Palmas' is a common short way of writing the city's name, and the analysis is designed to target 2023 (i.e. it uses data sources published then). The .yml file is a text file used to define region specific details, including which datasets used - eg cities from a particular region could share common excerpts of population and OpenStreetMap, potentially). + +Additional configuration files which may be modified can first be initialised by running: ```python 1_create_project_configuration_files.py``` @@ -30,7 +34,6 @@ The following configuration files will then be located in the `process/configura - config.yml (overall project configuration) - datasets.yml (defines datasets and metadata for OpenStreetMap and population, which can be referenced by regions) -- regions.yml (region specific details, including which datasets used - eg cities from a particular region could share common excerpts of population and OpenStreetMap, potentially) - indicators.yml (some aspects of indicators calculated can be modified here, although this isn't necessary; currently this is set up for our core indicators) - osm_destinations.csv (a table of key pair tags that collectively identify the kinds of destinations to be evaluated for accessibility) - osm_open_space.yml (definitions for identifying areas of open space using OpenStreetMap) @@ -70,7 +73,7 @@ Finally, spatial urban indicator summaries are aggregated for a small area grid To generate reports for the results, run -```python 3_generate_reports.py [CITY CODE NAME]``` +```python 3_generate_resources.py [CITY CODE NAME]``` This script is used to generate reports, optionally in multiple languages, for processed cities. It integrates the functionality previously located in the repository https://github.com/global-healthy-liveable-cities/global_scorecards, which was used to generate [city reports](https://doi.org/10.25439/rmt.c.6012649) for our 25 city study across 16 languages. These can be configured using the configuration file _report_configuration.xlsx in conjunction with the regions, indicators and policies configuration files. diff --git a/process/subprocesses/_00_create_database.py b/process/subprocesses/_00_create_database.py index 8569206a..1e04cd83 100644 --- a/process/subprocesses/_00_create_database.py +++ b/process/subprocesses/_00_create_database.py @@ -36,7 +36,7 @@ def main(): # SQL queries create_database = f""" -- Create database - CREATE DATABASE {db} + CREATE DATABASE "{db}" WITH OWNER = {admin_db} ENCODING = 'UTF8' TABLESPACE = pg_default @@ -54,7 +54,7 @@ def main(): print('Done.') comment_database = f""" - COMMENT ON DATABASE {db} IS '{dbComment}'; + COMMENT ON DATABASE "{db}" IS '{dbComment}'; """ print(f'Adding comment "{dbComment}"... ', end='', flush=True) curs.execute(comment_database) @@ -89,8 +89,8 @@ def main(): create_extensions = f""" CREATE EXTENSION IF NOT EXISTS postgis; CREATE EXTENSION IF NOT EXISTS postgis_raster; - ALTER DATABASE {db} SET postgis.enable_outdb_rasters = true; - ALTER DATABASE {db} SET postgis.gdal_enabled_drivers TO 'ENABLE_ALL'; + ALTER DATABASE "{db}" SET postgis.enable_outdb_rasters = true; + ALTER DATABASE "{db}" SET postgis.gdal_enabled_drivers TO 'ENABLE_ALL'; CREATE EXTENSION IF NOT EXISTS postgis_sfcgal; CREATE EXTENSION IF NOT EXISTS pgrouting; SELECT postgis_full_version(); diff --git a/process/subprocesses/_01_create_study_region.py b/process/subprocesses/_01_create_study_region.py index 6e82f2aa..62bbbf39 100644 --- a/process/subprocesses/_01_create_study_region.py +++ b/process/subprocesses/_01_create_study_region.py @@ -74,7 +74,7 @@ def main(): f' user={db_user} password={db_pwd}" ' f' "{boundary_data}" ' f' -lco geometry_name="geom" -lco precision=NO ' - f' -t_srs {crs_srid} -nln {study_region} ' + f' -t_srs {crs_srid} -nln "study_region_boundary" ' f' {query}' ) print(command) @@ -99,20 +99,20 @@ def main(): '{db}'::text AS "db", ST_Area(geom)/10^6 AS area_sqkm, geom - FROM {study_region}; + FROM "study_region_boundary"; CREATE INDEX IF NOT EXISTS {table}_gix ON {table} USING GIST (geom); """ with engine.begin() as connection: connection.execute(text(sql)) else: # get study region bounding box to be used to retrieve intersecting urban geometries - sql = f""" + sql = """ SELECT ST_Xmin(geom) xmin, ST_Ymin(geom) ymin, ST_Xmax(geom) xmax, ST_Ymax(geom) ymax - FROM {study_region}; + FROM "study_region_boundary"; """ with engine.begin() as connection: result = connection.execute(text(sql)) @@ -138,7 +138,7 @@ def main(): ST_Area(a.geom)/10^6 AS area_sqkm, a.geom FROM full_urban_region a, - {study_region} b + "study_region_boundary" b WHERE ST_Intersects(a.geom,b.geom); CREATE INDEX IF NOT EXISTS urban_region_gix ON urban_region USING GIST (geom); CREATE TABLE IF NOT EXISTS urban_study_region AS @@ -146,9 +146,9 @@ def main(): b."db", ST_Area(ST_Union(ST_Intersection(a.geom,b.geom)))/10^6 AS area_sqkm, ST_Union(ST_Intersection(a.geom,b.geom)) geom - FROM {study_region} a, + FROM "study_region_boundary" a, urban_region b - GROUP BY b."study_region", b."db"; + GROUP BY b."study_region_boundary", b."db"; CREATE INDEX IF NOT EXISTS urban_study_region_gix ON urban_study_region USING GIST (geom); """ with engine.begin() as connection: @@ -169,7 +169,7 @@ def main(): '{buffered_urban_study_region_extent}'::text AS "Study region buffer", ST_Buffer(geom,{study_buffer}) AS geom FROM urban_study_region ; - CREATE INDEX IF NOT EXISTS {study_region}_{study_buffer}{units}_gix ON + CREATE INDEX IF NOT EXISTS {buffered_urban_study_region}_gix ON {buffered_urban_study_region} USING GIST (geom); """ with engine.begin() as connection: @@ -177,7 +177,7 @@ def main(): print('Done.') print( f"""\nThe following layers have been created: - \n- {study_region}: To represent a policy-relevant administrative boundary (or proxy for this). + \n- study_region_boundary: To represent a policy-relevant administrative boundary (or proxy for this). \n- urban_region: Representing the urban area surrounding the study region. \n- urban_study_region: The urban portion of the policy-relevant study region. \n- {buffered_urban_study_region}: An analytical boundary extending {study_buffer} {units} further to mitigate edge effects. @@ -185,15 +185,15 @@ def main(): ) if ( - db_contents.has_table(study_region) + db_contents.has_table('study_region_boundary') and db_contents.has_table('urban_region') and db_contents.has_table('urban_study_region') and db_contents.has_table(buffered_urban_study_region) ): - return f"""Study region boundaries have previously been created ({study_region}, urban_region, urban_study_region and {buffered_urban_study_region}). If you wish to recreate these, please manually drop them (e.g. using psql) or optionally drop the {db} database and start again (e.g. using the subprocesses/_drop_study_region_database.py utility script.\n""" + return f"""Study region boundaries have previously been created (study_region_boundary, urban_region, urban_study_region and {buffered_urban_study_region}). If you wish to recreate these, please manually drop them (e.g. using psql) or optionally drop the {db} database and start again (e.g. using the subprocesses/_drop_study_region_database.py utility script.\n""" else: - raise ( - """Study region boundary creation failed; check configuration and log files to identify specific issues.""" + raise Exception( + """Study region boundary creation failed; check configuration and log files to identify specific issues.""", ) # output to completion log diff --git a/process/subprocesses/_03_create_network_resources.py b/process/subprocesses/_03_create_network_resources.py index 08bb15f5..af3500be 100644 --- a/process/subprocesses/_03_create_network_resources.py +++ b/process/subprocesses/_03_create_network_resources.py @@ -98,7 +98,15 @@ def main(): task = 'Create network resources' engine = create_engine( - f'postgresql://{db_user}:{db_pwd}@{db_host}/{db}', future=True, + f'postgresql://{db_user}:{db_pwd}@{db_host}/{db}', + future=True, + pool_pre_ping=True, + connect_args={ + 'keepalives': 1, + 'keepalives_idle': 30, + 'keepalives_interval': 10, + 'keepalives_count': 5, + }, ) db_contents = inspect(engine) if network['buffered_region']: @@ -114,6 +122,13 @@ def main(): print('\nGet networks and save as graphs.') ox.settings.use_cache = True ox.settings.log_console = True + # set OSMnx to retrieve filtered network to match OpenStreetMap publication date + osm_publication_date = f"""[date:"{datetime.strptime(str(regions_config['OpenStreetMap']['publication_date']), '%Y%m%d').strftime('%Y-%m-%d')}T00:00:00Z"]""" + ox.settings.overpass_settings = ( + '[out:json][timeout:{timeout}]' + + osm_publication_date + + '{maxsize}' + ) if not network['osmnx_retain_all']: print( """Note: "osmnx_retain_all = False" ie. only main network segment is retained. Please ensure this is appropriate for your study region (ie. networks on real islands may be excluded).""", diff --git a/process/subprocesses/_04_create_population_grid.py b/process/subprocesses/_04_create_population_grid.py index 38ac7f42..498e391e 100644 --- a/process/subprocesses/_04_create_population_grid.py +++ b/process/subprocesses/_04_create_population_grid.py @@ -28,18 +28,20 @@ def main(): start = time.time() script = os.path.basename(sys.argv[0]) task = 'Create population grid excerpt for city' - engine = create_engine(f'postgresql://{db_user}:{db_pwd}@{db_host}/{db}') + engine = create_engine( + f'postgresql://{db_user}:{db_pwd}@{db_host}/{db}', + pool_pre_ping=True, + connect_args={ + 'keepalives': 1, + 'keepalives_idle': 30, + 'keepalives_interval': 10, + 'keepalives_count': 5, + }, + ) db_contents = inspect(engine) - + db_tables = db_contents.get_table_names() # population raster set up population_stub = f'{region_dir}/{population_grid}_{codename}' - with engine.connect() as connection: - clipping_boundary = gpd.GeoDataFrame.from_postgis( - text(f"""SELECT geom FROM {buffered_urban_study_region}"""), - connection, - geom_col='geom', - ) - # construct virtual raster table vrt = f'{population["data_dir"]}/{population_grid}_{population["crs_srid"]}.vrt' population_raster_clipped = ( @@ -60,6 +62,12 @@ def main(): print(f' has already been indexed ({vrt}).') print('\nPopulation data clipped to region...', end='', flush=True) if not os.path.isfile(population_raster_clipped): + with engine.connect() as connection: + clipping_boundary = gpd.GeoDataFrame.from_postgis( + text(f"""SELECT geom FROM {buffered_urban_study_region}"""), + connection, + geom_col='geom', + ) # extract study region boundary in projection of tiles clipping = clipping_boundary.to_crs(population['crs_srid']) # get clipping boundary values in required order for gdal translate @@ -83,113 +91,123 @@ def main(): print(f' has now been created ({population_raster_projected}).') else: print(f' has already been created ({population_raster_projected}).') - print( - '\nPrepare population data grid for analysis (this may take a while)... ', - end='', - flush=True, - ) - # import raster to postgis and vectorise, as per http://www.brianmcgill.org/postgis_zonal.pdf - command = ( - f'raster2pgsql -d -s {crs["srid"]} -I -Y ' - f"-N {population['raster_nodata']} " - f'-t 1x1 {population_raster_projected} {population_grid} ' - f'| PGPASSWORD={db_pwd} psql -U postgres -h {db_host} -d {db} ' - '>> /dev/null' - ) - sp.call(command, shell=True) - print('Done.') - print( - 'Derive population grid variables and summaries... ', - end='', - flush=True, - ) - queries = [ - f""" - ALTER TABLE {population_grid} DROP COLUMN rid; - ALTER TABLE {population_grid} ADD grid_id bigserial; - ALTER TABLE {population_grid} ADD COLUMN IF NOT EXISTS pop_est int; - ALTER TABLE {population_grid} ADD COLUMN IF NOT EXISTS area_sqkm float; - ALTER TABLE {population_grid} ADD COLUMN IF NOT EXISTS pop_per_sqkm float; - ALTER TABLE {population_grid} ADD COLUMN IF NOT EXISTS intersection_count int; - ALTER TABLE {population_grid} ADD COLUMN IF NOT EXISTS intersections_per_sqkm float; - ALTER TABLE {population_grid} ADD COLUMN IF NOT EXISTS geom geometry; - """, - f"""DELETE FROM {population_grid} WHERE (ST_SummaryStats(rast)).sum IS NULL;""", - f"""UPDATE {population_grid} SET geom = ST_ConvexHull(rast);""", - f"""CREATE INDEX {population_grid}_ix ON {population_grid} (grid_id);""", - f"""CREATE INDEX {population_grid}_gix ON {population_grid} USING GIST(geom);""", - f""" - DELETE FROM {population_grid} - WHERE {population_grid}.grid_id NOT IN ( - SELECT p.grid_id - FROM - {population_grid} p, - {buffered_urban_study_region} b - WHERE ST_Intersects ( - p.geom, - b.geom - ) - ); - """, - f"""UPDATE {population_grid} SET area_sqkm = ST_Area(geom)/10^6;""", - f"""UPDATE {population_grid} SET pop_est = (ST_SummaryStats(rast)).sum;""", - f"""UPDATE {population_grid} SET pop_per_sqkm = pop_est/area_sqkm;""", - f"""ALTER TABLE {population_grid} DROP COLUMN rast;""", - f""" - CREATE MATERIALIZED VIEW pop_temp AS - SELECT h."grid_id", - COUNT(i.*) intersection_count - FROM {population_grid} h - LEFT JOIN {intersections_table} i - ON st_contains(h.geom,i.geom) - GROUP BY "grid_id"; - """, - f""" - UPDATE {population_grid} a - SET intersection_count = b.intersection_count, - intersections_per_sqkm = b.intersection_count/a.area_sqkm - FROM pop_temp b - WHERE a."grid_id" = b."grid_id"; - """, - """DROP MATERIALIZED VIEW pop_temp;""", - """ - ALTER TABLE urban_study_region ADD COLUMN IF NOT EXISTS area_sqkm double precision; - ALTER TABLE urban_study_region ADD COLUMN IF NOT EXISTS pop_est int; - ALTER TABLE urban_study_region ADD COLUMN IF NOT EXISTS pop_per_sqkm int; - ALTER TABLE urban_study_region ADD COLUMN IF NOT EXISTS intersection_count int; - ALTER TABLE urban_study_region ADD COLUMN IF NOT EXISTS intersections_per_sqkm double precision; - """, - f""" - UPDATE urban_study_region a - SET - area_sqkm = b.area_sqkm, - pop_est = b.pop_est, - pop_per_sqkm = b.pop_est/b.area_sqkm, - intersection_count = b.intersection_count, - intersections_per_sqkm = b.intersection_count/b.area_sqkm - FROM ( - SELECT - "study_region", - ST_Area(u.geom)/10^6 area_sqkm, - SUM(p.pop_est) pop_est, - SUM(p.intersection_count) intersection_count - FROM urban_study_region u, - {population_grid} p - WHERE ST_Intersects(u.geom,p.geom) - GROUP BY u."study_region",u.geom - ) b - WHERE a.study_region = b.study_region; - """, - ] - for sql in queries: - with engine.begin() as connection: - connection.execute(sql) + if population_grid not in db_tables: + print( + f'\nImport population grid {population_grid} to database... ', + end='', + flush=True, + ) + # import raster to postgis and vectorise, as per http://www.brianmcgill.org/postgis_zonal.pdf + command = ( + f'raster2pgsql -d -s {crs["srid"]} -I -Y ' + f"-N {population['raster_nodata']} " + f'-t 1x1 {population_raster_projected} {population_grid} ' + f'| PGPASSWORD={db_pwd} psql -U postgres -h {db_host} -d {db} ' + '>> /dev/null' + ) + sp.call(command, shell=True) + print('Done.') + else: + print(f'{population_grid} has been imported to database.') + if 'pop_est' not in [ + x['name'] for x in db_contents.get_columns('urban_study_region') + ]: + print( + 'Derive population grid variables and summaries... ', + end='', + flush=True, + ) + queries = [ + f""" + ALTER TABLE {population_grid} DROP COLUMN rid; + ALTER TABLE {population_grid} ADD grid_id bigserial; + ALTER TABLE {population_grid} ADD COLUMN IF NOT EXISTS pop_est int; + ALTER TABLE {population_grid} ADD COLUMN IF NOT EXISTS area_sqkm float; + ALTER TABLE {population_grid} ADD COLUMN IF NOT EXISTS pop_per_sqkm float; + ALTER TABLE {population_grid} ADD COLUMN IF NOT EXISTS intersection_count int; + ALTER TABLE {population_grid} ADD COLUMN IF NOT EXISTS intersections_per_sqkm float; + ALTER TABLE {population_grid} ADD COLUMN IF NOT EXISTS geom geometry; + """, + f"""DELETE FROM {population_grid} WHERE (ST_SummaryStats(rast)).sum IS NULL;""", + f"""UPDATE {population_grid} SET geom = ST_ConvexHull(rast);""", + f"""CREATE INDEX {population_grid}_ix ON {population_grid} (grid_id);""", + f"""CREATE INDEX {population_grid}_gix ON {population_grid} USING GIST(geom);""", + f""" + DELETE FROM {population_grid} + WHERE {population_grid}.grid_id NOT IN ( + SELECT p.grid_id + FROM + {population_grid} p, + {buffered_urban_study_region} b + WHERE ST_Intersects ( + p.geom, + b.geom + ) + ); + """, + f"""UPDATE {population_grid} SET area_sqkm = ST_Area(geom)/10^6;""", + f"""UPDATE {population_grid} SET pop_est = (ST_SummaryStats(rast)).sum;""", + f"""UPDATE {population_grid} SET pop_per_sqkm = pop_est/area_sqkm;""", + f"""ALTER TABLE {population_grid} DROP COLUMN rast;""", + f""" + CREATE MATERIALIZED VIEW pop_temp AS + SELECT h."grid_id", + COUNT(i.*) intersection_count + FROM {population_grid} h + LEFT JOIN {intersections_table} i + ON st_contains(h.geom,i.geom) + GROUP BY "grid_id"; + """, + f""" + UPDATE {population_grid} a + SET intersection_count = b.intersection_count, + intersections_per_sqkm = b.intersection_count/a.area_sqkm + FROM pop_temp b + WHERE a."grid_id" = b."grid_id"; + """, + """DROP MATERIALIZED VIEW pop_temp;""", + """ + ALTER TABLE urban_study_region ADD COLUMN IF NOT EXISTS area_sqkm double precision; + ALTER TABLE urban_study_region ADD COLUMN IF NOT EXISTS pop_est int; + ALTER TABLE urban_study_region ADD COLUMN IF NOT EXISTS pop_per_sqkm int; + ALTER TABLE urban_study_region ADD COLUMN IF NOT EXISTS intersection_count int; + ALTER TABLE urban_study_region ADD COLUMN IF NOT EXISTS intersections_per_sqkm double precision; + """, + f""" + UPDATE urban_study_region a + SET + area_sqkm = b.area_sqkm, + pop_est = b.pop_est, + pop_per_sqkm = b.pop_est/b.area_sqkm, + intersection_count = b.intersection_count, + intersections_per_sqkm = b.intersection_count/b.area_sqkm + FROM ( + SELECT + "study_region", + ST_Area(u.geom)/10^6 area_sqkm, + SUM(p.pop_est) pop_est, + SUM(p.intersection_count) intersection_count + FROM urban_study_region u, + {population_grid} p + WHERE ST_Intersects(u.geom,p.geom) + GROUP BY u."study_region",u.geom + ) b + WHERE a.study_region = b.study_region; + """, + ] + for sql in queries: + with engine.begin() as connection: + connection.execute(sql) - # grant access to the tables just created - with engine.begin() as connection: - connection.execute(grant_query) + # grant access to the tables just created + with engine.begin() as connection: + connection.execute(grant_query) - print('Done.') + print('Done.') + else: + print( + 'Population grid variables and summaries have previously been derived.', + ) # output to completion log script_running_log(script, task, start, codename) diff --git a/process/subprocesses/_06_open_space_areas_setup.py b/process/subprocesses/_06_open_space_areas_setup.py index a449ef64..556d9496 100644 --- a/process/subprocesses/_06_open_space_areas_setup.py +++ b/process/subprocesses/_06_open_space_areas_setup.py @@ -27,7 +27,6 @@ curs = conn.cursor() engine = create_engine(f'postgresql://{db_user}:{db_pwd}@{db_host}/{db}') -db_contents = inspect(engine) specific_inclusion = os_inclusion['criteria'] os_landuse = os_landuse['criteria'] @@ -70,15 +69,15 @@ aos_setup = [ f""" -- Create a 'Not Open Space' table -DROP TABLE IF EXISTS not_open_space; -CREATE TABLE not_open_space AS +-- DROP TABLE IF EXISTS not_open_space; +CREATE TABLE IF NOT EXISTS not_open_space AS SELECT ST_Union(geom) AS geom FROM {osm_prefix}_polygon p WHERE {exclusion_criteria}; """, f""" -- Create an 'Open Space' table -DROP TABLE IF EXISTS open_space; -CREATE TABLE open_space AS +-- DROP TABLE IF EXISTS open_space; +CREATE TABLE IF NOT EXISTS open_space AS SELECT p.* FROM {osm_prefix}_polygon p WHERE ({specific_inclusion} OR p.landuse IN ({os_landuse}) @@ -86,7 +85,7 @@ """, """ -- Create unique POS id and add indices -ALTER TABLE open_space ADD COLUMN os_id SERIAL PRIMARY KEY; +ALTER TABLE open_space ADD COLUMN IF NOT EXISTS os_id SERIAL PRIMARY KEY; CREATE INDEX open_space_idx ON open_space USING GIST (geom); CREATE INDEX not_open_space_idx ON not_open_space USING GIST (geom); """, @@ -101,12 +100,12 @@ """, """ -- Create variable for park size -ALTER TABLE open_space ADD COLUMN area_ha double precision; +ALTER TABLE open_space ADD COLUMN IF NOT EXISTS area_ha double precision; UPDATE open_space SET area_ha = ST_Area(geom)/10000.0; """, f""" -- Create variable for associated line tags -ALTER TABLE open_space ADD COLUMN tags_line jsonb; +ALTER TABLE open_space ADD COLUMN IF NOT EXISTS tags_line jsonb; WITH tags AS ( SELECT o.os_id, jsonb_strip_nulls(to_jsonb((SELECT d FROM (SELECT l.amenity,l.leisure,l."natural",l.tourism,l.waterway) d)))AS attributes @@ -125,7 +124,7 @@ """, f""" -- Create variable for associated point tags -ALTER TABLE open_space ADD COLUMN tags_point jsonb; +ALTER TABLE open_space ADD COLUMN IF NOT EXISTS tags_point jsonb; WITH tags AS ( SELECT o.os_id, jsonb_strip_nulls(to_jsonb((SELECT d FROM (SELECT l.amenity,l.leisure,l."natural",l.tourism,l.historic) d)))AS attributes @@ -144,7 +143,7 @@ """, f""" -- Create water feature indicator -ALTER TABLE open_space ADD COLUMN water_feature boolean; +ALTER TABLE open_space ADD COLUMN IF NOT EXISTS water_feature boolean; UPDATE open_space SET water_feature = FALSE; UPDATE open_space SET water_feature = TRUE WHERE "natural" IN ({water_features}) @@ -158,7 +157,7 @@ OR wetland IS NOT NULL; """, f""" -ALTER TABLE open_space ADD COLUMN linear_features boolean; +ALTER TABLE open_space ADD COLUMN IF NOT EXISTS linear_features boolean; UPDATE open_space SET linear_features = TRUE WHERE waterway IN ({linear_features}) OR "natural" IN ({linear_features}) @@ -167,31 +166,31 @@ """, """ -- Create variable for AOS water geometry -ALTER TABLE open_space ADD COLUMN water_geom geometry; +ALTER TABLE open_space ADD COLUMN IF NOT EXISTS water_geom geometry; UPDATE open_space SET water_geom = geom WHERE water_feature = TRUE; """, """ -ALTER TABLE open_space ADD COLUMN min_bounding_circle_area double precision; +ALTER TABLE open_space ADD COLUMN IF NOT EXISTS min_bounding_circle_area double precision; UPDATE open_space SET min_bounding_circle_area = ST_Area(ST_MinimumBoundingCircle(geom)); """, """ -ALTER TABLE open_space ADD COLUMN min_bounding_circle_diameter double precision; +ALTER TABLE open_space ADD COLUMN IF NOT EXISTS min_bounding_circle_diameter double precision; UPDATE open_space SET min_bounding_circle_diameter = 2*sqrt(min_bounding_circle_area / pi()); """, """ -ALTER TABLE open_space ADD COLUMN roundness double precision; +ALTER TABLE open_space ADD COLUMN IF NOT EXISTS roundness double precision; UPDATE open_space SET roundness = ST_Area(geom)/(ST_Area(ST_MinimumBoundingCircle(geom))); """, f""" -- Create indicator for linear features informed through EDA of OS topology -ALTER TABLE open_space ADD COLUMN linear_feature boolean; +ALTER TABLE open_space ADD COLUMN IF NOT EXISTS linear_feature boolean; UPDATE open_space SET linear_feature = FALSE; UPDATE open_space SET linear_feature = TRUE WHERE {linear_feature_criteria}; """, """ ---- Create 'Acceptable Linear Feature' indicator -ALTER TABLE open_space ADD COLUMN acceptable_linear_feature boolean; +ALTER TABLE open_space ADD COLUMN IF NOT EXISTS acceptable_linear_feature boolean; UPDATE open_space SET acceptable_linear_feature = FALSE WHERE linear_feature = TRUE; UPDATE open_space o SET acceptable_linear_feature = TRUE FROM (SELECT os_id,geom FROM open_space WHERE linear_feature = FALSE) nlf @@ -243,7 +242,7 @@ """, """ -- Check if area is within an indicated public access area - ALTER TABLE open_space ADD COLUMN within_public boolean; + ALTER TABLE open_space ADD COLUMN IF NOT EXISTS within_public boolean; UPDATE open_space SET within_public = FALSE; UPDATE open_space o SET within_public = TRUE @@ -287,8 +286,8 @@ -- the 'geom' attributes is the area within an AOS -- -- this is what we want to use to evaluate collective OS area within the AOS (aos_ha) -DROP TABLE IF EXISTS open_space_areas; -CREATE TABLE open_space_areas AS +-- DROP TABLE IF EXISTS open_space_areas; +CREATE TABLE IF NOT EXISTS open_space_areas AS WITH clusters AS( SELECT unnest(ST_ClusterWithin(open_space.geom, .001)) AS gc FROM open_space @@ -346,10 +345,10 @@ """, """ -- Create variable for AOS size -ALTER TABLE open_space_areas ADD COLUMN aos_ha_public double precision; -ALTER TABLE open_space_areas ADD COLUMN aos_ha_not_public double precision; -ALTER TABLE open_space_areas ADD COLUMN aos_ha double precision; -ALTER TABLE open_space_areas ADD COLUMN aos_ha_water double precision; +ALTER TABLE open_space_areas ADD COLUMN IF NOT EXISTS aos_ha_public double precision; +ALTER TABLE open_space_areas ADD COLUMN IF NOT EXISTS aos_ha_not_public double precision; +ALTER TABLE open_space_areas ADD COLUMN IF NOT EXISTS aos_ha double precision; +ALTER TABLE open_space_areas ADD COLUMN IF NOT EXISTS aos_ha_water double precision; """, """ -- Calculate total area of AOS in Ha @@ -361,7 +360,7 @@ """ -- Set water_feature as true where OS feature intersects a noted water feature -- wet by association -ALTER TABLE open_space_areas ADD COLUMN has_water_feature boolean; +ALTER TABLE open_space_areas ADD COLUMN IF NOT EXISTS has_water_feature boolean; UPDATE open_space_areas SET has_water_feature = FALSE; UPDATE open_space_areas o SET has_water_feature = TRUE FROM (SELECT * from open_space WHERE water_feature = TRUE) w @@ -369,14 +368,14 @@ """, """ -- Create variable for Water percent -ALTER TABLE open_space_areas ADD COLUMN water_percent numeric; +ALTER TABLE open_space_areas ADD COLUMN IF NOT EXISTS water_percent numeric; UPDATE open_space_areas SET water_percent = 0; UPDATE open_space_areas SET water_percent = 100 * aos_ha_water/aos_ha::numeric WHERE aos_ha > 0; """, f""" -- Create a linestring aos table -DROP TABLE IF EXISTS aos_line; -CREATE TABLE aos_line AS +-- DROP TABLE IF EXISTS aos_line; +CREATE TABLE IF NOT EXISTS aos_line AS WITH bounds AS (SELECT aos_id, ST_SetSRID(st_astext((ST_Dump(geom)).geom),{crs['srid']}) AS geom FROM open_space_areas) SELECT aos_id, ST_Length(geom)::numeric AS length, geom @@ -384,8 +383,8 @@ """, """ -- Generate a point every 20m along a park outlines: -DROP TABLE IF EXISTS aos_nodes; -CREATE TABLE aos_nodes AS +-- DROP TABLE IF EXISTS aos_nodes; +CREATE TABLE IF NOT EXISTS aos_nodes AS WITH aos AS (SELECT aos_id, length, @@ -397,13 +396,13 @@ FROM aos; CREATE INDEX aos_nodes_idx ON aos_nodes USING GIST (geom); -ALTER TABLE aos_nodes ADD COLUMN aos_entryid varchar; +ALTER TABLE aos_nodes ADD COLUMN IF NOT EXISTS aos_entryid varchar; UPDATE aos_nodes SET aos_entryid = aos_id::text || ',' || node::text; """, """ -- Create subset data for public_open_space_areas -DROP TABLE IF EXISTS aos_public_osm; -CREATE TABLE aos_public_osm AS +-- DROP TABLE IF EXISTS aos_public_osm; +CREATE TABLE IF NOT EXISTS aos_public_osm AS -- restrict to features > 10 sqm (e.g. 5m x 2m; this is very small, but plausible - and should be excluded) SELECT * FROM open_space_areas WHERE aos_ha_public > 0.001; CREATE INDEX aos_public_osm_idx ON aos_nodes (aos_id); @@ -412,8 +411,8 @@ """ -- Create table of points within 30m of lines (should be your road network) -- Distinct is used to avoid redundant duplication of points where they are within 20m of multiple roads -DROP TABLE IF EXISTS aos_public_any_nodes_30m_line; -CREATE TABLE aos_public_any_nodes_30m_line AS +-- DROP TABLE IF EXISTS aos_public_any_nodes_30m_line; +CREATE TABLE IF NOT EXISTS aos_public_any_nodes_30m_line AS SELECT DISTINCT n.* FROM aos_nodes n LEFT JOIN aos_public_osm a ON n.aos_id = a.aos_id, edges l @@ -424,8 +423,8 @@ """ -- Create table of points within 30m of lines (should be your road network) -- Distinct is used to avoid redundant duplication of points where they are within 20m of multiple roads -DROP TABLE IF EXISTS aos_public_large_nodes_30m_line; -CREATE TABLE aos_public_large_nodes_30m_line AS +-- DROP TABLE IF EXISTS aos_public_large_nodes_30m_line; +CREATE TABLE IF NOT EXISTS aos_public_large_nodes_30m_line AS SELECT DISTINCT n.* FROM aos_nodes n LEFT JOIN aos_public_osm a ON n.aos_id = a.aos_id, edges l @@ -438,12 +437,18 @@ def main(): - for sql in aos_setup: - query_start = time.time() - print(f'\nExecuting: {sql}') - curs.execute(sql) - conn.commit() - print(f'Executed in {(time.time() - query_start) / 60:04.2f} mins') + db_contents = inspect(engine) + if db_contents.has_table('aos_public_large_nodes_30m_line'): + print( + 'Areas of Open Space (AOS) for urban liveability indicators has previously been prepared for this region.\n', + ) + else: + for sql in aos_setup: + query_start = time.time() + print(f'\nExecuting: {sql}') + curs.execute(sql) + conn.commit() + print(f'Executed in {(time.time() - query_start) / 60:04.2f} mins') curs.execute(grant_query) conn.commit() diff --git a/process/subprocesses/_09_urban_covariates.py b/process/subprocesses/_09_urban_covariates.py index 47190f47..8a132c5b 100644 --- a/process/subprocesses/_09_urban_covariates.py +++ b/process/subprocesses/_09_urban_covariates.py @@ -68,7 +68,7 @@ def main(): CREATE TABLE urban_covariates AS SELECT '{continent}'::text "Continent", '{country}'::text "Country", - '{region}'::text "ISO 3166-1 alpha-2", + '{country_code}'::text "ISO 3166-1 alpha-2", u.study_region "City", u.area_sqkm "Area (sqkm)", u.pop_est "Population estimate", diff --git a/process/subprocesses/_13_aggregation.py b/process/subprocesses/_13_aggregation.py index 6551503d..d498c38a 100644 --- a/process/subprocesses/_13_aggregation.py +++ b/process/subprocesses/_13_aggregation.py @@ -31,7 +31,7 @@ def main(): # calc_grid_pct_sp_indicators take sample point stats within each city as # input and aggregate up to grid cell indicators by calculating the mean of # sample points stats within each hex - calc_grid_pct_sp_indicators(regions[codename], indicators) + calc_grid_pct_sp_indicators(region_config, indicators) print('Done.') print('Calculating city summary indicators... '), @@ -43,7 +43,7 @@ def main(): # in addition to the population weighted averages, unweighted averages are # also included to reflect the spatial distribution of key walkability # measures (regardless of population distribution) - calc_cities_pop_pct_indicators(regions[codename], indicators) + calc_cities_pop_pct_indicators(region_config, indicators) print('Done.') print( f'\nAggregation completed: {(time.time() - startTime)/60.0:.02f} mins', diff --git a/process/subprocesses/_project_setup.py b/process/subprocesses/_project_setup.py index b5e3ebb3..bad9bdce 100644 --- a/process/subprocesses/_project_setup.py +++ b/process/subprocesses/_project_setup.py @@ -80,43 +80,43 @@ def load_yaml( # Set up region data def region_data_setup( - region, regions, data, data_path=None, + region, region_config, data, data_path=None, ): """Check data configuration for regions and make paths absolute.""" try: if data not in datasets or datasets[data] is None: raise SystemExit( - f'An entry for at least one {data} dataset does not appear to have been defined in datasets.yml. This parameter is required for analysis, and is used to cross-reference a relevant dataset defined in datasets.yml with regions defined in regions.yml. Please update datasets.yml to proceed.', + f'An entry for at least one {data} dataset does not appear to have been defined in datasets.yml. This parameter is required for analysis, and is used to cross-reference a relevant dataset defined in datasets.yml with region configuration in {region}.yml. Please update datasets.yml to proceed.', ) - elif regions[region][data] is None: + elif region_config[data] is None: raise SystemExit( - f'The entry for {data} does not appear to have been defined in regions.yml {region}. This parameter is required for analysis, and is used to cross-reference a relevant dataset defined in datasets.yml. Please update regions.yml to proceed.', + f'The entry for {data} does not appear to have been defined in {region}.yml. This parameter is required for analysis, and is used to cross-reference a relevant dataset defined in datasets.yml. Please update {region}.yml to proceed.', ) else: - if 'citation' not in datasets[data][regions[region][data]]: + if 'citation' not in datasets[data][region_config[data]]: if data != 'OpenStreetMap': raise SystemExit( f'No citation record has been configured for the {data} dataset configured for this region. Please add this to its record in datasets.yml (see template datasets.yml for examples).', ) - elif 'source' not in regions[region]['OpenStreetMap']: - datasets[data][regions[region][data]][ + elif 'source' not in region_config['OpenStreetMap']: + datasets[data][region_config[data]][ 'citation' - ] = f'OpenStreetMap Contributors ({str(datasets[data][regions[region][data]]["publication_date"])[:4]}). {datasets[data][regions[region][data]]["url"]}' + ] = f'OpenStreetMap Contributors ({str(datasets[data][region_config[data]]["publication_date"])[:4]}). {datasets[data][region_config[data]]["url"]}' else: - datasets[data][regions[region][data]][ + datasets[data][region_config[data]][ 'citation' - ] = f'OpenStreetMap Contributors. {datasets[data][regions[region][data]]["source"]} ({str(datasets[data][regions[region][data]]["publication_date"])[:4]}). {datasets[data][regions[region][data]]["url"]}' - data_dictionary = datasets[data][regions[region][data]].copy() + ] = f'OpenStreetMap Contributors. {datasets[data][region_config[data]]["source"]} ({str(datasets[data][region_config[data]]["publication_date"])[:4]}). {datasets[data][region_config[data]]["url"]}' + data_dictionary = datasets[data][region_config[data]].copy() if ('data_dir' not in data_dictionary) or ( data_dictionary['data_dir'] is None ): raise SystemExit( - f"The 'data_dir' entry for {data} does not appear to have been defined in datasets.yml. This parameter is required for analysis of {region}, and is used to locate a required dataset cross-referenced in regions.yml. Please update datasets.yml to proceed.", + f"The 'data_dir' entry for {data} does not appear to have been defined in datasets.yml. This parameter is required for analysis of {region}, and is used to locate a required dataset cross-referenced in {region}.yml. Please update datasets.yml to proceed.", ) if data_path is not None: data_dictionary[ 'data_dir' - ] = f"{data_path}/{datasets[data][regions[region][data]]['data_dir']}" + ] = f"{data_path}/{datasets[data][region_config[data]]['data_dir']}" return data_dictionary except Exception: raise e @@ -135,10 +135,10 @@ def verify_data_dir(data_dir, verify_file_extension=None): # If False: f"A file having extension '{verify_file_extension}' could not be located within {data_dir}. Please check folder contents and configuration of datasets.yml." -def region_dictionary_setup(region, regions, config, folder_path): - r = regions[region].copy() +def region_dictionary_setup(region, region_config, config, folder_path): + r = region_config.copy() date = time.strftime('%Y-%m-%d') - r['study_region'] = f'{region}_{r["country_code"]}_{r["year"]}'.lower() + r['study_region'] = region study_buffer = config['project']['study_buffer'] units = config['project']['units'] buffered_urban_study_region = f'urban_study_region_{study_buffer}{units}' @@ -155,13 +155,13 @@ def region_dictionary_setup(region, regions, config, folder_path): 'data' ] = f"{data_path}/{r['study_region_boundary']['data']}" r['urban_region'] = region_data_setup( - region, regions, 'urban_region', data_path, + region, region_config, 'urban_region', data_path, ) r['buffered_urban_study_region'] = buffered_urban_study_region r['db'] = f'li_{region}_{r["year"]}'.lower() r['dbComment'] = f'Liveability indicator data for {region} {r["year"]}.' r['population'] = region_data_setup( - region, regions, 'population', data_path, + region, region_config, 'population', data_path, ) resolution = r['population']['resolution'].replace(' ', '') r['population'][ @@ -171,7 +171,7 @@ def region_dictionary_setup(region, regions, config, folder_path): 'population_grid' ] = f'population_{resolution}_{r["population"]["year_target"]}' r['OpenStreetMap'] = region_data_setup( - region, regions, 'OpenStreetMap', data_path, + region, region_config, 'OpenStreetMap', data_path, ) r['OpenStreetMap'][ 'osm_region' @@ -216,12 +216,16 @@ def region_dictionary_setup(region, regions, config, folder_path): raise Exception( "\n\nProject configuration file couldn't be located at process/configuration/config.yml. Please ensure project has been initialised and configured before commencing analysis.\n\n", ) -load_yaml(f'{config_path}/regions.yml') + load_yaml(f'{config_path}/datasets.yml', unnest=True) load_yaml(f'{config_path}/osm_open_space.yml', unnest=True) load_yaml(f'{config_path}/indicators.yml') load_yaml(f'{config_path}/policies.yml') -region_names = list(regions.keys()) +region_names = [ + x.split('.yml')[0] + for x in os.listdir(f'{config_path}/regions') + if x.endswith('.yml') +] # Set up date and time os.environ['TZ'] = analysis_timezone @@ -248,12 +252,13 @@ def region_dictionary_setup(region, regions, config, folder_path): elif any(['2_analyse_region.py' in f.filename for f in inspect.stack()[1:]]): sys.exit( f'\n{authors}, version {version}\n\n' - 'This script requires a study region code name corresponding to definitions ' - 'in configuration/regions.yml be provided as an argument (lower case, with ' - 'spaces instead of underscores). For example, for the demonstration city of Las Palmas de Gran Canaria for which data has been provided:\n\n' - 'python 1_create_project_configuration_files.py\n' - 'python 2_analyse_region.py example_las_palmas_2023\n' - 'python 3_generate_reports.py example_las_palmas_2023\n\n' + 'This script requires a study region code name corresponding to .yml files ' + 'in configuration/regions be provided as an argument. ' + 'For example, for Las Palmas de Gran Canaria, Spain (the provided example):\n\n' + 'python 1_create_project_configuration_files\n' + 'python 1_create_project_configuration_files example_ES_Las_Palmas_2023\n' + 'python 2_analyse_region.py example_ES_Las_Palmas_2023\n' + 'python 3_generate_resources.py example_ES_Las_Palmas_2023\n\n' f'The code names for currently configured regions are {region_names}\n', ) elif default_codename in region_names: @@ -262,25 +267,31 @@ def region_dictionary_setup(region, regions, config, folder_path): else: sys.exit( f'\n{authors}, version {version}\n\n' - 'This script requires a study region code name corresponding to definitions ' - 'in configuration/regions.yml be provided as an argument (lower case, with ' - 'spaces instead of underscores). For example, for Hong Kong:\n\n' - 'python 01_study_region_setup.py hong_kong\n' - 'python 02_neighbourhood_analysis.py hong_kong\n' - 'python 03_aggregation.py hong_kong\n\n' + 'This script requires a study region code name corresponding to .yml files ' + 'in configuration/regions be provided as an argument. ' + 'For example, for Las Palmas de Gran Canaria, Spain (the provided example):\n\n' + 'python 1_create_project_configuration_files\n' + 'python 2_analyse_region.py ES_Las_Palmas_2023\n' + 'python 3_generate_resources.py ES_Las_Palmas_2023\n\n' f'The code names for currently configured regions are {region_names}\n', ) # Data set up for region -for region in regions: - regions[region] = region_dictionary_setup( - region, regions, config, folder_path, +try: + load_yaml(f'{config_path}/regions/{codename}.yml', name='region_config') +except Exception as e: + sys.exit( + f'\n\nError: {e}\n\nLoading of study region configuration file for the specified city codename failed. Please confirm that configuration has been completed for this city (e.g. editing the file configuration/regions/{codename}.yml in a text editor), consulting the provided example configuration files as required.\n\nFurther assistance may be requested by logging an issue at:\nhttps://github.com/global-healthy-liveable-cities/global-indicators/issues\n\n', ) +region_config = region_dictionary_setup( + codename, region_config, config, folder_path, +) + # Add region variables for this study region to global variables -for var in regions[codename].keys(): - globals()[var] = regions[codename][var] +for var in region_config.keys(): + globals()[var] = region_config[var] # Check configured data exists for this specified region assert verify_data_dir(urban_region['data_dir'], verify_file_extension=None) diff --git a/process/subprocesses/_report_functions.py b/process/subprocesses/_report_functions.py index 82d677c2..eb65be29 100644 --- a/process/subprocesses/_report_functions.py +++ b/process/subprocesses/_report_functions.py @@ -1133,7 +1133,7 @@ def pdf_for_web( for x in range(1, 7): # check presence template[f'policy_urban_text{x}_response'] = policy_indicators[ - np.ceil(city_policy['Presence'][x - 1]) + city_policy['Presence'][x - 1] ] # format percentage units according to locale for gdp in ['middle', 'upper']: