Merge pull request #341 from global-healthy-liveable-cities/enhancements

Attempting a pull request to evaluate GH Actions workflow changes for more unittests

.test-compose.yml

@@ -16,6 +16,11 @@ services:
     user: "${UID}:${GID}"
     extra_hosts:
       - "gateway.docker.internal:host-gateway"
+    depends_on:
+      pgrouting:
+        condition: service_healthy
+    links:
+      - pgrouting
 
   pgrouting:
     image: pgrouting/pgrouting
@@ -26,9 +31,15 @@ services:
       - "5433:5433"
     volumes:
       - db-data:/var/lib/postgis:/postgresql/13/main
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready", "-q", "-d", "postgres", "-U", "postgres" ]
+      interval: 5s
+      timeout: 5s
+      retries: 5
     environment:
       POSTGRES_PASSWORD: ghscic
       POSTGRES_HOST: ghscic_postgis
+      PGPORT: 5433
 
 volumes:
   db-data:

process/subprocesses/_02_create_osm_resources.py

@@ -21,7 +21,7 @@ def create_poly_boundary_file(config):
     db_pwd = config['db_pwd']
     feature = f"""PG:"dbname={db} host={db_host} port={db_port} user={db_user} password={db_pwd}" {config['buffered_urban_study_region']}"""
     print('Create poly file, using command: '),
-    command = f'python ogr2poly.py {feature} -f "db"'
+    command = f'python /home/ghsci/process/subprocesses/ogr2poly.py {feature} -f "db"'
     print(command)
     sp.call(command, shell=True)
     command = f'mv {os.path.basename(config["codename_poly"])} {config["codename_poly"]}'

process/subprocesses/_11_neighbourhood_analysis.py

@@ -291,7 +291,6 @@ def calculate_sample_point_indicators(
                     / sample_points[columns].std()
                 ).sum(axis=1)
     # grid_id and edge_ogc_fid are integers
-    print(sample_points.query('grid_id.isna()'))
     sample_points[sample_points.columns[0:2]] = sample_points[
         sample_points.columns[0:2]
     ].astype(int)

process/subprocesses/ghsci.py

@@ -137,6 +137,94 @@ def drop(self):
 
         drop_resources(self)
 
+    def _create_database(self):
+        """Create database for this study region."""
+        from _00_create_database import create_database
+
+        create_database(self.codename)
+        return f"Database {self.config['db']} created."
+
+    def _create_study_region(self):
+        """Create study region boundaries for this study region."""
+        from _01_create_study_region import create_study_region
+
+        create_study_region(self.codename)
+        return 'Study region boundaries created.'
+
+    def _create_osm_resources(self):
+        """Create OSM resources for this study region."""
+        from _02_create_osm_resources import create_osm_resources
+
+        create_osm_resources(self.codename)
+        return 'OSM resources created.'
+
+    def _create_network_resources(self):
+        """Create network resources for this study region."""
+        from _03_create_network_resources import create_network_resources
+
+        create_network_resources(self.codename)
+        return 'Network resources created.'
+
+    def _create_population_grid(self):
+        """Create population grid for this study region."""
+        from _04_create_population_grid import create_population_grid
+
+        create_population_grid(self.codename)
+        return 'Population grid created.'
+
+    def _create_destinations(self):
+        """Compile destinations for this study region."""
+        from _05_compile_destinations import compile_destinations
+
+        compile_destinations(self.codename)
+        return 'Destinations compiled.'
+
+    def _create_open_space_areas(self):
+        """Create open space areas for this study region."""
+        from _06_open_space_areas_setup import open_space_areas_setup
+
+        open_space_areas_setup(self.codename)
+        return 'Open space areas created.'
+
+    def _create_neighbourhoods(self):
+        """Create neighbourhood relations between nodes for this study region."""
+        from _07_locate_origins_destinations import nearest_node_locations
+
+        nearest_node_locations(self.codename)
+        return 'Neighbourhoods created.'
+
+    def _create_destination_summary_tables(self):
+        """Create destination summary tables for this study region."""
+        from _08_destination_summary import destination_summary
+
+        destination_summary(self.codename)
+        return 'Destination summary tables created.'
+
+    def _link_urban_covariates(self):
+        """Link urban covariates to nodes for this study region."""
+        from _09_urban_covariates import link_urban_covariates
+
+        link_urban_covariates(self.codename)
+        return 'Urban covariates linked.'
+
+    def _gtfs_analysis(self):
+        from _10_gtfs_analysis import gtfs_analysis
+
+        gtfs_analysis(self.codename)
+        return 'GTFS analysis completed.'
+
+    def _neighbourhood_analysis(self):
+        from _11_neighbourhood_analysis import neighbourhood_analysis
+
+        neighbourhood_analysis(self.codename)
+        return 'Neighbourhood analysis completed.'
+
+    def _area_analysis(self):
+        from _12_aggregation import aggregate_study_region_indicators
+
+        aggregate_study_region_indicators(self.codename)
+        return 'Area analysis completed.'
+
     def get_engine(self):
         """Given configuration details, create a database engine."""
         engine = create_engine(

process/tests/tests.py

@@ -21,6 +21,7 @@
 Successful running of all tests may require running of tests within the global-indicators Docker container, hence the use of a custom .test-compose.yml for this purpose.
 """
 
+import os
 import sys
 import unittest
 
@@ -35,50 +36,75 @@
 class tests(unittest.TestCase):
     """A collection of tests to help ensure functionality."""
 
-    def test_global_indicators_shell(self):
+    def test_1_global_indicators_shell(self):
         """Unix shell script should only have unix-style line endings."""
         counts = calculate_line_endings('../global-indicators.sh')
         lf = counts.pop(b'\n')
         self.assertTrue(sum(counts.values()) == 0 and lf > 0)
 
-    def test_project_setup(self):
+    def test_2_project_setup(self):
         """Check if _project_setup.py imported successfully."""
         self.assertTrue(project_setup)
 
-    def test_load_example_region(self):
+    def test_3_load_example_region(self):
         """Load example region."""
         codename = 'example_ES_Las_Palmas_2023'
         r = ghsci.Region(codename)
 
-    # def test_example_analysis(self):
-    #     """Analyse example region."""
-    #     codename = 'example_ES_Las_Palmas_2023'
-    #     r = ghsci.Region(codename)
-    #     r.analysis()
-
-    # def test_example_generate(self):
-    #     """Generate resources for example region."""
-    #     codename = 'example_ES_Las_Palmas_2023'
-    #     r = ghsci.Region(codename)
-    #     r.generate()
-
-    # def test_sensitivity(self):
-    #     """Test sensitivity analysis of urban intersection parameter."""
-    #     reference = 'example_ES_Las_Palmas_2023'
-    #     comparison = 'ES_Las_Palmas_2023_test_not_urbanx'
-    #     # create modified version of reference configuration
-    #     with open(f'./configuration/regions/{reference}.yml') as file:
-    #         configuration = file.read()
-    #         configuration = configuration.replace(
-    #             'ghsl_urban_intersection: true',
-    #             'ghsl_urban_intersection: false',
-    #         )
-    #     with open(f'./configuration/regions/{comparison}.yml', 'w') as file:
-    #         file.write(configuration)
-    #     r = ghsci.Region(comparison)
-    #     r.analysis()
-    #     r.generate()
-    #     r.compare(reference)
+    def test_4_create_db(self):
+        """Load example region."""
+        codename = 'example_ES_Las_Palmas_2023'
+        r = ghsci.Region(codename)
+        r._create_database()
+
+    def test_5_example_analysis(self):
+        """Analyse example region."""
+        codename = 'example_ES_Las_Palmas_2023'
+        r = ghsci.Region(codename)
+        r.analysis()
+
+    def test_6_example_generate(self):
+        """Generate resources for example region."""
+        codename = 'example_ES_Las_Palmas_2023'
+        r = ghsci.Region(codename)
+        r.generate()
+
+    def test_7_sensitivity(self):
+        """Test sensitivity analysis of urban intersection parameter."""
+        reference = 'example_ES_Las_Palmas_2023'
+        comparison = 'ES_Las_Palmas_2023_test_not_urbanx'
+        # create modified version of reference configuration
+        with open(f'./configuration/regions/{reference}.yml') as file:
+            configuration = file.read()
+            configuration = configuration.replace(
+                'ghsl_urban_intersection: true',
+                'ghsl_urban_intersection: false',
+            )
+        with open(f'./configuration/regions/{comparison}.yml', 'w') as file:
+            file.write(configuration)
+        r_comparison = ghsci.Region(comparison)
+        # create output folder for comparison region
+        if not os.path.exists(
+            f'{ghsci.folder_path}/process/data/_study_region_outputs',
+        ):
+            os.makedirs(
+                f'{ghsci.folder_path}/process/data/_study_region_outputs',
+            )
+        if not os.path.exists(r_comparison.config['region_dir']):
+            os.makedirs(r_comparison.config['region_dir'])
+        with open(f'./configuration/regions/{comparison}.yml', 'w') as file:
+            file.write(configuration)
+        r = ghsci.Region(reference)
+        df = r.get_df('indicators_region')
+        df.drop(columns=['geom'], inplace=True)
+        df[df.columns[(df.dtypes == 'float64').values]] = df[
+            df.columns[(df.dtypes == 'float64').values]
+        ].astype(int)
+        df.to_csv(
+            f"{r_comparison.config['region_dir']}/{r_comparison.codename}_indicators_region.csv",
+            index=False,
+        )
+        r.compare(comparison)
 
 
 def calculate_line_endings(path):
@@ -105,20 +131,5 @@ def calculate_line_endings(path):
     return counts
 
 
-def suite():
-    suite = unittest.TestSuite()
-    for t in [
-        'test_global_indicators_shell',
-        'test_project_setup',
-        'test_load_example_region',
-        # 'test_example_analysis',
-        # 'test_example_generate',
-        # 'test_sensitivity',
-    ]:
-        suite.addTest(tests(t))
-    return suite
-
-
 if __name__ == '__main__':
-    runner = unittest.TextTestRunner()
-    runner.run(suite())
+    unittest.main(failfast=True)