Merge pull request #71 from azavea/feature/reorganize

Merge pfb-analysis into this repo (and reorganize it)

README.md

@@ -8,7 +8,7 @@ Requirements:
 - Vagrant 1.8+
 - VirtualBox 4.3+
 
-Run `./script/setup` to install project dependencies and prepare the development environment. Then, SSH into the VM:
+Run `./scripts/setup` to install project dependencies and prepare the development environment. Then, SSH into the VM:
 ```
 vagrant ssh
 ```
@@ -58,7 +58,9 @@ In this example, we configure the analysis to be run for Cambridge MA.
 
 Run:
 ```
+pushd pfb-analysis
 docker build -t pfb .
+popd
 
 docker run \
     -e PFB_SHPFILE=/data/neighborhood_boundary.shp \

pfb-analysis/.gitignore

@@ -0,0 +1,2 @@
+*.pyc
+*.log

pfb-analysis/Dockerfile

@@ -0,0 +1,53 @@
+FROM quay.io/azavea/postgis:postgres9.6-postgis2.3
+MAINTAINER Azavea
+
+ENV GIT_BRANCH_OSM2PGROUTING osm2pgrouting-2.1.0
+ENV GIT_BRANCH_OSM2PGSQL 0.90.1
+ENV GIT_BRANCH_QUANTILE master
+ENV GIT_BRANCH_TDGTOOLS nodes
+
+RUN set -xe && \
+    BUILD_DEPS=" \
+        postgresql-server-dev-$PG_MAJOR \
+        libexpat1-dev \
+        cmake \
+        libboost-all-dev make \
+        g++ \
+        zlib1g-dev \
+        libbz2-dev \
+        libpq-dev \
+        libgeos-dev \
+        libgeos++-dev \
+        libproj-dev \
+        git" \
+    DEPS=" \
+        ca-certificates \
+        liblua5.2-dev \
+        lua5.2 \
+        expat \
+        wget \
+        bc \
+        time \
+        parallel \
+        postgresql-plpython-$PG_MAJOR \
+        postgresql-$PG_MAJOR-pgrouting \
+        python-gdal \
+        unzip \
+        postgis" && \
+    apt-get update && apt-get install -y ${BUILD_DEPS} ${DEPS} --no-install-recommends && \
+    mkdir /tmp/build/ && cd /tmp/build && \
+      git clone --branch $GIT_BRANCH_OSM2PGROUTING https://github.com/pgRouting/osm2pgrouting.git && \
+        (cd osm2pgrouting && mkdir build && cmake -H. -Bbuild && cd build && make install) && \
+      git clone --branch $GIT_BRANCH_OSM2PGSQL https://github.com/openstreetmap/osm2pgsql.git && \
+        (cd osm2pgsql && mkdir build && cd build && cmake ../ && make install) && \
+      git clone --branch $GIT_BRANCH_QUANTILE https://github.com/tvondra/quantile.git && \
+        (cd quantile && make install) && \
+      git clone --branch $GIT_BRANCH_TDGTOOLS https://github.com/spencerrecneps/TDG-Tools.git && \
+        (cd TDG-Tools/TDG\ SQL\ Tools && make clean && make install) && \
+    cd /tmp/ && rm -rf /tmp/build/ /var/lib/apt/lists/* && \
+    apt-get purge -y --auto-remove ${BUILD_DEPS}
+
+COPY scripts/setup_database.sh /docker-entrypoint-initdb.d/setup_database.sh
+COPY ./ /pfb/
+
+ENTRYPOINT /pfb/scripts/entrypoint.sh

pfb-analysis/LICENSE

@@ -0,0 +1,13 @@
+Copyright 2017 Azavea, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

pfb-analysis/README.md

@@ -0,0 +1,32 @@
+# pfb
+
+## Docker
+
+To run the analysis in docker, first build the docker image:
+
+```bash
+docker build -t pfb .
+```
+
+Then run the analysis as follows:
+
+```bash
+docker run \
+    -e PFB_SHPFILE=/data/neighborhood_boundary.shp \
+    -e PFB_STATE=ma \
+    -e PFB_STATE_FIPS=25 \
+    -e NB_INPUT_SRID=2249 \
+    -e NB_BOUNDARY_BUFFER=11000 \
+    -v /vagrant/data/:/data/ \
+    pfb
+```
+
+The `-e` in this example sets environment variables, which will depend on the
+analysis you are running.
+
+The `-v` in this example mounts a local directory inside the docker container
+under `/data/`. The `PFB_SHPFILE` environment variable specifies the `.shp`
+file under this directory to use.
+
+This process of mounting a data directory with the input shapefile will be
+removed in favor of an import process.

pfb-analysis/connectivity/access_jobs.sql

@@ -0,0 +1,42 @@
+----------------------------------------
+-- INPUTS
+-- location: neighborhood
+----------------------------------------
+-- low stress access
+UPDATE neighborhood_census_blocks SET emp_low_stress = NULL;
+UPDATE  neighborhood_census_blocks
+SET     emp_low_stress = (
+            SELECT  SUM(blocks2.jobs)
+            FROM    neighborhood_census_block_jobs blocks2
+            WHERE   EXISTS (
+                        SELECT  1
+                        FROM    neighborhood_connected_census_blocks cb
+                        WHERE   cb.source_blockid10 = neighborhood_census_blocks.blockid10
+                        AND     cb.target_blockid10 = blocks2.blockid10
+                        AND     cb.low_stress
+            )
+        )
+WHERE   EXISTS (
+            SELECT  1
+            FROM    neighborhood_boundary AS b
+            WHERE   ST_Intersects(neighborhood_census_blocks.geom,b.geom)
+        );
+
+-- high stress access
+UPDATE neighborhood_census_blocks SET emp_high_stress = NULL;
+UPDATE  neighborhood_census_blocks
+SET     emp_high_stress = (
+            SELECT  SUM(blocks2.jobs)
+            FROM    neighborhood_census_block_jobs blocks2
+            WHERE   EXISTS (
+                        SELECT  1
+                        FROM    neighborhood_connected_census_blocks cb
+                        WHERE   cb.source_blockid10 = neighborhood_census_blocks.blockid10
+                        AND     cb.target_blockid10 = blocks2.blockid10
+            )
+        )
+WHERE   EXISTS (
+            SELECT  1
+            FROM    neighborhood_boundary AS b
+            WHERE   ST_Intersects(neighborhood_census_blocks.geom,b.geom)
+        );

pfb-analysis/connectivity/access_population.sql

@@ -0,0 +1,42 @@
+----------------------------------------
+-- INPUTS
+-- location: neighborhood
+----------------------------------------
+-- low stress access
+UPDATE neighborhood_census_blocks SET pop_low_stress = NULL;
+UPDATE  neighborhood_census_blocks
+SET     pop_low_stress = (
+            SELECT  SUM(blocks2.pop10)
+            FROM    neighborhood_census_blocks blocks2
+            WHERE   EXISTS (
+                        SELECT  1
+                        FROM    neighborhood_connected_census_blocks cb
+                        WHERE   cb.source_blockid10 = neighborhood_census_blocks.blockid10
+                        AND     cb.target_blockid10 = blocks2.blockid10
+                        AND     cb.low_stress
+            )
+        )
+WHERE   EXISTS (
+            SELECT  1
+            FROM    neighborhood_boundary AS b
+            WHERE   ST_Intersects(neighborhood_census_blocks.geom,b.geom)
+        );
+
+-- high stress access
+UPDATE neighborhood_census_blocks SET pop_high_stress = NULL;
+UPDATE  neighborhood_census_blocks
+SET     pop_high_stress = (
+            SELECT  SUM(blocks2.pop10)
+            FROM    neighborhood_census_blocks blocks2
+            WHERE   EXISTS (
+                        SELECT  1
+                        FROM    neighborhood_connected_census_blocks cb
+                        WHERE   cb.source_blockid10 = neighborhood_census_blocks.blockid10
+                        AND     cb.target_blockid10 = blocks2.blockid10
+            )
+        )
+WHERE   EXISTS (
+            SELECT  1
+            FROM    neighborhood_boundary AS b
+            WHERE   ST_Intersects(neighborhood_census_blocks.geom,b.geom)
+        );

pfb-analysis/connectivity/access_recreation.sql

@@ -0,0 +1,26 @@
+----------------------------------------
+-- INPUTS
+-- location: neighborhood
+----------------------------------------
+-- low stress access
+UPDATE  neighborhood_census_blocks
+SET     rec_low_stress = (
+            SELECT  COUNT(path_id)
+            FROM    neighborhood_paths
+            WHERE   EXISTS (
+                        SELECT  1
+                        FROM    neighborhood_census_block_roads cbr,
+                                neighborhood_reachable_roads_low_stress ls,
+                                neighborhood_ways,
+                                neighborhood_paths
+                        WHERE   cb.source_blockid10 = neighborhood_census_blocks.blockid10
+                        AND     cb.target_blockid10 = blocks2.blockid10
+                        AND     cb.low_stress
+            )
+        )
+WHERE   EXISTS (
+            SELECT  1
+            FROM    neighborhood_zip_codes zips
+            WHERE   ST_Intersects(neighborhood_census_blocks.geom,zips.geom)
+            AND     zips.zip_code = '02138'
+        );

pfb-analysis/connectivity/access_schools.sql

@@ -0,0 +1,65 @@
+----------------------------------------
+-- INPUTS
+-- location: neighborhood
+----------------------------------------
+-- low stress access
+UPDATE neighborhood_census_blocks SET schools_low_stress = NULL;
+UPDATE  neighborhood_census_blocks
+SET     schools_low_stress = (
+            SELECT  COUNT(cbs.id)
+            FROM    neighborhood_connected_census_blocks_schools cbs
+            WHERE   cbs.source_blockid10 = neighborhood_census_blocks.blockid10
+            AND     cbs.low_stress
+        )
+WHERE   EXISTS (
+            SELECT  1
+            FROM    neighborhood_boundary as b
+            WHERE   ST_Intersects(neighborhood_census_blocks.geom,b.geom)
+        );
+
+-- high stress access
+UPDATE neighborhood_census_blocks SET schools_high_stress = NULL;
+UPDATE  neighborhood_census_blocks
+SET     schools_high_stress = (
+            SELECT  COUNT(cbs.id)
+            FROM    neighborhood_connected_census_blocks_schools cbs
+            WHERE   cbs.source_blockid10 = neighborhood_census_blocks.blockid10
+        )
+WHERE   EXISTS (
+            SELECT  1
+            FROM    neighborhood_boundary as b
+            WHERE   ST_Intersects(neighborhood_census_blocks.geom,b.geom)
+        );
+
+-- low stress population shed for schools in neighborhood
+UPDATE neighborhood_schools SET pop_low_stress = NULL;
+UPDATE  neighborhood_schools
+SET     pop_low_stress = (
+            SELECT  SUM(cb.pop10)
+            FROM    neighborhood_census_blocks cb,
+                    neighborhood_connected_census_blocks_schools cbs
+            WHERE   cb.blockid10 = cbs.source_blockid10
+            AND     neighborhood_schools.id = cbs.target_school_id
+            AND     cbs.low_stress
+        )
+WHERE   EXISTS (
+            SELECT  1
+            FROM    neighborhood_boundary as b
+            WHERE   ST_Intersects(neighborhood_schools.geom_pt,b.geom)
+        );
+
+-- high stress population shed for schools in neighborhood
+UPDATE neighborhood_schools SET pop_high_stress = NULL;
+UPDATE  neighborhood_schools
+SET     pop_high_stress = (
+            SELECT  SUM(cb.pop10)
+            FROM    neighborhood_census_blocks cb,
+                    neighborhood_connected_census_blocks_schools cbs
+            WHERE   cb.blockid10 = cbs.source_blockid10
+            AND     neighborhood_schools.id = cbs.target_school_id
+        )
+WHERE   EXISTS (
+            SELECT  1
+            FROM    neighborhood_boundary as b
+            WHERE   ST_Intersects(neighborhood_schools.geom_pt,b.geom)
+        );

pfb-analysis/connectivity/census_block_jobs.sql

@@ -0,0 +1,58 @@
+----------------------------------------
+-- INPUTS
+-- location: neighborhood
+-- data downloaded from http://lehd.ces.census.gov/data/
+-- or http://lehd.ces.census.gov/data/lodes/LODES7/
+--     "ma_od_main_JT00_2014".csv
+--     ma_od_aux_JT00_2014.csv
+-- import to DB and check the block id to have 15 characters
+-- also aggregate so 1 block has 1 number of total jobs
+--     (total jobs comes from S000 field
+--     as per http://lehd.ces.census.gov/data/lodes/LODES7/LODESTechDoc7.2.pdf
+----------------------------------------
+
+-- process imported tables
+ALTER TABLE "state_od_aux_JT00_2014" ALTER COLUMN w_geocode TYPE VARCHAR(15);
+UPDATE "state_od_aux_JT00_2014" SET w_geocode = rpad(w_geocode,15,'0'); --just in case we lost any trailing zeros
+ALTER TABLE "state_od_main_JT00_2014" ALTER COLUMN w_geocode TYPE VARCHAR(15);
+UPDATE "state_od_main_JT00_2014" SET w_geocode = rpad(w_geocode,15,'0'); --just in case we lost any trailing zeros
+
+-- indexes
+CREATE INDEX IF NOT EXISTS tidx_auxjtw ON "state_od_aux_JT00_2014" (w_geocode);
+CREATE INDEX IF NOT EXISTS tidx_mainjtw ON "state_od_main_JT00_2014" (w_geocode);
+ANALYZE "state_od_aux_JT00_2014" (w_geocode);
+ANALYZE "state_od_main_JT00_2014" (w_geocode);
+
+-- create combined table
+DROP TABLE IF EXISTS generated.neighborhood_census_block_jobs;
+CREATE TABLE generated.neighborhood_census_block_jobs (
+    id SERIAL PRIMARY KEY,
+    blockid10 VARCHAR(15),
+    jobs INT
+);
+
+-- add blocks of interest
+INSERT INTO generated.neighborhood_census_block_jobs (blockid10)
+SELECT  blocks.blockid10
+FROM    neighborhood_census_blocks blocks;
+
+-- add main data
+UPDATE  generated.neighborhood_census_block_jobs
+SET     jobs = COALESCE((
+            SELECT  SUM(j."S000")
+            FROM    "state_od_main_JT00_2014" j
+            WHERE   j.w_geocode = neighborhood_census_block_jobs.blockid10
+        ),0);
+
+-- add aux data
+UPDATE  generated.neighborhood_census_block_jobs
+SET     jobs =  jobs +
+                COALESCE((
+                    SELECT  SUM(j."S000")
+                    FROM    "state_od_aux_JT00_2014" j
+                    WHERE   j.w_geocode = neighborhood_census_block_jobs.blockid10
+        ),0);
+
+-- indexes
+CREATE INDEX IF NOT EXISTS idx_neighborhood_blkjobs ON neighborhood_census_block_jobs (blockid10);
+ANALYZE neighborhood_census_block_jobs (blockid10);