Merge pull request #39 from koopjs/zips

Handle resources that are zipz

CHANGELOG.md

@@ -2,7 +2,10 @@
 All notable changes to this project will be documented in this file.
 This project adheres to [Semantic Versioning](http://semver.org/).
 
-## [Unrelased] - [Unreleased]
+## [0.4.0] - 2015-06-10
+### Added
+* Non-point spatial datasets are now supported
+
 ### Changed
 * Refactored processing logic
 
@@ -66,6 +69,7 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 * Keeping a legit changelog
 * Added tape testing with sinon stubs in the controller tests
 
+[0.4.0]: https://github.com/Esri/koop/releases/compare/v0.3.3...v0.4.0
 [0.3.4]: https://github.com/Esri/koop/releases/compare/v0.3.3...v0.3.4
 [0.3.3]: https://github.com/Esri/koop/releases/compare/v0.3.2...v0.3.3
 [0.3.2]: https://github.com/Esri/koop/releases/compare/v0.3.1...v0.3.2

controller/index.js

@@ -68,11 +68,11 @@ var Controller = function (Socrata, BaseController) {
         // Get the item
         Socrata.getResource(data.host, req.params.id, req.params.item, req.query, function (error, itemJson) {
           // return 502 when there are errors
-          if (itemJson && itemJson.length && itemJson[0].errors) {
+          if (itemJson && itemJson[0] && itemJson[0].errors) {
             return res.status(502).send(itemJson)
           }
           // return 202 when processing
-          if (itemJson && itemJson.length && itemJson[0].status === 'processing' && !itemJson[0].errors) {
+          if (itemJson && itemJson[0] && itemJson[0].status === 'processing' && !itemJson[0].errors) {
             Socrata.getCount(['Socrata', req.params.item, (req.query.layer || 0)].join(':'), req.query, function (err, count) {
               if (err) {
                 return res.status(202).json({status: 'processing'})

models/Socrata.js

@@ -1,7 +1,8 @@
 var request = require('request'),
   async = require('async'),
   JSONStream = require('JSONStream'),
-  es = require('event-stream')
+  es = require('event-stream'),
+  ogr2ogr = require('ogr2ogr')
 
 var Socrata = function (koop) {
   var socrata = koop.BaseModel(koop)
@@ -126,7 +127,23 @@ var Socrata = function (koop) {
             koop.log.info(info)
           }
         })
-        // if count is the only thing that failed we can still try to grab the dataset in one go
+        // maybe we have errors because this is a blob file, we can just request the whole thing if it's a zip
+      } else if (errors.length && meta.blobFilename && meta.blobFilename.split('.')[1] === 'zip') {
+        socrata.processZip(host, urlId, function (err, geojson) {
+          if (err) {
+            koop.log.error(err)
+          } else {
+            socrata.insert(key, meta, geojson, function (err, success) {
+              if (err) {
+                koop.log.error('Processing failed for zip resource: ' + 'Socrata:' + key + ':0', err)
+                // handle err
+              } else {
+                koop.log.info('Processing completed for zip resource: ' + 'Socrata:' + key + ':0')
+              }
+            })
+          }
+        })
+      // if count is the only thing that failed we can still try to grab the dataset in one go
       } else if (errors.length === 1 && errors[0].split('::')[0] === 'count') {
         socrata.ingestResourceFallback(host, urlId, meta, function (err, info) {
           if (err) {
@@ -176,6 +193,7 @@ var Socrata = function (koop) {
                     meta.location_field = info.location_field
                     meta.updated_at = info.updated_at
                     meta.fields = info.fields
+                    meta.blobFilename = info.blobFilename
                   }
                 }
             )
@@ -227,7 +245,12 @@ var Socrata = function (koop) {
       } else if (res.statusCode !== 200) {
         callback('count::' + options.url + '::' + res.statusCode, null)
       } else {
-        var rowCount = JSON.parse(body)[0].count
+        var rowCount
+        try {
+          rowCount = JSON.parse(body)[0].count
+        } catch (e) {
+          err = 'Could not parse count JSON'
+        }
         callback(err, rowCount)
       }
     })
@@ -244,6 +267,7 @@ var Socrata = function (koop) {
         callback('meta::' + options.url + '::' + res.statusCode)
       } else {
         var response = JSON.parse(body)
+        meta.blobFilename = response.blobFilename
         meta.updated_at = new Date(res.headers['last-modified']).getTime()
         meta.name = response.name
         meta.fields = []
@@ -294,6 +318,22 @@ var Socrata = function (koop) {
     return request(options)
   }
 
+  socrata.processZip = function (host, id, callback) {
+    var zipUrl = host + '/api/geospatial/' + id + '?method=export&format=Original'
+    var options = {url: zipUrl, gzip: true}
+    if (socrata.token) {
+      options.headers = {'X-App-Token': socrata.token}
+    }
+    socrata.ogrZip(request(options), callback)
+    // to do error handling
+  }
+
+  socrata.ogrZip = function (stream, callback) {
+    ogr2ogr(stream, 'ESRI Shapefile').exec(function (err, data) {
+      callback(err, data)
+    })
+  }
+
   socrata.processStream = function (dataStream, meta, callback) {
     var geojson = { type: 'FeatureCollection', features: [] }
     dataStream

package.json

@@ -1,6 +1,6 @@
 {
   "name": "koop-socrata",
-  "version": "0.3.4",
+  "version": "0.4.0",
   "description": "A socrata wrapper for koop ",
   "main": "index.js",
   "scripts": {
@@ -13,7 +13,8 @@
     "request": "^2.51.0",
     "sphericalmercator": "~1.0.2",
     "JSONStream": "~1.0.4",
-    "event-stream": "~3.3.1"
+    "event-stream": "~3.3.1",
+    "ogr2ogr": "~0.5.1"
   },
   "author": "Chris Helm",
   "contributors": ["Daniel Fenton", "Phil Holleran"],

test/fixtures/zip::count.json

@@ -0,0 +1 @@
+[ ]

test/fixtures/zip::first.json

@@ -0,0 +1,4 @@
+{
+  "error" : true,
+  "message" : "Unrecognized arguments [$order:id]"
+}

test/fixtures/zip::view.json

@@ -0,0 +1,98 @@
+{
+  "id" : "3rgj-95q8",
+  "name" : "TMDL Effectiveness Monitoring projects",
+  "attribution" : "Department of Ecology",
+  "attributionLink" : "http://www.ecy.wa.gov/",
+  "averageRating" : 0,
+  "blobFilename" : "EM_shapstate.zip",
+  "blobFileSize" : 1171335,
+  "blobId" : "bCcUN2kEX5mTN1tobXLjpH1_yt6GyN-7JsR3Ibxx0xA",
+  "blobMimeType" : "application/zip; charset=binary",
+  "category" : "Natural Resources & Environment",
+  "createdAt" : 1426468340,
+  "description" : "Map of Post-TMDL, or TMDL effectiveness monitoring studies and data.",
+  "displayType" : "map",
+  "downloadCount" : 6,
+  "indexUpdatedAt" : 1427685858,
+  "newBackend" : false,
+  "numberOfComments" : 0,
+  "oid" : 10612169,
+  "publicationAppendEnabled" : false,
+  "publicationDate" : 1427212102,
+  "publicationGroup" : 2489394,
+  "publicationStage" : "published",
+  "tableId" : 2489394,
+  "totalTimesRated" : 0,
+  "viewCount" : 63,
+  "viewLastModified" : 1427685702,
+  "viewType" : "geo",
+  "childViews" : [ "jc6v-v5jn" ],
+  "columns" : [ ],
+  "displayFormat" : {
+    "bkgdLayers" : [ {
+      "alias" : "Google",
+      "opacity" : 1,
+      "layerKey" : "Google Roadmap"
+    }, {
+      "alias" : "Bing",
+      "opacity" : 1,
+      "layerKey" : "Bing Road"
+    }, {
+      "alias" : "ESRI",
+      "opacity" : 1,
+      "layerKey" : "World Street Map (ESRI)"
+    } ],
+    "viewport" : {
+      "ymin" : 44.650181086072,
+      "ymax" : 49.486587996741,
+      "xmin" : -126.37856310264,
+      "xmax" : -113.72231310441
+    },
+    "plotStyle" : "point",
+    "distinctLegend" : true,
+    "exclusiveLayers" : true,
+    "viewDefinitions" : [ {
+      "uid" : "self"
+    } ]
+  },
+  "grants" : [ {
+    "inherited" : false,
+    "type" : "viewer",
+    "flags" : [ "public" ]
+  } ],
+  "metadata" : {
+    "renderTypeConfig" : {
+      "visible" : {
+        "map" : true
+      }
+    },
+    "availableDisplayTypes" : [ "map", "table", "fatrow", "page" ],
+    "geo" : {
+      "owsUrl" : "/api/geospatial/3rgj-95q8",
+      "bbox" : "-124.00511800000427,45.52546855905342,-117.53496719120704,49.006828340893726",
+      "layers" : "geo_3rgj-95q8-1",
+      "featureIdAttribute" : "_SocrataID",
+      "bboxCrs" : "EPSG:4326",
+      "namespace" : "geo_3rgj-95q8"
+    }
+  },
+  "owner" : {
+    "id" : "9t9n-gtbp",
+    "displayName" : "collyard",
+    "roleName" : "editor",
+    "screenName" : "collyard",
+    "rights" : [ "create_datasets", "view_domain", "create_pages", "view_goals", "view_dashboards" ]
+  },
+  "query" : {
+  },
+  "rights" : [ "read" ],
+  "tableAuthor" : {
+    "id" : "9t9n-gtbp",
+    "displayName" : "collyard",
+    "roleName" : "editor",
+    "screenName" : "collyard",
+    "rights" : [ "create_datasets", "view_domain", "create_pages", "view_goals", "view_dashboards" ]
+  },
+  "tags" : [ "effectiveness monitoring", "tmdls", "water quality", "ecology" ],
+  "flags" : [ "default" ]
+}

test/model.js

@@ -31,6 +31,12 @@ requests.get('/resource/filtered.json?$order=:id&$limit=1').reply(200, JSON.pars
 requests.get('/views/filtered.json').reply(200, JSON.parse(fs.readFileSync(__dirname + '/fixtures/filtered::view.json')))
 requests.get('/resource/filtered.json').reply(200, function (uri) {return fs.createReadStream(__dirname + '/fixtures/filtered::page.json')})
 
+// responses for a zip file resource
+requests.get('/resource/zip.json?$select=count(*)').reply(200, JSON.parse(fs.readFileSync(__dirname + '/fixtures/zip::count.json')))
+requests.get('/resource/zip.json?$order=:id&$limit=1').reply(200, JSON.parse(fs.readFileSync(__dirname + '/fixtures/zip::first.json')))
+requests.get('/views/zip.json').reply(200, JSON.parse(fs.readFileSync(__dirname + '/fixtures/zip::view.json')))
+requests.get('/api/geospatial/zip?method=export&format=Original').times(2).reply(200, function (uri) {return fs.createReadStream(__dirname + '/fixtures/zip::data.zip')})
+
 // use Koop's local cache as a db for tests
 koop.Cache = new koop.DataCache(koop)
 koop.Cache.db = koop.LocalDB
@@ -42,6 +48,11 @@ var id = 'seattle',
   host = 'https://data.seattle.gov',
   key = 'foobar'
 
+// stub out requests for a zip resource
+sinon.stub(socrata, 'ogrZip', function (stream, callback) {
+  callback(null, JSON.parse(fs.readFileSync(__dirname + '/fixtures/zip::geojson.json')))
+})
+
 test('adding a socrata instance', function (t) {
   socrata.register(id, host, function (err, success) {
     if (err) throw err
@@ -259,8 +270,33 @@ test('requesting a socrata dataset that does not exist', function (t) {
   })
 })
 
+test('processing a zip file', function (t) {
+  t.plan(1)
+  socrata.processZip(host, 'zip', function (err, geojson) {
+    if (err) throw err
+    t.deepEqual(geojson.features.length, 19)
+  })
+})
+
 // Integration tests
 
+test('fill the cache with a resource that is a zip', function (t) {
+  socrata.getResource(host, id, 'zip', {}, function (err, data) {
+    if (err) throw err
+    setTimeout(function () {
+      t.end()
+    }, 500)
+  })
+})
+
+test('requesting a resource that was a zip', function (t) {
+  t.plan(1)
+  socrata.getResource(host, id, 'zip', {layer: 0}, function (err, data) {
+    if (err) throw err
+    t.deepEqual(data[0].features.length, 19)
+  })
+})
+
 test('fill the cache with a resource that was filtered', function (t) {
   socrata.getResource(host, id, 'filtered', {}, function (err, data) {
     if (err) throw err
@@ -320,6 +356,7 @@ test('requesting a resource with a fully working resource', function (t) {
 })
 
 test('teardown', function (t) {
+  socrata.ogrZip.restore()
   t.end()
   process.exit(0)
 })