Skip to content

Commit

Permalink
feat(libpostal): Use libpostal service
Browse files Browse the repository at this point in the history
BREAKING CHANGE

Use microservice-wrapper to avoid having to load libpostal locally.

Note: this now requires a new configuration section in `pelias.json`, a
top-level `services` key with the usual properties. Here's an example
full `pelias.json`:

```
{
  "api": {
    "textAnalyzer": "libpostal"
  },
  "services": {
    "libpostal": {
      "url": "http://libpostal-service-url:8080",
      "timeout": 4000
    }
  }
}
```

Fixes #106
  • Loading branch information
orangejulius committed Sep 30, 2019
1 parent 4802095 commit d4ed36e
Show file tree
Hide file tree
Showing 6 changed files with 77 additions and 26 deletions.
24 changes: 2 additions & 22 deletions lib/analyze.js
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
const libpostal_service = require( './libpostal_wrapper' );
// constants for controlling how we parse ranges, eg: 'α-β'
// some ranges such as '1-7' are ambiguous; it could mean 'apt 7, no 1'; or
// it could mean 'apt 1, no 7'; or could even be a valid range 'one to seven'.
Expand All @@ -7,32 +8,11 @@ var MIN_RANGE = 1; // the miniumum amount β is higher than α
var MAX_RANGE = 6; // the maximum amount β is higher than α
var MIN_RANGE_HOUSENUMBER = 10; // the minimum acceptible value for both α and β

/*
* Return the appropriate version of node-postal
*/

var _nodepostal_module;
function get_libpostal() {
// lazy load this dependency; since it's large (~2GB RAM) and may be
// accidentally required by a process which doesn't use it.
if (!_nodepostal_module) {
// load the mock library if MOCK_LIBPOSTAL env var is set
if (process.env.MOCK_LIBPOSTAL) {
_nodepostal_module = require('../test/lib/mock_libpostal');
// otherwise load the real thing
} else {
_nodepostal_module = require('node-postal');
}
}

return _nodepostal_module;
}

/**
analyze input streetname string and return a list of expansions.
**/
function street( streetName ){
const postal = get_libpostal();
const postal = libpostal_service();

// use libpostal to expand the address
var expansions = postal.expand.expand_address( streetName );
Expand Down
22 changes: 22 additions & 0 deletions lib/libpostal_wrapper.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
const mock_libpostal = require('../test/lib/mock_libpostal');

// This module is a wrapper around the actual libpostal service library
// and the mock libpostal library
// it allows an environment variable to switch which library is used in application code

let libpostal_module;
function get_libpostal() {
// return the mock library if MOCK_LIBPOSTAL env var is set
if (process.env.MOCK_LIBPOSTAL) {
return mock_libpostal;
// otherwise return the actual service
} else {
// lazy load the libpostal module so that tests can skip configuring the service
if (!libpostal_module) {
libpostal_module = require( '../libpostal/service' );
}
return libpostal_module;
}
}

module.exports = get_libpostal;
51 changes: 51 additions & 0 deletions libpostal/service.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
// deasync is used to proved a sync-looking interface
// to the async call to the libpostal service
const deasync = require('deasync');
const microservice_wrapper = require('pelias-microservice-wrapper');
const pelias_config = require('pelias-config').generate();

const LibpostalServiceConfig = class extends microservice_wrapper.ServiceConfiguration {
constructor(configBlob) {
super('libpostal', configBlob);
}
getUrl(params) {
return this.baseUrl + params.endpoint;
}
getParameters(params) {
return {
address: params.address
};
}
};

// use the 'services.libpostal' config entry if available, otherwise fall back to 'api.services.libpostal'
const config_entry = pelias_config.get('services.libpostal') || pelias_config.get('api.services.libpostal');

if (!config_entry) {
throw new Error('Libpostal configuration not found in `services.libpostal` or `api.services.libpostal`');
}

// create an instance of the libpostal service, with a synchronous interface
const libpostal_service = deasync(microservice_wrapper.service(
new LibpostalServiceConfig(config_entry)
));

// create an object that looks like the interface to `node-postal` but uses a remote service
module.exports = {
expand: {
expand_address: function(param) {
const params = {
endpoint: 'expand',
address: param
};

// the libpostal service will not handle an empty parameter
// so return empty array immediately
if (!param) {
return [];
}
const result = libpostal_service(params);
return result;
}
}
};
1 change: 1 addition & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
"cheerio": "^1.0.0-rc.3",
"cli-table3": "^0.5.0",
"csv-parse": "^4.4.6",
"deasync": "^0.1.15",
"express": "^4.14.0",
"fs-extra": "^8.0.0",
"jsftp": "^2.0.0",
Expand Down
3 changes: 0 additions & 3 deletions readme.md
Original file line number Diff line number Diff line change
Expand Up @@ -238,7 +238,6 @@ see: [source](https://github.com/pelias/interpolation/blob/master/cmd/server.js)
# docker

### build docker image
this can take some time for the first build due to installing libpostal from source
```bash
docker build -t pelias/interpolation .
```
Expand Down Expand Up @@ -434,8 +433,6 @@ To use Interpolation service with the Pelias API, [configure the pelias config f

### install dependencies

*note:* [libpostal](https://github.com/openvenues/node-postal#troubleshooting) **must** be installed on your system before you continue!

The `Dockerfile` in this repo has complete instructions on how to install everything from scratch on Ubuntu.

### TIGER dependency on GDAL
Expand Down
2 changes: 1 addition & 1 deletion test/lib/mock_libpostal.js
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ module.exports.expand = {
// is enabled, lazy load real libpostal, and return the real response
} else if (use_real_libpostal) {
// lazy load libpostal only when needed
if (!real_libpostal) { real_libpostal = require('node-postal'); }
if (!real_libpostal) { real_libpostal = require('../../libpostal/service'); }

const real_response = real_libpostal.expand.expand_address(clean_string);
mock_responses[clean_string] = real_response;
Expand Down

0 comments on commit d4ed36e

Please sign in to comment.