From 51f19427f4bf190b8b762614c9d4245080681625 Mon Sep 17 00:00:00 2001 From: missinglink Date: Thu, 13 Feb 2020 13:57:47 +0100 Subject: [PATCH] nodejs: initial commit of nodejs port --- .gitignore | 2 ++ cmd/compose.js | 8 ++++++++ cmd/compose/down.js | 10 ++++++++++ cmd/compose/exec.js | 10 ++++++++++ cmd/compose/kill.js | 10 ++++++++++ cmd/compose/logs.js | 10 ++++++++++ cmd/compose/ps.js | 10 ++++++++++ cmd/compose/pull.js | 10 ++++++++++ cmd/compose/run.js | 17 +++++++++++++++++ cmd/compose/top.js | 10 ++++++++++ cmd/compose/up.js | 10 ++++++++++ cmd/download.js | 8 ++++++++ cmd/download/all.js | 23 +++++++++++++++++++++++ cmd/download/csv.js | 9 +++++++++ cmd/download/geonames.js | 9 +++++++++ cmd/download/oa.js | 9 +++++++++ cmd/download/osm.js | 9 +++++++++ cmd/download/tiger.js | 9 +++++++++ cmd/download/transit.js | 9 +++++++++ cmd/download/wof.js | 9 +++++++++ cmd/import.js | 8 ++++++++ cmd/import/all.js | 20 ++++++++++++++++++++ cmd/import/csv.js | 9 +++++++++ cmd/import/geonames.js | 9 +++++++++ cmd/import/oa.js | 9 +++++++++ cmd/import/osm.js | 9 +++++++++ cmd/import/polylines.js | 9 +++++++++ cmd/import/transit.js | 9 +++++++++ cmd/import/wof.js | 9 +++++++++ {cmd => cmd_old}/docker.sh | 1 - {cmd => cmd_old}/download.sh | 0 {cmd => cmd_old}/elastic.sh | 0 {cmd => cmd_old}/import.sh | 0 {cmd => cmd_old}/prepare.sh | 0 {cmd => cmd_old}/system.sh | 0 {cmd => cmd_old}/test.sh | 0 lib/env.js | 9 +++++++++ {lib => lib_old}/cli.sh | 0 {lib => lib_old}/env.sh | 0 package.json | 27 +++++++++++++++++++++++++++ pelias | 29 ++++++++--------------------- 41 files changed, 336 insertions(+), 22 deletions(-) create mode 100644 .gitignore create mode 100644 cmd/compose.js create mode 100644 cmd/compose/down.js create mode 100644 cmd/compose/exec.js create mode 100644 cmd/compose/kill.js create mode 100644 cmd/compose/logs.js create mode 100644 cmd/compose/ps.js create mode 100644 cmd/compose/pull.js create mode 100644 cmd/compose/run.js create mode 100644 cmd/compose/top.js create mode 100644 cmd/compose/up.js create mode 100644 cmd/download.js create mode 100644 cmd/download/all.js create mode 100644 cmd/download/csv.js create mode 100644 cmd/download/geonames.js create mode 100644 cmd/download/oa.js create mode 100644 cmd/download/osm.js create mode 100644 cmd/download/tiger.js create mode 100644 cmd/download/transit.js create mode 100644 cmd/download/wof.js create mode 100644 cmd/import.js create mode 100644 cmd/import/all.js create mode 100644 cmd/import/csv.js create mode 100644 cmd/import/geonames.js create mode 100644 cmd/import/oa.js create mode 100644 cmd/import/osm.js create mode 100644 cmd/import/polylines.js create mode 100644 cmd/import/transit.js create mode 100644 cmd/import/wof.js rename {cmd => cmd_old}/docker.sh (99%) rename {cmd => cmd_old}/download.sh (100%) rename {cmd => cmd_old}/elastic.sh (100%) rename {cmd => cmd_old}/import.sh (100%) rename {cmd => cmd_old}/prepare.sh (100%) rename {cmd => cmd_old}/system.sh (100%) rename {cmd => cmd_old}/test.sh (100%) create mode 100644 lib/env.js rename {lib => lib_old}/cli.sh (100%) rename {lib => lib_old}/env.sh (100%) create mode 100644 package.json diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..d5f19d89 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +node_modules +package-lock.json diff --git a/cmd/compose.js b/cmd/compose.js new file mode 100644 index 00000000..1fee4db4 --- /dev/null +++ b/cmd/compose.js @@ -0,0 +1,8 @@ +module.exports = { + command: 'compose ', + describe: 'shortcuts to running docker-compose directly', + builder: (yargs) => yargs + .commandDir('compose') + .usage('$0 [args]') + .demandCommand(1, '') +} diff --git a/cmd/compose/down.js b/cmd/compose/down.js new file mode 100644 index 00000000..ea8ef120 --- /dev/null +++ b/cmd/compose/down.js @@ -0,0 +1,10 @@ +const child = require('child_process') +const options = { stdio: 'inherit' } + +module.exports = { + command: 'down', + describe: 'stop all docker-compose service(s)', + handler: (argv) => { + child.spawnSync('docker-compose', ['down'], options) + } +} diff --git a/cmd/compose/exec.js b/cmd/compose/exec.js new file mode 100644 index 00000000..b8b4dd79 --- /dev/null +++ b/cmd/compose/exec.js @@ -0,0 +1,10 @@ +const child = require('child_process') +const options = { stdio: 'inherit' } + +module.exports = { + command: 'exec', + describe: 'execute an arbitrary docker-compose command', + handler: (argv) => { + child.spawnSync('docker-compose', argv._.slice(1), options) + } +} diff --git a/cmd/compose/kill.js b/cmd/compose/kill.js new file mode 100644 index 00000000..3c4fd29a --- /dev/null +++ b/cmd/compose/kill.js @@ -0,0 +1,10 @@ +const child = require('child_process') +const options = { stdio: 'inherit' } + +module.exports = { + command: 'kill', + describe: 'kill one or more docker-compose service(s)', + handler: (argv) => { + child.spawnSync('docker-compose', argv._.slice(1), options) + } +} diff --git a/cmd/compose/logs.js b/cmd/compose/logs.js new file mode 100644 index 00000000..ca6b8241 --- /dev/null +++ b/cmd/compose/logs.js @@ -0,0 +1,10 @@ +const child = require('child_process') +const options = { stdio: 'inherit' } + +module.exports = { + command: 'logs', + describe: 'display container logs', + handler: (argv) => { + child.spawnSync('docker-compose', argv._.slice(1), options) + } +} diff --git a/cmd/compose/ps.js b/cmd/compose/ps.js new file mode 100644 index 00000000..576e9cd5 --- /dev/null +++ b/cmd/compose/ps.js @@ -0,0 +1,10 @@ +const child = require('child_process') +const options = { stdio: 'inherit' } + +module.exports = { + command: 'ps', + describe: 'list containers', + handler: (argv) => { + child.spawnSync('docker-compose', argv._.slice(1), options) + } +} diff --git a/cmd/compose/pull.js b/cmd/compose/pull.js new file mode 100644 index 00000000..effd9935 --- /dev/null +++ b/cmd/compose/pull.js @@ -0,0 +1,10 @@ +const child = require('child_process') +const options = { stdio: 'inherit' } + +module.exports = { + command: 'pull', + describe: 'update all docker images', + handler: (argv) => { + child.spawnSync('docker-compose', ['pull'], options) + } +} diff --git a/cmd/compose/run.js b/cmd/compose/run.js new file mode 100644 index 00000000..c34f72e1 --- /dev/null +++ b/cmd/compose/run.js @@ -0,0 +1,17 @@ +const child = require('child_process') +const env = require('../../lib/env')() +const options = { stdio: 'inherit' } + +function net_init() { + const net = `${env.COMPOSE_PROJECT_NAME}_default` + child.spawnSync('docker', ['network', 'create', net], { stdio: 'ignore' }) +} + +module.exports = { + command: 'run', + describe: 'execute a docker-compose run command', + handler: (argv) => { + net_init() + child.spawnSync('docker-compose', ['run', '--rm', ...argv._.slice(2)], options) + } +} diff --git a/cmd/compose/top.js b/cmd/compose/top.js new file mode 100644 index 00000000..9285de5b --- /dev/null +++ b/cmd/compose/top.js @@ -0,0 +1,10 @@ +const child = require('child_process') +const options = { stdio: 'inherit' } + +module.exports = { + command: 'top', + describe: 'display the running processes of a container', + handler: (argv) => { + child.spawnSync('docker-compose', argv._.slice(1), options) + } +} diff --git a/cmd/compose/up.js b/cmd/compose/up.js new file mode 100644 index 00000000..feedc093 --- /dev/null +++ b/cmd/compose/up.js @@ -0,0 +1,10 @@ +const child = require('child_process') +const options = { stdio: 'inherit' } + +module.exports = { + command: 'up', + describe: 'start one or more docker-compose service(s)', + handler: (argv) => { + child.spawnSync('docker-compose', ['up', '-d', ...argv._.slice(2)], options) + } +} diff --git a/cmd/download.js b/cmd/download.js new file mode 100644 index 00000000..419d980c --- /dev/null +++ b/cmd/download.js @@ -0,0 +1,8 @@ +module.exports = { + command: 'download ', + describe: 'fetch and update geographic data from source', + builder: (yargs) => yargs + .commandDir('download') + .usage('$0 [args]') + .demandCommand(1, '') +} diff --git a/cmd/download/all.js b/cmd/download/all.js new file mode 100644 index 00000000..67762435 --- /dev/null +++ b/cmd/download/all.js @@ -0,0 +1,23 @@ +const env = require('../../lib/env')() + +module.exports = { + command: 'all', + describe: '(re)download all data', + handler: (argv) => { + + // @todo: make this parallel + // like it was with the bash script + + require('./wof').handler() + require('./oa').handler() + require('./osm').handler() + + if (env.ENABLE_GEONAMES === 'true'){ + require('./geonames').handler() + } + + require('./tiger').handler() + require('./transit').handler() + require('./csv').handler() + } +} diff --git a/cmd/download/csv.js b/cmd/download/csv.js new file mode 100644 index 00000000..21d58da0 --- /dev/null +++ b/cmd/download/csv.js @@ -0,0 +1,9 @@ +const run = require('../compose/run').handler + +module.exports = { + command: 'csv', + describe: '(re)download CSV data', + handler: (argv) => { + run({ _: ['compose', 'run', 'csv-importer', './bin/download'] }) + } +} diff --git a/cmd/download/geonames.js b/cmd/download/geonames.js new file mode 100644 index 00000000..70ddbfae --- /dev/null +++ b/cmd/download/geonames.js @@ -0,0 +1,9 @@ +const run = require('../compose/run').handler + +module.exports = { + command: 'geonames', + describe: '(re)download geonames data', + handler: (argv) => { + run({ _: ['compose', 'run', 'geonames', './bin/download'] }) + } +} diff --git a/cmd/download/oa.js b/cmd/download/oa.js new file mode 100644 index 00000000..90ae8036 --- /dev/null +++ b/cmd/download/oa.js @@ -0,0 +1,9 @@ +const run = require('../compose/run').handler + +module.exports = { + command: 'oa', + describe: '(re)download openaddresses data', + handler: (argv) => { + run({ _: ['compose', 'run', 'openaddresses', './bin/download'] }) + } +} diff --git a/cmd/download/osm.js b/cmd/download/osm.js new file mode 100644 index 00000000..bf2a911e --- /dev/null +++ b/cmd/download/osm.js @@ -0,0 +1,9 @@ +const run = require('../compose/run').handler + +module.exports = { + command: 'osm', + describe: '(re)download openstreetmap data', + handler: (argv) => { + run({ _: ['compose', 'run', 'openstreetmap', './bin/download'] }) + } +} diff --git a/cmd/download/tiger.js b/cmd/download/tiger.js new file mode 100644 index 00000000..f745363c --- /dev/null +++ b/cmd/download/tiger.js @@ -0,0 +1,9 @@ +const run = require('../compose/run').handler + +module.exports = { + command: 'tiger', + describe: '(re)download TIGER data', + handler: (argv) => { + run({ _: ['compose', 'run', 'interpolation', './bin/download-tiger'] }) + } +} diff --git a/cmd/download/transit.js b/cmd/download/transit.js new file mode 100644 index 00000000..41a29486 --- /dev/null +++ b/cmd/download/transit.js @@ -0,0 +1,9 @@ +const run = require('../compose/run').handler + +module.exports = { + command: 'transit', + describe: '(re)download transit data', + handler: (argv) => { + run({ _: ['compose', 'run', 'transit', './bin/download'] }) + } +} diff --git a/cmd/download/wof.js b/cmd/download/wof.js new file mode 100644 index 00000000..4f7499dd --- /dev/null +++ b/cmd/download/wof.js @@ -0,0 +1,9 @@ +const run = require('../compose/run').handler + +module.exports = { + command: 'wof', + describe: '(re)download whosonfirst data', + handler: (argv) => { + run({ _: ['compose', 'run', 'whosonfirst', './bin/download'] }) + } +} diff --git a/cmd/import.js b/cmd/import.js new file mode 100644 index 00000000..23101023 --- /dev/null +++ b/cmd/import.js @@ -0,0 +1,8 @@ +module.exports = { + command: 'import ', + describe: 'import source data into elasticsearch', + builder: (yargs) => yargs + .commandDir('import') + .usage('$0 [args]') + .demandCommand(1, '') +} diff --git a/cmd/import/all.js b/cmd/import/all.js new file mode 100644 index 00000000..7495a5cb --- /dev/null +++ b/cmd/import/all.js @@ -0,0 +1,20 @@ +const env = require('../../lib/env')() + +module.exports = { + command: 'all', + describe: '(re)import all data', + handler: (argv) => { + + require('./wof').handler() + require('./oa').handler() + require('./osm').handler() + require('./polylines').handler() + + if (env.ENABLE_GEONAMES === 'true'){ + require('./geonames').handler() + } + + require('./transit').handler() + require('./csv').handler() + } +} diff --git a/cmd/import/csv.js b/cmd/import/csv.js new file mode 100644 index 00000000..e58a7389 --- /dev/null +++ b/cmd/import/csv.js @@ -0,0 +1,9 @@ +const run = require('../compose/run').handler + +module.exports = { + command: 'csv', + describe: '(re)import CSV data', + handler: (argv) => { + run({ _: ['compose', 'run', 'csv-importer', './bin/start'] }) + } +} diff --git a/cmd/import/geonames.js b/cmd/import/geonames.js new file mode 100644 index 00000000..eca4a2e5 --- /dev/null +++ b/cmd/import/geonames.js @@ -0,0 +1,9 @@ +const run = require('../compose/run').handler + +module.exports = { + command: 'geonames', + describe: '(re)import geonames data', + handler: (argv) => { + run({ _: ['compose', 'run', 'geonames', './bin/start'] }) + } +} diff --git a/cmd/import/oa.js b/cmd/import/oa.js new file mode 100644 index 00000000..8ef52094 --- /dev/null +++ b/cmd/import/oa.js @@ -0,0 +1,9 @@ +const run = require('../compose/run').handler + +module.exports = { + command: 'oa', + describe: '(re)import openaddresses data', + handler: (argv) => { + run({ _: ['compose', 'run', 'openaddresses', './bin/start'] }) + } +} diff --git a/cmd/import/osm.js b/cmd/import/osm.js new file mode 100644 index 00000000..01a33faa --- /dev/null +++ b/cmd/import/osm.js @@ -0,0 +1,9 @@ +const run = require('../compose/run').handler + +module.exports = { + command: 'osm', + describe: '(re)import openstreetmap data', + handler: (argv) => { + run({ _: ['compose', 'run', 'openstreetmap', './bin/start'] }) + } +} diff --git a/cmd/import/polylines.js b/cmd/import/polylines.js new file mode 100644 index 00000000..f36208db --- /dev/null +++ b/cmd/import/polylines.js @@ -0,0 +1,9 @@ +const run = require('../compose/run').handler + +module.exports = { + command: 'polylines', + describe: '(re)import polylines data', + handler: (argv) => { + run({ _: ['compose', 'run', 'interpolation', './bin/start'] }) + } +} diff --git a/cmd/import/transit.js b/cmd/import/transit.js new file mode 100644 index 00000000..d687ba03 --- /dev/null +++ b/cmd/import/transit.js @@ -0,0 +1,9 @@ +const run = require('../compose/run').handler + +module.exports = { + command: 'transit', + describe: '(re)import transit data', + handler: (argv) => { + run({ _: ['compose', 'run', 'transit', './bin/start'] }) + } +} diff --git a/cmd/import/wof.js b/cmd/import/wof.js new file mode 100644 index 00000000..d3a78d8f --- /dev/null +++ b/cmd/import/wof.js @@ -0,0 +1,9 @@ +const run = require('../compose/run').handler + +module.exports = { + command: 'wof', + describe: '(re)import whosonfirst data', + handler: (argv) => { + run({ _: ['compose', 'run', 'whosonfirst', './bin/start'] }) + } +} diff --git a/cmd/docker.sh b/cmd_old/docker.sh similarity index 99% rename from cmd/docker.sh rename to cmd_old/docker.sh index 83cc4c6e..ff357ec3 100644 --- a/cmd/docker.sh +++ b/cmd_old/docker.sh @@ -31,4 +31,3 @@ register 'compose' 'kill' 'kill one or more docker-compose service(s)' compose_k function compose_down(){ docker-compose down; } register 'compose' 'down' 'stop all docker-compose service(s)' compose_down - diff --git a/cmd/download.sh b/cmd_old/download.sh similarity index 100% rename from cmd/download.sh rename to cmd_old/download.sh diff --git a/cmd/elastic.sh b/cmd_old/elastic.sh similarity index 100% rename from cmd/elastic.sh rename to cmd_old/elastic.sh diff --git a/cmd/import.sh b/cmd_old/import.sh similarity index 100% rename from cmd/import.sh rename to cmd_old/import.sh diff --git a/cmd/prepare.sh b/cmd_old/prepare.sh similarity index 100% rename from cmd/prepare.sh rename to cmd_old/prepare.sh diff --git a/cmd/system.sh b/cmd_old/system.sh similarity index 100% rename from cmd/system.sh rename to cmd_old/system.sh diff --git a/cmd/test.sh b/cmd_old/test.sh similarity index 100% rename from cmd/test.sh rename to cmd_old/test.sh diff --git a/lib/env.js b/lib/env.js new file mode 100644 index 00000000..2329c7b6 --- /dev/null +++ b/lib/env.js @@ -0,0 +1,9 @@ +const _ = require('lodash') +const dotenv = require('dotenv') + +module.exports = () => { + let env = Object.create(process.env) + _.extend(env, dotenv.config().parsed) + + return env +} diff --git a/lib/cli.sh b/lib_old/cli.sh similarity index 100% rename from lib/cli.sh rename to lib_old/cli.sh diff --git a/lib/env.sh b/lib_old/env.sh similarity index 100% rename from lib/env.sh rename to lib_old/env.sh diff --git a/package.json b/package.json new file mode 100644 index 00000000..20c19aac --- /dev/null +++ b/package.json @@ -0,0 +1,27 @@ +{ + "name": "docker", + "version": "1.0.0", + "description": "This repository contains a framework for downloading/preparing and building the [Pelias Geocoder](https://github.com/pelias/pelias) using Docker and [Docker Compose](https://github.com/docker/compose#docker-compose).", + "main": "index.js", + "directories": { + "lib": "lib" + }, + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/pelias/docker.git" + }, + "author": "", + "license": "ISC", + "bugs": { + "url": "https://github.com/pelias/docker/issues" + }, + "homepage": "https://github.com/pelias/docker#readme", + "dependencies": { + "dotenv": "^8.2.0", + "lodash": "^4.17.15", + "yargs": "^15.1.0" + } +} diff --git a/pelias b/pelias index 0f383f55..6cd95086 100755 --- a/pelias +++ b/pelias @@ -1,22 +1,9 @@ -#!/bin/bash -set -e +#!/usr/bin/env node -# OSX comes bundled with versions of readlink, sed, parallel etc which are not -# compatible with the linux tools. Force OSX users to install the GNU -# compatible versions (prefixed with 'g', such as 'greadlink', 'gsed' etc.). -export CMD_READLINK='readlink' -if [[ "$OSTYPE" == "darwin"* ]]; then - if [ -x "$(command -v greadlink)" ]; then - CMD_READLINK='greadlink'; - else - 2>&1 echo 'OSX: you must install the gnu standard tooling using:' - 2>&1 echo 'brew install coreutils' - fi -fi - -# resolve path to this file (following symlinks) and load libs -BASEDIR=$( dirname $( ${CMD_READLINK} -f "${BASH_SOURCE[0]}" ) ) -for f in ${BASEDIR}/lib/* ${BASEDIR}/cmd/*; do source $f; done - -# cli runner -cli "$@" +require('yargs') + .scriptName('pelias') + .commandDir('cmd') + .showHelpOnFail(true) + .demandCommand(1, '') + .help() + .parse()