From 558babb927f3f51a793ed88a694e3b3df79ba337 Mon Sep 17 00:00:00 2001 From: Jerroyd Moore Date: Mon, 30 Dec 2019 11:37:13 -0800 Subject: [PATCH] feat: load whoishiring data into the database --- .eslintrc | 3 +- container-populate-hn-data/.dockerignore | 2 + container-populate-hn-data/Dockerfile | 28 ++ .../__tests__}/index.test.js | 0 container-populate-hn-data/package-lock.json | 277 ++++++++++++++++++ container-populate-hn-data/package.json | 23 ++ .../src/hackerNewsApi.js | 65 ++++ container-populate-hn-data/src/index.js | 87 ++++++ .../src/models/index.js | 20 ++ .../src/models/posts.js | 47 +++ .../src/models/stories.js | 22 ++ container-populate-hn-data/src/transform.js | 42 +++ container-populate-hn-data/wait-for-it.sh | 178 +++++++++++ container-provision-db/Dockerfile | 19 ++ container-provision-db/index.js | 117 ++++++++ container-provision-db/package-lock.json | 119 ++++++++ container-provision-db/package.json | 19 ++ container-provision-db/wait-for-it.sh | 178 +++++++++++ docker-compose.yaml | 30 ++ package-lock.json | 179 +---------- package.json | 23 +- src/index.js | 1 - 22 files changed, 1296 insertions(+), 183 deletions(-) create mode 100644 container-populate-hn-data/.dockerignore create mode 100644 container-populate-hn-data/Dockerfile rename {__tests__ => container-populate-hn-data/__tests__}/index.test.js (100%) create mode 100644 container-populate-hn-data/package-lock.json create mode 100644 container-populate-hn-data/package.json create mode 100644 container-populate-hn-data/src/hackerNewsApi.js create mode 100644 container-populate-hn-data/src/index.js create mode 100644 container-populate-hn-data/src/models/index.js create mode 100644 container-populate-hn-data/src/models/posts.js create mode 100644 container-populate-hn-data/src/models/stories.js create mode 100644 container-populate-hn-data/src/transform.js create mode 100644 container-populate-hn-data/wait-for-it.sh create mode 100644 container-provision-db/Dockerfile create mode 100644 container-provision-db/index.js create mode 100644 container-provision-db/package-lock.json create mode 100644 container-provision-db/package.json create mode 100644 container-provision-db/wait-for-it.sh create mode 100644 docker-compose.yaml delete mode 100644 src/index.js diff --git a/.eslintrc b/.eslintrc index 1ec7e68..bbdf104 100644 --- a/.eslintrc +++ b/.eslintrc @@ -11,11 +11,12 @@ "node": true, "jest/globals": true }, + "ignorePatterns": [ "node_modules/"], "rules": { "import/no-commonjs": 0, "import/no-nodejs-modules": 0, "indent": [ "error", 2, { "SwitchCase": 1 } ], - "no-console": 1, + "no-console": 0, "no-process-exit": 0, "no-unused-expressions": 0, "no-unused-vars": ["error", { "argsIgnorePattern": "next" }], diff --git a/container-populate-hn-data/.dockerignore b/container-populate-hn-data/.dockerignore new file mode 100644 index 0000000..82fdcf5 --- /dev/null +++ b/container-populate-hn-data/.dockerignore @@ -0,0 +1,2 @@ +node_modules +__tests__ \ No newline at end of file diff --git a/container-populate-hn-data/Dockerfile b/container-populate-hn-data/Dockerfile new file mode 100644 index 0000000..9365342 --- /dev/null +++ b/container-populate-hn-data/Dockerfile @@ -0,0 +1,28 @@ +# This stage installs our modules +FROM mhart/alpine-node:12 + +ENV container=docker NODE_ENV=production +WORKDIR /app + +COPY package.json package-lock.json ./ +RUN npm ci --prod + +# Then we copy over the modules from above onto a `slim` image +FROM mhart/alpine-node:slim-12 + +COPY wait-for-it.sh . +RUN chmod +x ./wait-for-it.sh + +# If possible, run your container using `docker run --init` +# Otherwise, you can use `tini`: +RUN apk add --no-cache tini bash \ + && addgroup -g 1000 node \ + && adduser -u 1000 -G node -s /bin/sh -D node + +ENTRYPOINT ["/sbin/tini", "--"] +USER node + +COPY --from=0 /app . +COPY ./src ./src/ + +CMD ./wait-for-it.sh $PGHOST:$PGPORT -- node ./src/index.js \ No newline at end of file diff --git a/__tests__/index.test.js b/container-populate-hn-data/__tests__/index.test.js similarity index 100% rename from __tests__/index.test.js rename to container-populate-hn-data/__tests__/index.test.js diff --git a/container-populate-hn-data/package-lock.json b/container-populate-hn-data/package-lock.json new file mode 100644 index 0000000..8abf746 --- /dev/null +++ b/container-populate-hn-data/package-lock.json @@ -0,0 +1,277 @@ +{ + "name": "@whoishiring.work/populate-hn-data", + "version": "1.0.0", + "lockfileVersion": 1, + "requires": true, + "dependencies": { + "@types/node": { + "version": "13.1.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-13.1.1.tgz", + "integrity": "sha512-hx6zWtudh3Arsbl3cXay+JnkvVgCKzCWKv42C9J01N2T2np4h8w5X8u6Tpz5mj38kE3M9FM0Pazx8vKFFMnjLQ==" + }, + "any-promise": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/any-promise/-/any-promise-1.3.0.tgz", + "integrity": "sha1-q8av7tzqUugJzcA3au0845Y10X8=" + }, + "bluebird": { + "version": "3.7.2", + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", + "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==" + }, + "buffer-writer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-2.0.0.tgz", + "integrity": "sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw==" + }, + "cls-bluebird": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/cls-bluebird/-/cls-bluebird-2.1.0.tgz", + "integrity": "sha1-N+8eCAqP+1XC9BZPU28ZGeeWiu4=", + "requires": { + "is-bluebird": "^1.0.2", + "shimmer": "^1.1.0" + } + }, + "debug": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", + "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "requires": { + "ms": "^2.1.1" + } + }, + "dottie": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/dottie/-/dottie-2.0.2.tgz", + "integrity": "sha512-fmrwR04lsniq/uSr8yikThDTrM7epXHBAAjH9TbeH3rEA8tdCO7mRzB9hdmdGyJCxF8KERo9CITcm3kGuoyMhg==" + }, + "inflection": { + "version": "1.12.0", + "resolved": "https://registry.npmjs.org/inflection/-/inflection-1.12.0.tgz", + "integrity": "sha1-ogCTVlbW9fa8TcdQLhrstwMihBY=" + }, + "is-bluebird": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-bluebird/-/is-bluebird-1.0.2.tgz", + "integrity": "sha1-CWQ5Bg9KpBGr7hkUOoTWpVNG1uI=" + }, + "lodash": { + "version": "4.17.15", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz", + "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==" + }, + "moment": { + "version": "2.24.0", + "resolved": "https://registry.npmjs.org/moment/-/moment-2.24.0.tgz", + "integrity": "sha512-bV7f+6l2QigeBBZSM/6yTNq4P2fNpSWj/0e7jQcy87A8e7o2nAfP/34/2ky5Vw4B9S446EtIhodAzkFCcR4dQg==" + }, + "moment-timezone": { + "version": "0.5.27", + "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.27.tgz", + "integrity": "sha512-EIKQs7h5sAsjhPCqN6ggx6cEbs94GK050254TIJySD1bzoM5JTYDwAU1IoVOeTOL6Gm27kYJ51/uuvq1kIlrbw==", + "requires": { + "moment": ">= 2.9.0" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + }, + "node-fetch": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.0.tgz", + "integrity": "sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA==" + }, + "packet-reader": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/packet-reader/-/packet-reader-1.0.0.tgz", + "integrity": "sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ==" + }, + "pg": { + "version": "7.16.1", + "resolved": "https://registry.npmjs.org/pg/-/pg-7.16.1.tgz", + "integrity": "sha512-eN2WCYY8XBOKfZ2CZir4DCrPYEEK1g6yh5AIo3OMHMSkqNrk+OBJG4/e9EW54LfAkk96MWvtsvR1hbvDHhcVkQ==", + "requires": { + "buffer-writer": "2.0.0", + "packet-reader": "1.0.0", + "pg-connection-string": "0.1.3", + "pg-packet-stream": "^1.1.0", + "pg-pool": "^2.0.9", + "pg-types": "^2.1.0", + "pgpass": "1.x", + "semver": "4.3.2" + }, + "dependencies": { + "semver": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-4.3.2.tgz", + "integrity": "sha1-x6BxWKgL7dBSNVt3DYLWZA+AO+c=" + } + } + }, + "pg-connection-string": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-0.1.3.tgz", + "integrity": "sha1-2hhHsglA5C7hSSvq9l1J2RskXfc=" + }, + "pg-hstore": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/pg-hstore/-/pg-hstore-2.3.3.tgz", + "integrity": "sha512-qpeTpdkguFgfdoidtfeTho1Q1zPVPbtMHgs8eQ+Aan05iLmIs3Z3oo5DOZRclPGoQ4i68I1kCtQSJSa7i0ZVYg==", + "requires": { + "underscore": "^1.7.0" + } + }, + "pg-int8": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", + "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==" + }, + "pg-packet-stream": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/pg-packet-stream/-/pg-packet-stream-1.1.0.tgz", + "integrity": "sha512-kRBH0tDIW/8lfnnOyTwKD23ygJ/kexQVXZs7gEyBljw4FYqimZFxnMMx50ndZ8In77QgfGuItS5LLclC2TtjYg==" + }, + "pg-pool": { + "version": "2.0.9", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-2.0.9.tgz", + "integrity": "sha512-gNiuIEKNCT3OnudQM2kvgSnXsLkSpd6mS/fRnqs6ANtrke6j8OY5l9mnAryf1kgwJMWLg0C1N1cYTZG1xmEYHQ==" + }, + "pg-types": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", + "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", + "requires": { + "pg-int8": "1.0.1", + "postgres-array": "~2.0.0", + "postgres-bytea": "~1.0.0", + "postgres-date": "~1.0.4", + "postgres-interval": "^1.1.0" + } + }, + "pgpass": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.2.tgz", + "integrity": "sha1-Knu0G2BltnkH6R2hsHwYR8h3swY=", + "requires": { + "split": "^1.0.0" + } + }, + "postgres-array": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", + "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==" + }, + "postgres-bytea": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.0.tgz", + "integrity": "sha1-AntTPAqokOJtFy1Hz5zOzFIazTU=" + }, + "postgres-date": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.4.tgz", + "integrity": "sha512-bESRvKVuTrjoBluEcpv2346+6kgB7UlnqWZsnbnCccTNq/pqfj1j6oBaN5+b/NrDXepYUT/HKadqv3iS9lJuVA==" + }, + "postgres-interval": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", + "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", + "requires": { + "xtend": "^4.0.0" + } + }, + "retry-as-promised": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/retry-as-promised/-/retry-as-promised-3.2.0.tgz", + "integrity": "sha512-CybGs60B7oYU/qSQ6kuaFmRd9sTZ6oXSc0toqePvV74Ac6/IFZSI1ReFQmtCN+uvW1Mtqdwpvt/LGOiCBAY2Mg==", + "requires": { + "any-promise": "^1.3.0" + } + }, + "semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==" + }, + "sequelize": { + "version": "5.21.3", + "resolved": "https://registry.npmjs.org/sequelize/-/sequelize-5.21.3.tgz", + "integrity": "sha512-ptdeAxwTY0zbj7AK8m+SH3z52uHVrt/qmOTSIGo/kyfnSp3h5HeKlywkJf5GEk09kuRrPHfWARVSXH1W3IGU7g==", + "requires": { + "bluebird": "^3.5.0", + "cls-bluebird": "^2.1.0", + "debug": "^4.1.1", + "dottie": "^2.0.0", + "inflection": "1.12.0", + "lodash": "^4.17.15", + "moment": "^2.24.0", + "moment-timezone": "^0.5.21", + "retry-as-promised": "^3.2.0", + "semver": "^6.3.0", + "sequelize-pool": "^2.3.0", + "toposort-class": "^1.0.1", + "uuid": "^3.3.3", + "validator": "^10.11.0", + "wkx": "^0.4.8" + } + }, + "sequelize-pool": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/sequelize-pool/-/sequelize-pool-2.3.0.tgz", + "integrity": "sha512-Ibz08vnXvkZ8LJTiUOxRcj1Ckdn7qafNZ2t59jYHMX1VIebTAOYefWdRYFt6z6+hy52WGthAHAoLc9hvk3onqA==" + }, + "shimmer": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/shimmer/-/shimmer-1.2.1.tgz", + "integrity": "sha512-sQTKC1Re/rM6XyFM6fIAGHRPVGvyXfgzIDvzoq608vM+jeyVD0Tu1E6Np0Kc2zAIFWIj963V2800iF/9LPieQw==" + }, + "split": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/split/-/split-1.0.1.tgz", + "integrity": "sha512-mTyOoPbrivtXnwnIxZRFYRrPNtEFKlpB2fvjSnCQUiAA6qAZzqwna5envK4uk6OIeP17CsdF3rSBGYVBsU0Tkg==", + "requires": { + "through": "2" + } + }, + "through": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", + "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=" + }, + "toposort-class": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toposort-class/-/toposort-class-1.0.1.tgz", + "integrity": "sha1-f/0feMi+KMO6Rc1OGj9e4ZO9mYg=" + }, + "underscore": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.9.1.tgz", + "integrity": "sha512-5/4etnCkd9c8gwgowi5/om/mYO5ajCaOgdzj/oW+0eQV9WxKBDZw5+ycmKmeaTXjInS/W0BzpGLo2xR2aBwZdg==" + }, + "uuid": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.3.tgz", + "integrity": "sha512-pW0No1RGHgzlpHJO1nsVrHKpOEIxkGg1xB+v0ZmdNH5OAeAwzAVrCnI2/6Mtx+Uys6iaylxa+D3g4j63IKKjSQ==" + }, + "validator": { + "version": "10.11.0", + "resolved": "https://registry.npmjs.org/validator/-/validator-10.11.0.tgz", + "integrity": "sha512-X/p3UZerAIsbBfN/IwahhYaBbY68EN/UQBWHtsbXGT5bfrH/p4NQzUCG1kF/rtKaNpnJ7jAu6NGTdSNtyNIXMw==" + }, + "wkx": { + "version": "0.4.8", + "resolved": "https://registry.npmjs.org/wkx/-/wkx-0.4.8.tgz", + "integrity": "sha512-ikPXMM9IR/gy/LwiOSqWlSL3X/J5uk9EO2hHNRXS41eTLXaUFEVw9fn/593jW/tE5tedNg8YjT5HkCa4FqQZyQ==", + "requires": { + "@types/node": "*" + } + }, + "xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==" + } + } +} diff --git a/container-populate-hn-data/package.json b/container-populate-hn-data/package.json new file mode 100644 index 0000000..f54632b --- /dev/null +++ b/container-populate-hn-data/package.json @@ -0,0 +1,23 @@ +{ + "name": "@whoishiring.work/populate-hn-data", + "version": "1.0.0", + "private": true, + "description": "Populates whoishiring data from hacker news into the database", + "license": "MIT", + "repository": "https://github.com/jerroydmoore/whoishiring.work.git", + "type": "commonjs", + "engines": { + "node": ">=12.13.0" + }, + "scripts": { + "start": "node src/index.js" + }, + "dependencies": { + "debug": "^4.1.1", + "node-fetch": "^2.6.0", + "pg": "^7.16.1", + "pg-hstore": "^2.3.3", + "sequelize": "^5.21.3" + }, + "devDependencies": {} +} diff --git a/container-populate-hn-data/src/hackerNewsApi.js b/container-populate-hn-data/src/hackerNewsApi.js new file mode 100644 index 0000000..cb3df76 --- /dev/null +++ b/container-populate-hn-data/src/hackerNewsApi.js @@ -0,0 +1,65 @@ +const debug = require('debug')('app:hn-api'); +const { URL } = require('url'); +const fetch = require('node-fetch'); + +const HACKERNEWS_API_URL = 'http://hn.algolia.com/api/v1/search_by_date'; +const whoIsHiringRegex = /^Ask HN: Who is hiring\?/; + +module.exports = { + async *getItems(params, hitsPerPage = 100, page = 0) { + const url = new URL(HACKERNEWS_API_URL); + url.searchParams.set('hitsPerPage', hitsPerPage); + + for (const key of Object.keys(params)) { + url.searchParams.set(key, params[key]); + } + + let yieldCount = 0; + let max = -1; + + do { + url.searchParams.set('page', page++); + + debug(`getItems ${url.href}`); + const response = await fetch(url.href); + const body = await response.json(); + + max = body.nbHits; + if (body.hits.length === 0) { + debug('0 hits returned'); + return; + } + + yield* body.hits; + yieldCount += body.hits.length; + + debug(`page set exhausted. Refresh? ${yieldCount} < ${max}`); + } while (yieldCount < max); + debug('getItems exhausted'); + }, + async *getStoriesByAuthor(author, hitsPerPage = 100) { + const params = { tags: `author_${author},(story,poll)` }; + for await (const item of this.getItems(params, hitsPerPage)) { + yield item; + } + }, + + async *getWhoIsHiringStories() { + for await (const item of this.getStoriesByAuthor('whoishiring')) { + if (whoIsHiringRegex.test(item.title)) { + yield item; + } else { + debug(`getWhoIsHiringStories: skipping topic ${item.title}`); + } + } + }, + async *getPostsByStoryIds(storyIds) { + if (!Array.isArray(storyIds)) { + storyIds = [storyIds]; + } + const params = { tags: `comment,(${storyIds.map((id) => `story_${id}`).join(',')})` }; + for await (const item of this.getItems(params, 1000)) { + yield item; + } + }, +}; diff --git a/container-populate-hn-data/src/index.js b/container-populate-hn-data/src/index.js new file mode 100644 index 0000000..d524a3b --- /dev/null +++ b/container-populate-hn-data/src/index.js @@ -0,0 +1,87 @@ +const debug = require('debug')('app'); +const api = require('./hackerNewsApi'); +const transform = require('./transform'); +const { stories, posts, sequelize } = require('./models'); + +const HOWMANY_STORIES = parseInt(process.env.HOWMANY_STORIES, 10) || 1; + +async function loadStory(story) { + const [, isNewStory] = await stories.findOrCreate({ + where: { id: story.id }, + defaults: story, + }); + + const newStoryCount = isNewStory ? 1 : 0; + const promises = []; + let totalPostCount = 0; + let newPostCount = 0; + for await (const data of api.getPostsByStoryIds(story.id)) { + const post = transform.post(data); + if (post !== undefined) { + const p = posts + .findOrCreate({ + where: { id: post.id }, + defaults: post, + }) + // eslint-disable-next-line no-loop-func + .then(([, isNewPost]) => { + debug( + 'post(%d, new=%s): %s...', + ++totalPostCount, + isNewPost, + post.body.substr(0, 80).replace(/(?:\r\n|\r|\n)/g, ' ') + ); + if (isNewPost) { + newPostCount++; + } + }); + + promises.push(p); + } + } + await Promise.all(promises); + return [totalPostCount, newPostCount, newStoryCount]; +} + +async function main() { + try { + let totalStoryCount = 0; + const promises = []; + + for await (const data of api.getWhoIsHiringStories()) { + debug(`story: ${data.title} (ID: ${data.objectID})`); + const story = transform.story(data); + if (story !== undefined) { + // validate story, then insert post + promises.push(loadStory(story)); + + if (++totalStoryCount >= HOWMANY_STORIES) { + break; + } + } + } + + debug('Waiting for tasks to complete...'); + const results = await Promise.all(promises); + const [totalPostCount, newPostCount, newStoryCount] = results.reduce( + (total, current) => { + total[0] += current[0]; + total[1] += current[1]; + total[2] += current[2]; + return total; + }, + [0, 0, 0] + ); + + console.log(`story count (new/total): ${newStoryCount}/${totalStoryCount}`); + console.log(`post count (new/total): ${newPostCount}/${totalPostCount}`); + debug('Done!'); + } catch (err) { + console.error(`MAIN ERR ${err}`); + process.exit(1); + } finally { + sequelize.close(); + } +} + +main(); diff --git a/container-populate-hn-data/src/models/index.js b/container-populate-hn-data/src/models/index.js new file mode 100644 index 0000000..913d5e6 --- /dev/null +++ b/container-populate-hn-data/src/models/index.js @@ -0,0 +1,20 @@ +const debug = require('debug')('app:db'); +const Sequelize = require('sequelize'); + +const sequelize = new Sequelize(process.env.PGDATABASE, process.env.PGUSER, process.env.PGPASSWORD, { + host: process.env.PGHOST, + dialect: 'postgres', + + // see: https://github.com/sequelize/sequelize/issues/8615 + logging: debug.enabled && console.log, +}); + +const db = { + posts: require('./posts')(sequelize, Sequelize), + stories: require('./stories')(sequelize, Sequelize), + sequelize, +}; + +db.posts.belongsTo(db.stories, { as: 'story', foreignKey: 'storyId' }); + +module.exports = db; diff --git a/container-populate-hn-data/src/models/posts.js b/container-populate-hn-data/src/models/posts.js new file mode 100644 index 0000000..f985326 --- /dev/null +++ b/container-populate-hn-data/src/models/posts.js @@ -0,0 +1,47 @@ +module.exports = (sequelize, DataTypes) => + sequelize.define( + 'whoishiring_posts', + { + id: { + type: DataTypes.INTEGER, + allowNull: false, + primaryKey: true, + }, + author: { + type: DataTypes.STRING(50), + allowNull: false, + }, + body: { + type: DataTypes.TEXT, + allowNull: false, + }, + // automatically added with db.posts.belongsTo(db.topics) statement + // storyId: { + // type: DataTypes.INTEGER, + // allowNull: false, + // }, + postedDate: { + type: DataTypes.DATE, + allowNull: false, + }, + remoteFlag: { + type: DataTypes.BOOLEAN, + allowNull: false, + }, + onsiteFlag: { + type: DataTypes.BOOLEAN, + allowNull: false, + }, + internsFlag: { + type: DataTypes.BOOLEAN, + allowNull: false, + }, + visaFlag: { + type: DataTypes.BOOLEAN, + allowNull: false, + }, + }, + { + // no options + } + ); diff --git a/container-populate-hn-data/src/models/stories.js b/container-populate-hn-data/src/models/stories.js new file mode 100644 index 0000000..f7b6a15 --- /dev/null +++ b/container-populate-hn-data/src/models/stories.js @@ -0,0 +1,22 @@ +module.exports = (sequelize, DataTypes) => + sequelize.define( + 'whoishiring_stories', + { + id: { + type: DataTypes.INTEGER, + allowNull: false, + primaryKey: true, + }, + label: { + type: DataTypes.STRING(14), + allowNull: false, + }, + postedDate: { + type: DataTypes.DATE, + allowNull: false, + }, + }, + { + // no options + } + ); diff --git a/container-populate-hn-data/src/transform.js b/container-populate-hn-data/src/transform.js new file mode 100644 index 0000000..37af111 --- /dev/null +++ b/container-populate-hn-data/src/transform.js @@ -0,0 +1,42 @@ +const whoIsHiringRegex = /^Ask HN: Who is hiring\?\s*\(([^)]+)\)\s*$/; + +module.exports.story = function transformStory(story) { + const res = whoIsHiringRegex.exec(story.title); + const label = res[1]; + + if (!label) { + return; + } + + return { + id: parseInt(story.objectID, 10), + label, + postedDate: new Date(story.created_at), + }; +}; + +const remoteRegex = /\bremote\b/i; +const onsiteRegex = /\bon\s*site\b/i; +const internRegex = /\binterns?\b/i; +const visaRegex = /\bvisa\b/i; + +module.exports.post = function transformPost(post) { + const { author, comment_text: body, story_id: storyId, parent_id, created_at, objectID } = post; + + // only examine top level comments + if (storyId !== parent_id) { + return; + } + + return { + id: parseInt(objectID, 10), + author, + body, + storyId, + postedDate: new Date(created_at), + remoteFlag: remoteRegex.test(body), + onsiteFlag: onsiteRegex.test(body), + internsFlag: internRegex.test(body), + visaFlag: visaRegex.test(body), + }; +}; diff --git a/container-populate-hn-data/wait-for-it.sh b/container-populate-hn-data/wait-for-it.sh new file mode 100644 index 0000000..607a7d6 --- /dev/null +++ b/container-populate-hn-data/wait-for-it.sh @@ -0,0 +1,178 @@ +#!/usr/bin/env bash +# Use this script to test if a given TCP host/port are available + +WAITFORIT_cmdname=${0##*/} + +echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi } + +usage() +{ + cat << USAGE >&2 +Usage: + $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args] + -h HOST | --host=HOST Host or IP under test + -p PORT | --port=PORT TCP port under test + Alternatively, you specify the host and port as host:port + -s | --strict Only execute subcommand if the test succeeds + -q | --quiet Don't output any status messages + -t TIMEOUT | --timeout=TIMEOUT + Timeout in seconds, zero for no timeout + -- COMMAND ARGS Execute command with args after the test finishes +USAGE + exit 1 +} + +wait_for() +{ + if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then + echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" + else + echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout" + fi + WAITFORIT_start_ts=$(date +%s) + while : + do + if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then + nc -z $WAITFORIT_HOST $WAITFORIT_PORT + WAITFORIT_result=$? + else + (echo > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1 + WAITFORIT_result=$? + fi + if [[ $WAITFORIT_result -eq 0 ]]; then + WAITFORIT_end_ts=$(date +%s) + echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds" + break + fi + sleep 1 + done + return $WAITFORIT_result +} + +wait_for_wrapper() +{ + # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692 + if [[ $WAITFORIT_QUIET -eq 1 ]]; then + timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & + else + timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & + fi + WAITFORIT_PID=$! + trap "kill -INT -$WAITFORIT_PID" INT + wait $WAITFORIT_PID + WAITFORIT_RESULT=$? + if [[ $WAITFORIT_RESULT -ne 0 ]]; then + echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" + fi + return $WAITFORIT_RESULT +} + +# process arguments +while [[ $# -gt 0 ]] +do + case "$1" in + *:* ) + WAITFORIT_hostport=(${1//:/ }) + WAITFORIT_HOST=${WAITFORIT_hostport[0]} + WAITFORIT_PORT=${WAITFORIT_hostport[1]} + shift 1 + ;; + --child) + WAITFORIT_CHILD=1 + shift 1 + ;; + -q | --quiet) + WAITFORIT_QUIET=1 + shift 1 + ;; + -s | --strict) + WAITFORIT_STRICT=1 + shift 1 + ;; + -h) + WAITFORIT_HOST="$2" + if [[ $WAITFORIT_HOST == "" ]]; then break; fi + shift 2 + ;; + --host=*) + WAITFORIT_HOST="${1#*=}" + shift 1 + ;; + -p) + WAITFORIT_PORT="$2" + if [[ $WAITFORIT_PORT == "" ]]; then break; fi + shift 2 + ;; + --port=*) + WAITFORIT_PORT="${1#*=}" + shift 1 + ;; + -t) + WAITFORIT_TIMEOUT="$2" + if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi + shift 2 + ;; + --timeout=*) + WAITFORIT_TIMEOUT="${1#*=}" + shift 1 + ;; + --) + shift + WAITFORIT_CLI=("$@") + break + ;; + --help) + usage + ;; + *) + echoerr "Unknown argument: $1" + usage + ;; + esac +done + +if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then + echoerr "Error: you need to provide a host and port to test." + usage +fi + +WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15} +WAITFORIT_STRICT=${WAITFORIT_STRICT:-0} +WAITFORIT_CHILD=${WAITFORIT_CHILD:-0} +WAITFORIT_QUIET=${WAITFORIT_QUIET:-0} + +# check to see if timeout is from busybox? +WAITFORIT_TIMEOUT_PATH=$(type -p timeout) +WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH) +if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then + WAITFORIT_ISBUSY=1 + WAITFORIT_BUSYTIMEFLAG="-t" + +else + WAITFORIT_ISBUSY=0 + WAITFORIT_BUSYTIMEFLAG="" +fi + +if [[ $WAITFORIT_CHILD -gt 0 ]]; then + wait_for + WAITFORIT_RESULT=$? + exit $WAITFORIT_RESULT +else + if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then + wait_for_wrapper + WAITFORIT_RESULT=$? + else + wait_for + WAITFORIT_RESULT=$? + fi +fi + +if [[ $WAITFORIT_CLI != "" ]]; then + if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then + echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess" + exit $WAITFORIT_RESULT + fi + exec "${WAITFORIT_CLI[@]}" +else + exit $WAITFORIT_RESULT +fi \ No newline at end of file diff --git a/container-provision-db/Dockerfile b/container-provision-db/Dockerfile new file mode 100644 index 0000000..4c9560d --- /dev/null +++ b/container-provision-db/Dockerfile @@ -0,0 +1,19 @@ +FROM mhart/alpine-node:12 + +ENV container=docker NODE_ENV=production +WORKDIR /app + +RUN apk add --no-cache tini bash \ + && addgroup -g 1000 node \ + && adduser -u 1000 -G node -s /bin/sh -D node +ENTRYPOINT ["/sbin/tini", "--"] + +COPY package.json package-lock.json ./ +RUN npm ci --prod + +COPY wait-for-it.sh index.js ./ +RUN chmod +x wait-for-it.sh + +USER node + +CMD ./wait-for-it.sh $PGHOST:$PGPORT -- node index.js diff --git a/container-provision-db/index.js b/container-provision-db/index.js new file mode 100644 index 0000000..e33ff98 --- /dev/null +++ b/container-provision-db/index.js @@ -0,0 +1,117 @@ +const { Client } = require('pg'); + +async function createDatabaseIfNotExists(dbname) { + let pg; + try { + // must connect to maintenance db in case $dbname database does not exist + pg = new Client({ database: 'postgres' }); + await pg.connect(); + + const res = await pg.query(`SELECT FROM pg_database WHERE datname = '${dbname}'`); + + const exists = res.rowCount !== 0; + console.log(`Database "${dbname}" already exist? ${exists}`); + + if (!exists) { + await pg.query(`CREATE DATABASE ${dbname}`); + } + } finally { + if (pg) { + await pg.end(); + pg = undefined; + } + } +} + +async function createUserIfNotExists(pg, name, password, roles = []) { + const res = await pg.query('SELECT FROM pg_user WHERE usename = $1', [name]); + + const exists = res.rowCount !== 0; + console.log(`User "${name}" already exists? ${exists}`); + + if (!exists) { + await pg.query(`CREATE USER ${name} WITH ${roles.join(' ')} PASSWORD '${password}'`); + } +} + +async function createTableIfNotExists(pg, name, fields) { + const res = await pg.query('SELECT FROM pg_tables WHERE tablename = $1', [name]); + + const exists = res.rowCount !== 0; + console.log(`Table "${name}" already exists? ${exists}`); + + if (!exists) { + await pg.query(`CREATE TABLE "${name}" (${fields})`); + } +} + +function dropTable(pg, name) { + console.log(`Dropping table "${name}"...`); + return pg.query(`DROP TABLE IF EXISTS ${name}`); +} + +async function main() { + let pg; + const dbname = process.env.PGDATABASE; + + await createDatabaseIfNotExists(dbname); + + try { + console.log(`Connecting to database "${dbname}"...`); + pg = new Client(); + await pg.connect(); + + await createUserIfNotExists(pg, process.env.DB_USER, process.env.DB_USER_PASSWORD); + + if (process.env.DROP_TABLES) { + await dropTable(pg, 'whoishiring_posts'); + await dropTable(pg, 'whoishiring_stories'); + } + + await createTableIfNotExists( + pg, + 'whoishiring_stories', + `id INTEGER PRIMARY KEY, + "label" varchar(14) NOT NULL, + "postedDate" TIMESTAMP NOT NULL, + "createdAt" TIMESTAMP NOT NULL, + "updatedAt" TIMESTAMP NOT NULL` + ); + + await createTableIfNotExists( + pg, + 'whoishiring_posts', + `id INTEGER PRIMARY KEY, + "author" varchar(50) NOT NULL, + "body" text NOT NULL, + "storyId" INTEGER REFERENCES "whoishiring_stories"("id") ON DELETE RESTRICT NOT NULL, + "remoteFlag" BOOL NOT NULL, + "onsiteFlag" BOOL NOT NULL, + "internsFlag" BOOL NOT NULL, + "visaFlag" BOOL NOT NULL, + "postedDate" timestamp NOT NULL, + "createdAt" TIMESTAMP NOT NULL, + "updatedAt" TIMESTAMP NOT NULL` + ); + + // allow RW on tables for the DB_USER + await pg.query( + ['whoishiring_stories', 'whoishiring_posts'] + .map((x) => `GRANT ALL PRIVILEGES ON ${x} TO ${process.env.DB_USER}`) + .join(';') + ); + } catch (err) { + console.error(err); + process.exit(1); + } finally { + if (pg) { + console.log(`Closing database "${dbname}"...`); + pg.end(); + pg = undefined; + } + } + + console.log('Done!'); +} + +main(); diff --git a/container-provision-db/package-lock.json b/container-provision-db/package-lock.json new file mode 100644 index 0000000..fb8e776 --- /dev/null +++ b/container-provision-db/package-lock.json @@ -0,0 +1,119 @@ +{ + "name": "@whoishiring.work/provision-db", + "version": "1.0.0", + "lockfileVersion": 1, + "requires": true, + "dependencies": { + "buffer-writer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-2.0.0.tgz", + "integrity": "sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw==" + }, + "packet-reader": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/packet-reader/-/packet-reader-1.0.0.tgz", + "integrity": "sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ==" + }, + "pg": { + "version": "7.16.1", + "resolved": "https://registry.npmjs.org/pg/-/pg-7.16.1.tgz", + "integrity": "sha512-eN2WCYY8XBOKfZ2CZir4DCrPYEEK1g6yh5AIo3OMHMSkqNrk+OBJG4/e9EW54LfAkk96MWvtsvR1hbvDHhcVkQ==", + "requires": { + "buffer-writer": "2.0.0", + "packet-reader": "1.0.0", + "pg-connection-string": "0.1.3", + "pg-packet-stream": "^1.1.0", + "pg-pool": "^2.0.9", + "pg-types": "^2.1.0", + "pgpass": "1.x", + "semver": "4.3.2" + } + }, + "pg-connection-string": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-0.1.3.tgz", + "integrity": "sha1-2hhHsglA5C7hSSvq9l1J2RskXfc=" + }, + "pg-int8": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", + "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==" + }, + "pg-packet-stream": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/pg-packet-stream/-/pg-packet-stream-1.1.0.tgz", + "integrity": "sha512-kRBH0tDIW/8lfnnOyTwKD23ygJ/kexQVXZs7gEyBljw4FYqimZFxnMMx50ndZ8In77QgfGuItS5LLclC2TtjYg==" + }, + "pg-pool": { + "version": "2.0.9", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-2.0.9.tgz", + "integrity": "sha512-gNiuIEKNCT3OnudQM2kvgSnXsLkSpd6mS/fRnqs6ANtrke6j8OY5l9mnAryf1kgwJMWLg0C1N1cYTZG1xmEYHQ==" + }, + "pg-types": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", + "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", + "requires": { + "pg-int8": "1.0.1", + "postgres-array": "~2.0.0", + "postgres-bytea": "~1.0.0", + "postgres-date": "~1.0.4", + "postgres-interval": "^1.1.0" + } + }, + "pgpass": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.2.tgz", + "integrity": "sha1-Knu0G2BltnkH6R2hsHwYR8h3swY=", + "requires": { + "split": "^1.0.0" + } + }, + "postgres-array": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", + "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==" + }, + "postgres-bytea": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.0.tgz", + "integrity": "sha1-AntTPAqokOJtFy1Hz5zOzFIazTU=" + }, + "postgres-date": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.4.tgz", + "integrity": "sha512-bESRvKVuTrjoBluEcpv2346+6kgB7UlnqWZsnbnCccTNq/pqfj1j6oBaN5+b/NrDXepYUT/HKadqv3iS9lJuVA==" + }, + "postgres-interval": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", + "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", + "requires": { + "xtend": "^4.0.0" + } + }, + "semver": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-4.3.2.tgz", + "integrity": "sha1-x6BxWKgL7dBSNVt3DYLWZA+AO+c=" + }, + "split": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/split/-/split-1.0.1.tgz", + "integrity": "sha512-mTyOoPbrivtXnwnIxZRFYRrPNtEFKlpB2fvjSnCQUiAA6qAZzqwna5envK4uk6OIeP17CsdF3rSBGYVBsU0Tkg==", + "requires": { + "through": "2" + } + }, + "through": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", + "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=" + }, + "xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==" + } + } +} diff --git a/container-provision-db/package.json b/container-provision-db/package.json new file mode 100644 index 0000000..cdd8690 --- /dev/null +++ b/container-provision-db/package.json @@ -0,0 +1,19 @@ +{ + "name": "@whoishiring.work/provision-db", + "version": "1.0.0", + "private": true, + "description": "Provisions the whoishiring database and tables", + "license": "MIT", + "repository": "https://github.com/jerroydmoore/whoishiring.work.git", + "type": "commonjs", + "engines": { + "node": ">=12.13.0" + }, + "scripts": { + "start": "node ./index.js" + }, + "dependencies": { + "pg": "^7.16.1" + }, + "devDependencies": {} +} diff --git a/container-provision-db/wait-for-it.sh b/container-provision-db/wait-for-it.sh new file mode 100644 index 0000000..607a7d6 --- /dev/null +++ b/container-provision-db/wait-for-it.sh @@ -0,0 +1,178 @@ +#!/usr/bin/env bash +# Use this script to test if a given TCP host/port are available + +WAITFORIT_cmdname=${0##*/} + +echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi } + +usage() +{ + cat << USAGE >&2 +Usage: + $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args] + -h HOST | --host=HOST Host or IP under test + -p PORT | --port=PORT TCP port under test + Alternatively, you specify the host and port as host:port + -s | --strict Only execute subcommand if the test succeeds + -q | --quiet Don't output any status messages + -t TIMEOUT | --timeout=TIMEOUT + Timeout in seconds, zero for no timeout + -- COMMAND ARGS Execute command with args after the test finishes +USAGE + exit 1 +} + +wait_for() +{ + if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then + echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" + else + echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout" + fi + WAITFORIT_start_ts=$(date +%s) + while : + do + if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then + nc -z $WAITFORIT_HOST $WAITFORIT_PORT + WAITFORIT_result=$? + else + (echo > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1 + WAITFORIT_result=$? + fi + if [[ $WAITFORIT_result -eq 0 ]]; then + WAITFORIT_end_ts=$(date +%s) + echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds" + break + fi + sleep 1 + done + return $WAITFORIT_result +} + +wait_for_wrapper() +{ + # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692 + if [[ $WAITFORIT_QUIET -eq 1 ]]; then + timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & + else + timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & + fi + WAITFORIT_PID=$! + trap "kill -INT -$WAITFORIT_PID" INT + wait $WAITFORIT_PID + WAITFORIT_RESULT=$? + if [[ $WAITFORIT_RESULT -ne 0 ]]; then + echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" + fi + return $WAITFORIT_RESULT +} + +# process arguments +while [[ $# -gt 0 ]] +do + case "$1" in + *:* ) + WAITFORIT_hostport=(${1//:/ }) + WAITFORIT_HOST=${WAITFORIT_hostport[0]} + WAITFORIT_PORT=${WAITFORIT_hostport[1]} + shift 1 + ;; + --child) + WAITFORIT_CHILD=1 + shift 1 + ;; + -q | --quiet) + WAITFORIT_QUIET=1 + shift 1 + ;; + -s | --strict) + WAITFORIT_STRICT=1 + shift 1 + ;; + -h) + WAITFORIT_HOST="$2" + if [[ $WAITFORIT_HOST == "" ]]; then break; fi + shift 2 + ;; + --host=*) + WAITFORIT_HOST="${1#*=}" + shift 1 + ;; + -p) + WAITFORIT_PORT="$2" + if [[ $WAITFORIT_PORT == "" ]]; then break; fi + shift 2 + ;; + --port=*) + WAITFORIT_PORT="${1#*=}" + shift 1 + ;; + -t) + WAITFORIT_TIMEOUT="$2" + if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi + shift 2 + ;; + --timeout=*) + WAITFORIT_TIMEOUT="${1#*=}" + shift 1 + ;; + --) + shift + WAITFORIT_CLI=("$@") + break + ;; + --help) + usage + ;; + *) + echoerr "Unknown argument: $1" + usage + ;; + esac +done + +if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then + echoerr "Error: you need to provide a host and port to test." + usage +fi + +WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15} +WAITFORIT_STRICT=${WAITFORIT_STRICT:-0} +WAITFORIT_CHILD=${WAITFORIT_CHILD:-0} +WAITFORIT_QUIET=${WAITFORIT_QUIET:-0} + +# check to see if timeout is from busybox? +WAITFORIT_TIMEOUT_PATH=$(type -p timeout) +WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH) +if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then + WAITFORIT_ISBUSY=1 + WAITFORIT_BUSYTIMEFLAG="-t" + +else + WAITFORIT_ISBUSY=0 + WAITFORIT_BUSYTIMEFLAG="" +fi + +if [[ $WAITFORIT_CHILD -gt 0 ]]; then + wait_for + WAITFORIT_RESULT=$? + exit $WAITFORIT_RESULT +else + if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then + wait_for_wrapper + WAITFORIT_RESULT=$? + else + wait_for + WAITFORIT_RESULT=$? + fi +fi + +if [[ $WAITFORIT_CLI != "" ]]; then + if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then + echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess" + exit $WAITFORIT_RESULT + fi + exec "${WAITFORIT_CLI[@]}" +else + exit $WAITFORIT_RESULT +fi \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..cf9a44f --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,30 @@ +version: "3" + +services: + populate-hn-data: + build: ./container-populate-hn-data + environment: + - PGUSER=${DB_USER} + - PGPASSWORD=${DB_USER_PASSWORD} + - PGHOST + - PGDATABASE + - PGPORT + - HOWMANY_STORIES=2 + + postgres: + image: postgres:12 + environment: + - POSTGRES_PASSWORD=${DB_ADMIN_PASSWORD} + ports: + - "5432:5432" + + provision-db: + build: ./container-provision-db + environment: + - PGUSER=${DB_ADMIN} + - PGPASSWORD=${DB_ADMIN_PASSWORD} + - DB_USER + - DB_USER_PASSWORD + - PGHOST + - PGDATABASE + - PGPORT diff --git a/package-lock.json b/package-lock.json index 34e9cde..d983af6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,5 +1,5 @@ { - "name": "boilerplate-nodejs-app", + "name": "whoishiring.work", "version": "1.0.0", "lockfileVersion": 1, "requires": true, @@ -418,13 +418,6 @@ "@types/yargs": "^13.0.0" } }, - "@korzio/djv-draft-04": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/@korzio/djv-draft-04/-/djv-draft-04-2.0.1.tgz", - "integrity": "sha512-MeTVcNsfCIYxK6T7jW1sroC7dBAb4IfLmQe6RoCqlxHN5NFkzNpgdnBPR+/0D2wJDUJHM9s9NQv+ouhxKjvUjg==", - "dev": true, - "optional": true - }, "@nodelib/fs.scandir": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.3.tgz", @@ -556,9 +549,9 @@ "dev": true }, "@types/node": { - "version": "13.1.1", - "resolved": "https://registry.npmjs.org/@types/node/-/node-13.1.1.tgz", - "integrity": "sha512-hx6zWtudh3Arsbl3cXay+JnkvVgCKzCWKv42C9J01N2T2np4h8w5X8u6Tpz5mj38kE3M9FM0Pazx8vKFFMnjLQ==", + "version": "13.1.2", + "resolved": "https://registry.npmjs.org/@types/node/-/node-13.1.2.tgz", + "integrity": "sha512-B8emQA1qeKerqd1dmIsQYnXi+mmAzTB7flExjmy5X1aVAKFNNNDubkavwR13kR6JnpeLp3aLoJhwn9trWPAyFQ==", "dev": true }, "@types/normalize-package-data": { @@ -829,36 +822,6 @@ "integrity": "sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg==", "dev": true }, - "audit-resolve-core": { - "version": "1.1.7", - "resolved": "https://registry.npmjs.org/audit-resolve-core/-/audit-resolve-core-1.1.7.tgz", - "integrity": "sha512-9nLm9SgyMbMv86X5a/E6spcu3V+suceHF6Pg4BwjPqfxWBKDvISagJH9Ji592KihqBev4guKFO3BiNEVNnqh3A==", - "dev": true, - "requires": { - "concat-stream": "^1.6.2", - "debug": "^4.1.1", - "djv": "^2.1.2", - "spawn-shell": "^2.1.0", - "yargs-parser": "^10.1.0" - }, - "dependencies": { - "camelcase": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-4.1.0.tgz", - "integrity": "sha1-1UVjW+HjPFQmScaRc+Xeas+uNN0=", - "dev": true - }, - "yargs-parser": { - "version": "10.1.0", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-10.1.0.tgz", - "integrity": "sha512-VCIyR1wJoEBZUqk5PA+oOBF6ypbwh5aNB3I50guxAL/quggdfs4TtNHQrSazFA3fYZ+tEqfs0zIGlv0c/rgjbQ==", - "dev": true, - "requires": { - "camelcase": "^4.1.0" - } - } - } - }, "aws-sign2": { "version": "0.7.0", "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz", @@ -1431,18 +1394,6 @@ "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", "dev": true }, - "concat-stream": { - "version": "1.6.2", - "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz", - "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==", - "dev": true, - "requires": { - "buffer-from": "^1.0.0", - "inherits": "^2.0.3", - "readable-stream": "^2.2.2", - "typedarray": "^0.0.6" - } - }, "contains-path": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/contains-path/-/contains-path-0.1.0.tgz", @@ -1589,7 +1540,6 @@ "version": "4.1.1", "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", - "dev": true, "requires": { "ms": "^2.1.1" } @@ -1618,12 +1568,6 @@ "integrity": "sha1-s2nW+128E+7PUk+RsHD+7cNXzzQ=", "dev": true }, - "default-shell": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/default-shell/-/default-shell-1.0.1.tgz", - "integrity": "sha1-dSMEvdxhdPSespy5iP7qC4gTyLw=", - "dev": true - }, "define-properties": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz", @@ -1736,15 +1680,6 @@ } } }, - "djv": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/djv/-/djv-2.1.2.tgz", - "integrity": "sha512-ltQSINn+7aMTp7pKeQpfZg2ACd/Gy6VrL3LYuT25/plwPBb7xlGOekr463Luqn816AWJLuP7KZQGFct2JICyeA==", - "dev": true, - "requires": { - "@korzio/djv-draft-04": "^2.0.1" - } - }, "doctrine": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz", @@ -4253,12 +4188,6 @@ "integrity": "sha512-/2UGPSgmtqwo1ktx8NDHjuPwZWmHhO+gj0f93EkhLB5RgW9RZevWYYlIkS6zePc6U2WpOdQYIwHe9YC4DWEBVg==", "dev": true }, - "is-plain-obj": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-1.1.0.tgz", - "integrity": "sha1-caUMhCnfync8kqOQpKA7OfzVHT4=", - "dev": true - }, "is-plain-object": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz", @@ -4998,12 +4927,6 @@ } } }, - "jsonlines": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/jsonlines/-/jsonlines-0.1.1.tgz", - "integrity": "sha1-T80kbcXQ44aRkHxEqwAveC0dlMw=", - "dev": true - }, "jsprim": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz", @@ -5588,15 +5511,6 @@ "object-visit": "^1.0.0" } }, - "merge-options": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/merge-options/-/merge-options-1.0.1.tgz", - "integrity": "sha512-iuPV41VWKWBIOpBsjoxjDZw8/GbSfZ2mk7N1453bwMrfzdrIk7EzBd+8UVR6rkw67th7xnk9Dytl3J+lHPdxvg==", - "dev": true, - "requires": { - "is-plain-obj": "^1.1" - } - }, "merge-stream": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", @@ -5699,8 +5613,7 @@ "ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", - "dev": true + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, "mute-stream": { "version": "0.0.8", @@ -5752,6 +5665,11 @@ "integrity": "sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==", "dev": true }, + "node-fetch": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.0.tgz", + "integrity": "sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA==" + }, "node-int64": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", @@ -5814,21 +5732,6 @@ "remove-trailing-separator": "^1.0.1" } }, - "npm-audit-resolver": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/npm-audit-resolver/-/npm-audit-resolver-2.1.0.tgz", - "integrity": "sha512-8VaG7p3tbP0+JbpDKldQueZvh9oUcr3H/C2FIbcIhYBANAQ1kCIhUqYOxVFjG3RKEV9G1coIFzUOxBoPyyejNg==", - "dev": true, - "requires": { - "audit-resolve-core": "^1.1.7", - "chalk": "^2.4.2", - "djv": "^2.1.2", - "jsonlines": "^0.1.1", - "read": "^1.0.7", - "spawn-shell": "^2.1.0", - "yargs-parser": "^13.1.1" - } - }, "npm-run-path": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-2.0.2.tgz", @@ -6223,12 +6126,6 @@ } } }, - "process-nextick-args": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", - "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", - "dev": true - }, "progress": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", @@ -6279,15 +6176,6 @@ "integrity": "sha512-rPCkf/mWBtKc97aLL9/txD8DZdemK0vkA3JMLShjlJB3Pj3s+lpf1KaBzMfQrAmhMQB0n1cU/SUGgKKBCe837Q==", "dev": true }, - "read": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/read/-/read-1.0.7.tgz", - "integrity": "sha1-s9oZvQUkMal2cdRKQmNK33ELQMQ=", - "dev": true, - "requires": { - "mute-stream": "~0.0.4" - } - }, "read-pkg": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-2.0.0.tgz", @@ -6309,21 +6197,6 @@ "read-pkg": "^2.0.0" } }, - "readable-stream": { - "version": "2.3.6", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", - "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", - "dev": true, - "requires": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.3", - "isarray": "~1.0.0", - "process-nextick-args": "~2.0.0", - "safe-buffer": "~5.1.1", - "string_decoder": "~1.1.1", - "util-deprecate": "~1.0.1" - } - }, "realpath-native": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/realpath-native/-/realpath-native-1.1.0.tgz", @@ -6866,17 +6739,6 @@ "integrity": "sha1-PpNdfd1zYxuXZZlW1VEo6HtQhKM=", "dev": true }, - "spawn-shell": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/spawn-shell/-/spawn-shell-2.1.0.tgz", - "integrity": "sha512-mjlYAQbZPHd4YsoHEe+i0Xbp9sJefMKN09JPp80TqrjC5NSuo+y1RG3NBireJlzl1dDV2NIkIfgS6coXtyqN/A==", - "dev": true, - "requires": { - "default-shell": "^1.0.1", - "merge-options": "~1.0.1", - "npm-run-path": "^2.0.2" - } - }, "spdx-correct": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/spdx-correct/-/spdx-correct-3.1.0.tgz", @@ -7049,15 +6911,6 @@ "function-bind": "^1.1.1" } }, - "string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "dev": true, - "requires": { - "safe-buffer": "~5.1.0" - } - }, "stringify-object": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/stringify-object/-/stringify-object-3.3.0.tgz", @@ -7428,12 +7281,6 @@ "integrity": "sha512-4dbzIzqvjtgiM5rw1k5rEHtBANKmdudhGyBEajN01fEyhaAIhsoKNy6y7+IN93IfpFtwY9iqi7kD+xwKhQsNJA==", "dev": true }, - "typedarray": { - "version": "0.0.6", - "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", - "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=", - "dev": true - }, "uglify-js": { "version": "3.7.3", "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.7.3.tgz", @@ -7527,12 +7374,6 @@ "integrity": "sha512-cwESVXlO3url9YWlFW/TA9cshCEhtu7IKJ/p5soJ/gGpj7vbvFrAY/eIioQ6Dw23KjZhYgiIo8HOs1nQ2vr/oQ==", "dev": true }, - "util-deprecate": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", - "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=", - "dev": true - }, "util.promisify": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/util.promisify/-/util.promisify-1.0.0.tgz", diff --git a/package.json b/package.json index 40a56a7..402e292 100644 --- a/package.json +++ b/package.json @@ -1,31 +1,30 @@ { - "name": "boilerplate-nodejs-app", + "name": "whoishiring.work", "version": "1.0.0", "private": true, - "description": "Boilerplate Code for a NodeJS App", + "description": "Code for the website whoishiring.work", "license": "MIT", - "repository": "https://github.com/jerroydmoore/boilerplate-nodejs-app.git", + "repository": "https://github.com/jerroydmoore/whoishiring.work.git", "type": "commonjs", "engines": { "node": ">=12.13.0" }, "scripts": { - "audit": "check-audit --ignoreLow", - "audit:resolve": "resolve-audit --ignoreLow", - "lint": "eslint *.js src/ __tests__/", - "test": "jest", - "start": "node src/index.js" + "lint": "eslint *.js container-provision-db/ container-populate-hn-data/", + "test": "jest" + }, + "dependencies": { + "debug": "^4.1.1", + "node-fetch": "^2.6.0" }, - "dependencies": {}, "devDependencies": { "eslint": "^6.6.0", "eslint-config-node": "^4.0.0", "eslint-plugin-jest": "^23.0.4", "eslint-plugin-node": "^10.0.0", - "husky": "^3.0.9", + "husky": "^3.1.0", "jest": "^24.8.0", - "lint-staged": "^9.4.3", - "npm-audit-resolver": "^2.1.0", + "lint-staged": "^9.5.0", "prettier": "^1.19.1" }, "husky": { diff --git a/src/index.js b/src/index.js deleted file mode 100644 index 89a111c..0000000 --- a/src/index.js +++ /dev/null @@ -1 +0,0 @@ -process.stdout.write('Hello World!\n');