From c5800faf854271a4a44859e1f17d805cc2c21c85 Mon Sep 17 00:00:00 2001 From: Vincent Weevers Date: Sat, 9 Nov 2019 19:32:29 +0100 Subject: [PATCH 1/8] Rewrite as node cli using prebuild/docker-images --- .dockerignore | 4 - .gitignore | 3 +- .npmrc | 1 + Dockerfile | 20 ---- README.md | 100 ++++++------------ build | 256 ---------------------------------------------- build-in-docker | 19 ---- cli.js | 13 +++ guest.js | 69 +++++++++++++ index.js | 118 +++++++++++++++++++++ package-lock.json | 5 - package.json | 37 +++++-- 12 files changed, 259 insertions(+), 386 deletions(-) delete mode 100644 .dockerignore create mode 100644 .npmrc delete mode 100644 Dockerfile delete mode 100755 build delete mode 100755 build-in-docker create mode 100644 cli.js create mode 100644 guest.js create mode 100644 index.js delete mode 100644 package-lock.json diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index 9823984..0000000 --- a/.dockerignore +++ /dev/null @@ -1,4 +0,0 @@ -/examples -/output/ -/apt-cacher-ng/ -/.git diff --git a/.gitignore b/.gitignore index 43f6e73..239ecff 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,2 @@ -/output/ -/apt-cacher-ng/ node_modules +yarn.lock diff --git a/.npmrc b/.npmrc new file mode 100644 index 0000000..9cf9495 --- /dev/null +++ b/.npmrc @@ -0,0 +1 @@ +package-lock=false \ No newline at end of file diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index d11655a..0000000 --- a/Dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -ARG TARGET - -FROM dockcross/${TARGET} - -RUN apt-get -y update && \ - apt-get -y --no-install-recommends install \ - git curl gnupg apt-transport-https \ - && \ - curl -sSL https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - && \ - echo "deb https://deb.nodesource.com/node_10.x buster main" | tee /etc/apt/sources.list.d/nodesource.list && \ - echo "deb-src https://deb.nodesource.com/node_10.x buster main" | tee -a /etc/apt/sources.list.d/nodesource.list && \ - apt-get -y update && \ - apt-get -y install nodejs && \ - rm -rf /var/lib/apt/lists/* - -ENV STRIP ${CROSS_ROOT}/bin/${CROSS_TRIPLE}-strip - -COPY ./build-in-docker /app/ - -VOLUME ["/app/input"] diff --git a/README.md b/README.md index 735d6b9..f0c786a 100644 --- a/README.md +++ b/README.md @@ -1,105 +1,68 @@ # prebuildify-cross -cross-compile [prebuild](https://github.com/mafintosh/prebuildify)s +**Compile prebuilds in Docker, supporting Linux (including Debian 8, Ubuntu 14.04, RHEL 7, CentOS 7 and up), Alpine Linux, ARM Linux devices like the Raspberry Pi and mobile ARM devices like Android.** -## background +Runs [`prebuildify`](https://github.com/mafintosh/prebuildify) in the preconfigured [`prebuild/docker-images`](https://github.com/prebuild/docker-images) to compile and name prebuilds for a certain platform. This means you don't have to worry about GCC flags, environment variables or system dependencies. In addition, `prebuildify-cross` copies only npm package files to Docker (following the rules of `.npmignore` and `files`) and mounts `node_modules` removing the need for a repeated `npm install`. -i want to build native modules for [Scuttlebutt](https://scuttlebutt.nz) pubs, to use on ARM Linux devices like the Raspberry Pi. meanwhile, i also want to support mobile ARM devices like Android. +## Install -## how does it work? +Depends on having Docker installed, as well as `prebuildify` and `node-gyp`: -`prebuildify-cross` will: - -- build a Docker image for your intended target, based on the respective [`dockcross`](https://github.com/dockcross/dockcross) image -- create a Docker container with your input (default is `.`) mounted inside -- run `npm install --ignore-scripts` and `npm run prebuild` -- copy out `./prebuilds` from the container to your output (default is `./prebuilds`) - -## supported targets - -- `prebuildify-cross --platform=linux --arch=x32` -- `prebuildify-cross --platform=linux --arch=x64` -- `prebuildify-cross --platform=linux --arch=arm --arm-version=5` -- `prebuildify-cross --platform=linux --arch=arm --arm-version=6` -- `prebuildify-cross --platform=linux --arch=arm --arm-version=7` -- `prebuildify-cross --platform=linux --arch=arm64` -- `prebuildify-cross --platform=android --arch=arm --arm-version=7` -- `prebuildify-cross --platform=android --arch=arm64` - -## usage - -(note: `prebuildify-cross` depends on having Docker installed.) +``` +npm install --save-dev prebuildify node-gyp prebuildify-cross +``` -in the module you want to cross-compile prebuilds, +## Usage -ensure you have an npm script `prebuild`, like: +The `prebuildify-cross` cli wraps `prebuildify` and passes all command line arguments on to `prebuildify` or `npm run prebuild` if such a script exists in your `package.json`. For example, with the following setup you could run `prebuildify-cross` without arguments: -``` +```json { "scripts": { - "prebuild": "prebuildify --all --strip" + "prebuild": "prebuildify -t 8.14.0 --napi --strip" } } ``` -then install `prebuildify-cross` as a dev-dependency: +Without that script, you could run `prebuildify-cross -t 8.14.0 --napi --strip` to the same effect. If you combine both approaches, arguments will be concatenated. -``` -npm install --save-dev prebuildify-cross -``` - -then add new `prebuild:cross:${TARGET}` scripts for the targets you want to support: +By default `prebuildify-cross` makes prebuilds for [all platforms](#images). To override, pass one or more `--image` or `-i` arguments: ``` -{ - "scripts": { - "prebuild:cross:linux-armv7": "prebuildify-cross --platform linux --arch arm --arm-version 7", - "prebuild:cross:android-armv7": "prebuildify-cross --platform android --arch arm --arm-version 7" - } -} +prebuildify-cross -i linux -i alpine [..] ``` -then when you want to cross-compile prebuilds, `npm run` the appropriate script. +This is the only argument specific to `prebuildify-cross`, which does however respect the `--cwd` argument of `prebuildify`. -for the full command-line usage: +## Images -``` -Usage: - prebuildify-cross [options] - - Arguments: +### `centos7-devtoolset7` - --arch : **required** architecture (supported: x32, x64, arm, arm64) - --arm-version : if using arm architecture, **required* arm version (supported: 5, 6, 7, 8) - --platform: **required** platform (supported: linux, android) +Aliased as `linux`. - --input : _optional_ directory of input image spec (default: cwd) - --output : _optional_ directory to output build results (default: prebuilds) +Compile in CentOS 7, as a better alternative to (most commonly) Ubuntu on Travis. Makes the prebuilt binary compatible with Debian 8, Ubuntu 14.04, RHEL 7, CentOS 7 and other Linux flavors with an old glibc. - Flags: +By default the prebuild will be [tagged](https://github.com/prebuild/prebuildify#options) with the libc flavor to set it apart from Alpine prebuilds, e.g. `linux-x64/node.libc.node`. - -h, --help: show this usage +### `alpine` - Examples: +Compile in Alpine, which uses musl instead of glibc and therefore can't run regular linux prebuilds. Worse, it sometimes does successfully _load_ such a prebuild during `npm install` - which prevents a compilation fallback from kicking in - and then segfaults at runtime. - prebuildify-cross --platform linux --arch x64 +By default the prebuild will be [tagged](https://github.com/prebuild/prebuildify#options) with the libc flavor, e.g. `linux-x64/node.musl.node`. - prebuildify-cross --platform linux --arch arm --arm-version 7 +### `linux-armv7` and `linux-arm64` - prebuildify-cross --platform linux --arch arm64 +Cross-compile for Linux ARM. These images thinly wrap [`dockcross`](https://github.com/dockcross/dockcross) images. - prebuildify-cross --platform android --arch arm --arm-version 7 +By default the prebuild will be [tagged](https://github.com/prebuild/prebuildify#options) with the armv version (7 or 8, respectively). - prebuildify-cross --platform android --arch arm64 -``` +### `android-armv7` and `android-arm64` -you can also pass in environment variables instead of command-line arguments, e.g. +Cross-compile for Android ARM. These images thinly wrap [`dockcross`](https://github.com/dockcross/dockcross) images. -```shell -PLATFORM=linux ARCH=arm ARM_VERSION=7 prebuildify-cross -``` +By default the prebuild will be [tagged](https://github.com/prebuild/prebuildify#options) with the armv version (7 or 8, respectively). -## references +## References - [Debian multiarch tuples](https://wiki.debian.org/Multiarch/Tuples) - [Rust support tuples](https://forge.rust-lang.org/platform-support.html) @@ -107,8 +70,7 @@ PLATFORM=linux ARCH=arm ARM_VERSION=7 prebuildify-cross - [Arm options](https://gcc.gnu.org/onlinedocs/gcc/ARM-Options.html) - [Rust cross-compiling tutorial](https://github.com/japaric/rust-cross) - [Rust cross-compilation tool](https://github.com/rust-embedded/cross) -- [`leveldown` cross-compilation discussion](https://github.com/Level/leveldown/pull/572) -## license +## License GPL-3.0 diff --git a/build b/build deleted file mode 100755 index 1e1389e..0000000 --- a/build +++ /dev/null @@ -1,256 +0,0 @@ -#!/bin/bash - -CWD=$(pwd) -# https://stackoverflow.com/questions/59895/getting-the-source-directory-of-a-bash-script-from-within -DIRNAME="$(dirname "$(readlink -f "$0")")" - -# with help from https://stackoverflow.com/a/29754866, - -# saner programming env: these switches turn some bugs into errors -set -o errexit -o pipefail -o noclobber -o nounset - -DOCKER="docker" -set +e -if ! $DOCKER ps >/dev/null; then - echo "error connecting to docker:" - $DOCKER ps - exit 1 -fi -set -e - -! getopt --test > /dev/null -if [[ ${PIPESTATUS[0]} -ne 4 ]]; then - echo "GNU's enhanced getopt is required to run this script" - echo "You can usually find this in the util-linux package" - echo "On MacOS/OS X see homebrew's package: http://brewformulas.org/Gnu-getopt" - echo "For anyone else, build from source: http://frodo.looijaard.name/project/getopt" - exit 1 -fi - -OPTIONS=hi:o: -LONGOPTS=help,input:,output:,arch:,arm-version:,platform: - -# -use ! and PIPESTATUS to get exit code with errexit set -# -temporarily store output to be able to check for errors -# -activate quoting/enhanced mode (e.g. by writing out “--options”) -# -pass arguments only via -- "$@" to separate them correctly -! PARSED=$(getopt --options=$OPTIONS --longoptions=$LONGOPTS --name "$0" -- "$@") -if [[ ${PIPESTATUS[0]} -ne 0 ]]; then - # e.g. return value is 1 - # then getopt has complained about wrong arguments to stdout - exit 2 -fi -# read getopt’s output this way to handle the quoting right: -eval set -- "$PARSED" - -HELP=n -ARCH=${ARCH:-} ARM_VERSION=${ARM_VERSION:-} PLATFORM=${PLATFORM:-} -ARGS="" -INPUT="${CWD}" OUTPUT=prebuilds -while true; do - case "$1" in - -h|--help) - HELP=y - shift - ;; - -i|--input) - INPUT="$2" - shift 2 - ;; - -o|--output) - OUTPUT="$2" - shift 2 - ;; - --arch) - ARCH="$2" - shift 2 - ;; - --arm-version) - ARM_VERSION="$2" - shift 2 - ;; - --platform) - PLATFORM="$2" - shift 2 - ;; - --) - shift - ARGS="$@" - break - ;; - *) - echo "Programming error" - exit 3 - ;; - esac -done - -if - [ "${HELP}" == "y" ] || \ - [ "${ARCH}" == "" ] || \ - ( [ "${ARCH}" != "x32" ] && [ "${ARCH}" != "x64" ] && [ "${ARCH}" != "arm" ] && [ "${ARCH}" != "arm64" ] ) || \ - [ "${PLATFORM}" == "" ] || \ - ( [ "${PLATFORM}" != "android" ] && [ "${PLATFORM}" != "linux" ] ) || \ - [ "${ARCH}" == "arm" ] && [ "${ARM_VERSION}" == "" ] -then - cat >&2 <: **required** architecture (supported: x32, x64, arm, arm64) - --arm-version : if using arm architecture, **required* arm version (supported: 5, 6, 7, 8) - --platform: **required** platform (supported: linux, android) - - --input : _optional_ directory of input image spec (default: cwd) - --output : _optional_ directory to output build results (default: prebuilds) - - Flags: - - -h, --help: show this usage - - Examples: - - prebuildify-cross --platform linux --arch x64 - - prebuildify-cross --platform linux --arch arm --arm-version 7 - - prebuildify-cross --platform linux --arch arm64 - - prebuildify-cross --platform android --arch arm --arm-version 7 - - prebuildify-cross --platform android --arch arm64 -EOF - exit 1 -fi - -if [[ "${INPUT}" != /* ]] -then - INPUT="${CWD}/${INPUT}" -fi - -if [[ "${OUTPUT}" != /* ]] -then - OUTPUT="${CWD}/${OUTPUT}" -fi - -echo input $INPUT -echo output $OUTPUT - -if [ "${ARM_VERSION}" == "8" ] -then - ARCH=arm64 -fi - -if [ "${ARCH}" == "arm64" ] -then - ARM_VERSION=8 -fi - -function not_supported () { - if [ "${ARM_VERSION}" == "" ] - then - echo "platform=${PLATFORM}, arch=${ARCH}: not supported!" - else - echo "platform=${PLATFORM}, arch=${ARCH}, arm_version=${ARM_VERSION}: not supported!" - fi - exit 1 -} - -case "${PLATFORM}" in - "android") - case "${ARM_VERSION}" in - 7) - TARGET=android-arm - ;; - 8) - TARGET=android-arm64 - ;; - *) - not_supported - ;; - esac - ;; - "linux") - case "${ARM_VERSION}" in - 5|6|7) - TARGET="linux-armv${ARM_VERSION}" - ;; - 8) - TARGET=linux-arm64 - ;; - "") - TARGET=linux-${ARCH} - ;; - *) - not_supported - ;; - esac - ;; - *) - not_supported - ;; -esac - -echo args $ARGS - -IMAGE_NAME=${IMAGE_NAME:-prebuildify-cross} -CONTAINER_NAME=${CONTAINER_NAME:-prebuildify-cross-work-${TARGET}} - -CONTAINER_RUNNING=$($DOCKER ps --filter name="$CONTAINER_NAME" -q) -if [ "$CONTAINER_RUNNING" != "" ]; then - $DOCKER stop ${CONTAINER_NAME} > /dev/null -fi - -CONTAINER_EXISTS=$($DOCKER ps -a --filter name="$CONTAINER_NAME" -q) -if [ "$CONTAINER_EXISTS" != "" ]; then - $DOCKER rm -v "${CONTAINER_NAME}" -fi - -function cleanup { - echo 'got EXIT signal... please wait' - - CONTAINER_RUNNING=$($DOCKER ps --filter name="${CONTAINER_NAME}" -q) - if [ "${CONTAINER_RUNNING}" != "" ]; then - $DOCKER stop ${CONTAINER_NAME} > /dev/null - fi - - CONTAINER_EXISTS=$($DOCKER ps -a --filter name="${CONTAINER_NAME}" -q) - if [ "${CONTAINER_EXISTS}" != "" ]; then - $DOCKER rm -v ${CONTAINER_NAME} > /dev/null - fi -} - -trap cleanup EXIT - -echo ARCH "${ARCH}" -echo ARM_VERSION "${ARM_VERSION}" -echo PLATFORM "${PLATFORM}" -echo TARGET "${TARGET}" - -# build target image -$DOCKER build \ - --build-arg TARGET="${TARGET}" \ - -t ${IMAGE_NAME}:${TARGET} \ - ${DIRNAME} - -$DOCKER run \ - --name "${CONTAINER_NAME}" \ - -d \ - -v "${INPUT}:/app/input" \ - --env ARCH="${ARCH}" \ - --env ARM_VERSION="${ARM_VERSION}" \ - --env TARGET_PLATFORM="${PLATFORM}" \ - ${IMAGE_NAME}:${TARGET} \ - bash -e -o errexit -o pipefail -o noclobber -o nounset -c " - cd /app; ./build-in-docker ${ARGS}; - " -time $DOCKER logs "${CONTAINER_NAME}" --follow || true & -wait "$!" - -echo "copying results from output/" -$DOCKER cp "${CONTAINER_NAME}":/app/output/. "${OUTPUT}" -ls -lah "${OUTPUT}" - -echo "Done! Your prebuilds should be in ${OUTPUT}" diff --git a/build-in-docker b/build-in-docker deleted file mode 100755 index 8f1825c..0000000 --- a/build-in-docker +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -ARGUMENTS="$@" -CWD=$(pwd) - -# saner programming env: these switches turn some bugs into errors -set -o errexit -o pipefail -o noclobber -o nounset - -umask 022 - -cp -r "${CWD}/input" "${CWD}/work" - -cd "${CWD}/work" - -npm install --ignore-scripts - -PATH="$(npm bin):$PATH" prebuildify $ARGUMENTS - -cp -r "${CWD}/work/prebuilds" "${CWD}/output/" diff --git a/cli.js b/cli.js new file mode 100644 index 0000000..2330ddc --- /dev/null +++ b/cli.js @@ -0,0 +1,13 @@ +#!/usr/bin/env node +'use strict' + +const args = process.argv.slice(2) + +const opts = require('minimist')(args, { + string: ['image', 'cwd'], + alias: { image: 'i' } +}) + +require('.')({ args, ...opts }, function (err) { + if (err) throw err +}) diff --git a/guest.js b/guest.js new file mode 100644 index 0000000..5c379ef --- /dev/null +++ b/guest.js @@ -0,0 +1,69 @@ +'use strict' + +const fs = require('fs') +const path = require('path') +const cp = require('child_process') +const argv = process.argv.slice(process.argv[1] === __filename ? 2 : 1) +const mode = 0o755 +const fmod = 0o644 +const files = [] + +// Would prefer /app but that's owned by root in the prebuild images +const cwd = '/home/node/app' + +// Copy host files to working directory +for (const file of files) { + const a = path.join('/input', file) + const b = path.join(cwd, file) + + mkdirp(path.dirname(b), { mode }) + + fs.copyFileSync(a, b, fs.constants.COPYFILE_EXCL) + fs.chmodSync(b, fmod) +} + +// Use node_modules of host to avoid a second install step +fs.symlinkSync('/input/node_modules', path.join(cwd, 'node_modules')) + +const pkg = require(path.join(cwd, 'package.json')) +const scripts = pkg.scripts || {} + +process.umask(0o777 - mode) + +if (scripts.prebuild) { + cp.spawnSync('npm', ['run', 'prebuild', '--', ...argv], { cwd, stdio: 'inherit' }) +} else { + cp.spawnSync('npx', ['prebuildify', ...argv], { cwd, stdio: 'inherit' }) +} + +// Copy prebuilds to host +cpr(path.join(cwd, 'prebuilds'), '/input/prebuilds') + +function cpr (src, dst) { + mkdirp(dst, { mode }) + + for (const file of fs.readdirSync(src)) { + const a = path.join(src, file) + const b = path.join(dst, file) + + if (fs.statSync(a).isDirectory()) { + cpr(a, b) + } else { + fs.copyFileSync(a, b) // , fs.constants.COPYFILE_EXCL) + fs.chmodSync(b, fmod) + } + } +} + +// Simple version of mkdirp.sync() +function mkdirp (dir, opts) { + try { + fs.mkdirSync(dir, opts) + } catch (err) { + if (err.code === 'EEXIST') return + if (err.code !== 'ENOENT') throw err + const parent = path.dirname(dir) + if (parent !== dir) mkdirp(parent, opts) + fs.mkdirSync(dir, opts) + } +} diff --git a/index.js b/index.js new file mode 100644 index 0000000..0dee558 --- /dev/null +++ b/index.js @@ -0,0 +1,118 @@ +'use strict' + +// Also requires glob (npm/npm-packlist#42) +const packlist = require('npm-packlist') +const unixify = require('unixify') +const once = require('once') +const cp = require('child_process') +const path = require('path') +const fs = require('fs') + +module.exports = function (opts, callback) { + if (typeof opts === 'function') { + callback = opts + opts = null + } + + opts = opts || {} + callback = once(callback) + + const images = [].concat(opts.image || []) + const cwd = path.resolve(opts.cwd || '.') + + if (!images.length) { + images.push('centos7-devtoolset7') + images.push('alpine') + images.push('linux-armv7') + images.push('linux-arm64') + images.push('android-armv7') + images.push('android-arm64') + } + + const script = guestScript(packageFiles(cwd)) + const scriptArgs = opts.args || [] + + loop() + + function loop () { + let image = images.shift() + if (!image) return process.nextTick(callback) + if (image === 'linux') image = 'centos7-devtoolset7' + + const args = [ + 'run', + '--rm', + ...env(), + ...user(), + ...volume(cwd, '/input'), + 'prebuild/' + image, + ...command(script), + ...rest(scriptArgs, image) + ] + + console.error('Prebuild', image) + + cp.spawn('docker', args, { cwd, stdio: 'inherit' }) + .on('error', callback) + .on('exit', onexit) + } + + function onexit (code) { + if (code) return callback(new Error('Exited with code ' + code)) + loop() + } +} + +function packageFiles (dir) { + return packlist.sync({ dir }).filter(function (fp) { + return !/^(node_modules|prebuilds)[/\\]/i.test(fp) + }) +} + +function guestScript (files) { + return fs.readFileSync(require.resolve('./guest.js'), 'utf8') + .replace('const files = []', `const files = ${JSON.stringify(files)}`) +} + +function volume (host, guest) { + return ['-v', cygwin(host) + ':' + guest] +} + +function env () { + // Disable npm update check + return ['-e', 'NO_UPDATE_NOTIFIER=true'] +} + +function user () { + // TODO: test + // return ['-u', process.env.TRAVIS ? 'travis' : 'node'] + + return ['-u', 'node'] +} + +function command (script) { + return ['node', '-e', script] +} + +function rest (scriptArgs, image) { + const rest = scriptArgs.slice() + + if (/^(linux|android)-arm/.test(image)) rest.push('--tag-armv') + if (/^(centos|alpine)/.test(image)) rest.push('--tag-libc') + + return rest.length ? ['--'].concat(rest) : [] +} + +function cygwin (fp) { + if (process.platform !== 'win32') return fp + if (!truthy(process.env.COMPOSE_CONVERT_WINDOWS_PATHS)) return fp + + const unix = unixify(fp) + const drive = fp.match(/^([A-Z]):/i) + + return drive ? '/' + drive[1].toLowerCase() + unix : unix +} + +function truthy (str) { + return str === 'true' || str === '1' +} diff --git a/package-lock.json b/package-lock.json deleted file mode 100644 index 98c4d90..0000000 --- a/package-lock.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "name": "prebuildify-cross", - "version": "3.1.2", - "lockfileVersion": 1 -} diff --git a/package.json b/package.json index 99315e2..5877c92 100644 --- a/package.json +++ b/package.json @@ -1,21 +1,36 @@ { "name": "prebuildify-cross", "version": "3.1.2", - "description": "cross-compile prebuilds", - "bin": "./build", - "directories": { - "example": "examples" + "description": "Compile prebuilds in Docker", + "license": "GPL-3.0", + "bin": "./cli.js", + "scripts": { + "test": "standard" + }, + "dependencies": { + "glob": "^7.1.6", + "minimist": "^1.2.0", + "npm-packlist": "^2.0.0", + "once": "^1.4.0", + "unixify": "^1.0.0" + }, + "devDependencies": { + "standard": "^14.3.1" }, - "scripts": {}, "repository": { "type": "git", - "url": "git+ssh://git@github.com/ahdinosaur/prebuildify-cross.git" + "url": "https://github.com/prebuild/prebuildify-cross.git" }, - "keywords": [], - "author": "ahdinosaur", - "license": "GPL-3.0", "bugs": { - "url": "https://github.com/ahdinosaur/prebuildify-cross/issues" + "url": "https://github.com/prebuild/prebuildify-cross/issues" }, - "homepage": "https://github.com/ahdinosaur/prebuildify-cross#readme" + "homepage": "https://github.com/prebuild/prebuildify-cross", + "keywords": [ + "prebuildify", + "prebuild", + "dockcross" + ], + "engines": { + "node": ">=8" + } } From 03af93bc3c624b21941f292c59c7016706fb3cb3 Mon Sep 17 00:00:00 2001 From: Vincent Weevers Date: Sun, 10 Nov 2019 10:54:19 +0100 Subject: [PATCH 2/8] Write tarball to stdout, avoid messing with permissions --- guest.js | 45 +++++++++++++++------------------------------ index.js | 20 ++++++++++++-------- package.json | 1 + 3 files changed, 28 insertions(+), 38 deletions(-) diff --git a/guest.js b/guest.js index 5c379ef..1b0937c 100644 --- a/guest.js +++ b/guest.js @@ -4,8 +4,6 @@ const fs = require('fs') const path = require('path') const cp = require('child_process') const argv = process.argv.slice(process.argv[1] === __filename ? 2 : 1) -const mode = 0o755 -const fmod = 0o644 const files = [] // Would prefer /app but that's owned by root in the prebuild images @@ -16,54 +14,41 @@ for (const file of files) { const a = path.join('/input', file) const b = path.join(cwd, file) - mkdirp(path.dirname(b), { mode }) + mkdirp(path.dirname(b)) fs.copyFileSync(a, b, fs.constants.COPYFILE_EXCL) - fs.chmodSync(b, fmod) + fs.chmodSync(b, 0o644) } // Use node_modules of host to avoid a second install step fs.symlinkSync('/input/node_modules', path.join(cwd, 'node_modules')) +// TODO: bundle this +const tar = require(path.join(cwd, 'node_modules', 'tar-fs')) + const pkg = require(path.join(cwd, 'package.json')) const scripts = pkg.scripts || {} - -process.umask(0o777 - mode) +const stdio = ['ignore', 2, 2] if (scripts.prebuild) { - cp.spawnSync('npm', ['run', 'prebuild', '--', ...argv], { cwd, stdio: 'inherit' }) + cp.spawnSync('npm', ['run', 'prebuild', '--', ...argv], { cwd, stdio }) } else { - cp.spawnSync('npx', ['prebuildify', ...argv], { cwd, stdio: 'inherit' }) + cp.spawnSync('npx', ['prebuildify', ...argv], { cwd, stdio }) } -// Copy prebuilds to host -cpr(path.join(cwd, 'prebuilds'), '/input/prebuilds') - -function cpr (src, dst) { - mkdirp(dst, { mode }) - - for (const file of fs.readdirSync(src)) { - const a = path.join(src, file) - const b = path.join(dst, file) - - if (fs.statSync(a).isDirectory()) { - cpr(a, b) - } else { - fs.copyFileSync(a, b) // , fs.constants.COPYFILE_EXCL) - fs.chmodSync(b, fmod) - } - } -} +// Write tarball to stdout. With this approach we don't need +// a writable volume and can avoid messing with permissions. +tar.pack(path.join(cwd, 'prebuilds')).pipe(process.stdout) // Simple version of mkdirp.sync() -function mkdirp (dir, opts) { +function mkdirp (dir) { try { - fs.mkdirSync(dir, opts) + fs.mkdirSync(dir) } catch (err) { if (err.code === 'EEXIST') return if (err.code !== 'ENOENT') throw err const parent = path.dirname(dir) - if (parent !== dir) mkdirp(parent, opts) - fs.mkdirSync(dir, opts) + if (parent !== dir) mkdirp(parent) + fs.mkdirSync(dir) } } diff --git a/index.js b/index.js index 0dee558..98d430f 100644 --- a/index.js +++ b/index.js @@ -2,6 +2,7 @@ // Also requires glob (npm/npm-packlist#42) const packlist = require('npm-packlist') +const tar = require('tar-fs') const unixify = require('unixify') const once = require('once') const cp = require('child_process') @@ -52,20 +53,26 @@ module.exports = function (opts, callback) { console.error('Prebuild', image) - cp.spawn('docker', args, { cwd, stdio: 'inherit' }) + const stdio = ['ignore', 'pipe', 'inherit'] + const child = cp.spawn('docker', args, { cwd, stdio }) + + child.on('error', callback) + child.on('exit', onexit) + + child.stdout + .pipe(tar.extract('./prebuilds'), { dmode: 0o755, fmode: 0o644 }) + .on('finish', loop) .on('error', callback) - .on('exit', onexit) } function onexit (code) { if (code) return callback(new Error('Exited with code ' + code)) - loop() } } function packageFiles (dir) { return packlist.sync({ dir }).filter(function (fp) { - return !/^(node_modules|prebuilds)[/\\]/i.test(fp) + return !/^prebuilds[/\\]/i.test(fp) }) } @@ -75,7 +82,7 @@ function guestScript (files) { } function volume (host, guest) { - return ['-v', cygwin(host) + ':' + guest] + return ['-v', cygwin(host) + ':' + guest + ':ro'] } function env () { @@ -84,9 +91,6 @@ function env () { } function user () { - // TODO: test - // return ['-u', process.env.TRAVIS ? 'travis' : 'node'] - return ['-u', 'node'] } diff --git a/package.json b/package.json index 5877c92..07bb927 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,7 @@ "minimist": "^1.2.0", "npm-packlist": "^2.0.0", "once": "^1.4.0", + "tar-fs": "^2.0.0", "unixify": "^1.0.0" }, "devDependencies": { From dd45513ffe21d42346cdb61dbb0cae2c61c536ee Mon Sep 17 00:00:00 2001 From: Vincent Weevers Date: Sun, 10 Nov 2019 11:00:49 +0100 Subject: [PATCH 3/8] Fix path of prebuilds --- index.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/index.js b/index.js index 98d430f..ca42b27 100644 --- a/index.js +++ b/index.js @@ -55,12 +55,13 @@ module.exports = function (opts, callback) { const stdio = ['ignore', 'pipe', 'inherit'] const child = cp.spawn('docker', args, { cwd, stdio }) + const prebuilds = path.join(cwd, 'prebuilds') child.on('error', callback) child.on('exit', onexit) child.stdout - .pipe(tar.extract('./prebuilds'), { dmode: 0o755, fmode: 0o644 }) + .pipe(tar.extract(prebuilds), { dmode: 0o755, fmode: 0o644 }) .on('finish', loop) .on('error', callback) } From de6fa6e4db1bfab3da287ce3c43e21252aea89f1 Mon Sep 17 00:00:00 2001 From: Vincent Weevers Date: Sun, 10 Nov 2019 14:03:47 +0100 Subject: [PATCH 4/8] Use browserify and docker-run to bundle and run script --- guest.js | 12 ++++---- index.js | 77 ++++++++++++++++++++++------------------------------ package.json | 2 ++ 3 files changed, 40 insertions(+), 51 deletions(-) diff --git a/guest.js b/guest.js index 1b0937c..2079b38 100644 --- a/guest.js +++ b/guest.js @@ -3,11 +3,11 @@ const fs = require('fs') const path = require('path') const cp = require('child_process') -const argv = process.argv.slice(process.argv[1] === __filename ? 2 : 1) -const files = [] // Would prefer /app but that's owned by root in the prebuild images const cwd = '/home/node/app' +const files = JSON.parse(process.env.PREBUILDIFY_CROSS_FILES) +const argv = process.argv.slice(2) // Copy host files to working directory for (const file of files) { @@ -23,11 +23,9 @@ for (const file of files) { // Use node_modules of host to avoid a second install step fs.symlinkSync('/input/node_modules', path.join(cwd, 'node_modules')) -// TODO: bundle this -const tar = require(path.join(cwd, 'node_modules', 'tar-fs')) - -const pkg = require(path.join(cwd, 'package.json')) -const scripts = pkg.scripts || {} +const tar = require('tar-fs') +const pkg = fs.readFileSync(path.join(cwd, 'package.json'), 'utf8') +const scripts = JSON.parse(pkg).scripts || {} const stdio = ['ignore', 2, 2] if (scripts.prebuild) { diff --git a/index.js b/index.js index ca42b27..18f45a7 100644 --- a/index.js +++ b/index.js @@ -3,11 +3,11 @@ // Also requires glob (npm/npm-packlist#42) const packlist = require('npm-packlist') const tar = require('tar-fs') +const run = require('docker-run') +const browserify = require('browserify') const unixify = require('unixify') const once = require('once') -const cp = require('child_process') const path = require('path') -const fs = require('fs') module.exports = function (opts, callback) { if (typeof opts === 'function') { @@ -20,6 +20,8 @@ module.exports = function (opts, callback) { const images = [].concat(opts.image || []) const cwd = path.resolve(opts.cwd || '.') + const files = JSON.stringify(packageFiles(cwd)) + const prebuilds = path.join(cwd, 'prebuilds') if (!images.length) { images.push('centos7-devtoolset7') @@ -30,9 +32,6 @@ module.exports = function (opts, callback) { images.push('android-arm64') } - const script = guestScript(packageFiles(cwd)) - const scriptArgs = opts.args || [] - loop() function loop () { @@ -40,25 +39,30 @@ module.exports = function (opts, callback) { if (!image) return process.nextTick(callback) if (image === 'linux') image = 'centos7-devtoolset7' - const args = [ - 'run', - '--rm', - ...env(), - ...user(), - ...volume(cwd, '/input'), - 'prebuild/' + image, - ...command(script), - ...rest(scriptArgs, image) - ] - console.error('Prebuild', image) - const stdio = ['ignore', 'pipe', 'inherit'] - const child = cp.spawn('docker', args, { cwd, stdio }) - const prebuilds = path.join(cwd, 'prebuilds') + const child = run('prebuild/' + image, { + entrypoint: 'node', + argv: rest(opts.args || [], image), + volumes: { + [cygwin(cwd)]: '/input:ro' // mafintosh/docker-run#12 + }, + env: { + PREBUILDIFY_CROSS_FILES: files, + // Disable npm update check + NO_UPDATE_NOTIFIER: 'true' + } + }) + + child + .on('error', callback) + .on('exit', onexit) + + guestScript() + .pipe(child.stdin) - child.on('error', callback) - child.on('exit', onexit) + child.stderr + .pipe(process.stderr) child.stdout .pipe(tar.extract(prebuilds), { dmode: 0o755, fmode: 0o644 }) @@ -77,35 +81,20 @@ function packageFiles (dir) { }) } -function guestScript (files) { - return fs.readFileSync(require.resolve('./guest.js'), 'utf8') - .replace('const files = []', `const files = ${JSON.stringify(files)}`) -} - -function volume (host, guest) { - return ['-v', cygwin(host) + ':' + guest + ':ro'] -} - -function env () { - // Disable npm update check - return ['-e', 'NO_UPDATE_NOTIFIER=true'] -} - -function user () { - return ['-u', 'node'] -} - -function command (script) { - return ['node', '-e', script] +function guestScript () { + return browserify(require.resolve('./guest.js'), { + basedir: __dirname, + node: true + }).bundle() } -function rest (scriptArgs, image) { - const rest = scriptArgs.slice() +function rest (args, image) { + const rest = ['-'].concat(args) if (/^(linux|android)-arm/.test(image)) rest.push('--tag-armv') if (/^(centos|alpine)/.test(image)) rest.push('--tag-libc') - return rest.length ? ['--'].concat(rest) : [] + return rest } function cygwin (fp) { diff --git a/package.json b/package.json index 07bb927..b0b1dab 100644 --- a/package.json +++ b/package.json @@ -8,6 +8,8 @@ "test": "standard" }, "dependencies": { + "browserify": "^16.5.0", + "docker-run": "github:vweevers/docker-run#patch-1", "glob": "^7.1.6", "minimist": "^1.2.0", "npm-packlist": "^2.0.0", From 1a85bb0db7bd5bdc848aebd206b3b17b57efc73c Mon Sep 17 00:00:00 2001 From: Vincent Weevers Date: Sun, 10 Nov 2019 15:59:49 +0100 Subject: [PATCH 5/8] Pull image(s) --- README.md | 2 +- index.js | 34 +++++++++++++++++++++++++++++----- package.json | 3 +++ 3 files changed, 33 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index f0c786a..4733ab0 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ **Compile prebuilds in Docker, supporting Linux (including Debian 8, Ubuntu 14.04, RHEL 7, CentOS 7 and up), Alpine Linux, ARM Linux devices like the Raspberry Pi and mobile ARM devices like Android.** -Runs [`prebuildify`](https://github.com/mafintosh/prebuildify) in the preconfigured [`prebuild/docker-images`](https://github.com/prebuild/docker-images) to compile and name prebuilds for a certain platform. This means you don't have to worry about GCC flags, environment variables or system dependencies. In addition, `prebuildify-cross` copies only npm package files to Docker (following the rules of `.npmignore` and `files`) and mounts `node_modules` removing the need for a repeated `npm install`. +Runs [`prebuildify`](https://github.com/mafintosh/prebuildify) in preconfigured [`prebuild/docker-images`](https://github.com/prebuild/docker-images) containers to compile and name prebuilds for a certain platform. This means you don't have to worry about GCC flags, environment variables or system dependencies. In addition, `prebuildify-cross` copies only npm package files to Docker (following the rules of `.npmignore` and `files`) and mounts `node_modules` removing the need for a repeated `npm install`. ## Install diff --git a/index.js b/index.js index 18f45a7..7cfaecf 100644 --- a/index.js +++ b/index.js @@ -3,7 +3,10 @@ // Also requires glob (npm/npm-packlist#42) const packlist = require('npm-packlist') const tar = require('tar-fs') -const run = require('docker-run') +const dockerPull = require('docker-pull') +const dockerRun = require('docker-run') +const logger = require('log-update') +const bytes = require('pretty-bytes') const browserify = require('browserify') const unixify = require('unixify') const once = require('once') @@ -22,6 +25,7 @@ module.exports = function (opts, callback) { const cwd = path.resolve(opts.cwd || '.') const files = JSON.stringify(packageFiles(cwd)) const prebuilds = path.join(cwd, 'prebuilds') + const log = logger.create(process.stderr, { showCursor: true }) if (!images.length) { images.push('centos7-devtoolset7') @@ -38,10 +42,30 @@ module.exports = function (opts, callback) { let image = images.shift() if (!image) return process.nextTick(callback) if (image === 'linux') image = 'centos7-devtoolset7' + if (!image.includes('/')) image = 'prebuild/' + image - console.error('Prebuild', image) + dockerPull(image) + .on('progress', progress) + .on('error', callback) + .on('end', end) + + function progress () { + const count = `${this.layers} layers` + const ratio = `${bytes(this.transferred)} / ${bytes(this.length)}` + + log(`prebuildify-cross pull ${this.image}: ${count}, ${ratio}`) + } + + function end () { + log.done() + run(this.image) + } + } + + function run (image) { + console.error('prebuildify-cross run', image) - const child = run('prebuild/' + image, { + const child = dockerRun(image, { entrypoint: 'node', argv: rest(opts.args || [], image), volumes: { @@ -91,8 +115,8 @@ function guestScript () { function rest (args, image) { const rest = ['-'].concat(args) - if (/^(linux|android)-arm/.test(image)) rest.push('--tag-armv') - if (/^(centos|alpine)/.test(image)) rest.push('--tag-libc') + if (/^prebuild\/(linux|android)-arm/.test(image)) rest.push('--tag-armv') + if (/^prebuild\/(centos|alpine)/.test(image)) rest.push('--tag-libc') return rest } diff --git a/package.json b/package.json index b0b1dab..3ebb3c2 100644 --- a/package.json +++ b/package.json @@ -9,11 +9,14 @@ }, "dependencies": { "browserify": "^16.5.0", + "docker-pull": "^1.1.0", "docker-run": "github:vweevers/docker-run#patch-1", "glob": "^7.1.6", + "log-update": "^3.3.0", "minimist": "^1.2.0", "npm-packlist": "^2.0.0", "once": "^1.4.0", + "pretty-bytes": "^5.3.0", "tar-fs": "^2.0.0", "unixify": "^1.0.0" }, From b488dd6666f8067047a6da58132b87de82ef11fa Mon Sep 17 00:00:00 2001 From: Vincent Weevers Date: Sat, 16 Nov 2019 13:26:29 +0100 Subject: [PATCH 6/8] Simplify --- README.md | 30 ++++++++++++++---------------- cli.js | 6 +++--- guest.js | 31 +++++++------------------------ index.js | 39 ++++++++++++++++++++------------------- package.json | 3 ++- 5 files changed, 46 insertions(+), 63 deletions(-) diff --git a/README.md b/README.md index 4733ab0..3d3d626 100644 --- a/README.md +++ b/README.md @@ -14,39 +14,37 @@ npm install --save-dev prebuildify node-gyp prebuildify-cross ## Usage -The `prebuildify-cross` cli wraps `prebuildify` and passes all command line arguments on to `prebuildify` or `npm run prebuild` if such a script exists in your `package.json`. For example, with the following setup you could run `prebuildify-cross` without arguments: - -```json -{ - "scripts": { - "prebuild": "prebuildify -t 8.14.0 --napi --strip" - } -} -``` +The `prebuildify-cross` cli forwards all command line arguments to `prebuildify`, but adds an `--image` or `-i` argument. For example, the following command will invoke `prebuildify -t 8.14.0 --napi --strip` in a CentOS container and copy the resulting prebuild to `./prebuilds`: -Without that script, you could run `prebuildify-cross -t 8.14.0 --napi --strip` to the same effect. If you combine both approaches, arguments will be concatenated. +``` +prebuildify-cross -i centos7-devtoolset7 -t 8.14.0 --napi --strip +``` -By default `prebuildify-cross` makes prebuilds for [all platforms](#images). To override, pass one or more `--image` or `-i` arguments: +To build for more than one platform, multiple `--image` arguments may be passed: ``` -prebuildify-cross -i linux -i alpine [..] +prebuildify-cross -i linux-armv7 -i linux-arm64 -t .. ``` -This is the only argument specific to `prebuildify-cross`, which does however respect the `--cwd` argument of `prebuildify`. +Lastly, it's possible to use your own custom image with e.g. `-i my-namespace/my-image`. ## Images ### `centos7-devtoolset7` -Aliased as `linux`. +Compile in CentOS 7, as a better alternative to (commonly) Ubuntu 16.04 on Travis. Makes the prebuild compatible with Debian 8, Ubuntu 14.04, RHEL 7, CentOS 7 and other Linux flavors with an old glibc. -Compile in CentOS 7, as a better alternative to (most commonly) Ubuntu on Travis. Makes the prebuilt binary compatible with Debian 8, Ubuntu 14.04, RHEL 7, CentOS 7 and other Linux flavors with an old glibc. +> The neat thing about this is that you get to compile with gcc 7 but glibc 2.17, so binaries are compatible for \[among others] Ubuntu 14.04 and Debian 8. +> +> The RHEL folks put in a ton of work to make the devtoolsets work on their older base systems (libc mainly), which involves shipping a delta library that contains the new stuff that can be statically linked in where it's used. We use this method for building Node binary releases. +> +> \-- [**@rvagg**](https://github.com/rvagg) ([prebuild/docker-images#8](https://github.com/prebuild/docker-images/pull/8)) By default the prebuild will be [tagged](https://github.com/prebuild/prebuildify#options) with the libc flavor to set it apart from Alpine prebuilds, e.g. `linux-x64/node.libc.node`. ### `alpine` -Compile in Alpine, which uses musl instead of glibc and therefore can't run regular linux prebuilds. Worse, it sometimes does successfully _load_ such a prebuild during `npm install` - which prevents a compilation fallback from kicking in - and then segfaults at runtime. +Compile in Alpine, which uses musl instead of glibc and therefore can't run regular linux prebuilds. Worse, it sometimes does successfully _load_ such a prebuild during `npm install` - which prevents a compilation fallback from kicking in - and then segfaults at runtime. You can fix this situation in two ways: by shipping an `alpine` prebuild and/or by shipping a `centos7-devtoolset7` prebuild, because the latter will be skipped in Alpine thanks to the `libc` tag. By default the prebuild will be [tagged](https://github.com/prebuild/prebuildify#options) with the libc flavor, e.g. `linux-x64/node.musl.node`. diff --git a/cli.js b/cli.js index 2330ddc..a84f161 100644 --- a/cli.js +++ b/cli.js @@ -1,13 +1,13 @@ #!/usr/bin/env node 'use strict' -const args = process.argv.slice(2) +const argv = process.argv.slice(2) -const opts = require('minimist')(args, { +const opts = require('minimist')(argv, { string: ['image', 'cwd'], alias: { image: 'i' } }) -require('.')({ args, ...opts }, function (err) { +require('.')({ argv, ...opts }, function (err) { if (err) throw err }) diff --git a/guest.js b/guest.js index 2079b38..55d8438 100644 --- a/guest.js +++ b/guest.js @@ -3,8 +3,9 @@ const fs = require('fs') const path = require('path') const cp = require('child_process') +const mkdirp = require('mkdirp') -// Would prefer /app but that's owned by root in the prebuild images +// TODO: fix permissions of WORKDIR in prebuild images const cwd = '/home/node/app' const files = JSON.parse(process.env.PREBUILDIFY_CROSS_FILES) const argv = process.argv.slice(2) @@ -14,7 +15,7 @@ for (const file of files) { const a = path.join('/input', file) const b = path.join(cwd, file) - mkdirp(path.dirname(b)) + mkdirp.sync(path.dirname(b)) fs.copyFileSync(a, b, fs.constants.COPYFILE_EXCL) fs.chmodSync(b, 0o644) @@ -23,30 +24,12 @@ for (const file of files) { // Use node_modules of host to avoid a second install step fs.symlinkSync('/input/node_modules', path.join(cwd, 'node_modules')) -const tar = require('tar-fs') -const pkg = fs.readFileSync(path.join(cwd, 'package.json'), 'utf8') -const scripts = JSON.parse(pkg).scripts || {} const stdio = ['ignore', 2, 2] +const res = cp.spawnSync('npx', ['prebuildify', ...argv], { cwd, stdio }) -if (scripts.prebuild) { - cp.spawnSync('npm', ['run', 'prebuild', '--', ...argv], { cwd, stdio }) -} else { - cp.spawnSync('npx', ['prebuildify', ...argv], { cwd, stdio }) -} +if (res.status) process.exit(res.status) +if (res.error) throw res.error // Write tarball to stdout. With this approach we don't need // a writable volume and can avoid messing with permissions. -tar.pack(path.join(cwd, 'prebuilds')).pipe(process.stdout) - -// Simple version of mkdirp.sync() -function mkdirp (dir) { - try { - fs.mkdirSync(dir) - } catch (err) { - if (err.code === 'EEXIST') return - if (err.code !== 'ENOENT') throw err - const parent = path.dirname(dir) - if (parent !== dir) mkdirp(parent) - fs.mkdirSync(dir) - } -} +require('tar-fs').pack(path.join(cwd, 'prebuilds')).pipe(process.stdout) diff --git a/index.js b/index.js index 7cfaecf..588e2e0 100644 --- a/index.js +++ b/index.js @@ -27,21 +27,11 @@ module.exports = function (opts, callback) { const prebuilds = path.join(cwd, 'prebuilds') const log = logger.create(process.stderr, { showCursor: true }) - if (!images.length) { - images.push('centos7-devtoolset7') - images.push('alpine') - images.push('linux-armv7') - images.push('linux-arm64') - images.push('android-armv7') - images.push('android-arm64') - } - loop() function loop () { let image = images.shift() if (!image) return process.nextTick(callback) - if (image === 'linux') image = 'centos7-devtoolset7' if (!image.includes('/')) image = 'prebuild/' + image dockerPull(image) @@ -53,7 +43,7 @@ module.exports = function (opts, callback) { const count = `${this.layers} layers` const ratio = `${bytes(this.transferred)} / ${bytes(this.length)}` - log(`prebuildify-cross pull ${this.image}: ${count}, ${ratio}`) + log(`> prebuildify-cross pull ${this.image}: ${count}, ${ratio}`) } function end () { @@ -63,13 +53,17 @@ module.exports = function (opts, callback) { } function run (image) { - console.error('prebuildify-cross run', image) + const argv = prebuildifyArgv(opts.argv || [], image) + + console.error('> prebuildify-cross run %s', image) + console.error('> prebuildify %s\n', argv.join(' ')) const child = dockerRun(image, { entrypoint: 'node', - argv: rest(opts.args || [], image), + argv: ['-'].concat(argv), volumes: { - [cygwin(cwd)]: '/input:ro' // mafintosh/docker-run#12 + // Should but can't use :ro (mafintosh/docker-run#12) + [cygwin(cwd)]: '/input' }, env: { PREBUILDIFY_CROSS_FILES: files, @@ -112,13 +106,20 @@ function guestScript () { }).bundle() } -function rest (args, image) { - const rest = ['-'].concat(args) +function prebuildifyArgv (argv, image) { + argv = argv.slice() + + for (let i = 0; i < argv.length - 1; i++) { + if (/^(-i|--image)$/.test(argv[i]) && argv[i + 1][0] !== '-') { + argv.splice(i--, 2) + } + } - if (/^prebuild\/(linux|android)-arm/.test(image)) rest.push('--tag-armv') - if (/^prebuild\/(centos|alpine)/.test(image)) rest.push('--tag-libc') + // TODO: move this to the docker images? + if (/^prebuild\/(linux|android)-arm/.test(image)) argv.push('--tag-armv') + if (/^prebuild\/(centos|alpine)/.test(image)) argv.push('--tag-libc') - return rest + return argv } function cygwin (fp) { diff --git a/package.json b/package.json index 3ebb3c2..a000675 100644 --- a/package.json +++ b/package.json @@ -10,10 +10,11 @@ "dependencies": { "browserify": "^16.5.0", "docker-pull": "^1.1.0", - "docker-run": "github:vweevers/docker-run#patch-1", + "docker-run": "^3.1.0", "glob": "^7.1.6", "log-update": "^3.3.0", "minimist": "^1.2.0", + "mkdirp": "^0.5.1", "npm-packlist": "^2.0.0", "once": "^1.4.0", "pretty-bytes": "^5.3.0", From 2c3a8828808991181add4dc7ec76f20219a5cf40 Mon Sep 17 00:00:00 2001 From: Vincent Weevers Date: Sat, 16 Nov 2019 16:25:50 +0100 Subject: [PATCH 7/8] Skip pull progress indicator in CI --- index.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/index.js b/index.js index 588e2e0..b8c5f06 100644 --- a/index.js +++ b/index.js @@ -40,6 +40,11 @@ module.exports = function (opts, callback) { .on('end', end) function progress () { + if (process.env.CI) { + console.error(`> prebuildify-cross pull ${this.image}`) + return this.removeListener('progress', progress) + } + const count = `${this.layers} layers` const ratio = `${bytes(this.transferred)} / ${bytes(this.length)}` @@ -47,7 +52,6 @@ module.exports = function (opts, callback) { } function end () { - log.done() run(this.image) } } From 8696972350bb1edee9e2e8d008ab9de05ff4883a Mon Sep 17 00:00:00 2001 From: Vincent Weevers Date: Sat, 16 Nov 2019 17:35:32 +0100 Subject: [PATCH 8/8] Bump npm-packlist (with glob fix) --- index.js | 1 - package.json | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/index.js b/index.js index b8c5f06..906c945 100644 --- a/index.js +++ b/index.js @@ -1,6 +1,5 @@ 'use strict' -// Also requires glob (npm/npm-packlist#42) const packlist = require('npm-packlist') const tar = require('tar-fs') const dockerPull = require('docker-pull') diff --git a/package.json b/package.json index a000675..156689d 100644 --- a/package.json +++ b/package.json @@ -11,11 +11,10 @@ "browserify": "^16.5.0", "docker-pull": "^1.1.0", "docker-run": "^3.1.0", - "glob": "^7.1.6", "log-update": "^3.3.0", "minimist": "^1.2.0", "mkdirp": "^0.5.1", - "npm-packlist": "^2.0.0", + "npm-packlist": "^2.0.1", "once": "^1.4.0", "pretty-bytes": "^5.3.0", "tar-fs": "^2.0.0",