indexer-native: Install uses precompiled binaries hosted on GitHub

- Update install process to download precompiled binary,if that fails fallback to the current behavior and compile directly on user's machine - Add GitHub Actions workflow for pre-compiling the Node add-on module on various machines and binaries and pushing to the GitHub releases page
graphprotocol · Apr 26, 2022 · 896b2ad · 896b2ad
1 parent 9461c9d
commit 896b2ad
Show file tree

Hide file tree

Showing 4 changed files with 111 additions and 30 deletions.
diff --git a/.github/workflows/publish-native-binaries.yml b/.github/workflows/publish-native-binaries.yml
@@ -0,0 +1,56 @@
+# Creates and publishes assets for a new release.
+# To manually trigger this workflow, create a release in the GitHub UI.
+
+name: "Publish binaries"
+
+on:
+  release:
+    types:
+      - created
+
+jobs:
+  publish-github:
+    strategy:
+      # Allowing jobs to fail until 'node-pre-gyp-github' supports failing gracefully if file already exists
+      # (https://github.com/bchr02/node-pre-gyp-github/issues/42)
+      fail-fast: false
+      matrix:
+        node_version: [12, 14, 16, 17]
+        experimental: [false]
+        system:
+          - os: macos-latest
+            target: x86_64-apple-darwin
+          - os: ubuntu-latest
+            target: x86_64-unknown-linux-gnu
+        include:
+          - node_version: 16
+            system:
+              os: macos-latest
+              target: aarch64-apple-darwin
+            experimental: true
+    runs-on: ${{ matrix.system.os }}
+    steps:
+      - name: Checkout the repo
+        uses: actions/checkout@v2
+        with:
+          ref: ${{ github.event.release.tag_name }}
+      - name: Set up Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v2.1.5
+        with:
+          node-version: ${{ matrix.node_version }}
+      - name: Setup Rust
+        uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          target: ${{ matrix.system.target }}
+          override: true
+      - name: Install dependencies
+        working-directory: ./packages/indexer-native
+        run: yarn install --ignore-scripts
+      - name: Compile binary, test, package, and publish to Github release page
+        env:
+          NODE_PRE_GYP_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          CARGO_BUILD_TARGET: ${{ matrix.system.target }}
+        working-directory: ./packages/indexer-native
+        run: yarn build-test-pack-publish
diff --git a/packages/indexer-native/lib/index.js b/packages/indexer-native/lib/index.js
@@ -1,4 +1,4 @@
-var addon = require("../native");
+var addon = require("../binary");
 
 function promisify(f) {
   return new Promise((resolve, reject) =>

diff --git a/packages/indexer-native/package.json b/packages/indexer-native/package.json
@@ -10,27 +10,54 @@
     "!/native/target",
     "!/native/artifacts.json"
   ],
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/graphprotocol/indexer.git"
+  },
   "author": "Zac Burns <That3Percent@gmail.com>",
   "license": "MIT",
-  "dependencies": {
-    "@graphprotocol/common-ts": "1.8.3",
-    "neon-cli": "0.10.0"
-  },
+  "os": [
+    "darwin",
+    "linux"
+  ],
+  "cpu": [
+    "x64"
+  ],
   "scripts": {
-    "compile": "neon build --release",
+    "build": "cd native && cargo-cp-artifact -a cdylib indexer_native ../binary/index.node -- cargo build --message-format=json-render-diagnostics",
+    "build-debug": "yarn build --",
+    "build-release": "yarn build --release",
+    "pull-or-build": "node-pre-gyp install --fallback-to-build=false --update-binary || yarn build-release",
+    "package": "node-pre-gyp package",
+    "publish-github-draft": "node-pre-gyp-github publish",
+    "publish-github": "node-pre-gyp-github publish --release",
+    "build-test-pack-publish": "yarn build-release && yarn test && yarn package && yarn publish-github-draft",
     "format": "prettier --write 'lib/**/*.js'",
     "lint": "eslint .",
-    "prepare": "yarn format && yarn lint && yarn compile",
+    "prepare": "yarn format && yarn lint && yarn pull-or-build",
     "test": "jest --detectOpenHandles --verbose",
-    "clean": "rm -rf ./node_modules ./native/target ./native/artifacts.json ./native/index.node"
+    "clean": "rm -rf ./node_modules ./binary ./build ./coverage ./native/target ./native/artifacts.json ./native/index.node"
+  },
+  "dependencies": {
+    "@graphprotocol/common-ts": "1.8.3"
   },
   "devDependencies": {
+    "@mapbox/node-pre-gyp": "1.0.9",
     "bs58": "5.0.0",
+    "cargo-cp-artifact": "0.1.6",
     "eslint": "8.13.0",
     "eslint-config-prettier": "8.5.0",
     "ethers": "5.6.2",
     "jest": "27.5.1",
+    "node-pre-gyp-github": "1.4.4",
     "prettier": "2.6.2"
   },
+  "binary": {
+    "module_name": "index",
+    "module_path": "./binary",
+    "host": "https://github.com/graphprotocol/indexer/releases/download/",
+    "remote_path": "v{version}",
+    "package_name": "graphprotocol-indexer-native-v{version}-{node_abi}-{platform}-{arch}.tar.gz"
+  },
   "gitHead": "972ab96774007b2aee15b1da169d2ff4be9f9d27"
 }
diff --git a/yarn.lock b/yarn.lock
@@ -1844,7 +1844,7 @@
     npmlog "^4.1.2"
     write-file-atomic "^3.0.3"
 
-"@mapbox/node-pre-gyp@^1.0.0":
+"@mapbox/node-pre-gyp@1.0.9", "@mapbox/node-pre-gyp@^1.0.0":
   version "1.0.9"
   resolved "https://registry.npmjs.org/@mapbox/node-pre-gyp/-/node-pre-gyp-1.0.9.tgz#09a8781a3a036151cdebbe8719d6f8b25d4058bc"
   integrity sha512-aDF3S3rK9Q2gey/WAttUlISduDItz5BU3306M9Eyv6/oS40aMprnopshtlKTykxRNIBEZuRMaZAnbrQ4QtKGyw==
@@ -2051,7 +2051,7 @@
     node-fetch "^2.6.7"
     universal-user-agent "^6.0.0"
 
-"@octokit/rest@^18.1.0":
+"@octokit/rest@18.12.0", "@octokit/rest@^18.1.0":
   version "18.12.0"
   resolved "https://registry.npmjs.org/@octokit/rest/-/rest-18.12.0.tgz#f06bc4952fc87130308d810ca9d00e79f6988881"
   integrity sha512-gDPiOHlyGavxr72y0guQEhLsemgVjwRePayJ+FcKc2SJqKUbxbkvf5kAZEWA/MKvsfYlQAMVzNJE3ezQcxMJ2Q==
@@ -3668,6 +3668,11 @@ caniuse-lite@^1.0.30001317:
   resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001332.tgz#39476d3aa8d83ea76359c70302eafdd4a1d727dd"
   integrity sha512-10T30NYOEQtN6C11YGg411yebhvpnC6Z102+B95eAsN0oB6KUs01ivE8u+G6FMIRtIrVlYXhL+LUwQ3/hXwDWw==
 
+cargo-cp-artifact@0.1.6:
+  version "0.1.6"
+  resolved "https://registry.npmjs.org/cargo-cp-artifact/-/cargo-cp-artifact-0.1.6.tgz#df1bc9dad036ae0f4230639a869182e1d5850f89"
+  integrity sha512-CQw0doK/aaF7j041666XzuilHxqMxaKkn+I5vmBsd8SAwS0cO5CqVEVp0xJwOKstyqWZ6WK4Ww3O6p26x/Goyg==
+
 caseless@~0.12.0:
   version "0.12.0"
   resolved "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz#1b681c21ff84033c826543090689420d187151dc"
@@ -3899,6 +3904,11 @@ command-line-usage@^6.1.0:
     table-layout "^1.0.2"
     typical "^5.2.0"
 
+commander@7.2.0:
+  version "7.2.0"
+  resolved "https://registry.npmjs.org/commander/-/commander-7.2.0.tgz#a36cb57d0b501ce108e4d20559a150a391d97ab7"
+  integrity sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw==
+
 commander@^2.20.3:
   version "2.20.3"
   resolved "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz#fd485e84c03eb4881c20722ba48035e8531aeb33"
@@ -7704,26 +7714,6 @@ neo-async@^2.6.0:
   resolved "https://registry.npmjs.org/neo-async/-/neo-async-2.6.2.tgz#b4aafb93e3aeb2d8174ca53cf163ab7d7308305f"
   integrity sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==
 
-neon-cli@0.10.0:
-  version "0.10.0"
-  resolved "https://registry.npmjs.org/neon-cli/-/neon-cli-0.10.0.tgz#b588fceb0d5c0a5c1b140381f38511fc3be5f431"
-  integrity sha512-eiyn+XDlZEauVcPZkGSW+Iw2RwidrEoPrW0LfLntOkUecLHZgfm9dOainQI074ZMi75825VV/GCSTrv4J70TpA==
-  dependencies:
-    chalk "^4.1.0"
-    command-line-args "^5.1.1"
-    command-line-commands "^3.0.1"
-    command-line-usage "^6.1.0"
-    git-config "0.0.7"
-    handlebars "^4.7.6"
-    inquirer "^7.3.3"
-    make-promises-safe "^5.1.0"
-    rimraf "^3.0.2"
-    semver "^7.3.2"
-    toml "^3.0.0"
-    ts-typed-json "^0.3.2"
-    validate-npm-package-license "^3.0.4"
-    validate-npm-package-name "^3.0.0"
-
 neon-cli@^0.4.0:
   version "0.4.2"
   resolved "https://registry.npmjs.org/neon-cli/-/neon-cli-0.4.2.tgz#e5dfda11a918922ca24cc87abb1f7b1e89adcdb0"
@@ -7824,6 +7814,14 @@ node-int64@^0.4.0:
   resolved "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz#87a9065cdb355d3182d8f94ce11188b825c68a3b"
   integrity sha1-h6kGXNs1XTGC2PlM4RGIuCXGijs=
 
+node-pre-gyp-github@1.4.4:
+  version "1.4.4"
+  resolved "https://registry.npmjs.org/node-pre-gyp-github/-/node-pre-gyp-github-1.4.4.tgz#327da096266f35ecf4317d37ee201b448c14bb96"
+  integrity sha512-oE9JD1aXRi4+1jSH7Q+ybEhfujW5bJ66n4YMGpaUp/k2/X/8i09ouK1seznf3wOagcKjytRJCkf71DdEJx2zhA==
+  dependencies:
+    "@octokit/rest" "18.12.0"
+    commander "7.2.0"
+
 node-releases@^2.0.2:
   version "2.0.3"
   resolved "https://registry.npmjs.org/node-releases/-/node-releases-2.0.3.tgz#225ee7488e4a5e636da8da52854844f9d716ca96"