feat: add grpc server and client

Adds a server running a gRPC endpoint over websockets, a client to access the server and a `ipfs-client` module that uses the gRPC client with HTTP fallback.

So far only supports `ipfs.addAll` but the idea is to implement all streaming methods over websockets instead of HTTP, to give us bidirectional streaming and errors that work in the browser.

Fixes:

Depends on:

- [ ] ipfs/js-ipfsd-ctl#561

.travis.yml

@@ -164,6 +164,41 @@ jobs:
       script:
         - npm run test:interface:core -- $RUN_SINCE -- -- --bail -t electron-renderer --timeout 60000
 
+    - stage: test
+      name: js-ipfs interface tests - ipfs-client - node
+      script:
+        - npm run test:interface:client -- $RUN_SINCE -- -- --bail -t node
+
+    - stage: test
+      name: js-ipfs interface tests - ipfs-client - chrome
+      script:
+        - npm run test:interface:client -- $RUN_SINCE -- -- --bail -t browser
+
+    - stage: test
+      name: js-ipfs interface tests - ipfs-client - chrome webworker
+      script:
+        - npm run test:interface:client -- $RUN_SINCE -- -- --bail -t webworker --timeout 60000
+
+    - stage: test
+      name: js-ipfs interface tests - ipfs-client - firefox
+      script:
+        - npm run test:interface:client -- $RUN_SINCE -- -- --bail -t browser --browsers FirefoxHeadless
+
+    - stage: test
+      name: js-ipfs interface tests - ipfs-client - firefox webworker
+      script:
+        - npm run test:interface:client -- $RUN_SINCE -- -- --bail -t webworker --browsers FirefoxHeadless --timeout 60000
+
+    - stage: test
+      name: js-ipfs interface tests - ipfs-client - electron main
+      script:
+        - npm run test:interface:client -- $RUN_SINCE -- -- --bail -t electron-main --timeout 60000
+
+    - stage: test
+      name: js-ipfs interface tests - ipfs-client - electron renderer
+      script:
+        - npm run test:interface:client -- $RUN_SINCE -- -- --bail -t electron-renderer --timeout 60000
+
     - stage: test
       name: http-api-client interface tests vs go-ipfs - node
       script:

examples/browser-ipns-publish/package.json

@@ -27,7 +27,7 @@
   "devDependencies": {
     "delay": "^4.4.0",
     "execa": "^4.0.3",
-    "ipfsd-ctl": "^7.0.2",
+    "ipfsd-ctl": "ipfs/js-ipfsd-ctl#feat/expose-grpc-addr",
     "go-ipfs": "^0.7.0",
     "parcel-bundler": "^1.12.4",
     "path": "^0.12.7",

examples/explore-ethereum-blockchain/package.json

@@ -12,7 +12,7 @@
   "devDependencies": {
     "ipfs": "^0.52.1",
     "ipfs-http-client": "^48.1.1",
-    "ipfsd-ctl": "^7.0.2",
+    "ipfsd-ctl": "ipfs/js-ipfsd-ctl#feat/expose-grpc-addr",
     "ipld-ethereum": "^5.0.1",
     "test-ipfs-example": "^2.0.3"
   }

examples/http-client-browser-pubsub/package.json

@@ -21,7 +21,7 @@
     "execa": "^4.0.3",
     "go-ipfs": "^0.7.0",
     "ipfs": "^0.52.1",
-    "ipfsd-ctl": "^7.0.2",
+    "ipfsd-ctl": "ipfs/js-ipfsd-ctl#feat/expose-grpc-addr",
     "parcel-bundler": "^1.12.4",
     "test-ipfs-example": "^2.0.3"
   }

examples/http-client-bundle-webpack/package.json

@@ -25,7 +25,7 @@
     "copy-webpack-plugin": "^5.0.4",
     "execa": "^4.0.3",
     "ipfs": "^0.52.1",
-    "ipfsd-ctl": "^7.0.2",
+    "ipfsd-ctl": "ipfs/js-ipfsd-ctl#feat/expose-grpc-addr",
     "react-hot-loader": "^4.12.21",
     "rimraf": "^3.0.2",
     "test-ipfs-example": "^2.0.3",

examples/http-client-name-api/package.json

@@ -18,7 +18,7 @@
   "devDependencies": {
     "execa": "^4.0.3",
     "go-ipfs": "^0.7.0",
-    "ipfsd-ctl": "^7.0.2",
+    "ipfsd-ctl": "ipfs/js-ipfsd-ctl#feat/expose-grpc-addr",
     "parcel-bundler": "^1.12.4",
     "rimraf": "^3.0.2",
     "test-ipfs-example": "^2.0.3"

examples/ipfs-client-add-files/README.md

@@ -0,0 +1,21 @@
+# JS IPFS API - Example Browser - Name
+
+## Setup
+
+```sh
+npm install -g ipfs
+jsipfs init
+# Configure CORS to allow ipfs-http-client to access this IPFS node
+jsipfs config --json API.HTTPHeaders.Access-Control-Allow-Origin '["http://127.0.0.1:8888"]'
+# Start the IPFS node
+jsipfs daemon
+```
+
+Then in this folder run
+
+```bash
+> npm install
+> npm start
+```
+
+and open your browser at `http://127.0.0.1:8888`.

examples/ipfs-client-add-files/index.html

@@ -0,0 +1,36 @@
+<!doctype html>
+<html>
+  <head>
+    <meta charset="utf-8" />
+    <title>JS IPFS Client example</title>
+    <style>
+      .hidden {
+        opacity: 0;
+      }
+
+      form {
+        padding-bottom: 1em;
+      }
+    </style>
+  </head>
+
+  <body>
+    <h1>ipfs-client</h1>
+    <form id="connect-to-api">
+      <h3>Enter IPFS API details</h3>
+      <label for="grpc-input">
+        GRPC:
+        <input id="grpc-input" name="grpc-input" type="text" value="/ip4/127.0.0.1/tcp/5003" required>
+      </label>
+      <label for="http-input">
+        HTTP:
+        <input id="http-input" name="text" type="text" value="/ip4/127.0.0.1/tcp/5001" required>
+      </label>
+      <button id="connect-submit" type="submit">Connect</button>
+    </form>
+    <div id="output">
+    </div>
+
+    <script src="index.js"></script>
+  </body>
+</html>

examples/ipfs-client-add-files/index.js

@@ -0,0 +1,81 @@
+/* eslint-disable no-console */
+'use strict'
+
+const ipfsClient = require('ipfs-client')
+let ipfs
+
+const COLORS = {
+  active: 'blue',
+  success: 'green',
+  error: 'red'
+}
+
+const showStatus = (text, bg) => {
+  console.info(text)
+
+  const log = document.getElementById('output')
+
+  if (!log) {
+    return
+  }
+
+  const line = document.createElement('p')
+  line.innerText = text
+  line.style.color = bg
+
+  log.appendChild(line)
+}
+
+async function * streamFiles () {
+  for (let i = 0; i < 100; i++) {
+    await new Promise((resolve) => {
+      setTimeout(() => resolve(), 100)
+    })
+
+    showStatus(`Sending /file-${i}.txt`, COLORS.active)
+
+    yield {
+      path: `/file-${i}.txt`,
+      content: `file ${i}`
+    }
+  }
+}
+
+async function main (grpcApi, httpApi) {
+  showStatus(`Connecting to ${grpcApi} using ${httpApi} as fallback`, COLORS.active)
+
+  ipfs = ipfsClient({
+    grpc: grpcApi,
+    http: httpApi
+  })
+
+  const id = await ipfs.id()
+  showStatus(`Daemon active\nID: ${id.id}`, COLORS.success)
+
+  for await (const file of ipfs.addAll(streamFiles(), {
+    wrapWithDirectory: true,
+    // this is just to show the interleaving of uploads and progress events
+    // otherwise we'd have to upload 50 files before we see any response from
+    // the server. do not specify this so low in production as you'll have
+    // greatly degraded import performance
+    fileImportConcurrency: 1,
+    progress: (bytes, file) => {
+      showStatus(`File progress ${file} ${bytes}`, COLORS.active)
+    }
+  })) {
+    showStatus(`Added file: ${file.path} ${file.cid}`, COLORS.success)
+  }
+
+  showStatus('Finished!', COLORS.success)
+}
+
+// Event listeners
+document.getElementById('connect-submit').onclick = (e) => {
+  e.preventDefault()
+
+  main(document.getElementById('grpc-input').value, document.getElementById('http-input').value)
+    .catch(err => {
+      showStatus(err.message, COLORS.error)
+      console.error(err)
+    })
+}

examples/ipfs-client-add-files/package.json

@@ -0,0 +1,27 @@
+{
+  "name": "example-ipfs-client-add-files",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.js",
+  "private": true,
+  "scripts": {
+    "clean": "rimraf ./dist",
+    "build": "parcel build index.html --public-url '.'",
+    "start": "parcel index.html -p 8888",
+    "test": "test-ipfs-example"
+  },
+  "dependencies": {
+    "ipfs-client": "^0.1.0"
+  },
+  "devDependencies": {
+    "execa": "^4.0.3",
+    "ipfs": "^0.52.0",
+    "ipfsd-ctl": "ipfs/js-ipfsd-ctl#feat/expose-grpc-addr",
+    "parcel-bundler": "^1.12.4",
+    "rimraf": "^3.0.2",
+    "test-ipfs-example": "^2.0.3"
+  },
+  "browserslist": [
+    "last 2 versions and not dead and > 2%"
+  ]
+}

examples/ipfs-client-add-files/test.js

@@ -0,0 +1,82 @@
+'use strict'
+
+const path = require('path')
+const execa = require('execa')
+const { createFactory } = require('ipfsd-ctl')
+const df = createFactory({
+  ipfsClientModule: require('ipfs-client'),
+  ipfsBin: require.resolve('ipfs/src/cli.js')
+})
+const {
+  startServer
+} = require('test-ipfs-example/utils')
+const pkg = require('./package.json')
+
+async function testUI (url, http, grpc, id) {
+  const proc = execa(require.resolve('test-ipfs-example/node_modules/.bin/nightwatch'), ['--config', require.resolve('test-ipfs-example/nightwatch.conf.js'), path.join(__dirname, 'test.js')], {
+    cwd: path.resolve(__dirname, '../'),
+    env: {
+      ...process.env,
+      CI: true,
+      IPFS_EXAMPLE_TEST_URL: url,
+      IPFS_GRPC_API_MULTIADDR: grpc,
+      IPFS_HTTP_API_MULTIADDR: http
+    },
+    all: true
+  })
+  proc.all.on('data', (data) => {
+    process.stdout.write(data)
+  })
+
+  await proc
+}
+
+async function runTest () {
+  const app = await startServer(__dirname)
+  const daemon = await df.spawn({
+    type: 'js',
+    test: true,
+    ipfsOptions: {
+      config: {
+        Addresses: {
+          API: '/ip4/127.0.0.1/tcp/0',
+          RPC: '/ip4/127.0.0.1/tcp/0'
+        },
+        API: {
+          HTTPHeaders: {
+            'Access-Control-Allow-Origin': [
+              app.url
+            ]
+          }
+        }
+      }
+    }
+  })
+
+  try {
+    await testUI(app.url, daemon.apiAddr, daemon.grpcAddr, daemon.api.peerId.id)
+  } finally {
+    await daemon.stop()
+    await app.stop()
+  }
+}
+
+module.exports = runTest
+
+module.exports[pkg.name] = function (browser) {
+  browser
+    .url(process.env.IPFS_EXAMPLE_TEST_URL)
+    .waitForElementVisible('#grpc-input')
+    .clearValue('#grpc-input')
+    .setValue('#grpc-input', process.env.IPFS_GRPC_API_MULTIADDR)
+    .pause(1000)
+    .waitForElementVisible('#http-input')
+    .clearValue('#http-input')
+    .setValue('#http-input', process.env.IPFS_HTTP_API_MULTIADDR)
+    .pause(1000)
+    .click('#connect-submit')
+
+  browser.expect.element('#output').text.to.contain('Added file: file-0.txt QmUDLiEJwL3vUhhXNXDF2RrCnVkSB2LemWYffpCCPcQCeU')
+
+  browser.end()
+}

package.json

@@ -16,6 +16,7 @@
     "test:external": "lerna run test:external",
     "test:cli": "lerna run test:cli",
     "test:interop": "lerna run test:interop",
+    "test:interface:client": "lerna run test:interface:client",
     "test:interface:core": "lerna run test:interface:core",
     "test:interface:http-go": "lerna run test:interface:http-go",
     "test:interface:http-js": "lerna run test:interface:http-js",

packages/ipfs-cli/package.json

@@ -40,6 +40,7 @@
     "get-folder-size": "^2.0.1",
     "ipfs-core": "^0.2.1",
     "ipfs-core-utils": "^0.5.2",
+    "ipfs-grpc-server": "0.0.0",
     "ipfs-http-client": "^48.1.1",
     "ipfs-http-gateway": "^0.1.2",
     "ipfs-http-server": "^0.1.2",

packages/ipfs-cli/src/commands/daemon.js

@@ -85,8 +85,10 @@ module.exports = {
       await daemon.start()
       // @ts-ignore - _httpApi is possibly undefined
       daemon._httpApi._apiServers.forEach(apiServer => {
-        print(`API listening on ${apiServer.info.ma}`)
+        print(`HTTP API listening on ${apiServer.info.ma}`)
       })
+      // @ts-ignore - _httpApi is possibly undefined
+      print(`gRPC listening on ${daemon._grpcServer.multiaddr}`)
       // @ts-ignore - _httpGateway is possibly undefined
       daemon._httpGateway._gatewayServers.forEach(gatewayServer => {
         print(`Gateway (read only) listening on ${gatewayServer.info.ma}`)

packages/ipfs-cli/src/daemon.js

@@ -11,6 +11,7 @@ const ipfsHttpClient = require('ipfs-http-client')
 const IPFS = require('ipfs-core')
 const HttpApi = require('ipfs-http-server')
 const HttpGateway = require('ipfs-http-gateway')
+const gRPCServer = require('ipfs-grpc-server')
 const createRepo = require('ipfs-core/src/runtime/repo-nodejs')
 
 class Daemon {
@@ -58,6 +59,8 @@ class Daemon {
       await repo.apiAddr.set(this._httpApi._apiServers[0].info.ma)
     }
 
+    this._grpcServer = await gRPCServer(ipfs, ipfsOpts)
+
     log('started')
     return this
   }

packages/ipfs-cli/tsconfig.json

@@ -14,6 +14,9 @@
     {
       "path": "../ipfs-core-utils"
     },
+    {
+      "path": "../ipfs-grpc-server"
+    },
     {
       "path": "../ipfs-http-client"
     },

packages/ipfs-client/.aegir.js

@@ -0,0 +1,5 @@
+'use strict'
+
+module.exports = {
+  bundlesize: { maxSize: '81kB' }
+}