From 5c07b65144a56cd112dbfae3cf90705678579f47 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Wed, 4 Nov 2020 16:52:55 +0000 Subject: [PATCH] feat: add grpc server and client Adds a server running a gRPC endpoint over websockets, a client to access the server and a `ipfs-client` module that uses the gRPC client with HTTP fallback. So far only supports `ipfs.addAll` but the idea is to implement all streaming methods over websockets instead of HTTP, to give us bidirectional streaming and errors that work in the browser. Fixes: Depends on: - [ ] https://github.com/ipfs/js-ipfsd-ctl/pull/561 --- .travis.yml | 35 ++++ examples/browser-ipns-publish/package.json | 2 +- .../explore-ethereum-blockchain/package.json | 2 +- .../http-client-browser-pubsub/package.json | 2 +- .../http-client-bundle-webpack/package.json | 2 +- examples/http-client-name-api/package.json | 2 +- examples/ipfs-client-add-files/README.md | 21 ++ examples/ipfs-client-add-files/index.html | 36 ++++ examples/ipfs-client-add-files/index.js | 81 ++++++++ examples/ipfs-client-add-files/package.json | 27 +++ examples/ipfs-client-add-files/test.js | 82 ++++++++ package.json | 3 +- packages/ipfs-cli/package.json | 1 + packages/ipfs-cli/src/commands/daemon.js | 8 +- packages/ipfs-cli/src/daemon.js | 3 + packages/ipfs-cli/tsconfig.json | 3 + packages/ipfs-client/.aegir.js | 7 + packages/ipfs-client/README.md | 55 ++++++ packages/ipfs-client/package.json | 48 +++++ packages/ipfs-client/src/index.js | 28 +++ packages/ipfs-client/tsconfig.json | 10 + packages/ipfs-core/.aegir.js | 2 +- packages/ipfs-core/package.json | 2 +- .../ipfs-core/src/runtime/config-browser.js | 1 + .../ipfs-core/src/runtime/config-nodejs.js | 1 + packages/ipfs-grpc-client/.aegir.js | 7 + packages/ipfs-grpc-client/README.md | 57 ++++++ packages/ipfs-grpc-client/package.json | 62 ++++++ .../ipfs-grpc-client/src/core-api/add-all.js | 121 ++++++++++++ .../ipfs-grpc-client/src/core-api/files/ls.js | 35 ++++ .../src/core-api/files/write.js | 48 +++++ packages/ipfs-grpc-client/src/core-api/id.js | 26 +++ .../src/grpc/transport.browser.js | 5 + .../ipfs-grpc-client/src/grpc/transport.js | 125 ++++++++++++ packages/ipfs-grpc-client/src/index.js | 37 ++++ .../src/utils/bidi-to-duplex.js | 63 ++++++ .../src/utils/client-stream-to-promise.js | 28 +++ .../src/utils/load-services.js | 84 ++++++++ .../src/utils/server-stream-to-iterator.js | 23 +++ .../ipfs-grpc-client/src/utils/to-headers.js | 21 ++ .../src/utils/unary-to-promise.js | 24 +++ packages/ipfs-grpc-client/test/utils.spec.js | 96 +++++++++ packages/ipfs-grpc-client/tsconfig.json | 21 ++ packages/ipfs-grpc-protocol/README.md | 3 + packages/ipfs-grpc-protocol/package.json | 31 +++ packages/ipfs-grpc-protocol/src/common.proto | 8 + packages/ipfs-grpc-protocol/src/mfs.proto | 33 ++++ packages/ipfs-grpc-protocol/src/root.proto | 49 +++++ packages/ipfs-grpc-protocol/tsconfig.json | 10 + packages/ipfs-grpc-server/README.md | 182 ++++++++++++++++++ packages/ipfs-grpc-server/package.json | 62 ++++++ .../ipfs-grpc-server/src/endpoints/add.js | 114 +++++++++++ packages/ipfs-grpc-server/src/endpoints/id.js | 16 ++ .../ipfs-grpc-server/src/endpoints/mfs/ls.js | 27 +++ .../src/endpoints/mfs/write.js | 38 ++++ packages/ipfs-grpc-server/src/index.js | 111 +++++++++++ .../src/utils/load-services.js | 77 ++++++++ .../src/utils/snake-to-camel.js | 13 ++ .../src/utils/web-socket-message-channel.js | 139 +++++++++++++ .../src/utils/web-socket-server.js | 88 +++++++++ packages/ipfs-grpc-server/test/add.spec.js | 95 +++++++++ packages/ipfs-grpc-server/test/id.spec.js | 65 +++++++ packages/ipfs-grpc-server/test/mfs/ls.spec.js | 80 ++++++++ .../ipfs-grpc-server/test/mfs/write.spec.js | 46 +++++ .../ipfs-grpc-server/test/utils/channel.js | 48 +++++ .../ipfs-grpc-server/test/utils/server.js | 23 +++ packages/ipfs-grpc-server/tsconfig.json | 15 ++ packages/ipfs-http-client/package.json | 2 +- packages/ipfs/.aegir.js | 5 +- packages/ipfs/package.json | 4 +- packages/ipfs/test/interface-client.js | 147 ++++++++++++++ tsconfig.json | 12 ++ 72 files changed, 2876 insertions(+), 14 deletions(-) create mode 100644 examples/ipfs-client-add-files/README.md create mode 100644 examples/ipfs-client-add-files/index.html create mode 100644 examples/ipfs-client-add-files/index.js create mode 100644 examples/ipfs-client-add-files/package.json create mode 100644 examples/ipfs-client-add-files/test.js create mode 100644 packages/ipfs-client/.aegir.js create mode 100644 packages/ipfs-client/README.md create mode 100644 packages/ipfs-client/package.json create mode 100644 packages/ipfs-client/src/index.js create mode 100644 packages/ipfs-client/tsconfig.json create mode 100644 packages/ipfs-grpc-client/.aegir.js create mode 100644 packages/ipfs-grpc-client/README.md create mode 100644 packages/ipfs-grpc-client/package.json create mode 100644 packages/ipfs-grpc-client/src/core-api/add-all.js create mode 100644 packages/ipfs-grpc-client/src/core-api/files/ls.js create mode 100644 packages/ipfs-grpc-client/src/core-api/files/write.js create mode 100644 packages/ipfs-grpc-client/src/core-api/id.js create mode 100644 packages/ipfs-grpc-client/src/grpc/transport.browser.js create mode 100644 packages/ipfs-grpc-client/src/grpc/transport.js create mode 100644 packages/ipfs-grpc-client/src/index.js create mode 100644 packages/ipfs-grpc-client/src/utils/bidi-to-duplex.js create mode 100644 packages/ipfs-grpc-client/src/utils/client-stream-to-promise.js create mode 100644 packages/ipfs-grpc-client/src/utils/load-services.js create mode 100644 packages/ipfs-grpc-client/src/utils/server-stream-to-iterator.js create mode 100644 packages/ipfs-grpc-client/src/utils/to-headers.js create mode 100644 packages/ipfs-grpc-client/src/utils/unary-to-promise.js create mode 100644 packages/ipfs-grpc-client/test/utils.spec.js create mode 100644 packages/ipfs-grpc-client/tsconfig.json create mode 100644 packages/ipfs-grpc-protocol/README.md create mode 100644 packages/ipfs-grpc-protocol/package.json create mode 100644 packages/ipfs-grpc-protocol/src/common.proto create mode 100644 packages/ipfs-grpc-protocol/src/mfs.proto create mode 100644 packages/ipfs-grpc-protocol/src/root.proto create mode 100644 packages/ipfs-grpc-protocol/tsconfig.json create mode 100644 packages/ipfs-grpc-server/README.md create mode 100644 packages/ipfs-grpc-server/package.json create mode 100644 packages/ipfs-grpc-server/src/endpoints/add.js create mode 100644 packages/ipfs-grpc-server/src/endpoints/id.js create mode 100644 packages/ipfs-grpc-server/src/endpoints/mfs/ls.js create mode 100644 packages/ipfs-grpc-server/src/endpoints/mfs/write.js create mode 100644 packages/ipfs-grpc-server/src/index.js create mode 100644 packages/ipfs-grpc-server/src/utils/load-services.js create mode 100644 packages/ipfs-grpc-server/src/utils/snake-to-camel.js create mode 100644 packages/ipfs-grpc-server/src/utils/web-socket-message-channel.js create mode 100644 packages/ipfs-grpc-server/src/utils/web-socket-server.js create mode 100644 packages/ipfs-grpc-server/test/add.spec.js create mode 100644 packages/ipfs-grpc-server/test/id.spec.js create mode 100644 packages/ipfs-grpc-server/test/mfs/ls.spec.js create mode 100644 packages/ipfs-grpc-server/test/mfs/write.spec.js create mode 100644 packages/ipfs-grpc-server/test/utils/channel.js create mode 100644 packages/ipfs-grpc-server/test/utils/server.js create mode 100644 packages/ipfs-grpc-server/tsconfig.json create mode 100644 packages/ipfs/test/interface-client.js diff --git a/.travis.yml b/.travis.yml index 9561a8e1a1..e18b74ad1e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -164,6 +164,41 @@ jobs: script: - npm run test:interface:core -- $RUN_SINCE -- -- --bail -t electron-renderer --timeout 60000 + - stage: test + name: js-ipfs interface tests - ipfs-client - node + script: + - npm run test:interface:client -- $RUN_SINCE -- -- --bail -t node + + - stage: test + name: js-ipfs interface tests - ipfs-client - chrome + script: + - npm run test:interface:client -- $RUN_SINCE -- -- --bail -t browser + + - stage: test + name: js-ipfs interface tests - ipfs-client - chrome webworker + script: + - npm run test:interface:client -- $RUN_SINCE -- -- --bail -t webworker --timeout 60000 + + - stage: test + name: js-ipfs interface tests - ipfs-client - firefox + script: + - npm run test:interface:client -- $RUN_SINCE -- -- --bail -t browser --browsers FirefoxHeadless + + - stage: test + name: js-ipfs interface tests - ipfs-client - firefox webworker + script: + - npm run test:interface:client -- $RUN_SINCE -- -- --bail -t webworker --browsers FirefoxHeadless --timeout 60000 + + - stage: test + name: js-ipfs interface tests - ipfs-client - electron main + script: + - npm run test:interface:client -- $RUN_SINCE -- -- --bail -t electron-main --timeout 60000 + + - stage: test + name: js-ipfs interface tests - ipfs-client - electron renderer + script: + - npm run test:interface:client -- $RUN_SINCE -- -- --bail -t electron-renderer --timeout 60000 + - stage: test name: http-api-client interface tests vs go-ipfs - node script: diff --git a/examples/browser-ipns-publish/package.json b/examples/browser-ipns-publish/package.json index 988c6d2184..399029f5d7 100644 --- a/examples/browser-ipns-publish/package.json +++ b/examples/browser-ipns-publish/package.json @@ -27,7 +27,7 @@ "devDependencies": { "delay": "^4.4.0", "execa": "^4.0.3", - "ipfsd-ctl": "^7.0.2", + "ipfsd-ctl": "ipfs/js-ipfsd-ctl#feat/expose-grpc-addr", "go-ipfs": "^0.7.0", "parcel-bundler": "^1.12.4", "path": "^0.12.7", diff --git a/examples/explore-ethereum-blockchain/package.json b/examples/explore-ethereum-blockchain/package.json index 28ae6c4a39..a63d07b004 100644 --- a/examples/explore-ethereum-blockchain/package.json +++ b/examples/explore-ethereum-blockchain/package.json @@ -12,7 +12,7 @@ "devDependencies": { "ipfs": "^0.52.1", "ipfs-http-client": "^48.1.1", - "ipfsd-ctl": "^7.0.2", + "ipfsd-ctl": "ipfs/js-ipfsd-ctl#feat/expose-grpc-addr", "ipld-ethereum": "^5.0.1", "test-ipfs-example": "^2.0.3" } diff --git a/examples/http-client-browser-pubsub/package.json b/examples/http-client-browser-pubsub/package.json index 0b04ed1f63..52e6409e05 100644 --- a/examples/http-client-browser-pubsub/package.json +++ b/examples/http-client-browser-pubsub/package.json @@ -21,7 +21,7 @@ "execa": "^4.0.3", "go-ipfs": "^0.7.0", "ipfs": "^0.52.1", - "ipfsd-ctl": "^7.0.2", + "ipfsd-ctl": "ipfs/js-ipfsd-ctl#feat/expose-grpc-addr", "parcel-bundler": "^1.12.4", "test-ipfs-example": "^2.0.3" } diff --git a/examples/http-client-bundle-webpack/package.json b/examples/http-client-bundle-webpack/package.json index 5401879b70..90aa4732cb 100644 --- a/examples/http-client-bundle-webpack/package.json +++ b/examples/http-client-bundle-webpack/package.json @@ -25,7 +25,7 @@ "copy-webpack-plugin": "^5.0.4", "execa": "^4.0.3", "ipfs": "^0.52.1", - "ipfsd-ctl": "^7.0.2", + "ipfsd-ctl": "ipfs/js-ipfsd-ctl#feat/expose-grpc-addr", "react-hot-loader": "^4.12.21", "rimraf": "^3.0.2", "test-ipfs-example": "^2.0.3", diff --git a/examples/http-client-name-api/package.json b/examples/http-client-name-api/package.json index 28587a19f2..1bef2f5cbb 100644 --- a/examples/http-client-name-api/package.json +++ b/examples/http-client-name-api/package.json @@ -18,7 +18,7 @@ "devDependencies": { "execa": "^4.0.3", "go-ipfs": "^0.7.0", - "ipfsd-ctl": "^7.0.2", + "ipfsd-ctl": "ipfs/js-ipfsd-ctl#feat/expose-grpc-addr", "parcel-bundler": "^1.12.4", "rimraf": "^3.0.2", "test-ipfs-example": "^2.0.3" diff --git a/examples/ipfs-client-add-files/README.md b/examples/ipfs-client-add-files/README.md new file mode 100644 index 0000000000..1fbd3bb0fd --- /dev/null +++ b/examples/ipfs-client-add-files/README.md @@ -0,0 +1,21 @@ +# JS IPFS API - Example Browser - Name + +## Setup + +```sh +npm install -g ipfs +jsipfs init +# Configure CORS to allow ipfs-http-client to access this IPFS node +jsipfs config --json API.HTTPHeaders.Access-Control-Allow-Origin '["http://127.0.0.1:8888"]' +# Start the IPFS node +jsipfs daemon +``` + +Then in this folder run + +```bash +> npm install +> npm start +``` + +and open your browser at `http://127.0.0.1:8888`. diff --git a/examples/ipfs-client-add-files/index.html b/examples/ipfs-client-add-files/index.html new file mode 100644 index 0000000000..619eb6b919 --- /dev/null +++ b/examples/ipfs-client-add-files/index.html @@ -0,0 +1,36 @@ + + + + + JS IPFS Client example + + + + +

ipfs-client

+
+

Enter IPFS API details

+ + + +
+
+
+ + + + diff --git a/examples/ipfs-client-add-files/index.js b/examples/ipfs-client-add-files/index.js new file mode 100644 index 0000000000..d66a40ad4a --- /dev/null +++ b/examples/ipfs-client-add-files/index.js @@ -0,0 +1,81 @@ +/* eslint-disable no-console */ +'use strict' + +const ipfsClient = require('ipfs-client') +let ipfs + +const COLORS = { + active: 'blue', + success: 'green', + error: 'red' +} + +const showStatus = (text, bg) => { + console.info(text) + + const log = document.getElementById('output') + + if (!log) { + return + } + + const line = document.createElement('p') + line.innerText = text + line.style.color = bg + + log.appendChild(line) +} + +async function * streamFiles () { + for (let i = 0; i < 100; i++) { + await new Promise((resolve) => { + setTimeout(() => resolve(), 100) + }) + + showStatus(`Sending /file-${i}.txt`, COLORS.active) + + yield { + path: `/file-${i}.txt`, + content: `file ${i}` + } + } +} + +async function main (grpcApi, httpApi) { + showStatus(`Connecting to ${grpcApi} using ${httpApi} as fallback`, COLORS.active) + + ipfs = ipfsClient({ + grpc: grpcApi, + http: httpApi + }) + + const id = await ipfs.id() + showStatus(`Daemon active\nID: ${id.id}`, COLORS.success) + + for await (const file of ipfs.addAll(streamFiles(), { + wrapWithDirectory: true, + // this is just to show the interleaving of uploads and progress events + // otherwise we'd have to upload 50 files before we see any response from + // the server. do not specify this so low in production as you'll have + // greatly degraded import performance + fileImportConcurrency: 1, + progress: (bytes, file) => { + showStatus(`File progress ${file} ${bytes}`, COLORS.active) + } + })) { + showStatus(`Added file: ${file.path} ${file.cid}`, COLORS.success) + } + + showStatus('Finished!', COLORS.success) +} + +// Event listeners +document.getElementById('connect-submit').onclick = (e) => { + e.preventDefault() + + main(document.getElementById('grpc-input').value, document.getElementById('http-input').value) + .catch(err => { + showStatus(err.message, COLORS.error) + console.error(err) + }) +} diff --git a/examples/ipfs-client-add-files/package.json b/examples/ipfs-client-add-files/package.json new file mode 100644 index 0000000000..41f52fc623 --- /dev/null +++ b/examples/ipfs-client-add-files/package.json @@ -0,0 +1,27 @@ +{ + "name": "example-ipfs-client-add-files", + "version": "1.0.0", + "description": "", + "main": "index.js", + "private": true, + "scripts": { + "clean": "rimraf ./dist", + "build": "parcel build index.html --public-url '.'", + "start": "parcel index.html -p 8888", + "test": "test-ipfs-example" + }, + "dependencies": { + "ipfs-client": "^0.1.0" + }, + "devDependencies": { + "execa": "^4.0.3", + "ipfs": "^0.52.0", + "ipfsd-ctl": "ipfs/js-ipfsd-ctl#feat/expose-grpc-addr", + "parcel-bundler": "^1.12.4", + "rimraf": "^3.0.2", + "test-ipfs-example": "^2.0.3" + }, + "browserslist": [ + "last 2 versions and not dead and > 2%" + ] +} diff --git a/examples/ipfs-client-add-files/test.js b/examples/ipfs-client-add-files/test.js new file mode 100644 index 0000000000..84f6892d9d --- /dev/null +++ b/examples/ipfs-client-add-files/test.js @@ -0,0 +1,82 @@ +'use strict' + +const path = require('path') +const execa = require('execa') +const { createFactory } = require('ipfsd-ctl') +const df = createFactory({ + ipfsClientModule: require('ipfs-client'), + ipfsBin: require.resolve('ipfs/src/cli.js') +}) +const { + startServer +} = require('test-ipfs-example/utils') +const pkg = require('./package.json') + +async function testUI (url, http, grpc, id) { + const proc = execa(require.resolve('test-ipfs-example/node_modules/.bin/nightwatch'), ['--config', require.resolve('test-ipfs-example/nightwatch.conf.js'), path.join(__dirname, 'test.js')], { + cwd: path.resolve(__dirname, '../'), + env: { + ...process.env, + CI: true, + IPFS_EXAMPLE_TEST_URL: url, + IPFS_GRPC_API_MULTIADDR: grpc, + IPFS_HTTP_API_MULTIADDR: http + }, + all: true + }) + proc.all.on('data', (data) => { + process.stdout.write(data) + }) + + await proc +} + +async function runTest () { + const app = await startServer(__dirname) + const daemon = await df.spawn({ + type: 'js', + test: true, + ipfsOptions: { + config: { + Addresses: { + API: '/ip4/127.0.0.1/tcp/0', + RPC: '/ip4/127.0.0.1/tcp/0' + }, + API: { + HTTPHeaders: { + 'Access-Control-Allow-Origin': [ + app.url + ] + } + } + } + } + }) + + try { + await testUI(app.url, daemon.apiAddr, daemon.grpcAddr, daemon.api.peerId.id) + } finally { + await daemon.stop() + await app.stop() + } +} + +module.exports = runTest + +module.exports[pkg.name] = function (browser) { + browser + .url(process.env.IPFS_EXAMPLE_TEST_URL) + .waitForElementVisible('#grpc-input') + .clearValue('#grpc-input') + .setValue('#grpc-input', process.env.IPFS_GRPC_API_MULTIADDR) + .pause(1000) + .waitForElementVisible('#http-input') + .clearValue('#http-input') + .setValue('#http-input', process.env.IPFS_HTTP_API_MULTIADDR) + .pause(1000) + .click('#connect-submit') + + browser.expect.element('#output').text.to.contain('Added file: file-0.txt QmUDLiEJwL3vUhhXNXDF2RrCnVkSB2LemWYffpCCPcQCeU') + + browser.end() +} diff --git a/package.json b/package.json index 73868f983e..57764be71f 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,7 @@ "version": "1.0.0", "description": "JavaScript implementation of the IPFS specification", "scripts": { - "postinstall": "lerna bootstrap", + "postinstall": "lerna bootstrap && npm run build -- --scope=ipfs-grpc-protocol", "link": "lerna link", "reset": "lerna run clean && rimraf packages/*/node_modules node_modules", "test": "lerna run test", @@ -16,6 +16,7 @@ "test:external": "lerna run test:external", "test:cli": "lerna run test:cli", "test:interop": "lerna run test:interop", + "test:interface:client": "lerna run test:interface:client", "test:interface:core": "lerna run test:interface:core", "test:interface:http-go": "lerna run test:interface:http-go", "test:interface:http-js": "lerna run test:interface:http-js", diff --git a/packages/ipfs-cli/package.json b/packages/ipfs-cli/package.json index d3b1d0818f..51d588c91b 100644 --- a/packages/ipfs-cli/package.json +++ b/packages/ipfs-cli/package.json @@ -40,6 +40,7 @@ "get-folder-size": "^2.0.1", "ipfs-core": "^0.2.1", "ipfs-core-utils": "^0.5.2", + "ipfs-grpc-server": "0.0.0", "ipfs-http-client": "^48.1.1", "ipfs-http-gateway": "^0.1.2", "ipfs-http-server": "^0.1.2", diff --git a/packages/ipfs-cli/src/commands/daemon.js b/packages/ipfs-cli/src/commands/daemon.js index 1843dc3897..a0c13c9f0b 100644 --- a/packages/ipfs-cli/src/commands/daemon.js +++ b/packages/ipfs-cli/src/commands/daemon.js @@ -83,15 +83,17 @@ module.exports = { try { await daemon.start() - // @ts-ignore - _httpApi is possibly undefined + // @ts-ignore - _apiServers is possibly undefined daemon._httpApi._apiServers.forEach(apiServer => { - print(`API listening on ${apiServer.info.ma}`) + print(`HTTP API listening on ${apiServer.info.ma}`) }) + // @ts-ignore - _grpcServer is possibly undefined + print(`gRPC listening on ${daemon._grpcServer.multiaddr}`) // @ts-ignore - _httpGateway is possibly undefined daemon._httpGateway._gatewayServers.forEach(gatewayServer => { print(`Gateway (read only) listening on ${gatewayServer.info.ma}`) }) - // @ts-ignore - _httpApi is possibly undefined + // @ts-ignore - _apiServers is possibly undefined daemon._httpApi._apiServers.forEach(apiServer => { print(`Web UI available at ${toUri(apiServer.info.ma)}/webui`) }) diff --git a/packages/ipfs-cli/src/daemon.js b/packages/ipfs-cli/src/daemon.js index be461a6c4b..630986140b 100644 --- a/packages/ipfs-cli/src/daemon.js +++ b/packages/ipfs-cli/src/daemon.js @@ -11,6 +11,7 @@ const ipfsHttpClient = require('ipfs-http-client') const IPFS = require('ipfs-core') const HttpApi = require('ipfs-http-server') const HttpGateway = require('ipfs-http-gateway') +const gRPCServer = require('ipfs-grpc-server') const createRepo = require('ipfs-core/src/runtime/repo-nodejs') class Daemon { @@ -58,6 +59,8 @@ class Daemon { await repo.apiAddr.set(this._httpApi._apiServers[0].info.ma) } + this._grpcServer = await gRPCServer(ipfs, ipfsOpts) + log('started') return this } diff --git a/packages/ipfs-cli/tsconfig.json b/packages/ipfs-cli/tsconfig.json index f84b5f655c..2111e28905 100644 --- a/packages/ipfs-cli/tsconfig.json +++ b/packages/ipfs-cli/tsconfig.json @@ -14,6 +14,9 @@ { "path": "../ipfs-core-utils" }, + { + "path": "../ipfs-grpc-server" + }, { "path": "../ipfs-http-client" }, diff --git a/packages/ipfs-client/.aegir.js b/packages/ipfs-client/.aegir.js new file mode 100644 index 0000000000..b55dc70587 --- /dev/null +++ b/packages/ipfs-client/.aegir.js @@ -0,0 +1,7 @@ +'use strict' + +module.exports = { + bundlesize: { + maxSize: '112kB' + } +} diff --git a/packages/ipfs-client/README.md b/packages/ipfs-client/README.md new file mode 100644 index 0000000000..23d7f8500e --- /dev/null +++ b/packages/ipfs-client/README.md @@ -0,0 +1,55 @@ +# ipfs-client + +> A client for [ipfs][] daemons + +This module combines the [ipfs-grpc-client][] and [ipfs-http-client][] modules to give you a client that is capable of bidirectional streaming in the browser as well as node. + +## Install + +```console +$ npm install ipfs-client +``` + +## API + +The client object created by the `createClient` function supports the [IPFS Core API](https://github.com/ipfs/js-ipfs/tree/master/docs/core-api), see the docs for more. + +### `createClient([options])` + +### Parameters + +None + +### Options + +An optional object which may have the following keys: + +| Name | Type | Default | Description | +| ---- | ---- | ------- | ----------- | +| grpc | `Multiaddr` or `string` or `URL` | `undefined` | The address of a [ipfs-grpc-server][] to connect to | +| http | `Multiaddr` or `string` or `URL` | `undefined` | The address of a [ipfs-http-server][] to connect to | + +### Returns + +| Type | Description | +| -------- | -------- | +| `object` | An instance of the client | + +### Example + +```js +const createClient = require('ipfs-client') + +const client = createClient({ + grpc: '/ipv4/127.0.0.1/tcp/5003/ws', + http: '/ipv4/127.0.0.1/tcp/5002/http' +}) + +const id = await client.id() +``` + +[ipfs]: https://www.npmjs.com/package/ipfs +[ipfs-grpc-client]: https://www.npmjs.com/package/ipfs-grpc-client +[ipfs-http-client]: https://www.npmjs.com/package/ipfs-http-client +[ipfs-grpc-server]: https://www.npmjs.com/package/ipfs-grpc-server +[ipfs-http-server]: https://www.npmjs.com/package/ipfs-http-server diff --git a/packages/ipfs-client/package.json b/packages/ipfs-client/package.json new file mode 100644 index 0000000000..de7d99825d --- /dev/null +++ b/packages/ipfs-client/package.json @@ -0,0 +1,48 @@ +{ + "name": "ipfs-client", + "version": "0.1.0", + "description": "A client library to talk to local IPFS daemons", + "keywords": [ + "ipfs" + ], + "homepage": "https://github.com/ipfs/js-ipfs/tree/master/packages/ipfs-client#readme", + "bugs": "https://github.com/ipfs/js-ipfs/issues", + "license": "(Apache-2.0 OR MIT)", + "leadMaintainer": "Alex Potsides ", + "files": [ + "src", + "dist" + ], + "main": "src/index.js", + "typesVersions": { + "*": { + "*": [ + "dist/*" + ] + } + }, + "repository": { + "type": "git", + "url": "git+https://github.com/ipfs/js-ipfs.git" + }, + "scripts": { + "test": "aegir test", + "lint": "aegir lint", + "build": "npm run build:js && npm run build:types", + "build:js": "aegir build", + "build:types": "tsc --build", + "coverage": "npx nyc -r html npm run test:node -- --bail", + "clean": "rimraf ./dist", + "dep-check": "aegir dep-check -i aegir -i typescript -i rimraf" + }, + "dependencies": { + "ipfs-grpc-client": "0.0.0", + "ipfs-http-client": "^48.1.0", + "merge-options": "^2.0.0" + }, + "devDependencies": { + "aegir": "^28.2.0", + "rimraf": "^3.0.2", + "typescript": "^4.0.3" + } +} diff --git a/packages/ipfs-client/src/index.js b/packages/ipfs-client/src/index.js new file mode 100644 index 0000000000..2e4edee9b2 --- /dev/null +++ b/packages/ipfs-client/src/index.js @@ -0,0 +1,28 @@ +'use strict' + +const httpClient = require('ipfs-http-client') +const grpcClient = require('ipfs-grpc-client') +const mergeOptions = require('merge-options') + +module.exports = function createClient (opts = {}) { + opts = opts || {} + + const clients = [] + + if (opts.http) { + clients.push(httpClient({ + ...opts, + url: opts.http + })) + } + + if (opts.grpc) { + clients.push(grpcClient({ + ...opts, + url: opts.grpc + })) + } + + // override http methods with grpc if address is supplied + return mergeOptions.apply({ ignoreUndefined: true }, clients) +} diff --git a/packages/ipfs-client/tsconfig.json b/packages/ipfs-client/tsconfig.json new file mode 100644 index 0000000000..979a39adab --- /dev/null +++ b/packages/ipfs-client/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "outDir": "dist" + }, + "include": [ + "src", + "package.json" + ] +} diff --git a/packages/ipfs-core/.aegir.js b/packages/ipfs-core/.aegir.js index a11af408f4..41e0cda3b0 100644 --- a/packages/ipfs-core/.aegir.js +++ b/packages/ipfs-core/.aegir.js @@ -8,7 +8,7 @@ let preloadNode = MockPreloadNode.createNode() let ipfsdServer module.exports = { - bundlesize: { maxSize: '524kB' }, + bundlesize: { maxSize: '560kB' }, karma: { files: [{ pattern: 'node_modules/interface-ipfs-core/test/fixtures/**/*', diff --git a/packages/ipfs-core/package.json b/packages/ipfs-core/package.json index 04e6c576eb..fb261ab132 100644 --- a/packages/ipfs-core/package.json +++ b/packages/ipfs-core/package.json @@ -122,7 +122,7 @@ "delay": "^4.4.0", "go-ipfs": "^0.7.0", "interface-ipfs-core": "^0.142.1", - "ipfsd-ctl": "^7.0.2", + "ipfsd-ctl": "ipfs/js-ipfsd-ctl#feat/expose-grpc-addr", "ipld-git": "^0.6.1", "iso-url": "^1.0.0", "nanoid": "^3.1.12", diff --git a/packages/ipfs-core/src/runtime/config-browser.js b/packages/ipfs-core/src/runtime/config-browser.js index d3d37da198..7420ca8a0c 100644 --- a/packages/ipfs-core/src/runtime/config-browser.js +++ b/packages/ipfs-core/src/runtime/config-browser.js @@ -6,6 +6,7 @@ module.exports = () => ({ ], API: '', Gateway: '', + RPC: '', Delegates: [ '/dns4/node0.delegate.ipfs.io/tcp/443/https', '/dns4/node1.delegate.ipfs.io/tcp/443/https', diff --git a/packages/ipfs-core/src/runtime/config-nodejs.js b/packages/ipfs-core/src/runtime/config-nodejs.js index 483b9b72ac..2f264b62b2 100644 --- a/packages/ipfs-core/src/runtime/config-nodejs.js +++ b/packages/ipfs-core/src/runtime/config-nodejs.js @@ -8,6 +8,7 @@ module.exports = () => ({ ], API: '/ip4/127.0.0.1/tcp/5002', Gateway: '/ip4/127.0.0.1/tcp/9090', + RPC: '/ip4/127.0.0.1/tcp/5003', Delegates: [ '/dns4/node0.delegate.ipfs.io/tcp/443/https', '/dns4/node1.delegate.ipfs.io/tcp/443/https', diff --git a/packages/ipfs-grpc-client/.aegir.js b/packages/ipfs-grpc-client/.aegir.js new file mode 100644 index 0000000000..4173cb261e --- /dev/null +++ b/packages/ipfs-grpc-client/.aegir.js @@ -0,0 +1,7 @@ +'use strict' + +module.exports = { + bundlesize: { + maxSize: '48kB' + } +} diff --git a/packages/ipfs-grpc-client/README.md b/packages/ipfs-grpc-client/README.md new file mode 100644 index 0000000000..1c2be2b0d9 --- /dev/null +++ b/packages/ipfs-grpc-client/README.md @@ -0,0 +1,57 @@ +# ipfs-grpc-client + +> A client for the [ipfs-grpc-server][] module + +This module implements part of the [IPFS Core API](https://github.com/ipfs/js-ipfs/tree/master/docs/core-api) using gRPC over websockets to achieve the bidirectional streaming necessary to have full duplex streams running in the browser. + +It's not recommended you use this directly, instead use the [ipfs-client](https://www.npmjs.com/package/ipfs-client) to combine this with the [ipfs-http-client](https://www.npmjs.com/package/ipfs-http-client) in order to have HTTP fallback for the missing parts of the API. + +## Why? + +The [fetch](https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API) and [XHR](https://developer.mozilla.org/en-US/docs/Web/API/XMLHttpRequest) APIs do not allow for full-duplex streaming, that is, allowing the client to receive bytes from the response while also adding more bytes to the outgoing request. + +This limits what we can do in browsers in terms of the API, for example streaming arbitrarily sized payloads or exposing libp2p duplex streams. + +gPRC over websockets has no such limitations so allows us to harness the full power of a remote IPFS node in the browser without the need to work around browser behaviour. + +## Install + +```console +$ npm install ipfs-grpc-client +``` + +## API + +### `createClient([options])` + +### Parameters + +None + +### Options + +An optional object which may have the following keys: + +| Name | Type | Default | Description | +| ---- | ---- | ------- | ----------- | +| url | `Multiaddr` or `string` or `URL` | `undefined` | The address of a [ipfs-grpc-server][] to connect to | + +### Returns + +| Type | Description | +| -------- | -------- | +| `object` | An instance of the client | + +### Example + +```js +const createClient = require('ipfs-gprc-client') + +const client = createClient({ + url: '/ipv4/127.0.0.1/tcp/1234/ws' +}) + +const id = await client.id() +``` + +[ipfs-grpc-server]: https://www.npmjs.com/package/ipfs-grpc-server diff --git a/packages/ipfs-grpc-client/package.json b/packages/ipfs-grpc-client/package.json new file mode 100644 index 0000000000..8e179cf1d0 --- /dev/null +++ b/packages/ipfs-grpc-client/package.json @@ -0,0 +1,62 @@ +{ + "name": "ipfs-grpc-client", + "version": "0.0.0", + "description": "A client library for the IPFS gRPC API", + "keywords": [ + "ipfs" + ], + "homepage": "https://github.com/ipfs/js-ipfs/tree/master/packages/ipfs-grpc-client#readme", + "bugs": "https://github.com/ipfs/js-ipfs/issues", + "license": "(Apache-2.0 OR MIT)", + "leadMaintainer": "Alex Potsides ", + "files": [ + "src", + "dist" + ], + "main": "src/index.js", + "browser": { + "./src/grpc/transport.js": "./src/grpc/transport.browser.js" + }, + "typesVersions": { + "*": { + "*": [ + "dist/*" + ] + } + }, + "repository": { + "type": "git", + "url": "git+https://github.com/ipfs/js-ipfs.git" + }, + "scripts": { + "test": "aegir test", + "lint": "aegir lint", + "build": "npm run build:js && npm run build:types", + "build:js": "aegir build", + "build:types": "tsc --build", + "coverage": "npx nyc -r html npm run test:node -- --bail", + "clean": "rimraf ./dist", + "dep-check": "aegir dep-check -i aegir -i typescript -i rimraf -i ipfs-grpc-protocol" + }, + "dependencies": { + "@improbable-eng/grpc-web": "^0.13.0", + "change-case": "^4.1.1", + "cids": "^1.0.0", + "debug": "^4.1.1", + "err-code": "^2.0.3", + "ipfs-core-utils": "^0.5.0", + "ipfs-grpc-protocol": "0.0.0", + "it-first": "^1.0.4", + "it-pushable": "^1.4.0", + "protobufjs": "^6.10.2", + "multiaddr": "^8.0.0", + "ws": "^7.3.1" + }, + "devDependencies": { + "aegir": "^28.2.0", + "it-all": "^1.0.4", + "rimraf": "^3.0.2", + "sinon": "^9.0.3", + "typescript": "^4.0.3" + } +} diff --git a/packages/ipfs-grpc-client/src/core-api/add-all.js b/packages/ipfs-grpc-client/src/core-api/add-all.js new file mode 100644 index 0000000000..fedd58c55e --- /dev/null +++ b/packages/ipfs-grpc-client/src/core-api/add-all.js @@ -0,0 +1,121 @@ +'use strict' + +const normaliseInput = require('ipfs-core-utils/src/files/normalise-input') +const CID = require('cids') +const bidiToDuplex = require('../utils/bidi-to-duplex') +const withTimeoutOption = require('ipfs-core-utils/src/with-timeout-option') + +function sendDirectory (index, source, path, mode, mtime) { + const message = { + index, + type: 'DIRECTORY', + path + } + + if (mtime) { + message.mtime = mtime + } + + if (mode != null) { + message.mode = mode + } + + source.push(message) +} + +async function sendFile (index, source, content, path, mode, mtime) { + for await (const buf of content) { + const message = { + index, + type: 'FILE', + path + } + + if (mtime) { + message.mtime = mtime.secs + message.mtimeNsecs = mtime.nsecs + } + + if (mode != null) { + message.mode = mode + } + + message.content = new Uint8Array(buf, buf.byteOffset, buf.byteLength) + + source.push(message) + } + + // signal that the file data has finished + const message = { + index, + type: 'FILE', + path + } + + source.push(message) +} + +async function sendFiles (stream, source, options) { + let i = 1 + + for await (const { path, content, mode, mtime } of normaliseInput(stream)) { + const index = i + i++ + + if (content) { + await sendFile(index, source, content, path, mode, mtime) + } else { + sendDirectory(index, source, path, mode, mtime) + } + } +} + +module.exports = function grpcAddAll (grpc, service, opts = {}) { + opts = opts || {} + + async function * addAll (stream, options = {}) { + const { + source, + sink + } = bidiToDuplex(grpc, service, { + host: opts.url, + debug: Boolean(process.env.DEBUG), + metadata: options + }) + + setTimeout(() => { + sendFiles(stream, source, options) + .catch(err => { + source.end(err) + }) + .finally(() => { + source.end() + }) + }, 0) + + for await (const result of sink) { + // received progress result + if (result.type === 'PROGRESS') { + if (options.progress) { + options.progress(result.bytes, result.path) + } + + continue + } + + // received file/dir import result + yield { + path: result.path, + cid: new CID(result.cid), + mode: result.mode, + mtime: { + secs: result.mtime || 0, + nsecs: result.mtimeNsecs || 0 + }, + size: result.size + } + } + } + + return withTimeoutOption(addAll) +} diff --git a/packages/ipfs-grpc-client/src/core-api/files/ls.js b/packages/ipfs-grpc-client/src/core-api/files/ls.js new file mode 100644 index 0000000000..bbd38b14f5 --- /dev/null +++ b/packages/ipfs-grpc-client/src/core-api/files/ls.js @@ -0,0 +1,35 @@ +'use strict' + +const CID = require('cids') +const serverStreamToIterator = require('../../utils/server-stream-to-iterator') +const withTimeoutOption = require('ipfs-core-utils/src/with-timeout-option') + +module.exports = function grpcMfsLs (grpc, service, opts = {}) { + opts = opts || {} + + async function * mfsLs (path, options = {}) { + const request = { + path + } + + for await (const result of serverStreamToIterator(grpc, service, request, { + host: opts.url, + debug: Boolean(process.env.DEBUG), + metadata: options + })) { + yield { + name: result.name, + type: result.type.toLowerCase(), + size: result.size, + cid: new CID(result.cid), + mode: result.mode, + mtime: { + secs: result.mtime || 0, + nsecs: result.mtimeNsecs || 0 + } + } + } + } + + return withTimeoutOption(mfsLs) +} diff --git a/packages/ipfs-grpc-client/src/core-api/files/write.js b/packages/ipfs-grpc-client/src/core-api/files/write.js new file mode 100644 index 0000000000..820775bbac --- /dev/null +++ b/packages/ipfs-grpc-client/src/core-api/files/write.js @@ -0,0 +1,48 @@ +'use strict' + +const clientStreamToPromise = require('../../utils/client-stream-to-promise') +const withTimeoutOption = require('ipfs-core-utils/src/with-timeout-option') +const normaliseInput = require('ipfs-core-utils/src/files/normalise-input') +const { mtimeToObject, modeToNumber } = require('ipfs-core-utils/src/files/normalise-input/utils') + +module.exports = function grpcMfsWrite (grpc, service, opts = {}) { + opts = opts || {} + + async function mfsWrite (path, content, options = {}) { + const stream = async function * () { + for await (const { content: bufs } of normaliseInput(content)) { + if (!bufs) { + return + } + + for await (const content of bufs) { + yield { path, content } + } + } + } + + const mtime = mtimeToObject(options.mtime) + + if (mtime != null) { + options = { + ...options, + mtime: mtime.secs, + mtimeNsecs: mtime.nsecs + } + } + + const mode = modeToNumber(options.mode) + + if (mode != null) { + options.mode = mode + } + + await clientStreamToPromise(grpc, service, stream(), { + host: opts.url, + debug: Boolean(process.env.DEBUG), + metadata: options + }) + } + + return withTimeoutOption(mfsWrite) +} diff --git a/packages/ipfs-grpc-client/src/core-api/id.js b/packages/ipfs-grpc-client/src/core-api/id.js new file mode 100644 index 0000000000..172caddaaa --- /dev/null +++ b/packages/ipfs-grpc-client/src/core-api/id.js @@ -0,0 +1,26 @@ +'use strict' + +const withTimeoutOption = require('ipfs-core-utils/src/with-timeout-option') +const toHeaders = require('../utils/to-headers') +const unaryToPromise = require('../utils/unary-to-promise') +const multiaddr = require('multiaddr') + +module.exports = function grpcId (grpc, service, opts = {}) { + opts = opts || {} + + async function id (options = {}) { + const request = {} + + const res = await unaryToPromise(grpc, service, request, { + host: opts.url, + metadata: toHeaders(options) + }) + + return { + ...res, + addresses: res.addresses.map(multiaddr) + } + } + + return withTimeoutOption(id) +} diff --git a/packages/ipfs-grpc-client/src/grpc/transport.browser.js b/packages/ipfs-grpc-client/src/grpc/transport.browser.js new file mode 100644 index 0000000000..4981987221 --- /dev/null +++ b/packages/ipfs-grpc-client/src/grpc/transport.browser.js @@ -0,0 +1,5 @@ +'use strict' + +const { grpc } = require('@improbable-eng/grpc-web') + +module.exports = grpc.WebsocketTransport diff --git a/packages/ipfs-grpc-client/src/grpc/transport.js b/packages/ipfs-grpc-client/src/grpc/transport.js new file mode 100644 index 0000000000..21a3dd53aa --- /dev/null +++ b/packages/ipfs-grpc-client/src/grpc/transport.js @@ -0,0 +1,125 @@ +'use strict' + +// copied from https://github.com/improbable-eng/grpc-web/blob/master/client/grpc-web/src/transports/websocket/websocket.ts +// but uses the ws implementation of WebSockets +// see: https://github.com/improbable-eng/grpc-web/issues/796 + +const WebSocket = require('ws') +const debug = require('debug')('ipfs:grpc-client:websocket-transport') + +const WebsocketSignal = { + FINISH_SEND: 1 +} + +const finishSendFrame = new Uint8Array([1]) + +function WebsocketTransport () { + return (opts) => { + return websocketRequest(opts) + } +} + +function websocketRequest (options) { + const webSocketAddress = constructWebSocketAddress(options.url) + + let sendQueue = [] + let ws + + function sendToWebsocket (toSend) { + if (toSend === WebsocketSignal.FINISH_SEND) { + ws.send(finishSendFrame) + } else { + const byteArray = toSend + const c = new Int8Array(byteArray.byteLength + 1) + c.set(new Uint8Array([0])) + c.set(byteArray, 1) + + ws.send(c) + } + } + + return { + sendMessage: (msgBytes) => { + if (!ws || ws.readyState === ws.CONNECTING) { + sendQueue.push(msgBytes) + } else { + sendToWebsocket(msgBytes) + } + }, + finishSend: () => { + if (!ws || ws.readyState === ws.CONNECTING) { + sendQueue.push(WebsocketSignal.FINISH_SEND) + } else { + sendToWebsocket(WebsocketSignal.FINISH_SEND) + } + }, + start: (metadata) => { + ws = new WebSocket(webSocketAddress, ['grpc-websockets']) + ws.binaryType = 'arraybuffer' + ws.onopen = function () { + options.debug && debug('websocketRequest.onopen') + ws.send(headersToBytes(metadata)) + + // send any messages that were passed to sendMessage before the connection was ready + sendQueue.forEach(toSend => { + sendToWebsocket(toSend) + }) + sendQueue = [] + } + + ws.onclose = function (closeEvent) { + options.onEnd() + } + + ws.onerror = function (error) { + options.debug && debug('websocketRequest.onerror', error) + } + + ws.onmessage = function (e) { + options.onChunk(new Uint8Array(e.data, 0, e.data.byteLength)) + } + }, + cancel: () => { + ws.close() + } + } +} + +function constructWebSocketAddress (url) { + if (url.startsWith('wss://') || url.startsWith('ws://')) { + return url + } else if (url.substr(0, 8) === 'https://') { + return `wss://${url.substr(8)}` + } else if (url.substr(0, 7) === 'http://') { + return `ws://${url.substr(7)}` + } + throw new Error('Websocket transport constructed with non-https:// or http:// host.') +} + +function headersToBytes (headers) { + let asString = '' + headers.forEach((key, values) => { + asString += `${key}: ${values.join(', ')}\r\n` + }) + return encodeASCII(asString) +} + +function encodeASCII (input) { + const encoded = new Uint8Array(input.length) + for (let i = 0; i !== input.length; ++i) { + const charCode = input.charCodeAt(i) + if (!isValidHeaderAscii(charCode)) { + throw new Error('Metadata contains invalid ASCII') + } + encoded[i] = charCode + } + return encoded +} + +const isAllowedControlChars = (char) => char === 0x9 || char === 0xa || char === 0xd + +function isValidHeaderAscii (val) { + return isAllowedControlChars(val) || (val >= 0x20 && val <= 0x7e) +} + +module.exports = WebsocketTransport diff --git a/packages/ipfs-grpc-client/src/index.js b/packages/ipfs-grpc-client/src/index.js new file mode 100644 index 0000000000..0c4f7b1780 --- /dev/null +++ b/packages/ipfs-grpc-client/src/index.js @@ -0,0 +1,37 @@ +'use strict' + +const transport = require('./grpc/transport') +const toUrlString = require('ipfs-core-utils/src/to-url-string') +const loadServices = require('./utils/load-services') +const { grpc } = require('@improbable-eng/grpc-web') +grpc.setDefaultTransport(transport()) + +const service = loadServices() + +const protocols = { + 'ws://': 'http://', + 'wss://': 'https://' +} + +module.exports = function createClient (opts = {}) { + opts = opts || {} + opts.url = toUrlString(opts.url) + + // @improbable-eng/grpc-web requires http:// protocol URLs, not ws:// + Object.keys(protocols).forEach(protocol => { + if (opts.url.startsWith(protocol)) { + opts.url = protocols[protocol] + opts.url.substring(protocol.length) + } + }) + + const client = { + addAll: require('./core-api/add-all')(grpc, service.Root.add, opts), + id: require('./core-api/id')(grpc, service.Root.id, opts), + files: { + ls: require('./core-api/files/ls')(grpc, service.MFS.ls, opts), + write: require('./core-api/files/write')(grpc, service.MFS.write, opts) + } + } + + return client +} diff --git a/packages/ipfs-grpc-client/src/utils/bidi-to-duplex.js b/packages/ipfs-grpc-client/src/utils/bidi-to-duplex.js new file mode 100644 index 0000000000..e711f0f02a --- /dev/null +++ b/packages/ipfs-grpc-client/src/utils/bidi-to-duplex.js @@ -0,0 +1,63 @@ +'use strict' + +const pushable = require('it-pushable') +const errCode = require('err-code') +const toHeaders = require('./to-headers') + +/** + * @param {object} grpc - an @improbable-eng/grpc-web instance + * @param {object} service - an @improbable-eng/grpc-web service + * @param {object} options - RPC options + * @param {string} options.host - The remote host + * @param {boolean} [options.debug] - Whether to print debug messages + * @param {object} [options.metadata] - Metadata sent as headers + * @returns {{ source: { push: Function, end: Function }, sink: AsyncIterable }} + **/ +module.exports = function bidiToDuplex (grpc, service, options) { + // @ts-ignore + const source = pushable() + + // @ts-ignore + const sink = pushable() + + const client = grpc.client(service, options) + client.onMessage(message => { + sink.push(message) + }) + client.onEnd((status, message, trailers) => { + let err + + if (status) { + const error = new Error(message) + + err = errCode(error, trailers.get('grpc-code')[0], { + status + }) + + err.stack = trailers.get('grpc-stack')[0] || error.stack + } + + sink.end(err) + }) + + setTimeout(async () => { + try { + for await (const obj of source) { + client.send({ + serializeBinary: () => service.requestType.serializeBinary(obj) + }) + } + } catch (err) { + sink.end(err) + } finally { + client.finishSend() + } + }, 0) + + client.start(toHeaders(options.metadata)) + + return { + source, + sink + } +} diff --git a/packages/ipfs-grpc-client/src/utils/client-stream-to-promise.js b/packages/ipfs-grpc-client/src/utils/client-stream-to-promise.js new file mode 100644 index 0000000000..a691b3bedf --- /dev/null +++ b/packages/ipfs-grpc-client/src/utils/client-stream-to-promise.js @@ -0,0 +1,28 @@ +'use strict' + +const first = require('it-first') +const bidiToDuplex = require('./bidi-to-duplex') + +/** + * @param {object} grpc - an @improbable-eng/grpc-web instance + * @param {object} service - an @improbable-eng/grpc-web service + * @param {AsyncIterable} source - a source of objects to send + * @param {object} options - RPC options + * @param {string} options.host - The remote host + * @param {boolean} [options.debug] - Whether to print debug messages + * @param {object} [options.metadata] - Metadata sent as headers + * @returns {Promise} - A promise that resolves to a response object + **/ +module.exports = async function clientStreamToPromise (grpc, service, source, options) { + const { + source: serverSource, sink + } = bidiToDuplex(grpc, service, options) + + for await (const obj of source) { + serverSource.push(obj) + } + + serverSource.end() + + return first(sink) +} diff --git a/packages/ipfs-grpc-client/src/utils/load-services.js b/packages/ipfs-grpc-client/src/utils/load-services.js new file mode 100644 index 0000000000..00af934b61 --- /dev/null +++ b/packages/ipfs-grpc-client/src/utils/load-services.js @@ -0,0 +1,84 @@ +'use strict' + +// @ts-ignore +const protocol = require('ipfs-grpc-protocol') +const protobuf = require('protobufjs/light') +const { Service } = protobuf + +const CONVERSION_OPTS = { + keepCase: false, + // longs: String, // long.js is required + enums: String, + defaults: false, + oneofs: true +} + +module.exports = function loadServices () { + const root = protobuf.Root.fromJSON(protocol) + const output = {} + + Object + // @ts-ignore + .keys(root.nested.ipfs) + // @ts-ignore + .filter(key => root.nested.ipfs[key] instanceof Service) + // @ts-ignore + .map(key => root.nested.ipfs[key]) + .forEach(service => { + const serviceDef = {} + + output[service.name] = serviceDef + + Object.keys(service.methods) + .forEach(methodName => { + const method = service.methods[methodName].resolve() + + serviceDef[methodName] = { + service: { + serviceName: `ipfs.${service.name}` + }, + methodName, + requestStream: method.requestStream, + responseStream: method.responseStream, + requestType: { + serializeBinary: (obj) => { + const message = method.resolvedRequestType.fromObject(obj) + return method.resolvedRequestType.encode(message).finish() + }, + deserializeBinary: (buf) => { + const message = method.resolvedRequestType.decode(buf) + const obj = method.resolvedRequestType.toObject(message, CONVERSION_OPTS) + + Object.defineProperty(obj, 'toObject', { + enumerable: false, + configurable: false, + value: () => obj + }) + + return obj + } + }, + responseType: { + serializeBinary: (obj) => { + const message = method.resolvedResponseType.fromObject(obj) + return method.resolvedResponseType.encode(message).finish() + }, + deserializeBinary: (buf) => { + const message = method.resolvedResponseType.decode(buf) + const obj = method.resolvedResponseType.toObject(message, CONVERSION_OPTS) + + Object.defineProperty(obj, 'toObject', { + enumerable: false, + configurable: false, + value: () => obj + }) + + return obj + } + } + } + }) + }) + + return output +} diff --git a/packages/ipfs-grpc-client/src/utils/server-stream-to-iterator.js b/packages/ipfs-grpc-client/src/utils/server-stream-to-iterator.js new file mode 100644 index 0000000000..d82fbd3441 --- /dev/null +++ b/packages/ipfs-grpc-client/src/utils/server-stream-to-iterator.js @@ -0,0 +1,23 @@ +'use strict' + +const bidiToDuplex = require('./bidi-to-duplex') + +/** + * @param {object} grpc - an @improbable-eng/grpc-web instance + * @param {object} service - an @improbable-eng/grpc-web service + * @param {object} request - a request object + * @param {object} options - RPC options + * @param {string} options.host - The remote host + * @param {boolean} [options.debug] - Whether to print debug messages + * @param {object} [options.metadata] - Metadata sent as headers + * @returns {AsyncIterable} + **/ +module.exports = function serverStreamToIterator (grpc, service, request, options) { + const { + source, sink + } = bidiToDuplex(grpc, service, options) + + source.push(request) + + return sink +} diff --git a/packages/ipfs-grpc-client/src/utils/to-headers.js b/packages/ipfs-grpc-client/src/utils/to-headers.js new file mode 100644 index 0000000000..9cffb3c766 --- /dev/null +++ b/packages/ipfs-grpc-client/src/utils/to-headers.js @@ -0,0 +1,21 @@ +'use strict' + +const { paramCase } = require('change-case') + +/** + * @param {object} [object] - key/value pairs to turn into HTTP headers + * @returns {object} - HTTP headers + **/ +module.exports = (object) => { + const output = {} + + Object.keys(object || {}).forEach(key => { + if (typeof object[key] === 'function') { + return + } + + output[paramCase(key)] = object[key] + }) + + return output +} diff --git a/packages/ipfs-grpc-client/src/utils/unary-to-promise.js b/packages/ipfs-grpc-client/src/utils/unary-to-promise.js new file mode 100644 index 0000000000..409b4753f3 --- /dev/null +++ b/packages/ipfs-grpc-client/src/utils/unary-to-promise.js @@ -0,0 +1,24 @@ +'use strict' + +const first = require('it-first') +const bidiToDuplex = require('./bidi-to-duplex') + +/** + * @param {object} grpc - an @improbable-eng/grpc-web instance + * @param {object} service - an @improbable-eng/grpc-web service + * @param {object} request - a request object + * @param {object} options - RPC options + * @param {string} options.host - The remote host + * @param {boolean} [options.debug] - Whether to print debug messages + * @param {object} [options.metadata] - Metadata sent as headers + * @returns {Promise} - A promise that resolves to a response object + **/ +module.exports = function unaryToPromise (grpc, service, request, options) { + const { + source, sink + } = bidiToDuplex(grpc, service, options) + + source.push(request) + + return first(sink) +} diff --git a/packages/ipfs-grpc-client/test/utils.spec.js b/packages/ipfs-grpc-client/test/utils.spec.js new file mode 100644 index 0000000000..cc3866e5ef --- /dev/null +++ b/packages/ipfs-grpc-client/test/utils.spec.js @@ -0,0 +1,96 @@ +/* eslint-env mocha */ +'use strict' + +const { expect } = require('aegir/utils/chai') +const all = require('it-all') +const bidiToDuplex = require('../src/utils/bidi-to-duplex') +const toHeaders = require('../src/utils/to-headers') +const sinon = require('sinon') + +describe('utils', () => { + describe('bidi-to-duplex', () => { + it('should transform a bidirectional client into an async iterable', async () => { + const service = 'service' + const options = { + metadata: { + foo: 'bar' + } + } + + const client = { + onMessage: sinon.stub(), + onEnd: sinon.stub(), + start: sinon.stub() + } + + const grpc = { + client: sinon.stub().withArgs(service, options).returns(client) + } + + const { + sink + } = bidiToDuplex(grpc, service, options) + + expect(client.start.calledWith(options.metadata)).to.be.true() + + client.onMessage.getCall(0).args[0]('hello') + client.onMessage.getCall(0).args[0]('world') + client.onEnd.getCall(0).args[0]() + + await expect(all(sink)).to.eventually.deep.equal(['hello', 'world']) + }) + + it('should propagate client errors', async () => { + const service = 'service' + const options = { + metadata: { + foo: 'bar' + } + } + + const client = { + onMessage: sinon.stub(), + onEnd: sinon.stub(), + start: sinon.stub() + } + + const grpc = { + client: sinon.stub().withArgs(service, options).returns(client) + } + + const { + sink + } = bidiToDuplex(grpc, service, options) + + expect(client.start.calledWith(options.metadata)).to.be.true() + + client.onEnd.getCall(0).args[0](1, 'Erp!', { get: () => [] }) + + await expect(all(sink)).to.eventually.be.rejectedWith(/Erp!/) + }) + }) + + describe('to-headers', () => { + it('should rename property fields', () => { + const input = { + propSimple: 'foo' + } + + const output = toHeaders(input) + + expect(output.propSimple).to.be.undefined() + expect(output['prop-simple']).to.equal(input.propSimple) + }) + + it('should remove function fields', () => { + const input = { + funcProp: () => {} + } + + const output = toHeaders(input) + + expect(output.funcProp).to.be.undefined() + expect(output['func-prop']).to.be.undefined() + }) + }) +}) diff --git a/packages/ipfs-grpc-client/tsconfig.json b/packages/ipfs-grpc-client/tsconfig.json new file mode 100644 index 0000000000..8783effb6f --- /dev/null +++ b/packages/ipfs-grpc-client/tsconfig.json @@ -0,0 +1,21 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "outDir": "dist" + }, + "include": [ + "src", + "package.json" + ], + "references": [ + { + "path": "../ipfs-core" + }, + { + "path": "../ipfs-core-utils" + }, + { + "path": "../ipfs-grpc-protocol" + } + ] +} diff --git a/packages/ipfs-grpc-protocol/README.md b/packages/ipfs-grpc-protocol/README.md new file mode 100644 index 0000000000..75e7e478de --- /dev/null +++ b/packages/ipfs-grpc-protocol/README.md @@ -0,0 +1,3 @@ +# ipfs-grpc-protocol + +Contains `.proto` files that define the IPFS gRPC interface. diff --git a/packages/ipfs-grpc-protocol/package.json b/packages/ipfs-grpc-protocol/package.json new file mode 100644 index 0000000000..ca7c9d1275 --- /dev/null +++ b/packages/ipfs-grpc-protocol/package.json @@ -0,0 +1,31 @@ +{ + "name": "ipfs-grpc-protocol", + "version": "0.0.0", + "description": "Protobuf definitions for the IPFS gRPC API", + "keywords": [ + "ipfs" + ], + "main": "dist/ipfs.json", + "homepage": "https://github.com/ipfs/js-ipfs/tree/master/packages/ipfs-grpc-protocol#readme", + "bugs": "https://github.com/ipfs/js-ipfs/issues", + "license": "(Apache-2.0 OR MIT)", + "leadMaintainer": "Alex Potsides ", + "files": [ + "src", + "dist" + ], + "repository": { + "type": "git", + "url": "git+https://github.com/ipfs/js-ipfs.git" + }, + "scripts": { + "test": "echo 'No tests here'", + "clean": "rimraf ./dist", + "build": "npm run clean && mkdirp ./dist && pbjs ./src/*.proto -t json -o ./dist/ipfs.json" + }, + "devDependencies": { + "mkdirp": "^1.0.4", + "protobufjs": "^6.10.2", + "rimraf": "^3.0.2" + } +} diff --git a/packages/ipfs-grpc-protocol/src/common.proto b/packages/ipfs-grpc-protocol/src/common.proto new file mode 100644 index 0000000000..175f672e77 --- /dev/null +++ b/packages/ipfs-grpc-protocol/src/common.proto @@ -0,0 +1,8 @@ +syntax = "proto3"; + +package ipfs; + +enum FileType { + DIRECTORY = 0; + FILE = 1; +} diff --git a/packages/ipfs-grpc-protocol/src/mfs.proto b/packages/ipfs-grpc-protocol/src/mfs.proto new file mode 100644 index 0000000000..8a9652933e --- /dev/null +++ b/packages/ipfs-grpc-protocol/src/mfs.proto @@ -0,0 +1,33 @@ +syntax = "proto3"; + +import "common.proto"; + +package ipfs; + +service MFS { + rpc ls (LsRequest) returns (stream LsResponse) {} + rpc write (stream WriteRequest) returns (WriteResponse) {} +} + +message LsRequest { + string path = 1; +} + +message LsResponse { + string name = 1; + FileType type = 2; + uint32 size = 3; + string cid = 4; + uint32 mode = 5; + int32 mtime = 6; + uint32 mtime_nsecs = 7; +} + +message WriteRequest { + string path = 1; + bytes content = 2; +} + +message WriteResponse { + +} diff --git a/packages/ipfs-grpc-protocol/src/root.proto b/packages/ipfs-grpc-protocol/src/root.proto new file mode 100644 index 0000000000..c5eca6a18c --- /dev/null +++ b/packages/ipfs-grpc-protocol/src/root.proto @@ -0,0 +1,49 @@ +syntax = "proto3"; + +import "common.proto"; + +package ipfs; + +service Root { + rpc id (IdRequest) returns (IdResponse) {} + rpc add (stream AddRequest) returns (stream AddResponse) {} +} + +enum AddResponseType { + PROGRESS = 0; + RESULT = 1; +} + +message IdRequest { + string peer_id = 1; +} + +message IdResponse { + string id = 1; + string public_key = 2; + repeated string addresses = 3; + string agent_version = 4; + string protocol_version = 5; + repeated string protocols = 6; +} + +message AddRequest { + int32 index = 1; + FileType type = 2; + string path = 3; + uint32 mode = 4; + int32 mtime = 5; + uint32 mtime_nsecs = 6; + bytes content = 7; +} + +message AddResponse { + AddResponseType type = 1; + string path = 2; + int32 bytes = 3; + string cid = 4; + uint32 mode = 5; + int32 mtime = 6; + uint32 mtime_nsecs = 7; + uint32 size = 8; +} diff --git a/packages/ipfs-grpc-protocol/tsconfig.json b/packages/ipfs-grpc-protocol/tsconfig.json new file mode 100644 index 0000000000..979a39adab --- /dev/null +++ b/packages/ipfs-grpc-protocol/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "outDir": "dist" + }, + "include": [ + "src", + "package.json" + ] +} diff --git a/packages/ipfs-grpc-server/README.md b/packages/ipfs-grpc-server/README.md new file mode 100644 index 0000000000..40ccb63bff --- /dev/null +++ b/packages/ipfs-grpc-server/README.md @@ -0,0 +1,182 @@ +# ipfs-grpc-server + +> A gRPC server that runs over a websocket + +## Why? + +[gRPC-web](https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-WEB.md) allows us to form HTTP requests out of gRPC invocations, but the [official implementation](https://github.com/grpc/grpc-web) supports only unary calls and server streaming, which in terms of functionality doesn't give us much over the existing [ipfs-http-client](https://www.npmjs.com/package/ipfs-http-client). + +In order to support streaming file uploads with errors, pubsub, etc, bi-directional streaming is required. We can either use Websockets for this, or use two connections, one for upload and one for download though this involves two requests for every operation and some orchestration on the server side to match one up with the other. + +Websockets are a cheaper and simpler way of accomplishing the same thing though sadly the official gRPC implementation has [no plans](https://github.com/grpc/grpc-web/blob/master/doc/streaming-roadmap.md#issues-with-websockets) to implement full-duplex streaming in this way. + +This module implements a Websocket proxy for a gRPC-web server. It's a js port of the [grpcwebproxy](https://github.com/improbable-eng/grpc-web/tree/master/go/grpcwebproxy) project from [improbable-eng/grpc-web](https://github.com/improbable-eng/grpc-web). + +## Protocol + +Every RPC invocation opens a new WebSocket connection, the invocation is completed and the socket connection is closed. + +The connection is opened against the path of the RPC method the client wishes to invoke. The path is created from the protobuf service definition package, service and procedure name. + +E.g. given the following service definition: + +```protobuf +package ipfs; + +service Root { + rpc id (Req) returns (Res) {} +} +``` + +A path of `/ipfs.Root/id` would be created. + +There are three parts to the communication, metadata, messages and trailers. Communication is symmetrical; that is, the client sends metadata, one or more messages and finally some trailers and the server responds with metadata, one or more messages and finally some trailers. + +The amount of messages a client/server can send is dictated by if the RPC method is unary or streaming and if so in which direction. + +Unary will result in one message sent and one received, client streaming is many sent and one received, server streaming is one sent and many received and finally bidirectional is many sent and many received. + +### 1. Metadata + +Metadata is sent as the first websocket message. It is a utf8 encoded list in the same format as [HTTP Headers][] + +### 2. Messages + +One ore more messages will be sent. Messages are sent as a single websocket message and contain a signal, a header, some message data and an optional trailer. + +Every message sent to or received from the server will have the following format: + +| byte index | Notes | +|---|---| +| 0 | Signal | +| 1-5 | Header | +| n1-n2 | Message data | +| n3-n3+5 | Trailer + +#### Signal + +A one-byte field. + +| Value | Meaning | +|---|---| +| 0 | START_SEND: Further messages will be sent as part of this context | +| 1 | FINISH_SEND: This is the final message, no further data will be sent | + +#### Header + +A five-byte field that contains one byte signifying if it's a Header or a Trailer and four bytes that contain the length of the following data. + +| byte index | Meaning | +|--------------|---| +| 0 | 0: This is a header, 128: This is a footer | +| 1-4 | An unsigned big-endian 32-bit integer that specifies the length of the message | + +#### Message data + +A protocol buffer message, the length of which is defined in the header + +#### Trailer + +A five-byte field that contains one byte signifying if it's a Header or a Trailer and four bytes that contain the length of the following data. + +| byte index | Meaning | +|--------------|---| +| 0 | 0: This is a header, 128: This is a footer | +| 1-4 | A big-endian 32-bit integer that specifies the length of the trailer | + +The trailer contains [HTTP headers][] as a utf8 encoded string in the same way as invocation metadata. + +## Handlers + +Method handlers come in four flavours - unary, server streaming, client streaming, bidirectional streaming and accept metadata as an argument. + +### Metadata + +All methods accept metadata which are sent as the equivalent of HTTP headers as part of every request. These are accepted by the client as options to a given method. + +E.g.: + +```js +ipfs.addAll(source, options) +// `source` will be turned into a message stream +// `options` will be sent as metadata +``` + +### Unary + +The simplest case, one request message and one response message. + +```javascript +module.exports = function grpcFunction (ipfs, options = {}) { + async function handler (request, metadata) { + const response = { + //... some fields here + } + + return response + } + + return handler +} +``` + +### Server streaming + +Where the server sends repeated messages. `sink` is an [it-pushable][]. + +```javascript +module.exports = function grpcFunction (ipfs, options = {}) { + async function serverStreamingHandler (request, sink, metadata) { + sink.push(..) + sink.push(..) + + sink.end() + } + + return clientStreamingHandler +} +``` + +### Client streaming + +Where the client sends repeated messages. `source` is an [AsyncIterator][]. + +```javascript +module.exports = function grpcFunction (ipfs, options = {}) { + async function clientStreamingHandler (source, metadata) { + const response = { + //... some fields here + } + + for await (const thing of source) { + // do something with `thing` + } + + return response + } + + return handler +} +``` + +### Bidirectional streaming + +The simplest case, one request message and one response message. `source` is an [AsyncIterator][] and `sink` is an [it-pushable][]. + +```javascript +module.exports = function grpcFunction (ipfs, options = {}) { + async function bidirectionalHandler (source, sink, metadata) { + for await (const thing of source) { + sink.push(sink) + } + + sink.end() + } + + return bidirectionalHandler +} +``` + +[HTTP headers]: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers +[it-pushable]: https://www.npmjs.com/package/it-pushable +[AsyncIterator]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Symbol/asyncIterator diff --git a/packages/ipfs-grpc-server/package.json b/packages/ipfs-grpc-server/package.json new file mode 100644 index 0000000000..fc50914786 --- /dev/null +++ b/packages/ipfs-grpc-server/package.json @@ -0,0 +1,62 @@ +{ + "name": "ipfs-grpc-server", + "version": "0.0.0", + "description": "A server library for the IPFS gRPC API", + "keywords": [ + "ipfs" + ], + "homepage": "https://github.com/ipfs/js-ipfs/tree/master/packages/ipfs-grpc-server#readme", + "bugs": "https://github.com/ipfs/js-ipfs/issues", + "license": "(Apache-2.0 OR MIT)", + "leadMaintainer": "Alex Potsides ", + "files": [ + "src", + "dist" + ], + "main": "src/index.js", + "browser": {}, + "typesVersions": { + "*": { + "*": [ + "dist/*" + ] + } + }, + "repository": { + "type": "git", + "url": "git+https://github.com/ipfs/js-ipfs.git" + }, + "scripts": { + "lint": "aegir lint", + "build": "npm run build:types", + "build:types": "tsc --build", + "test": "aegir test -t node", + "coverage": "nyc --reporter=text --reporter=lcov npm run test:node", + "clean": "rimraf ./dist", + "dep-check": "aegir dep-check -i ipfs-grpc-protocol -i typescript -i aegir -i ipfs-core -i rimraf" + }, + "dependencies": { + "@grpc/grpc-js": "^1.1.8", + "protobufjs": "^6.10.2", + "change-case": "^4.1.1", + "coercer": "^1.1.2", + "debug": "^4.1.1", + "ipfs-grpc-protocol": "0.0.0", + "it-first": "^1.0.4", + "it-map": "^1.0.4", + "it-peekable": "^1.0.1", + "it-pipe": "^1.1.0", + "it-pushable": "^1.4.0", + "ws": "^7.3.1" + }, + "devDependencies": { + "aegir": "^28.2.0", + "ipfs-core": "^0.2.1", + "it-all": "^1.0.4", + "it-drain": "^1.0.3", + "rimraf": "^3.0.2", + "sinon": "^9.0.3", + "typescript": "^4.0.3", + "uint8arrays": "^1.1.0" + } +} diff --git a/packages/ipfs-grpc-server/src/endpoints/add.js b/packages/ipfs-grpc-server/src/endpoints/add.js new file mode 100644 index 0000000000..30987e21dc --- /dev/null +++ b/packages/ipfs-grpc-server/src/endpoints/add.js @@ -0,0 +1,114 @@ +'use strict' + +// @ts-ignore +const pushable = require('it-pushable') +const { pipe } = require('it-pipe') + +module.exports = function grpcAdd (ipfs, options = {}) { + async function add (source, sink, metadata) { + const opts = { + ...metadata, + progress: (bytes = 0, path = '') => { + sink.push({ + type: 'PROGRESS', + bytes, + path + }) + } + } + + await pipe( + async function * toInput () { + // @ts-ignore + const fileInputStream = pushable() + + setTimeout(async () => { + const streams = [] + + try { + for await (const { index, type, path, mode, mtime, mtimeNsecs, content } of source) { + let mtimeObj + + if (mtime != null) { + mtimeObj = { + secs: mtime, + nsecs: undefined + } + + if (mtimeNsecs != null) { + mtimeObj.nsecs = mtimeNsecs + } + } + + if (!type || type === 'DIRECTORY') { + // directory + fileInputStream.push({ + path, + mode: mode !== 0 ? mode : undefined, + mtime: mtimeObj + }) + + continue + } + + let stream = streams[index] + + if (!stream) { + // start of new file + // @ts-ignore + stream = streams[index] = pushable() + + fileInputStream.push({ + path, + mode: mode !== 0 ? mode : undefined, + mtime: mtimeObj, + content: stream + }) + } + + if (content && content.length) { + // file is in progress + stream.push(content) + } else { + // file is finished + stream.end() + + streams[index] = null + } + } + + fileInputStream.end() + } catch (err) { + fileInputStream.end(err) + } finally { + // clean up any open streams + streams.filter(Boolean).forEach(stream => stream.end()) + } + }, 0) + + yield * fileInputStream + }, + async function (source) { + for await (const result of ipfs.addAll(source, opts)) { + result.cid = result.cid.toString() + + if (!result.mtime) { + delete result.mtime + } else { + result.mtime_nsecs = result.mtime.nsecs + result.mtime = result.mtime.secs + } + + sink.push({ + type: 'RESULT', + ...result + }) + } + + sink.end() + } + ) + } + + return add +} diff --git a/packages/ipfs-grpc-server/src/endpoints/id.js b/packages/ipfs-grpc-server/src/endpoints/id.js new file mode 100644 index 0000000000..8bdd210fce --- /dev/null +++ b/packages/ipfs-grpc-server/src/endpoints/id.js @@ -0,0 +1,16 @@ +'use strict' + +const { callbackify } = require('util') + +module.exports = function grpcId (ipfs, options = {}) { + function id (request, metadata) { + const opts = { + ...request, + ...metadata + } + + return ipfs.id(opts) + } + + return callbackify(id) +} diff --git a/packages/ipfs-grpc-server/src/endpoints/mfs/ls.js b/packages/ipfs-grpc-server/src/endpoints/mfs/ls.js new file mode 100644 index 0000000000..c223505936 --- /dev/null +++ b/packages/ipfs-grpc-server/src/endpoints/mfs/ls.js @@ -0,0 +1,27 @@ +'use strict' + +module.exports = function grpcMfsLs (ipfs, options = {}) { + async function mfsLs (request, sink, metadata) { + const opts = { + ...metadata + } + + for await (const result of ipfs.files.ls(request.path, opts)) { + result.cid = result.cid.toString() + result.type = result.type.toUpperCase() + + if (!result.mtime) { + delete result.mtime + } else { + result.mtime_nsecs = result.mtime.nsecs + result.mtime = result.mtime.secs + } + + sink.push(result) + } + + sink.end() + } + + return mfsLs +} diff --git a/packages/ipfs-grpc-server/src/endpoints/mfs/write.js b/packages/ipfs-grpc-server/src/endpoints/mfs/write.js new file mode 100644 index 0000000000..92eb61b600 --- /dev/null +++ b/packages/ipfs-grpc-server/src/endpoints/mfs/write.js @@ -0,0 +1,38 @@ +'use strict' + +const peekable = require('it-peekable') +const map = require('it-map') +const { callbackify } = require('util') + +module.exports = function grpcMfsWrite (ipfs, options = {}) { + async function mfsWrite (source, metadata) { + const opts = { + ...metadata + } + + if (opts.mtime) { + opts.mtime = { + secs: opts.mtime, + nsecs: opts.mtimeNsecs + } + } + + // path is sent with content messages + const content = peekable(source) + const result = await content.peek() + const { + value: { + // @ts-ignore + path + } + } = result + content.push(result.value) + + // @ts-ignore + await ipfs.files.write(path, map(content, ({ content }) => content), opts) + + return {} + } + + return callbackify(mfsWrite) +} diff --git a/packages/ipfs-grpc-server/src/index.js b/packages/ipfs-grpc-server/src/index.js new file mode 100644 index 0000000000..1a98a79950 --- /dev/null +++ b/packages/ipfs-grpc-server/src/index.js @@ -0,0 +1,111 @@ +'use strict' + +const grpc = require('@grpc/grpc-js') +const first = require('it-first') +const debug = require('debug')('ipfs:grpc-server') +const webSocketServer = require('./utils/web-socket-server') +const loadServices = require('./utils/load-services') + +const { + Root, + MFS +} = loadServices() + +module.exports = async function createServer (ipfs, options = {}) { + options = options || {} + + const server = new grpc.Server() + server.addService(Root, { + add: require('./endpoints/add')(ipfs, options), + // @ts-ignore + id: require('./endpoints/id')(ipfs, options) + }) + server.addService(MFS, { + ls: require('./endpoints/mfs/ls')(ipfs, options), + // @ts-ignore + write: require('./endpoints/mfs/write')(ipfs, options) + }) + + const socket = options.socket || await webSocketServer(ipfs, options) + + socket.on('error', (error) => debug(error)) + + socket.on('data', async ({ path, metadata, channel }) => { + // @ts-ignore + const handler = server.handlers.get(path) + + if (!handler) { + channel.end(new Error(`Request path ${path} unimplemented`)) + return + } + + channel.handler = handler + + switch (handler.type) { + case 'bidi': + handler.func(channel.source, channel.sink, metadata) + .catch(err => { + channel.end(err) + }) + + channel.sendMetadata({}) + + for await (const output of channel.sink) { + channel.sendMessage(output) + } + + channel.end() + + break + case 'unary': + handler.func(await first(channel.source), metadata, (err, value, metadata, flags) => { + if (err) { + return channel.end(err) + } + + channel.sendMetadata(metadata || {}) + + if (value) { + channel.sendMessage(value) + } + + channel.end() + }) + break + case 'clientStream': + handler.func(channel.source, metadata, (err, value, metadata, flags) => { + if (err) { + return channel.end(err) + } + + channel.sendMetadata(metadata || {}) + + if (value) { + channel.sendMessage(value) + } + + channel.end() + }) + break + case 'serverStream': + handler.func(await first(channel.source), channel.sink, metadata) + .catch(err => { + channel.end(err) + }) + + channel.sendMetadata({}) + + for await (const output of channel.sink) { + channel.sendMessage(output) + } + + channel.end() + + break + default: + debug(`Invalid handler type ${handler.type}`) + } + }) + + return socket +} diff --git a/packages/ipfs-grpc-server/src/utils/load-services.js b/packages/ipfs-grpc-server/src/utils/load-services.js new file mode 100644 index 0000000000..ef5d1d7d59 --- /dev/null +++ b/packages/ipfs-grpc-server/src/utils/load-services.js @@ -0,0 +1,77 @@ +'use strict' + +// @ts-ignore +const protocol = require('ipfs-grpc-protocol') +const protobuf = require('protobufjs/light') +const { Service } = protobuf + +const CONVERSION_OPTS = { + keepCase: false, + // longs: String, // long.js is required + enums: String, + defaults: false, + oneofs: true +} + +module.exports = function loadServices () { + const root = protobuf.Root.fromJSON(protocol) + const output = {} + + Object + // @ts-ignore + .keys(root.nested.ipfs) + // @ts-ignore + .filter(key => root.nested.ipfs[key] instanceof Service) + // @ts-ignore + .map(key => root.nested.ipfs[key]) + .forEach(service => { + const serviceDef = {} + + output[service.name] = serviceDef + + Object.keys(service.methods) + .forEach(methodName => { + const method = service.methods[methodName].resolve() + + serviceDef[methodName] = { + path: `/ipfs.${service.name}/${methodName}`, + requestStream: method.requestStream, + responseStream: method.responseStream, + requestSerialize: (obj) => { + const message = method.resolvedRequestType.fromObject(obj) + return method.resolvedRequestType.encode(message).finish() + }, + requestDeserialize: (buf) => { + const message = method.resolvedRequestType.decode(buf) + const obj = method.resolvedRequestType.toObject(message, CONVERSION_OPTS) + + Object.defineProperty(obj, 'toObject', { + enumerable: false, + configurable: false, + value: () => obj + }) + + return obj + }, + responseSerialize: (obj) => { + const message = method.resolvedResponseType.fromObject(obj) + return method.resolvedResponseType.encode(message).finish() + }, + responseDeserialize: (buf) => { + const message = method.resolvedResponseType.decode(buf) + const obj = method.resolvedResponseType.toObject(message, CONVERSION_OPTS) + + Object.defineProperty(obj, 'toObject', { + enumerable: false, + configurable: false, + value: () => obj + }) + + return obj + } + } + }) + }) + + return output +} diff --git a/packages/ipfs-grpc-server/src/utils/snake-to-camel.js b/packages/ipfs-grpc-server/src/utils/snake-to-camel.js new file mode 100644 index 0000000000..380afa1bb5 --- /dev/null +++ b/packages/ipfs-grpc-server/src/utils/snake-to-camel.js @@ -0,0 +1,13 @@ +'use strict' + +module.exports = (string) => { + return string.split('_') + .map((str, index) => { + if (index === 0) { + return str + } + + return str.substring(0, 1).toUpperCase() + str.substring(1) + }) + .join('') +} diff --git a/packages/ipfs-grpc-server/src/utils/web-socket-message-channel.js b/packages/ipfs-grpc-server/src/utils/web-socket-message-channel.js new file mode 100644 index 0000000000..1322f94b7a --- /dev/null +++ b/packages/ipfs-grpc-server/src/utils/web-socket-message-channel.js @@ -0,0 +1,139 @@ +'use strict' + +// @ts-ignore +const pushable = require('it-pushable') +const { paramCase } = require('change-case') + +const WebsocketSignal = { + START_SEND: 0, + FINISH_SEND: 1 +} + +const HEADER_SIZE = 5 +const TRAILER_BYTES = 0x80 + +/** + * @param {object} object - key/value pairs to turn into HTTP headers + * @returns {Uint8Array} - HTTP headers + **/ +const objectToHeaders = (object) => { + const output = {} + + Object.keys(object).forEach(key => { + if (typeof object[key] === 'function') { + return + } + + output[paramCase(key)] = object[key] + }) + + return Buffer.from( + Object.entries(object) + .filter(([, value]) => value != null) + .map(([key, value]) => `${key}: ${JSON.stringify(value)}`) + .join('\r\n') + ) +} + +class WebsocketMessageChannel { + constructor (ws) { + this._ws = ws + + this.handler = { + deserialize: (buf) => ({}), + serialize: (message) => Buffer.from([]) + } + + // @ts-ignore + this.source = pushable() + + // @ts-ignore + this.sink = pushable() + + ws.on('message', (buf) => { + const flag = buf[0] + + if (flag === WebsocketSignal.FINISH_SEND) { + this.source.end() + + return + } + + let offset = 1 + + if (buf.length < (HEADER_SIZE + offset)) { + return + } + + const header = buf.slice(offset, HEADER_SIZE + offset) + const length = header.readUInt32BE(1, 4) + offset += HEADER_SIZE + + if (buf.length < (length + offset)) { + return + } + + const message = buf.slice(offset, offset + length) + const deserialized = this.handler.deserialize(message) + this.source.push(deserialized) + }) + + ws.once('end', () => { + this.source.end() + this.sink.end() + }) + } + + sendMetadata (metadata) { + this._ws.send(objectToHeaders(metadata)) + } + + /** + * @param {object} message - A message object to send to the client + * @returns {void} + */ + sendMessage (message) { + const response = this.handler.serialize(message) + + const header = new DataView(new ArrayBuffer(HEADER_SIZE)) + header.setUint32(1, response.byteLength) + + this._ws.send( + Buffer.concat([ + new Uint8Array(header.buffer, header.byteOffset, header.byteLength), + response + ], header.byteLength + response.byteLength) + ) + + this.sendTrailer() + } + + sendTrailer (err) { + const trailerBuffer = objectToHeaders({ + 'grpc-status': err ? 1 : 0, + 'grpc-message': err ? err.message : undefined, + 'grpc-stack': err ? err.stack : undefined, + 'grpc-code': err ? err.code : undefined + }) + + const trailer = new DataView(new ArrayBuffer(HEADER_SIZE)) + trailer.setUint8(0, TRAILER_BYTES) + trailer.setUint32(1, trailerBuffer.byteLength) + + this._ws.send( + Buffer.concat([ + new Uint8Array(trailer.buffer, trailer.byteOffset, trailer.byteLength), + trailerBuffer + ], trailer.byteLength + trailerBuffer.byteLength) + ) + } + + end (err) { + this.sendTrailer(err) + this.source.end() + this.sink.end() + this._ws.close() + } +} + +module.exports = WebsocketMessageChannel diff --git a/packages/ipfs-grpc-server/src/utils/web-socket-server.js b/packages/ipfs-grpc-server/src/utils/web-socket-server.js new file mode 100644 index 0000000000..e6302383ce --- /dev/null +++ b/packages/ipfs-grpc-server/src/utils/web-socket-server.js @@ -0,0 +1,88 @@ +'use strict' + +const { Server: WebSocketServer } = require('ws') +const EventEmitter = require('events').EventEmitter +const WebSocketMessageChannel = require('./web-socket-message-channel') +const debug = require('debug')('ipfs:grpc-server:utils:web-socket-server') +const coerce = require('coercer') +const { camelCase } = require('change-case') + +/** + * @param {Buffer} buf - e.g. `Buffer.from('foo-bar: baz\r\n')` + * @returns {object} - e.g. `{ foorBar: 'baz' }` + **/ +const fromHeaders = (buf) => { + const headers = buf.toString('utf8') + .trim() + .split('\r\n') + .map(s => s.split(':').map(s => s.trim())) + .reduce((acc, curr) => { + if (curr[0] !== 'content-type' && curr[0] !== 'x-grpc-web') { + acc[camelCase(curr[0])] = curr[1] + } + + return acc + }, {}) + + return coerce(headers) +} + +class Messages extends EventEmitter { + constructor (wss) { + super() + + this._wss = wss + this.multiaddr = '' + + wss.on('connection', (ws, request) => { + ws.on('error', error => debug(`WebSocket Error: ${error.stack}`)) + + ws.once('message', (buf) => { + const path = request.url + const metadata = fromHeaders(buf) + const channel = new WebSocketMessageChannel(ws) + + this.emit('data', { + path, + metadata, + channel + }) + }) + }) + + wss.on('error', error => this.emit('error', error)) + } + + stop () { + return new Promise((resolve) => { + this._wss.close(() => resolve()) + }) + } + + ready () { + return new Promise((resolve) => { + this._wss.on('listening', () => { + this.multiaddr = `/ip4/${this._wss.address().address}/tcp/${this._wss.address().port}/ws` + + resolve(this) + }) + }) + } +} + +module.exports = async (ipfs, options = {}) => { + const config = await ipfs.config.getAll() + const grpcAddr = config.Addresses.RPC + const [,, host, , port] = grpcAddr.split('/') + + debug(`starting ws server on ${host}:${port}`) + + const wss = new WebSocketServer({ + host, + port + }) + + const messages = new Messages(wss) + + return messages.ready() +} diff --git a/packages/ipfs-grpc-server/test/add.spec.js b/packages/ipfs-grpc-server/test/add.spec.js new file mode 100644 index 0000000000..5262a62118 --- /dev/null +++ b/packages/ipfs-grpc-server/test/add.spec.js @@ -0,0 +1,95 @@ +/* eslint-env mocha */ +'use strict' + +const sinon = require('sinon') +const server = require('./utils/server') +const { expect } = require('aegir/utils/chai') +const uint8ArrayFromString = require('uint8arrays/from-string') +const all = require('it-all') +const drain = require('it-drain') + +describe('Root.add', () => { + let ipfs + let socket + + before(() => { + ipfs = { + addAll: sinon.stub() + } + socket = server({ ipfs }) + }) + + it('should add files', async () => { + const path1 = '/path/file-1.txt' + const cid1 = 'cid-1' + const path2 = '/path/file-1.txt' + const cid2 = 'cid-2' + + const results = [{ + type: 'RESULT', + path: path1, + cid: cid1 + }, { + type: 'RESULT', + path: path2, + cid: cid2 + }] + + ipfs.addAll.returns(results) + + const requests = [ + { index: 1, type: 'FILE', path: path1, content: uint8ArrayFromString('hello world') }, + { index: 1, type: 'FILE', path: path1 }, + { index: 2, type: 'FILE', path: path2, content: uint8ArrayFromString('hello world') }, + { index: 2, type: 'FILE', path: path2 } + ] + + const channel = socket.send('/ipfs.Root/add', {}) + requests.forEach(request => channel.clientSend(request)) + channel.clientEnd() + + await expect(all(channel.clientSink)).to.eventually.deep.equal(results) + }) + + it('should propagate error when adding files', async () => { + const path = '/path' + const err = new Error('halp!') + + ipfs.addAll.throws(err) + + const channel = socket.send('/ipfs.Root/add', {}) + channel.clientSend({ + index: 1, + type: 'DIRECTORY', + path + }) + channel.clientEnd() + + await expect(drain(channel.clientSink)).to.eventually.be.rejectedWith(/halp!/) + }) + + it('should propagate async error when adding files', async () => { + const path = '/path' + const err = new Error('halp!') + + ipfs.addAll.returns(async function * () { + yield { + type: 'file', + name: 'name', + cid: 'cid' + } + await Promise.resolve(true) + throw err + }()) + + const channel = socket.send('/ipfs.Root/add', {}) + channel.clientSend({ + index: 1, + type: 'DIRECTORY', + path + }) + channel.clientEnd() + + await expect(drain(channel.clientSink)).to.eventually.be.rejectedWith(/halp!/) + }) +}) diff --git a/packages/ipfs-grpc-server/test/id.spec.js b/packages/ipfs-grpc-server/test/id.spec.js new file mode 100644 index 0000000000..761170e853 --- /dev/null +++ b/packages/ipfs-grpc-server/test/id.spec.js @@ -0,0 +1,65 @@ +/* eslint-env mocha */ +'use strict' + +const sinon = require('sinon') +const server = require('./utils/server') +const { expect } = require('aegir/utils/chai') +const all = require('it-all') +const drain = require('it-drain') + +describe('Root.id', () => { + let ipfs + let socket + + beforeEach(() => { + ipfs = { + id: sinon.stub() + } + socket = server({ ipfs }) + }) + + it('should get the node id', async () => { + const id = 'hello world ' + Math.random() + + ipfs.id.withArgs({}).resolves(id) + + const channel = socket.send('/ipfs.Root/id', {}) + channel.clientSend({}) + channel.clientEnd() + + const messages = await all(channel.clientSink) + expect(messages).to.have.lengthOf(1) + expect(messages).to.have.nested.property('[0]', id) + }) + + it('should get a different node id', async () => { + const peerId = 'peer-id ' + Math.random() + const id = 'hello world ' + Math.random() + + ipfs.id.withArgs({ + peerId + }).resolves(id) + + const channel = socket.send('/ipfs.Root/id', {}) + channel.clientSend({ + peerId + }) + channel.clientEnd() + + const messages = await all(channel.clientSink) + expect(messages).to.have.lengthOf(1) + expect(messages).to.have.nested.property('[0]', id) + }) + + it('should propagate error when getting id', async () => { + const err = new Error('halp!') + + ipfs.id.rejects(err) + + const channel = socket.send('/ipfs.Root/id', {}) + channel.clientSend({}) + channel.clientEnd() + + await expect(drain(channel.clientSink)).to.eventually.be.rejectedWith(/halp!/) + }) +}) diff --git a/packages/ipfs-grpc-server/test/mfs/ls.spec.js b/packages/ipfs-grpc-server/test/mfs/ls.spec.js new file mode 100644 index 0000000000..37aca5fd4f --- /dev/null +++ b/packages/ipfs-grpc-server/test/mfs/ls.spec.js @@ -0,0 +1,80 @@ +/* eslint-env mocha */ +'use strict' + +const sinon = require('sinon') +const server = require('../utils/server') +const { expect } = require('aegir/utils/chai') +const all = require('it-all') +const drain = require('it-drain') + +describe('MFS.ls', () => { + let ipfs + let socket + + before(() => { + ipfs = { + files: { + ls: sinon.stub() + } + } + socket = server({ ipfs }) + }) + + it('should list files', async () => { + const path = '/path' + const results = [{ + type: 'file', + name: 'file', + cid: 'cid-1' + }, { + type: 'directory', + name: 'dir', + cid: 'cid-2' + }] + + ipfs.files.ls.withArgs(path).returns(results) + + const channel = socket.send('/ipfs.MFS/ls', {}) + channel.clientSend({ path }) + channel.clientEnd() + + await expect(all(channel.clientSink)).to.eventually.deep.equal(results.map(result => ({ + ...result, + type: result.type.toUpperCase() + }))) + }) + + it('should propagate error when listing files', async () => { + const path = '/path' + const err = new Error('halp!') + + ipfs.files.ls.withArgs(path).throws(err) + + const channel = socket.send('/ipfs.MFS/ls', {}) + channel.clientSend({ path }) + channel.clientEnd() + + await expect(drain(channel.clientSink)).to.eventually.be.rejectedWith(/halp!/) + }) + + it('should propagate async error when listing files', async () => { + const path = '/path' + const err = new Error('halp!') + + ipfs.files.ls.withArgs(path).returns(async function * () { + yield { + type: 'file', + name: 'name', + cid: 'cid' + } + await Promise.resolve(true) + throw err + }()) + + const channel = socket.send('/ipfs.MFS/ls', {}) + channel.clientSend({ path }) + channel.clientEnd() + + await expect(drain(channel.clientSink)).to.eventually.be.rejectedWith(/halp!/) + }) +}) diff --git a/packages/ipfs-grpc-server/test/mfs/write.spec.js b/packages/ipfs-grpc-server/test/mfs/write.spec.js new file mode 100644 index 0000000000..7bc9bb9343 --- /dev/null +++ b/packages/ipfs-grpc-server/test/mfs/write.spec.js @@ -0,0 +1,46 @@ +/* eslint-env mocha */ +'use strict' + +const sinon = require('sinon') +const server = require('../utils/server') +const { expect } = require('aegir/utils/chai') +const drain = require('it-drain') + +describe('MFS.write', () => { + let ipfs + let socket + + before(() => { + ipfs = { + files: { + write: sinon.stub() + } + } + socket = server({ ipfs }) + }) + + it('should write a file', async () => { + const path = '/path' + + const channel = socket.send('/ipfs.MFS/write', {}) + channel.clientSend({ path }) + channel.clientEnd() + + await drain(channel.clientSink) + + expect(ipfs.files.write.calledWith(path)).to.be.true() + }) + + it('should propagate error when writing files', async () => { + const path = '/path' + const err = new Error('halp!') + + ipfs.files.write.withArgs(path).throws(err) + + const channel = socket.send('/ipfs.MFS/write', {}) + channel.clientSend({ path }) + channel.clientEnd() + + await expect(drain(channel.clientSink)).to.eventually.be.rejectedWith(/halp!/) + }) +}) diff --git a/packages/ipfs-grpc-server/test/utils/channel.js b/packages/ipfs-grpc-server/test/utils/channel.js new file mode 100644 index 0000000000..7258fb62a4 --- /dev/null +++ b/packages/ipfs-grpc-server/test/utils/channel.js @@ -0,0 +1,48 @@ +'use strict' + +const pushable = require('it-pushable') + +class MessageChannel { + constructor () { + this.source = pushable() + this.sink = pushable() + + this.clientSink = pushable() + } + + sendMetadata (metadata) { + this.metadata = metadata + } + + sendMessage (message) { + setTimeout(() => { + this.clientSink.push(message) + }, 0) + } + + sendTrailers (trailers) { + this.trailers = trailers + } + + end (error) { + setTimeout(() => { + this.clientSink.end(error) + }, 0) + } + + clientSend (message) { + setTimeout(() => { + this.source.push(message) + }, 0) + } + + clientEnd (err) { + setTimeout(() => { + this.source.end(err) + }, 0) + } +} + +module.exports = () => { + return new MessageChannel() +} diff --git a/packages/ipfs-grpc-server/test/utils/server.js b/packages/ipfs-grpc-server/test/utils/server.js new file mode 100644 index 0000000000..d2b0b45945 --- /dev/null +++ b/packages/ipfs-grpc-server/test/utils/server.js @@ -0,0 +1,23 @@ +'use strict' + +const createServer = require('../../src') +const EventEmitter = require('events').EventEmitter +const createChannel = require('./channel') + +module.exports = ({ ipfs, options }) => { + const socket = new EventEmitter() + + createServer(ipfs, { + socket + }) + + return { + send: (path, metadata) => { + const channel = createChannel() + + socket.emit('data', { path, metadata, channel }) + + return channel + } + } +} diff --git a/packages/ipfs-grpc-server/tsconfig.json b/packages/ipfs-grpc-server/tsconfig.json new file mode 100644 index 0000000000..449e8148c0 --- /dev/null +++ b/packages/ipfs-grpc-server/tsconfig.json @@ -0,0 +1,15 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "outDir": "dist" + }, + "include": [ + "src", + "package.json" + ], + "references": [ + { + "path": "../ipfs-grpc-protocol" + } + ] +} diff --git a/packages/ipfs-http-client/package.json b/packages/ipfs-http-client/package.json index 9f8984d5cc..05c9d30b3c 100644 --- a/packages/ipfs-http-client/package.json +++ b/packages/ipfs-http-client/package.json @@ -80,7 +80,7 @@ "aegir": "^28.2.0", "go-ipfs": "^0.7.0", "ipfs-core": "^0.2.1", - "ipfsd-ctl": "^7.0.2", + "ipfsd-ctl": "ipfs/js-ipfsd-ctl#feat/expose-grpc-addr", "it-all": "^1.0.4", "it-concat": "^1.0.1", "nock": "^13.0.2", diff --git a/packages/ipfs/.aegir.js b/packages/ipfs/.aegir.js index 622a7ac601..4a3cf8c709 100644 --- a/packages/ipfs/.aegir.js +++ b/packages/ipfs/.aegir.js @@ -15,7 +15,7 @@ let sigServerB let ipfsdServer module.exports = { - bundlesize: { maxSize: '530kB' }, + bundlesize: { maxSize: '560kB' }, karma: { files: [{ pattern: 'node_modules/interface-ipfs-core/test/fixtures/**/*', @@ -86,6 +86,9 @@ module.exports = { }, { go: { ipfsBin: require('go-ipfs').path() + }, + js: { + ipfsClientModule: require('ipfs-client') } }).start() diff --git a/packages/ipfs/package.json b/packages/ipfs/package.json index 6266639a8c..4b674dc1e7 100644 --- a/packages/ipfs/package.json +++ b/packages/ipfs/package.json @@ -28,6 +28,7 @@ "build:types": "tsc --build", "test": "echo 'Only interface tests live here'", "test:interface:core": "aegir test -f test/interface-core.js", + "test:interface:client": "aegir test -f test/interface-client.js", "test:interface:http-js": "aegir test -f test/interface-http-js.js", "test:interface:http-go": "aegir test -f test/interface-http-go.js", "test:interop": "cross-env IPFS_JS_EXEC=$PWD/src/cli.js IPFS_JS_MODULE=$PWD IPFS_JS_HTTP_MODULE=$PWD/../ipfs-http-client IPFS_REUSEPORT=false ipfs-interop", @@ -50,10 +51,11 @@ "electron-webrtc": "^0.3.0", "go-ipfs": "^0.7.0", "interface-ipfs-core": "^0.142.1", + "ipfs-client": "^0.1.0", "ipfs-http-client": "^48.1.1", "ipfs-interop": "^3.0.0", "ipfs-utils": "^5.0.0", - "ipfsd-ctl": "^7.0.2", + "ipfsd-ctl": "ipfs/js-ipfsd-ctl#feat/expose-grpc-addr", "iso-url": "^1.0.0", "libp2p-webrtc-star": "^0.20.1", "merge-options": "^2.0.0", diff --git a/packages/ipfs/test/interface-client.js b/packages/ipfs/test/interface-client.js new file mode 100644 index 0000000000..ff6ae19223 --- /dev/null +++ b/packages/ipfs/test/interface-client.js @@ -0,0 +1,147 @@ +/* eslint-env mocha, browser */ +'use strict' + +const tests = require('interface-ipfs-core') +const { isNode } = require('ipfs-utils/src/env') +const factory = require('./utils/factory') + +describe('interface-ipfs-core ipfs-client tests', () => { + const commonFactory = factory({ + type: 'js', + ipfsClientModule: require('ipfs-client') + }) + + tests.files(commonFactory, { + skip: [{ + name: '.files.chmod', + reason: 'not implemented' + }, { + name: '.files.cp', + reason: 'not implemented' + }, { + name: '.files.mkdir', + reason: 'not implemented' + }, { + name: '.files.stat', + reason: 'not implemented' + }, { + name: '.files.touch', + reason: 'not implemented' + }, { + name: '.files.rm', + reason: 'not implemented' + }, { + name: '.files.read', + reason: 'not implemented' + }, { + name: '.files.mv', + reason: 'not implemented' + }, { + name: '.files.flush', + reason: 'not implemented' + }].concat(isNode ? [] : [{ + name: 'should make directory and specify mtime as hrtime', + reason: 'Not designed to run in the browser' + }, { + name: 'should set mtime as hrtime', + reason: 'Not designed to run in the browser' + }, { + name: 'should write file and specify mtime as hrtime', + reason: 'Not designed to run in the browser' + }]) + }) + + tests.root(commonFactory, { + skip: [ + { + name: 'add', + reason: 'not implemented' + }, + { + name: 'should add with only-hash=true', + reason: 'ipfs.object.get is not implemented' + }, + { + name: 'should add a directory with only-hash=true', + reason: 'ipfs.object.get is not implemented' + }, + { + name: 'should add with mtime as hrtime', + reason: 'process.hrtime is not a function in browser' + }, + { + name: 'should add from a URL with only-hash=true', + reason: 'ipfs.object.get is not implemented' + }, + { + name: 'should cat with a Uint8Array multihash', + reason: 'Passing CID as Uint8Array is not supported' + }, + { + name: 'should add from a HTTP URL', + reason: 'https://github.com/ipfs/js-ipfs/issues/3195' + }, + { + name: 'should add from a HTTP URL with redirection', + reason: 'https://github.com/ipfs/js-ipfs/issues/3195' + }, + { + name: 'should add from a URL with only-hash=true', + reason: 'https://github.com/ipfs/js-ipfs/issues/3195' + }, + { + name: 'should add from a URL with wrap-with-directory=true', + reason: 'https://github.com/ipfs/js-ipfs/issues/3195' + }, + { + name: 'should add from a URL with wrap-with-directory=true and URL-escaped file name', + reason: 'https://github.com/ipfs/js-ipfs/issues/3195' + }, + { + name: 'should not add from an invalid url', + reason: 'https://github.com/ipfs/js-ipfs/issues/3195' + }, + { + name: 'should be able to add dir without sharding', + reason: 'Cannot spawn IPFS with different args' + }, + { + name: 'with sharding', + reason: 'TODO: allow spawning new daemons with different config' + }, + { + name: 'get', + reason: 'Not implemented' + }, + { + name: 'refs', + reason: 'Not implemented' + }, + { + name: 'refsLocal', + reason: 'Not implemented' + } + ] + }) + + tests.miscellaneous(commonFactory, { + skip: [ + { + name: '.dns', + reason: 'Not implemented' + }, + { + name: '.resolve', + reason: 'Not implemented' + }, + { + name: '.stop', + reason: 'Not implemented' + }, + { + name: '.version', + reason: 'Not implemented' + } + ] + }) +}) diff --git a/tsconfig.json b/tsconfig.json index 61fe44e93d..9143b1f782 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -35,12 +35,24 @@ "ipfs-cli/*": [ "ipfs-cli/*" ], + "ipfs-client/*": [ + "ipfs-client/*" + ], "ipfs-core/*": [ "ipfs-core/*" ], "ipfs-core-utils/*": [ "ipfs-core-utils/*" ], + "ipfs-grpc-client/*": [ + "ipfs-grpc-client/*" + ], + "ipfs-grpc-protocol/*": [ + "ipfs-grpc-protocol/*" + ], + "ipfs-grpc-server/*": [ + "ipfs-grpc-server/*" + ], "ipfs-http-client/*": [ "ipfs-http-client/*" ],