From 0f5021c1ec841a8995bd53a7e03051208efdc448 Mon Sep 17 00:00:00 2001 From: Jacob Heun Date: Thu, 14 Feb 2019 13:05:05 +0100 Subject: [PATCH] feat: add createRepo utility License: MIT Signed-off-by: Jacob Heun --- README.md | 17 +++++ examples/full-s3-repo/index.js | 42 ++---------- examples/full-s3-repo/package.json | 4 +- package.json | 14 ++-- src/index.js | 4 ++ src/s3-repo.js | 100 +++++++++++++++++++++++++++++ test/index.spec.js | 20 +++++- 7 files changed, 156 insertions(+), 45 deletions(-) create mode 100644 src/s3-repo.js diff --git a/README.md b/README.md index 3d502a0..84db78f 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,23 @@ const store = new S3Store('.ipfs/datastore', { }) ``` +### Create a Repo +You can quickly create an S3 backed repo using the `createRepo` convenience function. + +```js +const IPFS = require('ipfs') +const { createRepo } = require('datastore-s3') + +const ipfs = new IPFS({ + repo: createRepo({ + path: '/my/ipfs', + createIfMissing: false + }, { + bucket: 'MyS3Bucket' + }) +}) +``` + ### Examples You can see examples of S3 backed ipfs in the [examples folder](examples/) diff --git a/examples/full-s3-repo/index.js b/examples/full-s3-repo/index.js index 2f0bfcf..7168ed3 100644 --- a/examples/full-s3-repo/index.js +++ b/examples/full-s3-repo/index.js @@ -1,48 +1,20 @@ 'use strict' const IPFS = require('ipfs') -const Repo = require('ipfs-repo') -const S3 = require('aws-sdk').S3 -const S3Store = require('datastore-s3') -const S3Lock = require('./s3-lock') +const { createRepo } = require('datastore-s3') -// Initialize the AWS S3 instance -const s3 = new S3({ - params: { - Bucket: 'my-bucket' - }, +// Create the repo +const s3Repo = createRepo({ + path: '/tmp/test/.ipfs' +}, { + bucket: 'my-bucket', accessKeyId: 'myaccesskey', secretAccessKey: 'mysecretkey' }) -// Create our custom lock -const s3Store = new S3Store('/tmp/test/.ipfs', { s3 }) -const s3Lock = new S3Lock(s3Store) - -// Create the IPFS Repo, full backed by S3 -const repo = new Repo('/tmp/test/.ipfs', { - storageBackends: { - root: S3Store, - blocks: S3Store, - keys: S3Store, - datastore: S3Store - }, - storageBackendOptions: { - root: { s3 }, - blocks: { s3 }, - keys: { s3 }, - datastore: { s3 } - }, - lock: s3Lock -}) - // Create a new IPFS node with our S3 backed Repo let node = new IPFS({ - repo, - config: { - Discovery: { MDNS: { Enabled: false }, webRTCStar: { Enabled: false } }, - Bootstrap: [] - } + repo: s3Repo }) console.log('Start the node') diff --git a/examples/full-s3-repo/package.json b/examples/full-s3-repo/package.json index cd59071..204d0bf 100644 --- a/examples/full-s3-repo/package.json +++ b/examples/full-s3-repo/package.json @@ -10,8 +10,8 @@ "author": "", "license": "ISC", "dependencies": { - "async": "^2.6.1", - "aws-sdk": "^2.394.0", + "async": "^2.6.2", + "aws-sdk": "^2.402.0", "datastore-s3": "../../", "ipfs": "~0.34.4", "ipfs-repo": "~0.26.1" diff --git a/package.json b/package.json index d85cf1a..e38dd8c 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "datastore-s3", "version": "0.2.1", - "description": "Datastore implementation backed by s3", + "description": "IPFS datastore implementation backed by s3", "leadMaintainer": "Jacob Heun", "main": "src/index.js", "files": [ @@ -37,7 +37,7 @@ }, "homepage": "https://github.com/ipfs/js-datastore-s3#readme", "dependencies": { - "async": "^2.6.1", + "async": "^2.6.2", "datastore-core": "~0.6.0", "interface-datastore": "~0.6.0", "once": "^1.4.0", @@ -46,14 +46,18 @@ "upath": "^1.1.0" }, "devDependencies": { - "aegir": "^18.0.3", - "aws-sdk": "^2.394.0", + "aegir": "^18.1.0", + "aws-sdk": "^2.402.0", "chai": "^4.2.0", "dirty-chai": "^2.0.1", - "flow-bin": "~0.91.0", + "flow-bin": "~0.93.0", "flow-typed": "^2.5.1", + "ipfs-repo": "~0.26.1", "stand-in": "^4.2.0" }, + "peerDependencies": { + "ipfs-repo": "0.x" + }, "contributors": [ "Jacob Heun ", "Justin Chase " diff --git a/src/index.js b/src/index.js index 21cd10a..ff995a8 100644 --- a/src/index.js +++ b/src/index.js @@ -14,6 +14,7 @@ const asyncSort = require('interface-datastore').utils.asyncSort const IDatastore = require('interface-datastore') const Key = IDatastore.Key const Errors = IDatastore.Errors +const createRepo = require('./s3-repo') const Deferred = require('pull-defer') const pull = require('pull-stream') @@ -384,3 +385,6 @@ class S3Datastore { } module.exports = S3Datastore +module.exports.createRepo = (...args) => { + return createRepo(S3Datastore, ...args) +} diff --git a/src/s3-repo.js b/src/s3-repo.js new file mode 100644 index 0000000..07d5449 --- /dev/null +++ b/src/s3-repo.js @@ -0,0 +1,100 @@ +'use strict' + +const S3 = require('aws-sdk/clients/s3') +const IPFSRepo = require('ipfs-repo') + +// A mock lock +const notALock = { + getLockfilePath: () => {}, + lock: (_, cb) => { + cb(null, notALock.getCloser()) + }, + getCloser: (_) => { + return { + close: (cb) => { + cb() + } + } + }, + locked: (_, cb) => { + cb(null, false) + } +} + +/** + * @typedef {Object} S3Options + * @property {string} bucket + * @property {string} region + * @property {string} accessKeyId + * @property {string} secretAccessKey + */ + +/** + * @typedef {Object} RepoOptions + * @property {string} path The path inside the bucket to create the repo + * @property {boolean} createIfMissing If the repo should be created if it's missing + * @property {RepoLock} lock An optional lock for the repo + */ + +/** + * A convenience method for creating an S3 backed IPFS repo + * @param {S3Store} S3Store + * @param {RepoOptions} options + * @param {S3Options} s3Options + * @returns {IPFSRepo} + */ +const createRepo = (S3Store, options, s3Options) => { + const { + bucket, + region, + accessKeyId, + secretAccessKey + } = s3Options + + let { + path, + createIfMissing, + lock + } = options + + const storeConfig = { + s3: new S3({ + params: { + Bucket: bucket + }, + region, + accessKeyId, + secretAccessKey + }), + createIfMissing + } + + const store = new S3Store(path, storeConfig) + + class Store { + constructor () { + return store + } + } + + // If no lock is given, create a mock lock + lock = lock || notALock + + return new IPFSRepo(path, { + storageBackends: { + root: Store, + blocks: Store, + keys: Store, + datastore: Store + }, + storageBackendconfig: { + root: storeConfig, + blocks: storeConfig, + keys: storeConfig, + datastore: storeConfig + }, + lock: lock + }) +} + +module.exports = createRepo diff --git a/test/index.spec.js b/test/index.spec.js index 256d7bc..feb7683 100644 --- a/test/index.spec.js +++ b/test/index.spec.js @@ -6,13 +6,13 @@ const chai = require('chai') chai.use(require('dirty-chai')) const expect = chai.expect +const standin = require('stand-in') const Key = require('interface-datastore').Key - const S3 = require('aws-sdk').S3 -const S3Mock = require('./utils/s3-mock') -const standin = require('stand-in') +const S3Mock = require('./utils/s3-mock') const S3Store = require('../src') +const { createRepo } = require('../src') describe('S3Datastore', () => { describe('construction', () => { @@ -183,6 +183,20 @@ describe('S3Datastore', () => { }) }) + describe('createRepo', () => { + it('should be able to create a repo', () => { + const path = '.ipfs/datastore' + const repo = createRepo({ + path + }, { + bucket: 'my-ipfs-bucket' + }) + + expect(repo).to.exist() + expect(repo.path).to.eql(path) + }) + }) + describe('interface-datastore', () => { require('interface-datastore/src/tests')({ setup (callback) {