refactor: peristent peer-store extended class and disabled by defaul

.aegir.js

@@ -45,7 +45,7 @@ const after = async () => {
 }
 
 module.exports = {
-  bundlesize: { maxSize: '179kB' },
+  bundlesize: { maxSize: '185kB' },
   hooks: {
     pre: before,
     post: after

doc/API.md

@@ -74,6 +74,7 @@ Creates an instance of Libp2p.
 | [options.dialer] | `object` | libp2p Dialer configuration
 | [options.metrics] | `object` | libp2p Metrics configuration
 | [options.peerId] | [`PeerId`][peer-id] | peerId instance (it will be created if not provided) |
+| [options.peerStore] | [`PeerId`][peer-id] | libp2p PeerStore configuration |
 
 For Libp2p configurations and modules details read the [Configuration Document](./CONFIGURATION.md).
 

doc/CONFIGURATION.md

@@ -505,6 +505,32 @@ const node = await Libp2p.create({
 })
 ```
 
+#### Configuring PeerStore
+
+PeerStore persistence is disabled in libp2p by default. You can enable and configure it as follows. Aside from enabled being `false` by default, it will need an implementation of a [datastore](https://github.com/ipfs/interface-datastore).
+
+```js
+const Libp2p = require('libp2p')
+const TCP = require('libp2p-tcp')
+const MPLEX = require('libp2p-mplex')
+const SECIO = require('libp2p-secio')
+
+const LevelStore = require('datastore-level')
+
+const node = await Libp2p.create({
+  modules: {
+    transport: [TCP],
+    streamMuxer: [MPLEX],
+    connEncryption: [SECIO]
+  },
+  datastore: new LevelStore('path/to/store'),
+  peerStore: {
+    persistence: true,
+    threshold: 5
+  }
+})
+```
+
 #### Customizing Transports
 
 Some Transports can be passed additional options when they are created. For example, `libp2p-webrtc-star` accepts an optional, custom `wrtc` implementation. In addition to libp2p passing itself and an `Upgrader` to handle connection upgrading, libp2p will also pass the options, if they are provided, from `config.transport`.

src/config.js

@@ -21,7 +21,8 @@ const DefaultConfig = {
     enabled: false
   },
   peerStore: {
-    persistence: true
+    persistence: false,
+    threshold: 5
   },
   config: {
     dht: {

src/index.js

@@ -6,7 +6,6 @@ const globalThis = require('ipfs-utils/src/globalthis')
 const log = debug('libp2p')
 log.error = debug('libp2p:error')
 
-const { MemoryDatastore } = require('interface-datastore')
 const PeerId = require('peer-id')
 
 const peerRouting = require('./peer-routing')
@@ -24,6 +23,7 @@ const Metrics = require('./metrics')
 const TransportManager = require('./transport-manager')
 const Upgrader = require('./upgrader')
 const PeerStore = require('./peer-store')
+const PersistentPeerStore = require('./peer-store/persistent')
 const Registrar = require('./registrar')
 const ping = require('./ping')
 const {
@@ -45,11 +45,14 @@ class Libp2p extends EventEmitter {
     this._options = validateConfig(_options)
 
     this.peerId = this._options.peerId
-    this.datastore = this._options.datastore || new MemoryDatastore()
-    this.peerStore = new PeerStore({
-      datastore: this.datastore,
-      ...this._options.peerStore
-    })
+    this.datastore = this._options.datastore
+
+    this.peerStore = !(this.datastore && this._options.peerStore.persistence)
+      ? new PeerStore()
+      : new PersistentPeerStore({
+        datastore: this.datastore,
+        ...this._options.peerStore
+      })
 
     // Addresses {listen, announce, noAnnounce}
     this.addresses = this._options.addresses
@@ -223,7 +226,8 @@ class Libp2p extends EventEmitter {
 
       this._discovery = new Map()
 
-      this.connectionManager.stop()
+      await this.peerStore.stop()
+      await this.connectionManager.stop()
 
       await Promise.all([
         this.pubsub && this.pubsub.stop(),
@@ -398,7 +402,7 @@ class Libp2p extends EventEmitter {
     await this.transportManager.listen()
 
     // Start PeerStore
-    await this.peerStore.load()
+    await this.peerStore.start()
 
     if (this._config.pubsub.enabled) {
       this.pubsub && this.pubsub.start()

src/peer-store/README.md

@@ -85,15 +85,11 @@ Access to its underlying books:
 
 ## Data Persistence
 
-The data stored in the PeerStore will be persisted by default. Keeping a record of the peers already discovered by the peer, as well as their known data aims to improve the efficiency of peers joining the network after being offline.
+The data stored in the PeerStore can be persisted if configured appropriately. Keeping a record of the peers already discovered by the peer, as well as their known data aims to improve the efficiency of peers joining the network after being offline.
 
----
-TODO: Discuss if we should make it persisted by default now. Taking into consideration that we will use a MemoryDatastore by default, unless the user configures a datastore to use, it will be worthless. It might make sense to make it disabled by default until we work on improving configuration and provide good defauls for each environment.
----
+The libp2p node will need to receive a [datastore](https://github.com/ipfs/interface-datastore), in order to store this data in a persistent way. A [datastore](https://github.com/ipfs/interface-datastore) stores its data in a key-value fashion. As a result, we need coherent keys so that we do not overwrite data.
 
-The libp2p node will need to receive a [datastore](https://github.com/ipfs/interface-datastore), in order to store this data in a persistent way. Otherwise, it will be stored on a [memory datastore](https://github.com/ipfs/interface-datastore/blob/master/src/memory.js).
-
-A [datastore](https://github.com/ipfs/interface-datastore) stores its data in a key-value fashion. As a result, we need coherent keys so that we do not overwrite data.
+The PeerStore should not be continuously updating the datastore with the new data observed. Accordingly, it should only store new data after reaching a certain threshold of "dirty" peers, as well as when the node is stopped.
 
 Taking into account that a datastore allows queries using a key prefix, we can find all the information if we define a consistent namespace that allow us to find the content without having any information. The namespaces were defined as follows:
 

src/peer-store/address-book.js

@@ -9,7 +9,6 @@ const multiaddr = require('multiaddr')
 const PeerId = require('peer-id')
 
 const Book = require('./book')
-const Protobuf = require('./pb/address-book.proto')
 
 const {
   codes: { ERR_INVALID_PARAMETERS }
@@ -18,8 +17,6 @@ const {
 /**
  * The AddressBook is responsible for keeping the known multiaddrs
  * of a peer.
- * This data will be persisted in the PeerStore datastore as follows:
- * /peers/addrs/<b32 peer id no padding>
  */
 class AddressBook extends Book {
   /**
@@ -40,24 +37,9 @@ class AddressBook extends Book {
      */
     super({
       peerStore,
-      event: {
-        name: 'change:multiaddrs',
-        property: 'multiaddrs',
-        transformer: (data) => data.map((address) => address.multiaddr)
-      },
-      ds: {
-        prefix: '/peers/addrs/',
-        setTransformer: (data) => Protobuf.encode({
-          addrs: data.map((address) => address.multiaddr.buffer)
-        }),
-        getTransformer: (encData) => {
-          const data = Protobuf.decode(encData)
-
-          return data.addrs.map((a) => ({
-            multiaddr: multiaddr(a)
-          }))
-        }
-      }
+      eventName: 'change:multiaddrs',
+      eventProperty: 'multiaddrs',
+      eventTransformer: (data) => data.map((address) => address.multiaddr)
     })
 
     /**
@@ -145,6 +127,7 @@ class AddressBook extends Book {
     }
 
     this._setData(peerId, addresses)
+
     log(`added provided multiaddrs for ${id}`)
 
     // Notify the existance of a new peer

src/peer-store/book.js

@@ -1,11 +1,6 @@
 'use strict'
 
 const errcode = require('err-code')
-const debug = require('debug')
-const log = debug('libp2p:peer-store:book')
-log.error = debug('libp2p:peer-store:book:error')
-
-const { Key } = require('interface-datastore')
 const PeerId = require('peer-id')
 
 const {
@@ -16,31 +11,21 @@ const passthrough = data => data
 
 /**
  * The Book is the skeleton for the PeerStore books.
- * It handles the PeerStore persistence and events.
  */
 class Book {
   /**
    * @constructor
    * @param {Object} properties
    * @param {PeerStore} properties.peerStore PeerStore instance.
-   * @param {Object} [properties.event] Event properties. If not provided, no events will be emitted.
-   * @param {string} [properties.event.name] Name of the event to emit by the PeerStore.
-   * @param {string} [properties.event.property] Name of the property to emit by the PeerStore.
-   * @param {function} [properties.events.transformer] Transformer function of the provided data for being emitted.
-   * @param {Object} [properties.ds] Datastore properties. If not provided, no data will be persisted.
-   * @param {String} [properties.ds.prefix] Prefix of the Datastore Key
-   * @param {String} [properties.ds.suffix = ''] Suffix of the Datastore Key
-   * @param {function} [properties.ds.setTransformer] Transformer function of the provided data for being persisted.
-   * @param {function} [properties.ds.getTransformer] Transformer function of the persisted data to be loaded.
+   * @param {string} properties.eventName Name of the event to emit by the PeerStore.
+   * @param {string} properties.eventProperty Name of the property to emit by the PeerStore.
+   * @param {function} [properties.eventTransformer] Transformer function of the provided data for being emitted.
    */
-  constructor ({
-    peerStore,
-    event,
-    ds
-  }) {
+  constructor ({ peerStore, eventName, eventProperty, eventTransformer = passthrough }) {
     this._ps = peerStore
-    this.event = event
-    this.ds = ds
+    this.eventName = eventName
+    this.eventProperty = eventProperty
+    this.eventTransformer = eventTransformer
 
     /**
      * Map known peers to their data.
@@ -50,38 +35,12 @@ class Book {
   }
 
   /**
-   * Load data from peerStore datastore into the books datastructures.
-   * This will not persist the replicated data nor emit modify events.
-   * @private
-   * @return {Promise<void>}
+   * Set known data of a provided peer.
+   * @param {PeerId} peerId
+   * @param {Array<Data>|Data} data
    */
-  async _loadData () {
-    if (!this._ps._datastore || !this._ps._enabledPersistance || !this.ds) {
-      return
-    }
-
-    const prefix = this.ds.prefix || ''
-    const suffix = this.ds.suffix || ''
-    const transformer = this.ds.getTransformer || passthrough
-
-    for await (const { key, value } of this._ps._datastore.query({ prefix })) {
-      try {
-        // PeerId to add to the book
-        const b32key = key.toString()
-          .replace(prefix, '') // remove prefix from key
-          .replace(suffix, '') // remove suffix from key
-        const peerId = PeerId.createFromCID(b32key)
-        // Data in the format to add to the book
-        const data = transformer(value)
-        // Add the book without persist the replicated data and emit modify
-        this._setData(peerId, data, {
-          persist: false,
-          emit: false
-        })
-      } catch (err) {
-        log.error(err)
-      }
-    }
+  set (peerId, data) {
+    throw errcode(new Error('set must be implemented by the subclass'), 'ERR_NOT_IMPLEMENTED')
   }
 
   /**
@@ -90,65 +49,21 @@ class Book {
    * @param {PeerId} peerId peerId of the data to store
    * @param {Array<*>} data data to store.
    * @param {Object} [options] storing options.
-   * @param {boolean} [options.persist = true] persist the provided data.
    * @param {boolean} [options.emit = true] emit the provided data.
-   * @return {Promise<void>}
+   * @return {void}
    */
-  async _setData (peerId, data, { persist = true, emit = true } = {}) {
+  _setData (peerId, data, { emit = true } = {}) {
     const b58key = peerId.toB58String()
 
     // Store data in memory
     this.data.set(b58key, data)
     this._setPeerId(peerId)
 
     // Emit event
-    if (this.event && emit) {
-      const transformer = this.event.transformer || passthrough
-
-      this._ps.emit(this.event.name, {
-        peerId,
-        [this.event.property]: transformer(data)
-      })
-    }
-
-    // Add to Persistence datastore
-    persist && await this._persistData(peerId, data)
-  }
-
-  /**
-   * Persist data on the datastore
-   * @private
-   * @param {PeerId} peerId peerId of the data to persist
-   * @param {Array<*>} data data to persist
-   * @return {Promise<void>}
-   */
-  async _persistData (peerId, data) {
-    if (!this._ps._datastore || !this._ps._enabledPersistance || !this.ds) {
-      return
-    }
-
-    const prefix = this.ds.prefix || ''
-    const suffix = this.ds.suffix || ''
-    const transformer = this.ds.setTransformer || passthrough
-
-    const b32key = peerId.toString()
-    const k = `${prefix}${b32key}${suffix}`
-    try {
-      const value = transformer(data)
-
-      await this._ps._datastore.put(new Key(k), value)
-    } catch (err) {
-      log.error(err)
-    }
-  }
-
-  /**
-   * Set known data of a provided peer.
-   * @param {PeerId} peerId
-   * @param {Array<Data>|Data} data
-   */
-  set (peerId, data) {
-    throw errcode(new Error('set must be implemented by the subclass'), 'ERR_NOT_IMPLEMENTED')
+    emit && this._ps.emit(this.eventName, {
+      peerId,
+      [this.eventProperty]: this.eventTransformer(data)
+    })
   }
 
   /**
@@ -189,22 +104,11 @@ class Book {
       return false
     }
 
-    // Emit event
-    this.event && this._ps.emit(this.event.name, {
+    this._ps.emit(this.eventName, {
       peerId,
-      [this.event.property]: []
+      [this.eventProperty]: []
     })
 
-    // Update Persistence datastore
-    if (this._ps._datastore && this._ps._enabledPersistance && this.ds) {
-      const prefix = this.ds.prefix || ''
-      const suffix = this.ds.suffix || ''
-      const b32key = peerId.toString()
-
-      const k = `${prefix}${b32key}${suffix}`
-      this._ps._datastore.delete(new Key(k))
-    }
-
     return true
   }