datastore: plugin to replicate datastore commands before next release.

See ElementsProject/lightning#4674
(I manually tested that it passes all the tests there, too!)

When they finally upgrade the node, this automatically puts any
datastore data into the inbuilt datastore and shuts down.

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>

datastore/README.md

@@ -0,0 +1,47 @@
+# Datastore Plugin
+
+The next version of c-lightning (0.10.2?) has a built-in datastore.
+
+Until then, this plugin serves the same purpose, and has the same
+interface.  When it detects that you have upgraded, it will place
+its datastore into the built-in one and shut down.
+
+The plugin (`datastore.py`) is a bit weird, in that it loads the
+*real* plugin if it detects that it is needed.  This is because a
+plugin cannot replace an existing command, so it would fail badly when
+you finally upgrade.
+
+## Usage
+
+Just add it to your .lightning/config file as
+"plugin=/path/to/plugin/datastore.py".
+
+This plugin is usually used by *other* plugins to store and retreive
+data.  The commands, briefly, are:
+
+### **datastore** *key* [*string*] [*hex*] [*mode*]
+
+There can only be one entry for each *key*, so prefixing with the
+plugin name (e.g. `summary.`) is recommended.
+
+*mode* is one of "must-create" (default, fails it it already exists),
+"must-replace" (fails it it doesn't already exist),
+"create-or-replace" (never fails), "must-append" (must already exist,
+append this to what's already there) or "create-or-append" (append if
+anything is there, otherwise create).
+
+### **deldatastore** *key*
+
+The command fails if the *key* isn't present.
+
+### **listdatastore** [*key*]
+
+Fetch data which was stored in the database.
+
+All entries are returned in *key* isn't present; if *key* is present,
+zero or one entries are returned.
+
+## Author
+
+Rusty Russell wrote this so he can use it before the next release, and
+plugins can start relying on storing data.

datastore/datastore-plugin.py

@@ -0,0 +1,91 @@
+#!/usr/bin/env python3
+"""This does the actual datastore work, if the main plugin says there's no
+datastore support.  We can't even load this if there's real datastore support.
+"""
+from pyln.client import Plugin, RpcError
+import shelve
+
+
+plugin = Plugin()
+
+
+def datastore_entry(key, data):
+    """Return a dict representing the entry"""
+
+    ret = {'key': key, 'hex': data.hex()}
+
+    # FFS, Python3 seems happy with \0 in UTF-8.
+    if 0 not in data:
+        try:
+            ret['string'] = data.decode('utf8')
+        except UnicodeDecodeError:
+            pass
+    return ret
+
+
+@plugin.method("datastore")
+def datastore(plugin, key, string=None, hex=None, mode="must-create"):
+    """Add a {key} and {hex}/{string} data to the data store"""
+
+    if string is not None:
+        if hex is not None:
+            raise RpcError("datastore", {'key': key},
+                           {'message': "Cannot specify both string or hex"})
+        data = bytes(string, encoding="utf8")
+    elif hex is None:
+        raise RpcError("datastore", {'key': key},
+                       {'message': "Must specify string or hex"})
+    else:
+        data = bytes.fromhex(hex)
+
+    print("key={}, data={}, mode={}".format(key, data, mode))
+    if mode == "must-create":
+        if key in plugin.datastore:
+            raise RpcError("datastore", {'key': key},
+                           {'message': "already exists"})
+    elif mode == "must-replace":
+        if key not in plugin.datastore:
+            raise RpcError("datastore", {'key': key},
+                           {'message': "does not exist"})
+    elif mode == "create-or-replace":
+        pass
+    elif mode == "must-append":
+        if key not in plugin.datastore:
+            raise RpcError("datastore", {'key': key},
+                           {'message': "does not exist"})
+        data = plugin.datastore[key] + data
+    elif mode == "create-or-append":
+        data = plugin.datastore.get(key, bytes()) + data
+    else:
+        raise RpcError("datastore", {'key': key}, {'message': "invalid mode"})
+
+    plugin.datastore[key] = data
+    return datastore_entry(key, data)
+
+
+@plugin.method("deldatastore")
+def deldatastore(plugin, key):
+    """Remove a {key} from the data store"""
+
+    ret = datastore_entry(key, plugin.datastore[key])
+    del plugin.datastore[key]
+    return ret
+
+
+@plugin.method("listdatastore")
+def listdatastore(plugin, key=None):
+    """List datastore entries"""
+    if key is None:
+        return {'datastore': [datastore_entry(k, d)
+                              for k, d in plugin.datastore.items()]}
+    if key in plugin.datastore:
+        return {'datastore': [datastore_entry(key, plugin.datastore[key])]}
+    return {'datastore': []}
+
+
+@plugin.init()
+def init(options, configuration, plugin):
+    plugin.datastore = shelve.open('datastore.dat')
+
+
+plugin.run()

datastore/datastore.py

@@ -0,0 +1,45 @@
+#!/usr/bin/env python3
+from pyln.client import Plugin, RpcError
+import shelve
+import os
+
+
+plugin = Plugin()
+
+
+def unload_store(plugin):
+    """When we have a real store, we transfer our contents into it"""
+    try:
+        datastore = shelve.open('datastore.dat', 'r')
+    except:
+        return
+
+    plugin.log("Emptying store into main store!", level='unusual')
+    for k, d in datastore.items():
+        try:
+            plugin.rpc.datastore(k, d.hex())
+        except RpcError as e:
+            plugin.log("Failed to put {} into store: {}".format(k, e),
+                       level='broken')
+    datastore.close()
+    plugin.log("Erasing our store", level='unusual')
+    os.unlink('datastore.dat')
+
+
+@plugin.init()
+def init(options, configuration, plugin):
+    # If we have real datastore commands, don't load plugin.
+    try:
+        plugin.rpc.help('datastore')
+        unload_store(plugin)
+        return {'disable': 'there is a real datastore command'}
+    except RpcError:
+        pass
+
+    # Start up real plugin now
+    plugin.rpc.plugin_start(os.path.join(os.path.dirname(__file__),
+                                         "datastore-plugin.py"))
+    return {'disable': 'no builtin-datastore: plugin loaded'}
+
+
+plugin.run()

datastore/requirements.txt

@@ -0,0 +1 @@
+pyln-client