From fd6172aa85c82e0eb2e6e95c4a168e9437e04805 Mon Sep 17 00:00:00 2001 From: darosior Date: Sun, 2 Feb 2020 14:20:23 +0100 Subject: [PATCH 01/21] libplugin: add a 'still_pending' helper --- plugins/libplugin.c | 6 ++++++ plugins/libplugin.h | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/plugins/libplugin.c b/plugins/libplugin.c index 99c38707c458..993e458931c8 100644 --- a/plugins/libplugin.c +++ b/plugins/libplugin.c @@ -217,6 +217,12 @@ command_finished(struct command *cmd, struct json_stream *response) return command_complete(cmd, response); } +struct command_result *WARN_UNUSED_RESULT +command_still_pending(struct command *cmd) +{ + return &pending; +} + struct json_out *json_out_obj(const tal_t *ctx, const char *fieldname, const char *str) diff --git a/plugins/libplugin.h b/plugins/libplugin.h index 829a6ab8e6a8..c47163ddd587 100644 --- a/plugins/libplugin.h +++ b/plugins/libplugin.h @@ -138,6 +138,10 @@ struct json_stream *jsonrpc_stream_fail_data(struct command *cmd, struct command_result *WARN_UNUSED_RESULT command_finished(struct command *cmd, struct json_stream *response); +/* Helper for a command that'll be finished in a callback. */ +struct command_result *WARN_UNUSED_RESULT +command_still_pending(struct command *cmd); + /* Helper to create a zero or single-value JSON object; if @str is NULL, * object is empty. */ struct json_out *json_out_obj(const tal_t *ctx, From 22780f87dde499f5280293f2d9a1f6bab12236b4 Mon Sep 17 00:00:00 2001 From: darosior Date: Mon, 3 Feb 2020 22:27:46 +0100 Subject: [PATCH 02/21] libplugin: generalize the plugin_timer callback type We don't take the callback result into account, so it can better be void. Having a general callback parameter is handy, because for bcli we want to pass it the struct bcli. --- plugins/autoclean.c | 14 ++++++++------ plugins/libplugin.c | 9 ++++++--- plugins/libplugin.h | 3 ++- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/plugins/autoclean.c b/plugins/autoclean.c index fff1b41e805a..57dba720cf9b 100644 --- a/plugins/autoclean.c +++ b/plugins/autoclean.c @@ -9,7 +9,7 @@ static u64 cycle_seconds = 0, expired_by = 86400; static struct plugin_timer *cleantimer; -static struct command_result *do_clean(struct plugin *p); +static void do_clean(void *cb_arg); static struct command_result *ignore(struct command *timer, const char *buf, @@ -17,19 +17,20 @@ static struct command_result *ignore(struct command *timer, void *arg) { struct plugin *p = arg; - cleantimer = plugin_timer(p, time_from_sec(cycle_seconds), do_clean); + cleantimer = plugin_timer(p, time_from_sec(cycle_seconds), do_clean, p); return timer_complete(p); } -static struct command_result *do_clean(struct plugin *p) +static void do_clean(void *cb_arg) { + struct plugin *p = cb_arg; /* FIXME: delexpiredinvoice should be in our plugin too! */ struct out_req *req = jsonrpc_request_start(p, NULL, "delexpiredinvoice", ignore, ignore, p); json_add_u64(req->js, "maxexpirytime", time_now().ts.tv_sec - expired_by); - return send_outreq(p, req); + send_outreq(p, req); } static struct command_result *json_autocleaninvoice(struct command *cmd, @@ -53,7 +54,8 @@ static struct command_result *json_autocleaninvoice(struct command *cmd, return command_success_str(cmd, "Autoclean timer disabled"); } tal_free(cleantimer); - cleantimer = plugin_timer(cmd->plugin, time_from_sec(cycle_seconds), do_clean); + cleantimer = plugin_timer(cmd->plugin, time_from_sec(cycle_seconds), + do_clean, cmd->plugin); return command_success_str(cmd, tal_fmt(cmd, "Autocleaning %"PRIu64 @@ -68,7 +70,7 @@ static void init(struct plugin *p, if (cycle_seconds) { plugin_log(p, LOG_INFORM, "autocleaning every %"PRIu64" seconds", cycle_seconds); cleantimer = plugin_timer(p, time_from_sec(cycle_seconds), - do_clean); + do_clean, p); } else plugin_log(p, LOG_DBG, "autocleaning not active"); } diff --git a/plugins/libplugin.c b/plugins/libplugin.c index 993e458931c8..b3baefafc5b6 100644 --- a/plugins/libplugin.c +++ b/plugins/libplugin.c @@ -25,7 +25,8 @@ bool deprecated_apis; struct plugin_timer { struct timer timer; - struct command_result *(*cb)(struct plugin *p); + void (*cb)(void *cb_arg); + void *cb_arg; }; struct rpc_conn { @@ -820,7 +821,7 @@ static void call_plugin_timer(struct plugin *p, struct timer *timer) p->in_timer++; /* Free this if they don't. */ tal_steal(tmpctx, t); - t->cb(p); + t->cb(t->cb_arg); } static void destroy_plugin_timer(struct plugin_timer *timer, struct plugin *p) @@ -829,10 +830,12 @@ static void destroy_plugin_timer(struct plugin_timer *timer, struct plugin *p) } struct plugin_timer *plugin_timer(struct plugin *p, struct timerel t, - struct command_result *(*cb)(struct plugin *p)) + void (*cb)(void *cb_arg), + void *cb_arg) { struct plugin_timer *timer = tal(NULL, struct plugin_timer); timer->cb = cb; + timer->cb_arg = cb_arg; timer_init(&timer->timer); timer_addrel(&p->timers, &timer->timer, t); tal_add_destructor2(timer, destroy_plugin_timer, p); diff --git a/plugins/libplugin.h b/plugins/libplugin.h index c47163ddd587..6c4343f9dc2c 100644 --- a/plugins/libplugin.h +++ b/plugins/libplugin.h @@ -212,7 +212,8 @@ struct command_result *timer_complete(struct plugin *p); */ struct plugin_timer *plugin_timer(struct plugin *p, struct timerel t, - struct command_result *(*cb)(struct plugin *p)); + void (*cb)(void *cb_arg), + void *cb_arg); /* Log something */ void plugin_log(struct plugin *p, enum log_level l, const char *fmt, ...) PRINTF_FMT(3, 4); From f5999fd447cd594838936117900e2e7480d621db Mon Sep 17 00:00:00 2001 From: darosior Date: Thu, 2 Jan 2020 21:04:03 +0100 Subject: [PATCH 03/21] plugins/bcli: a new plugin for gathering Bitcoin data Most is taken from lightningd/bitcoind and adapted. This currently exposes 5 commands: - `getchaininfo`, currently called at startup to check the network and whether we are on IBD. - `getrawblockbyheight`, which basically does the `getblockhash` + `getblock` trick. - `getfeerate` - `sendrawtransaction` - `getutxout`, used to gather infos about an output and currently used by `getfilteredblock` in `lightningd/bitcoind`. --- Makefile | 2 +- common/jsonrpc_errors.h | 3 + plugins/Makefile | 11 +- plugins/bcli.c | 755 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 767 insertions(+), 4 deletions(-) create mode 100644 plugins/bcli.c diff --git a/Makefile b/Makefile index 749e6ddabc62..7f49817de936 100644 --- a/Makefile +++ b/Makefile @@ -503,7 +503,7 @@ PKGLIBEXEC_PROGRAMS = \ lightningd/lightning_hsmd \ lightningd/lightning_onchaind \ lightningd/lightning_openingd -PLUGINS=plugins/pay plugins/autoclean plugins/fundchannel +PLUGINS=plugins/pay plugins/autoclean plugins/fundchannel plugins/bcli install-program: installdirs $(BIN_PROGRAMS) $(PKGLIBEXEC_PROGRAMS) $(PLUGINS) @$(NORMAL_INSTALL) diff --git a/common/jsonrpc_errors.h b/common/jsonrpc_errors.h index 791cdf3fccac..0e4f206f8fa5 100644 --- a/common/jsonrpc_errors.h +++ b/common/jsonrpc_errors.h @@ -51,6 +51,9 @@ static const errcode_t FUNDING_UNKNOWN_PEER = 306; static const errcode_t CONNECT_NO_KNOWN_ADDRESS = 400; static const errcode_t CONNECT_ALL_ADDRESSES_FAILED = 401; +/* bitcoin-cli plugin errors */ +#define BCLI_ERROR 400 + /* Errors from `invoice` command */ static const errcode_t INVOICE_LABEL_ALREADY_EXISTS = 900; static const errcode_t INVOICE_PREIMAGE_ALREADY_EXISTS = 901; diff --git a/plugins/Makefile b/plugins/Makefile index fae47b922cbc..441438262878 100644 --- a/plugins/Makefile +++ b/plugins/Makefile @@ -7,6 +7,9 @@ PLUGIN_AUTOCLEAN_OBJS := $(PLUGIN_AUTOCLEAN_SRC:.c=.o) PLUGIN_FUNDCHANNEL_SRC := plugins/fundchannel.c PLUGIN_FUNDCHANNEL_OBJS := $(PLUGIN_FUNDCHANNEL_SRC:.c=.o) +PLUGIN_BCLI_SRC := plugins/bcli.c +PLUGIN_BCLI_OBJS := $(PLUGIN_BCLI_SRC:.c=.o) + PLUGIN_LIB_SRC := plugins/libplugin.c PLUGIN_LIB_HEADER := plugins/libplugin.h PLUGIN_LIB_OBJS := $(PLUGIN_LIB_SRC:.c=.o) @@ -51,11 +54,13 @@ plugins/autoclean: bitcoin/chainparams.o $(PLUGIN_AUTOCLEAN_OBJS) $(PLUGIN_LIB_O plugins/fundchannel: common/addr.o $(PLUGIN_FUNDCHANNEL_OBJS) $(PLUGIN_LIB_OBJS) $(PLUGIN_COMMON_OBJS) $(JSMN_OBJS) $(CCAN_OBJS) -$(PLUGIN_PAY_OBJS) $(PLUGIN_AUTOCLEAN_OBJS) $(PLUGIN_FUNDCHANNEL_OBJS) $(PLUGIN_LIB_OBJS): $(PLUGIN_LIB_HEADER) +plugins/bcli: bitcoin/chainparams.o $(PLUGIN_BCLI_OBJS) $(PLUGIN_LIB_OBJS) $(PLUGIN_COMMON_OBJS) $(JSMN_OBJS) $(CCAN_OBJS) + +$(PLUGIN_PAY_OBJS) $(PLUGIN_AUTOCLEAN_OBJS) $(PLUGIN_FUNDCHANNEL_OBJS) $(PLUGIN_BCLI_OBJS) $(PLUGIN_LIB_OBJS): $(PLUGIN_LIB_HEADER) # Make sure these depend on everything. -ALL_PROGRAMS += plugins/pay plugins/autoclean plugins/fundchannel -ALL_OBJS += $(PLUGIN_PAY_OBJS) $(PLUGIN_AUTOCLEAN_OBJS) $(PLUGIN_FUNDCHANNEL_OBJS) $(PLUGIN_LIB_OBJS) +ALL_PROGRAMS += plugins/pay plugins/autoclean plugins/fundchannel plugins/bcli +ALL_OBJS += $(PLUGIN_PAY_OBJS) $(PLUGIN_AUTOCLEAN_OBJS) $(PLUGIN_FUNDCHANNEL_OBJS) $(PLUGIN_BCLI_OBJS) $(PLUGIN_LIB_OBJS) check-source: $(PLUGIN_PAY_SRC:%=check-src-include-order/%) $(PLUGIN_AUTOCLEAN_SRC:%=check-src-include-order/%) $(PLUGIN_FUNDCHANNEL_SRC:%=check-src-include-order/%) check-source-bolt: $(PLUGIN_PAY_SRC:%=bolt-check/%) $(PLUGIN_AUTOCLEAN_SRC:%=bolt-check/%) $(PLUGIN_FUNDCHANNEL_SRC:%=bolt-check/%) diff --git a/plugins/bcli.c b/plugins/bcli.c new file mode 100644 index 000000000000..34019035f465 --- /dev/null +++ b/plugins/bcli.c @@ -0,0 +1,755 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Bitcoind's web server has a default of 4 threads, with queue depth 16. + * It will *fail* rather than queue beyond that, so we must not stress it! + * + * This is how many request for each priority level we have. + */ +#define BITCOIND_MAX_PARALLEL 4 + +enum bitcoind_prio { + BITCOIND_LOW_PRIO, + BITCOIND_HIGH_PRIO +}; +#define BITCOIND_NUM_PRIO (BITCOIND_HIGH_PRIO+1) + +struct bitcoind { + /* eg. "bitcoin-cli" */ + char *cli; + + /* -datadir arg for bitcoin-cli. */ + char *datadir; + + /* Is bitcoind synced? If not, we retry. */ + bool synced; + + /* How many high/low prio requests are we running (it's ratelimited) */ + size_t num_requests[BITCOIND_NUM_PRIO]; + + /* Pending requests (high and low prio). */ + struct list_head pending[BITCOIND_NUM_PRIO]; + + /* If non-zero, time we first hit a bitcoind error. */ + unsigned int error_count; + struct timemono first_error_time; + + /* How long to keep trying to contact bitcoind + * before fatally exiting. */ + u64 retry_timeout; + + /* Passthrough parameters for bitcoin-cli */ + char *rpcuser, *rpcpass, *rpcconnect, *rpcport; +}; + +static struct bitcoind *bitcoind; + +struct bitcoin_cli { + struct list_node list; + int fd; + int *exitstatus; + pid_t pid; + const char **args; + struct timeabs start; + enum bitcoind_prio prio; + char *output; + size_t output_bytes; + size_t new_output; + struct command_result *(*process)(struct bitcoin_cli *); + struct command *cmd; + /* Used to stash content between multiple calls */ + void *stash; +}; + +/* Add the n'th arg to *args, incrementing n and keeping args of size n+1 */ +static void add_arg(const char ***args, const char *arg) +{ + tal_arr_expand(args, arg); +} + +static const char **gather_args(const tal_t *ctx, const char *cmd, const char **cmd_args) +{ + const char **args = tal_arr(ctx, const char *, 1); + + args[0] = bitcoind->cli ? bitcoind->cli : chainparams->cli; + if (chainparams->cli_args) + add_arg(&args, chainparams->cli_args); + if (bitcoind->datadir) + add_arg(&args, tal_fmt(args, "-datadir=%s", bitcoind->datadir)); + if (bitcoind->rpcconnect) + add_arg(&args, + tal_fmt(args, "-rpcconnect=%s", bitcoind->rpcconnect)); + if (bitcoind->rpcport) + add_arg(&args, + tal_fmt(args, "-rpcport=%s", bitcoind->rpcport)); + if (bitcoind->rpcuser) + add_arg(&args, tal_fmt(args, "-rpcuser=%s", bitcoind->rpcuser)); + if (bitcoind->rpcpass) + add_arg(&args, + tal_fmt(args, "-rpcpassword=%s", bitcoind->rpcpass)); + + add_arg(&args, cmd); + for (size_t i = 0; i < tal_count(cmd_args); i++) + add_arg(&args, cmd_args[i]); + add_arg(&args, NULL); + + return args; +} + +static struct io_plan *read_more(struct io_conn *conn, struct bitcoin_cli *bcli) +{ + bcli->output_bytes += bcli->new_output; + if (bcli->output_bytes == tal_count(bcli->output)) + tal_resize(&bcli->output, bcli->output_bytes * 2); + return io_read_partial(conn, bcli->output + bcli->output_bytes, + tal_count(bcli->output) - bcli->output_bytes, + &bcli->new_output, read_more, bcli); +} + +static struct io_plan *output_init(struct io_conn *conn, struct bitcoin_cli *bcli) +{ + bcli->output_bytes = bcli->new_output = 0; + bcli->output = tal_arr(bcli, char, 100); + return read_more(conn, bcli); +} + +static void next_bcli(enum bitcoind_prio prio); + +/* For printing: simple string of args (no secrets!) */ +static char *args_string(const tal_t *ctx, const char **args) +{ + size_t i; + char *ret = tal_strdup(ctx, args[0]); + + for (i = 1; args[i]; i++) { + ret = tal_strcat(ctx, take(ret), " "); + if (strstarts(args[i], "-rpcpassword")) { + ret = tal_strcat(ctx, take(ret), "-rpcpassword=..."); + } else if (strstarts(args[i], "-rpcuser")) { + ret = tal_strcat(ctx, take(ret), "-rpcuser=..."); + } else { + ret = tal_strcat(ctx, take(ret), args[i]); + } + } + return ret; +} + +static char *bcli_args(struct bitcoin_cli *bcli) +{ + return args_string(bcli, bcli->args); +} + +static void retry_bcli(void *cb_arg) +{ + struct bitcoin_cli *bcli = cb_arg; + list_add_tail(&bitcoind->pending[bcli->prio], &bcli->list); + next_bcli(bcli->prio); +} + +/* We allow 60 seconds of spurious errors, eg. reorg. */ +static void bcli_failure(struct bitcoin_cli *bcli, + int exitstatus) +{ + struct timerel t; + + if (!bitcoind->error_count) + bitcoind->first_error_time = time_mono(); + + t = timemono_between(time_mono(), bitcoind->first_error_time); + if (time_greater(t, time_from_sec(bitcoind->retry_timeout))) + plugin_err(bcli->cmd->plugin, + "%s exited %u (after %u other errors) '%.*s'; " + "we have been retrying command for " + "--bitcoin-retry-timeout=%"PRIu64" seconds; " + "bitcoind setup or our --bitcoin-* configs broken?", + bcli_args(bcli), + exitstatus, + bitcoind->error_count, + (int)bcli->output_bytes, + bcli->output, + bitcoind->retry_timeout); + + plugin_log(bcli->cmd->plugin, LOG_UNUSUAL, "%s exited with status %u", + bcli_args(bcli), exitstatus); + bitcoind->error_count++; + + /* Retry in 1 second (not a leak!) */ + plugin_timer(bcli->cmd->plugin, time_from_sec(1), retry_bcli, bcli); +} + +static void bcli_finished(struct io_conn *conn UNUSED, struct bitcoin_cli *bcli) +{ + int ret, status; + struct command_result *res; + enum bitcoind_prio prio = bcli->prio; + u64 msec = time_to_msec(time_between(time_now(), bcli->start)); + + /* If it took over 10 seconds, that's rather strange. */ + if (msec > 10000) + plugin_log(bcli->cmd->plugin, LOG_UNUSUAL, + "bitcoin-cli: finished %s (%"PRIu64" ms)", + bcli_args(bcli), msec); + + assert(bitcoind->num_requests[prio] > 0); + + /* FIXME: If we waited for SIGCHILD, this could never hang! */ + while ((ret = waitpid(bcli->pid, &status, 0)) < 0 && errno == EINTR); + if (ret != bcli->pid) + plugin_err(bcli->cmd->plugin, "%s %s", bcli_args(bcli), + ret == 0 ? "not exited?" : strerror(errno)); + + if (!WIFEXITED(status)) + plugin_err(bcli->cmd->plugin, "%s died with signal %i", + bcli_args(bcli), + WTERMSIG(status)); + + /* Implicit nonzero_exit_ok == false */ + if (!bcli->exitstatus) { + if (WEXITSTATUS(status) != 0) { + bcli_failure(bcli, WEXITSTATUS(status)); + bitcoind->num_requests[prio]--; + goto done; + } + } else + *bcli->exitstatus = WEXITSTATUS(status); + + if (WEXITSTATUS(status) == 0) + bitcoind->error_count = 0; + + bitcoind->num_requests[bcli->prio]--; + + res = bcli->process(bcli); + if (!res) + bcli_failure(bcli, WEXITSTATUS(status)); + else + tal_free(bcli); + +done: + next_bcli(prio); +} + +static void next_bcli(enum bitcoind_prio prio) +{ + struct bitcoin_cli *bcli; + struct io_conn *conn; + + if (bitcoind->num_requests[prio] >= BITCOIND_MAX_PARALLEL) + return; + + bcli = list_pop(&bitcoind->pending[prio], struct bitcoin_cli, list); + if (!bcli) + return; + + bcli->pid = pipecmdarr(NULL, &bcli->fd, &bcli->fd, + cast_const2(char **, bcli->args)); + if (bcli->pid < 0) + plugin_err(bcli->cmd->plugin, "%s exec failed: %s", + bcli->args[0], strerror(errno)); + + bcli->start = time_now(); + + bitcoind->num_requests[prio]++; + + conn = io_new_conn(bcli, bcli->fd, output_init, bcli); + io_set_finish(conn, bcli_finished, bcli); +} + +/* If ctx is non-NULL, and is freed before we return, we don't call process(). + * process returns false() if it's a spurious error, and we should retry. */ +static void +start_bitcoin_cli(const tal_t *ctx, + struct command *cmd, + struct command_result *(*process)(struct bitcoin_cli *), + bool nonzero_exit_ok, + enum bitcoind_prio prio, + char *method, const char **method_args, + void *stash) +{ + struct bitcoin_cli *bcli = tal(bitcoind, struct bitcoin_cli); + + bcli->process = process; + bcli->cmd = cmd; + bcli->prio = prio; + + if (nonzero_exit_ok) + bcli->exitstatus = tal(bcli, int); + else + bcli->exitstatus = NULL; + + bcli->args = gather_args(bcli, method, method_args); + bcli->stash = stash; + + list_add_tail(&bitcoind->pending[bcli->prio], &bcli->list); + next_bcli(bcli->prio); +} + +static struct command_result *process_getutxout(struct bitcoin_cli *bcli) +{ + const jsmntok_t *tokens, *valuetok, *scriptpubkeytok, *hextok; + struct json_stream *response; + bool valid; + struct bitcoin_tx_output output; + char *err; + + /* As of at least v0.15.1.0, bitcoind returns "success" but an empty + string on a spent txout. */ + if (*bcli->exitstatus != 0 || bcli->output_bytes == 0) { + response = jsonrpc_stream_success(bcli->cmd); + json_add_null(response, "amount"); + json_add_null(response, "script"); + + return command_finished(bcli->cmd, response); + } + + tokens = json_parse_input(bcli->output, bcli->output, + bcli->output_bytes, &valid); + if (!tokens) { + err = tal_fmt(bcli, "%s: %s response", bcli_args(bcli), + valid ? "partial" : "invalid"); + return command_done_err(bcli->cmd, BCLI_ERROR, err, NULL); + } + + if (tokens[0].type != JSMN_OBJECT) { + err = tal_fmt(bcli, "%s: gave non-object (%.*s)?", + bcli_args(bcli), (int)bcli->output_bytes, + bcli->output); + return command_done_err(bcli->cmd, BCLI_ERROR, err, NULL); + } + + valuetok = json_get_member(bcli->output, tokens, "value"); + if (!valuetok) { + err = tal_fmt(bcli,"%s: had no value member (%.*s)?", + bcli_args(bcli), (int)bcli->output_bytes, + bcli->output); + return command_done_err(bcli->cmd, BCLI_ERROR, err, NULL); + } + + if (!json_to_bitcoin_amount(bcli->output, valuetok, &output.amount.satoshis)) {/* Raw: talking to bitcoind */ + err = tal_fmt(bcli, "%s: had bad value (%.*s)?", + bcli_args(bcli), (int)bcli->output_bytes, + bcli->output); + return command_done_err(bcli->cmd, BCLI_ERROR, err, NULL); + } + + scriptpubkeytok = json_get_member(bcli->output, tokens, "scriptPubKey"); + if (!scriptpubkeytok) { + err = tal_fmt(bcli, "%s: had no scriptPubKey member (%.*s)?", + bcli_args(bcli), (int)bcli->output_bytes, + bcli->output); + return command_done_err(bcli->cmd, BCLI_ERROR, err, NULL); + } + + hextok = json_get_member(bcli->output, scriptpubkeytok, "hex"); + if (!hextok) { + err = tal_fmt(bcli, "%s: had no scriptPubKey->hex member (%.*s)?", + bcli_args(bcli), (int)bcli->output_bytes, + bcli->output); + return command_done_err(bcli->cmd, BCLI_ERROR, err, NULL); + } + + output.script = tal_hexdata(bcli, bcli->output + hextok->start, + hextok->end - hextok->start); + if (!output.script) { + err = tal_fmt(bcli, "%s: scriptPubKey->hex invalid hex (%.*s)?", + bcli_args(bcli), (int)bcli->output_bytes, + bcli->output); + return command_done_err(bcli->cmd, BCLI_ERROR, err, NULL); + } + + response = jsonrpc_stream_success(bcli->cmd); + json_add_amount_sat_only(response, "amount", output.amount); + json_add_string(response, "script", + tal_hexstr(response, output.script, sizeof(output.script))); + + return command_finished(bcli->cmd, response); +} + +static struct command_result *process_getblockchaininfo(struct bitcoin_cli *bcli) +{ + const jsmntok_t *tokens, *chaintok, *headerstok, *blockstok, *ibdtok; + struct json_stream *response; + bool valid, ibd; + u32 headers, blocks; + char *err; + + tokens = json_parse_input(bcli, bcli->output, bcli->output_bytes, + &valid); + if (!tokens) { + err = tal_fmt(bcli->cmd, "%s: %s response (%.*s)", + bcli_args(bcli), valid ? "partial" : "invalid", + (int)bcli->output_bytes, bcli->output); + return command_done_err(bcli->cmd, BCLI_ERROR, err, NULL); + } + + if (tokens[0].type != JSMN_OBJECT) { + err = tal_fmt(bcli->cmd, "%s: gave non-object (%.*s)", + bcli_args(bcli), (int)bcli->output_bytes, + bcli->output); + return command_done_err(bcli->cmd, BCLI_ERROR, err, NULL); + } + + chaintok = json_get_member(bcli->output, tokens, "chain"); + if (!chaintok) { + err = tal_fmt(bcli->cmd, "%s: bad 'chain' field (%.*s)", + bcli_args(bcli), (int)bcli->output_bytes, + bcli->output); + return command_done_err(bcli->cmd, BCLI_ERROR, err, NULL); + } + + headerstok = json_get_member(bcli->output, tokens, "headers"); + if (!headerstok || !json_to_number(bcli->output, headerstok, &headers)) { + err = tal_fmt(bcli->cmd, "%s: bad 'headers' field (%.*s)", + bcli_args(bcli), (int)bcli->output_bytes, + bcli->output); + return command_done_err(bcli->cmd, BCLI_ERROR, err, NULL); + } + + blockstok = json_get_member(bcli->output, tokens, "blocks"); + if (!blockstok || !json_to_number(bcli->output, blockstok, &blocks)) { + err = tal_fmt(bcli->cmd, "%s: bad 'blocks' field (%.*s)", + bcli_args(bcli), (int)bcli->output_bytes, + bcli->output); + return command_done_err(bcli->cmd, BCLI_ERROR, err, NULL); + } + + ibdtok = json_get_member(bcli->output, tokens, "initialblockdownload"); + if (!ibdtok || !json_to_bool(bcli->output, ibdtok, &ibd)) { + err = tal_fmt(bcli->cmd, "%s: bad 'initialblockdownload' field (%.*s)", + bcli_args(bcli), (int)bcli->output_bytes, + bcli->output); + return command_done_err(bcli->cmd, BCLI_ERROR, err, NULL); + } + + response = jsonrpc_stream_success(bcli->cmd); + json_add_string(response, "chain", + json_strdup(response, bcli->output, chaintok)); + json_add_u32(response, "headercount", headers); + json_add_u32(response, "blockcount", blocks); + json_add_bool(response, "ibd", ibd); + + return command_finished(bcli->cmd, response); +} + +static struct command_result *process_estimatefee(struct bitcoin_cli *bcli) +{ + const jsmntok_t *tokens, *feeratetok = NULL; + struct json_stream *response; + bool valid; + u64 feerate; + char *err; + + if (*bcli->exitstatus != 0) + goto end; + + tokens = json_parse_input(bcli->output, bcli->output, + (int)bcli->output_bytes, &valid); + if (!tokens) { + err = tal_fmt(bcli->cmd, "%s: %s response (%.*s)", + bcli_args(bcli), valid ? "partial" : "invalid", + (int)bcli->output_bytes, bcli->output); + return command_done_err(bcli->cmd, BCLI_ERROR, err, NULL); + } + + if (tokens[0].type != JSMN_OBJECT) { + err = tal_fmt(bcli->cmd, "%s: gave non-object (%.*s)", + bcli_args(bcli), (int)bcli->output_bytes, + bcli->output); + return command_done_err(bcli->cmd, BCLI_ERROR, err, NULL); + } + + feeratetok = json_get_member(bcli->output, tokens, "feerate"); + if (feeratetok && + !json_to_bitcoin_amount(bcli->output, feeratetok, &feerate)) { + err = tal_fmt(bcli->cmd, "%s: bad 'feerate' field (%.*s)", + bcli_args(bcli), (int)bcli->output_bytes, + bcli->output); + return command_done_err(bcli->cmd, BCLI_ERROR, err, NULL); + } + +end: + response = jsonrpc_stream_success(bcli->cmd); + if (feeratetok) + json_add_u64(response, "feerate", feerate); + else + json_add_null(response, "feerate"); + + return command_finished(bcli->cmd, response); +} + +static struct command_result *process_sendrawtransaction(struct bitcoin_cli *bcli) +{ + struct json_stream *response; + + plugin_log(bcli->cmd->plugin, LOG_DBG, "sendrawtx exit %i (%s)", + *bcli->exitstatus, bcli_args(bcli)); + + response = jsonrpc_stream_success(bcli->cmd); + json_add_bool(response, "success", *bcli->exitstatus == 0); + json_add_string(response, "errmsg", + bcli->exitstatus ? tal_strndup(bcli, bcli->output, + bcli->output_bytes-1) : ""); + + return command_finished(bcli->cmd, response); +} + +struct getrawblock_stash { + const char *block_hash; + u32 block_height; + const char *block_hex; +}; + +static struct command_result *process_getrawblock(struct bitcoin_cli *bcli) +{ + struct json_stream *response; + struct getrawblock_stash *stash = bcli->stash; + + /* -1 to strip \n. */ + stash->block_hex = tal_fmt(stash, "%.*s", + (int)bcli->output_bytes-1, bcli->output); + + response = jsonrpc_stream_success(bcli->cmd); + json_add_string(response, "blockhash", stash->block_hash); + json_add_string(response, "block", stash->block_hex); + + return command_finished(bcli->cmd, response); +} + +static struct command_result * +getrawblockbyheight_notfound(struct bitcoin_cli *bcli) +{ + struct json_stream *response; + + response = jsonrpc_stream_success(bcli->cmd); + json_add_null(response, "blockhash"); + json_add_null(response, "block"); + + return command_finished(bcli->cmd, response); +} + +static struct command_result *process_getblockhash(struct bitcoin_cli *bcli) +{ + const char *err, **params; + struct getrawblock_stash *stash = bcli->stash; + + /* If it failed with error 8, give an empty response. */ + if (bcli->exitstatus && *bcli->exitstatus != 0) { + /* Other error means we have to retry. */ + if (*bcli->exitstatus != 8) + return NULL; + return getrawblockbyheight_notfound(bcli); + } + + /* `-1` to strip the newline character. */ + stash->block_hash = tal_strndup(stash, bcli->output, + bcli->output_bytes-1); + if (!stash->block_hash || strlen(stash->block_hash) != 64) { + err = tal_fmt(bcli->cmd, "%s: bad blockhash '%s'", + bcli_args(bcli), stash->block_hash); + return command_done_err(bcli->cmd, BCLI_ERROR, err, NULL); + } + + params = tal_arr(bcli->cmd, const char *, 2); + params[0] = stash->block_hash; + /* Non-verbose: raw block. */ + params[1] = "0"; + start_bitcoin_cli(NULL, bcli->cmd, process_getrawblock, false, + BITCOIND_HIGH_PRIO, "getblock", params, stash); + + return command_still_pending(bcli->cmd); +} + +/* Get a raw block given its height. + * Calls `getblockhash` then `getblock` to retrieve it from bitcoin_cli. + * Will return early with null fields if block isn't known (yet). + */ +static struct command_result *getrawblockbyheight(struct command *cmd, + const char *buf, + const jsmntok_t *toks) +{ + struct getrawblock_stash *stash; + u32 *height; + const char **params; + + /* bitcoin-cli wants a string. */ + if (!param(cmd, buf, toks, + p_req("height", param_number, &height), + NULL)) + return command_param_failed(); + + stash = tal(cmd, struct getrawblock_stash); + stash->block_height = *height; + + params = tal_arr(cmd, const char *, 1); + params[0] = tal_fmt(params, "%u", *height); + start_bitcoin_cli(NULL, cmd, process_getblockhash, true, + BITCOIND_LOW_PRIO, "getblockhash", params, stash); + + return command_still_pending(cmd); +} + +/* Get infos about the block chain. + * Calls `getblockchaininfo` and returns headers count, blocks count, + * the chain id, and whether this is initialblockdownload. + */ +static struct command_result *getchaininfo(struct command *cmd, + const char *buf UNUSED, + const jsmntok_t *toks UNUSED) +{ + if (!param(cmd, buf, toks, NULL)) + return command_param_failed(); + + start_bitcoin_cli(NULL, cmd, process_getblockchaininfo, false, + BITCOIND_HIGH_PRIO, "getblockchaininfo", NULL, NULL); + + return command_still_pending(cmd); +} + +/* Get current feerate. + * Calls `estimatesmartfee` and returns the feerate as btc/k*VBYTE*. + */ +static struct command_result *getfeerate(struct command *cmd, + const char *buf UNUSED, + const jsmntok_t *toks UNUSED) +{ + u32 *blocks; + const char **params = tal_arr(cmd, const char *, 2); + + if (!param(cmd, buf, toks, + p_req("blocks", param_number, &blocks), + p_req("mode", param_string, ¶ms[1]), + NULL)) + return command_param_failed(); + + params[0] = tal_fmt(params, "%u", *blocks); + start_bitcoin_cli(NULL, cmd, process_estimatefee, true, + BITCOIND_LOW_PRIO, "estimatesmartfee", params, NULL); + + return command_still_pending(cmd); +} + +/* Send a transaction to the Bitcoin network. + * Calls `sendrawtransaction` using the first parameter as the raw tx. + */ +static struct command_result *sendrawtransaction(struct command *cmd, + const char *buf, + const jsmntok_t *toks) +{ + const char **params = tal_arr(cmd, const char *, 1); + + /* bitcoin-cli wants strings. */ + if (!param(cmd, buf, toks, + p_req("tx", param_string, ¶ms[0]), + NULL)) + return command_param_failed(); + + start_bitcoin_cli(NULL, cmd, process_sendrawtransaction, true, + BITCOIND_HIGH_PRIO, "sendrawtransaction", params, NULL); + + return command_still_pending(cmd); +} + +static struct command_result *getutxout(struct command *cmd, + const char *buf, + const jsmntok_t *toks) +{ + const char **params = tal_arr(cmd, const char *, 2); + + /* bitcoin-cli wants strings. */ + if (!param(cmd, buf, toks, + p_req("txid", param_string, ¶ms[0]), + p_req("vout", param_string, ¶ms[1]), + NULL)) + return command_param_failed(); + + start_bitcoin_cli(NULL, cmd, process_getutxout, true, + BITCOIND_HIGH_PRIO, "gettxout", params, NULL); + + return command_still_pending(cmd); +} + +/* Initialize the global context when handshake is done. */ +static void init(struct plugin *p UNUSED, const char *buffer UNUSED, + const jsmntok_t *config UNUSED) +{ + bitcoind = tal(NULL, struct bitcoind); + + bitcoind->cli = NULL; + bitcoind->datadir = NULL; + for (size_t i = 0; i < BITCOIND_NUM_PRIO; i++) { + bitcoind->num_requests[i] = 0; + list_head_init(&bitcoind->pending[i]); + } + bitcoind->error_count = 0; + bitcoind->retry_timeout = 60; + bitcoind->rpcuser = NULL; + bitcoind->rpcpass = NULL; + bitcoind->rpcconnect = NULL; + bitcoind->rpcport = NULL; +} + +static const struct plugin_command commands[] = { + { + "getrawblockbyheight", + "bitcoin", + "Get the bitcoin block at a given height", + "", + getrawblockbyheight + }, + { + "getchaininfo", + "bitcoin", + "Get the chain id, the header count, the block count," + " and whether this is IBD.", + "", + getchaininfo + }, + { + "getfeerate", + "bitcoin", + "Get the Bitcoin feerate in btc/kilo-vbyte.", + "", + getfeerate + }, + { + "sendrawtransaction", + "bitcoin", + "Send a raw transaction to the Bitcoin network.", + "", + sendrawtransaction + }, + { + "getutxout", + "bitcoin", + "Get informations about an output, identified by a {txid} an a {vout}", + "", + getutxout + }, +}; + +int main(int argc, char *argv[]) +{ + setup_locale(); + + /* FIXME: handle bitcoind options at init */ + plugin_main(argv, init, PLUGIN_STATIC, commands, ARRAY_SIZE(commands), + NULL, 0, NULL, 0, NULL); +} From 44d408cc87beb854bdaf60c72108bb4492cfb34b Mon Sep 17 00:00:00 2001 From: darosior Date: Sat, 4 Jan 2020 20:56:58 +0100 Subject: [PATCH 04/21] plugins/bcli: wait for bitcoind to be warmed up at init This is also taken and adapted from lightningd/bitcoind. The call to 'getblockchaininfo' is replaced by 'echo' as we don't make use of the result and the former can sometimes be slow (e.g. on IBD). --- plugins/bcli.c | 70 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 69 insertions(+), 1 deletion(-) diff --git a/plugins/bcli.c b/plugins/bcli.c index 34019035f465..604bbf65c347 100644 --- a/plugins/bcli.c +++ b/plugins/bcli.c @@ -686,8 +686,72 @@ static struct command_result *getutxout(struct command *cmd, return command_still_pending(cmd); } +static void bitcoind_failure(struct plugin *p, const char *error_message) +{ + const char **cmd = gather_args(bitcoind, "echo", NULL); + const char *err = + tal_fmt(bitcoind, "\n%s\n\n" + "Make sure you have bitcoind running and that bitcoin-cli" + " is able to connect to bitcoind.\n\n" + "You can verify that your Bitcoin Core installation is" + " ready for use by running:\n\n" + " $ %s 'hello world'\n", error_message, + args_string(cmd, cmd)); + plugin_err(p, err); +} + +static void wait_for_bitcoind(struct plugin *p) +{ + int from, status, ret; + pid_t child; + const char **cmd = gather_args(bitcoind, "echo", NULL); + bool printed = false; + + for (;;) { + child = pipecmdarr(NULL, &from, &from, cast_const2(char **,cmd)); + if (child < 0) { + if (errno == ENOENT) + bitcoind_failure(p, "bitcoin-cli not found. Is bitcoin-cli " + "(part of Bitcoin Core) available in your PATH?"); + plugin_err(p, "%s exec failed: %s", cmd[0], strerror(errno)); + } + + char *output = grab_fd(cmd, from); + + while ((ret = waitpid(child, &status, 0)) < 0 && errno == EINTR); + if (ret != child) + bitcoind_failure(p, tal_fmt(bitcoind, "Waiting for %s: %s", + cmd[0], strerror(errno))); + if (!WIFEXITED(status)) + bitcoind_failure(p, tal_fmt(bitcoind, "Death of %s: signal %i", + cmd[0], WTERMSIG(status))); + + if (WEXITSTATUS(status) == 0) + break; + + /* bitcoin/src/rpc/protocol.h: + * RPC_IN_WARMUP = -28, //!< Client still warming up + */ + if (WEXITSTATUS(status) != 28) { + if (WEXITSTATUS(status) == 1) + bitcoind_failure(p, "Could not connect to bitcoind using" + " bitcoin-cli. Is bitcoind running?"); + bitcoind_failure(p, tal_fmt(bitcoind, "%s exited with code %i: %s", + cmd[0], WEXITSTATUS(status), output)); + } + + if (!printed) { + plugin_log(p, LOG_UNUSUAL, + "Waiting for bitcoind to warm up..."); + printed = true; + } + sleep(1); + } + tal_free(cmd); +} + /* Initialize the global context when handshake is done. */ -static void init(struct plugin *p UNUSED, const char *buffer UNUSED, +static void init(struct plugin *p, const char *buffer UNUSED, const jsmntok_t *config UNUSED) { bitcoind = tal(NULL, struct bitcoind); @@ -704,6 +768,10 @@ static void init(struct plugin *p UNUSED, const char *buffer UNUSED, bitcoind->rpcpass = NULL; bitcoind->rpcconnect = NULL; bitcoind->rpcport = NULL; + + wait_for_bitcoind(p); + plugin_log(p, LOG_INFORM, + "bitcoin-cli initialized and connected to bitcoind."); } static const struct plugin_command commands[] = { From 8261ff1fc32ad49fb8ece260b4e9f0c8b3fbfb87 Mon Sep 17 00:00:00 2001 From: darosior Date: Sat, 4 Jan 2020 21:49:06 +0100 Subject: [PATCH 05/21] plugins/bcli: register Bitcoin-related options --- plugins/bcli.c | 65 ++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 47 insertions(+), 18 deletions(-) diff --git a/plugins/bcli.c b/plugins/bcli.c index 604bbf65c347..e0133eee5a76 100644 --- a/plugins/bcli.c +++ b/plugins/bcli.c @@ -750,25 +750,9 @@ static void wait_for_bitcoind(struct plugin *p) tal_free(cmd); } -/* Initialize the global context when handshake is done. */ static void init(struct plugin *p, const char *buffer UNUSED, const jsmntok_t *config UNUSED) { - bitcoind = tal(NULL, struct bitcoind); - - bitcoind->cli = NULL; - bitcoind->datadir = NULL; - for (size_t i = 0; i < BITCOIND_NUM_PRIO; i++) { - bitcoind->num_requests[i] = 0; - list_head_init(&bitcoind->pending[i]); - } - bitcoind->error_count = 0; - bitcoind->retry_timeout = 60; - bitcoind->rpcuser = NULL; - bitcoind->rpcpass = NULL; - bitcoind->rpcconnect = NULL; - bitcoind->rpcport = NULL; - wait_for_bitcoind(p); plugin_log(p, LOG_INFORM, "bitcoin-cli initialized and connected to bitcoind."); @@ -817,7 +801,52 @@ int main(int argc, char *argv[]) { setup_locale(); - /* FIXME: handle bitcoind options at init */ + /* Initialize our global context object here to handle startup options. */ + bitcoind = tal(NULL, struct bitcoind); + + bitcoind->cli = NULL; + bitcoind->datadir = NULL; + for (size_t i = 0; i < BITCOIND_NUM_PRIO; i++) { + bitcoind->num_requests[i] = 0; + list_head_init(&bitcoind->pending[i]); + } + bitcoind->error_count = 0; + bitcoind->retry_timeout = 60; + bitcoind->rpcuser = NULL; + bitcoind->rpcpass = NULL; + bitcoind->rpcconnect = NULL; + bitcoind->rpcport = NULL; + plugin_main(argv, init, PLUGIN_STATIC, commands, ARRAY_SIZE(commands), - NULL, 0, NULL, 0, NULL); + NULL, 0, NULL, 0, + plugin_option("bitcoin-datadir", + "string", + "-datadir arg for bitcoin-cli", + charp_option, &bitcoind->datadir), + plugin_option("bitcoin-cli", + "string", + "bitcoin-cli pathname", + charp_option, &bitcoind->cli), + plugin_option("bitcoin-rpcuser", + "string", + "bitcoind RPC username", + charp_option, &bitcoind->rpcuser), + plugin_option("bitcoin-rpcpassword", + "string", + "bitcoind RPC password", + charp_option, &bitcoind->rpcpass), + plugin_option("bitcoin-rpcconnect", + "string", + "bitcoind RPC host to connect to", + charp_option, &bitcoind->rpcconnect), + plugin_option("bitcoin-rpcport", + "string", + "bitcoind RPC host's port", + charp_option, &bitcoind->rpcport), + plugin_option("bitcoin-retry-timeout", + "string", + "how long to keep retrying to contact bitcoind" + " before fatally exiting", + u64_option, &bitcoind->retry_timeout), + NULL); } From 983d2e3ca4b49ea6e42620e24a6e872cf80054ea Mon Sep 17 00:00:00 2001 From: darosior Date: Tue, 7 Jan 2020 18:03:10 +0100 Subject: [PATCH 06/21] plugins/libplugin: don't crash if 'lightning-rpc' doesnt exist (yet) We are going to initialize a plugin before its creation, so log as UNUSUAL instead. Also, `pay` and `fundchannel` inits are using rpc_delve(), so we need to io_new_conn() (which sets the socket as non blocking) after calling the plugin's init. --- plugins/libplugin.c | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/plugins/libplugin.c b/plugins/libplugin.c index b3baefafc5b6..afa02a670d90 100644 --- a/plugins/libplugin.c +++ b/plugins/libplugin.c @@ -719,6 +719,7 @@ static struct command_result *handle_init(struct command *cmd, char *dir, *network; struct json_out *param_obj; struct plugin *p = cmd->plugin; + bool with_rpc = true; configtok = json_delve(buf, params, ".configuration"); @@ -742,16 +743,19 @@ static struct command_result *handle_init(struct command *cmd, addr.sun_path[rpctok->end - rpctok->start] = '\0'; addr.sun_family = AF_UNIX; - if (connect(p->rpc_conn->fd, (struct sockaddr *)&addr, sizeof(addr)) != 0) - plugin_err(p, "Connecting to '%.*s': %s", + if (connect(p->rpc_conn->fd, (struct sockaddr *)&addr, sizeof(addr)) != 0) { + with_rpc = false; + plugin_log(p, LOG_UNUSUAL, "Could not connect to '%.*s': %s", rpctok->end - rpctok->start, buf + rpctok->start, strerror(errno)); + } else { + param_obj = json_out_obj(NULL, "config", "allow-deprecated-apis"); + deprecated_apis = streq(rpc_delve(tmpctx, p, "listconfigs", + take(param_obj), + ".allow-deprecated-apis"), + "true"); + } - param_obj = json_out_obj(NULL, "config", "allow-deprecated-apis"); - deprecated_apis = streq(rpc_delve(tmpctx, p, "listconfigs", - take(param_obj), - ".allow-deprecated-apis"), - "true"); opttok = json_get_member(buf, params, "options"); json_for_each_obj(i, t, opttok) { char *opt = json_strdup(NULL, buf, t); @@ -772,7 +776,8 @@ static struct command_result *handle_init(struct command *cmd, if (p->init) p->init(p, buf, configtok); - io_new_conn(p, p->rpc_conn->fd, rpc_conn_init, p); + if (with_rpc) + io_new_conn(p, p->rpc_conn->fd, rpc_conn_init, p); return command_success_str(cmd, NULL); } From fbc92b0014e6da31c7dab6ec17e2aa620a38a9c0 Mon Sep 17 00:00:00 2001 From: darosior Date: Tue, 7 Jan 2020 11:59:18 +0100 Subject: [PATCH 07/21] chaintopology: check bitcoin plugin commands at startup Exit early if we won't be able to fully communicate with our Bitcoin backend. --- lightningd/bitcoind.c | 23 +++++++++++++++++++++++ lightningd/bitcoind.h | 2 ++ lightningd/chaintopology.c | 3 +++ lightningd/plugin.c | 16 +++++++++------- lightningd/plugin.h | 7 +++++++ 5 files changed, 44 insertions(+), 7 deletions(-) diff --git a/lightningd/bitcoind.c b/lightningd/bitcoind.c index b9c0131c83a0..5df163d64a51 100644 --- a/lightningd/bitcoind.c +++ b/lightningd/bitcoind.c @@ -7,10 +7,12 @@ #include "bitcoind.h" #include "lightningd.h" #include "log.h" +#include #include #include #include #include +#include #include #include #include @@ -22,6 +24,7 @@ #include #include #include +#include /* Bitcoind's web server has a default of 4 threads, with queue depth 16. * It will *fail* rather than queue beyond that, so we must not stress it! @@ -30,6 +33,26 @@ */ #define BITCOIND_MAX_PARALLEL 4 +/* The names of the request we can make to our Bitcoin backend. */ +static const char *methods[] = {"getchaininfo", "getrawblockbyheight", + "sendrawtransaction", "getutxout", + "getfeerate"}; + +void bitcoind_check_commands(struct bitcoind *bitcoind) +{ + size_t i; + struct plugin *p; + + for (i = 0; i < ARRAY_SIZE(methods); i++) { + p = find_plugin_for_command(bitcoind->ld, methods[i]); + if (p == NULL) { + fatal("Could not access the plugin for %s, is a " + "Bitcoin plugin (by default plugins/bcli) " + "registered ?", methods[i]); + } + } +} + /* Add the n'th arg to *args, incrementing n and keeping args of size n+1 */ static void add_arg(const char ***args, const char *arg) { diff --git a/lightningd/bitcoind.h b/lightningd/bitcoind.h index 7812f6e065ac..5b1e22966ea3 100644 --- a/lightningd/bitcoind.h +++ b/lightningd/bitcoind.h @@ -182,4 +182,6 @@ void bitcoind_gettxout(struct bitcoind *bitcoind, void bitcoind_getclientversion(struct bitcoind *bitcoind); +void bitcoind_check_commands(struct bitcoind *bitcoind); + #endif /* LIGHTNING_LIGHTNINGD_BITCOIND_H */ diff --git a/lightningd/chaintopology.c b/lightningd/chaintopology.c index 9b5c9ff7a8e8..fa7837ae354b 100644 --- a/lightningd/chaintopology.c +++ b/lightningd/chaintopology.c @@ -955,6 +955,9 @@ void setup_topology(struct chain_topology *topo, topo->min_blockheight = min_blockheight; topo->max_blockheight = max_blockheight; + /* This waits for bitcoind. */ + bitcoind_check_commands(topo->bitcoind); + /* Make sure bitcoind is started, and ready */ wait_for_bitcoind(topo->bitcoind); diff --git a/lightningd/plugin.c b/lightningd/plugin.c index 1dc9f2d8c383..fbfd6caada36 100644 --- a/lightningd/plugin.c +++ b/lightningd/plugin.c @@ -611,21 +611,21 @@ static void plugin_rpcmethod_cb(const char *buffer, command_raw_complete(cmd, response); } -static struct plugin *find_plugin_for_command(struct command *cmd) +struct plugin *find_plugin_for_command(struct lightningd *ld, + const char *cmd_name) { - struct plugins *plugins = cmd->ld->plugins; + struct plugins *plugins = ld->plugins; struct plugin *plugin; /* Find the plugin that registered this RPC call */ list_for_each(&plugins->plugins, plugin, list) { for (size_t i=0; imethods); i++) { - if (streq(cmd->json_cmd->name, plugin->methods[i])) + if (streq(cmd_name, plugin->methods[i])) return plugin; } } - /* This should never happen, it'd mean that a plugin didn't - * cleanup after dying */ - abort(); + + return NULL; } static struct command_result *plugin_rpcmethod_dispatch(struct command *cmd, @@ -641,7 +641,9 @@ static struct command_result *plugin_rpcmethod_dispatch(struct command *cmd, if (cmd->mode == CMD_CHECK) return command_param_failed(); - plugin = find_plugin_for_command(cmd); + plugin = find_plugin_for_command(cmd->ld, cmd->json_cmd->name); + if (!plugin) + fatal("No plugin for %s ?", cmd->json_cmd->name); /* Find ID again (We've parsed them before, this should not fail!) */ idtok = json_get_member(buffer, toks, "id"); diff --git a/lightningd/plugin.h b/lightningd/plugin.h index aa89f902bc63..97ba82e3bcf0 100644 --- a/lightningd/plugin.h +++ b/lightningd/plugin.h @@ -180,6 +180,13 @@ bool plugin_remove(struct plugins *plugins, const char *name); */ void plugin_kill(struct plugin *plugin, char *fmt, ...) PRINTF_FMT(2,3); +/** + * Returns the plugin which registers the command with name {cmd_name} + */ +struct plugin *find_plugin_for_command(struct lightningd *ld, + const char *cmd_name); + + /** * Send the configure message to all plugins. * From a3eb36bb3282fd32693e42909f914cb5d8f6a11e Mon Sep 17 00:00:00 2001 From: darosior Date: Tue, 7 Jan 2020 16:09:59 +0100 Subject: [PATCH 08/21] bitcoind: allow commands to be registered by different plugins An strmap is convenient to get a plugin given a method. Populate it while checking commands. --- lightningd/bitcoind.c | 2 ++ lightningd/bitcoind.h | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/lightningd/bitcoind.c b/lightningd/bitcoind.c index 5df163d64a51..713bd6ab815b 100644 --- a/lightningd/bitcoind.c +++ b/lightningd/bitcoind.c @@ -1011,6 +1011,7 @@ static bool process_getblockchaininfo(struct bitcoin_cli *bcli) static void destroy_bitcoind(struct bitcoind *bitcoind) { + strmap_clear(&bitcoind->pluginsmap); /* Suppresses the callbacks from bcli_finished as we free conns. */ bitcoind->shutdown = true; } @@ -1146,6 +1147,7 @@ struct bitcoind *new_bitcoind(const tal_t *ctx, { struct bitcoind *bitcoind = tal(ctx, struct bitcoind); + strmap_init(&bitcoind->pluginsmap); bitcoind->cli = NULL; bitcoind->datadir = NULL; bitcoind->ld = ld; diff --git a/lightningd/bitcoind.h b/lightningd/bitcoind.h index 5b1e22966ea3..f39af0b7c9d4 100644 --- a/lightningd/bitcoind.h +++ b/lightningd/bitcoind.h @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -61,6 +62,10 @@ struct bitcoind { char *rpcuser, *rpcpass, *rpcconnect, *rpcport; struct list_head pending_getfilteredblock; + + /* Map each method to a plugin, so we can have multiple plugins + * handling different functionalities. */ + STRMAP(struct plugin *) pluginsmap; }; /* A single outpoint in a filtered block */ From c21d902d61e417e6021e578a42a0fd13f4835bf2 Mon Sep 17 00:00:00 2001 From: darosior Date: Tue, 7 Jan 2020 18:08:03 +0100 Subject: [PATCH 09/21] bitcoind: initialize Bitcoin-backend plugin early We need our Bitcoin backend to be initialized, but the plugins have not yet been started at this point. --- lightningd/bitcoind.c | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/lightningd/bitcoind.c b/lightningd/bitcoind.c index 713bd6ab815b..6bb8f548749b 100644 --- a/lightningd/bitcoind.c +++ b/lightningd/bitcoind.c @@ -38,6 +38,43 @@ static const char *methods[] = {"getchaininfo", "getrawblockbyheight", "sendrawtransaction", "getutxout", "getfeerate"}; +static void plugin_config_cb(const char *buffer, + const jsmntok_t *toks, + const jsmntok_t *idtok, + struct plugin *plugin) +{ + plugin->plugin_state = CONFIGURED; + io_break(plugin); +} + +static void config_plugin(struct plugin *plugin) +{ + struct jsonrpc_request *req; + + req = jsonrpc_request_start(plugin, "init", plugin->log, + plugin_config_cb, plugin); + plugin_populate_init_request(plugin, req); + jsonrpc_request_end(req); + plugin_request_send(plugin, req); + io_loop_with_timers(plugin->plugins->ld); +} + +static void wait_plugin(struct bitcoind *bitcoind, const char *method, + struct plugin *p) +{ + /* We need our Bitcoin backend to be initialized, but the plugins have + * not yet been started at this point. + * So send `init` to each plugin which registered for a Bitcoin method + * and wait for its response, which we take as an ACK that it is + * operational (i.e. bcli will wait for `bitcoind` to be warmed up + * before responding to `init`). + * Note that lightningd/plugin will not send `init` to an already + * configured plugin. */ + if (p->plugin_state != CONFIGURED) + config_plugin(p); + strmap_add(&bitcoind->pluginsmap, method, p); +} + void bitcoind_check_commands(struct bitcoind *bitcoind) { size_t i; @@ -50,6 +87,7 @@ void bitcoind_check_commands(struct bitcoind *bitcoind) "Bitcoin plugin (by default plugins/bcli) " "registered ?", methods[i]); } + wait_plugin(bitcoind, methods[i], p); } } From ed2a7147e9fa0a5772fd06bc719ece955e198b98 Mon Sep 17 00:00:00 2001 From: darosior Date: Wed, 8 Jan 2020 13:07:51 +0100 Subject: [PATCH 10/21] chaintopology: dont check bitcoin-core version at startup The Bitcoin backend is generalized through the Bitcoin plugin and this was specific to core. --- lightningd/bitcoind.c | 60 -------------------------------------- lightningd/chaintopology.c | 2 -- 2 files changed, 62 deletions(-) diff --git a/lightningd/bitcoind.c b/lightningd/bitcoind.c index 6bb8f548749b..72c09c4966ce 100644 --- a/lightningd/bitcoind.c +++ b/lightningd/bitcoind.c @@ -890,66 +890,6 @@ void bitcoind_getfilteredblock_(struct bitcoind *bitcoind, u32 height, bitcoind_getblockhash(bitcoind, height, process_getfilteredblock_step1, call); } -static bool extract_numeric_version(struct bitcoin_cli *bcli, - const char *output, size_t output_bytes, - u64 *version) -{ - const jsmntok_t *tokens, *versiontok; - bool valid; - - tokens = json_parse_input(output, output, output_bytes, &valid); - if (!tokens) - fatal("%s: %s response", - bcli_args(tmpctx, bcli), - valid ? "partial" : "invalid"); - - if (tokens[0].type != JSMN_OBJECT) { - log_unusual(bcli->bitcoind->log, - "%s: gave non-object (%.*s)?", - bcli_args(tmpctx, bcli), - (int)output_bytes, output); - return false; - } - - versiontok = json_get_member(output, tokens, "version"); - if (!versiontok) - return false; - - return json_to_u64(output, versiontok, version); -} - -static bool process_getclientversion(struct bitcoin_cli *bcli) -{ - u64 version; - u64 min_version = chainparams->cli_min_supported_version; - - if (!extract_numeric_version(bcli, bcli->output, - bcli->output_bytes, - &version)) { - fatal("%s: Unable to getclientversion (%.*s)", - bcli_args(tmpctx, bcli), - (int)bcli->output_bytes, - bcli->output); - } - - if (version < min_version) - fatal("Unsupported bitcoind version? bitcoind version: %"PRIu64"," - " supported minimum version: %"PRIu64"", - version, min_version); - - return true; -} - -void bitcoind_getclientversion(struct bitcoind *bitcoind) -{ - /* `getnetworkinfo` was added in v0.14.0. The older version would - * return non-zero exitstatus. */ - start_bitcoin_cli(bitcoind, NULL, process_getclientversion, false, - BITCOIND_HIGH_PRIO, - NULL, NULL, - "getnetworkinfo", NULL); -} - /* Mutual recursion */ static bool process_getblockchaininfo(struct bitcoin_cli *bcli); diff --git a/lightningd/chaintopology.c b/lightningd/chaintopology.c index fa7837ae354b..a9bdb48541ce 100644 --- a/lightningd/chaintopology.c +++ b/lightningd/chaintopology.c @@ -961,8 +961,6 @@ void setup_topology(struct chain_topology *topo, /* Make sure bitcoind is started, and ready */ wait_for_bitcoind(topo->bitcoind); - bitcoind_getclientversion(topo->bitcoind); - bitcoind_getblockcount(topo->bitcoind, get_init_blockhash, topo); tal_add_destructor(topo, destroy_chain_topology); From 3515cffea4d2c06464cb738088b1a15203844781 Mon Sep 17 00:00:00 2001 From: darosior Date: Wed, 8 Jan 2020 17:53:31 +0100 Subject: [PATCH 11/21] lightningd/chaintopology: use plugin backend to setup topology This adds `getchaininfo` and `getrawblockbyheight` handling lightningd-side, and use them in setup_topology(). We then remove legacy bitcoind_getblockcount() (we already get the count in `getchaininfo`), bitcoind_getblockchaininfo() (it was only used in setup_topology()), and wait_for_bitcoind() (this was specific to bitcoin-core and we assume our Bitcoin backend to be functional if the plugin responds to `init`). --- lightningd/bitcoind.c | 442 +++++++++++++++++-------------------- lightningd/bitcoind.h | 50 +++-- lightningd/chaintopology.c | 134 +++++++---- 3 files changed, 323 insertions(+), 303 deletions(-) diff --git a/lightningd/bitcoind.c b/lightningd/bitcoind.c index 72c09c4966ce..f1bf7b7fec3a 100644 --- a/lightningd/bitcoind.c +++ b/lightningd/bitcoind.c @@ -556,34 +556,211 @@ void bitcoind_getrawblock_(struct bitcoind *bitcoind, "getblock", hex, "false", NULL); } -static bool process_getblockcount(struct bitcoin_cli *bcli) +/* Our Bitcoin backend plugin gave us a bad response. We can't recover. */ +static void bitcoin_plugin_error(struct bitcoind *bitcoind, const char *buf, + const jsmntok_t *toks, const char *method, + const char *reason) { - u32 blockcount; - char *p, *end; + struct plugin *p = strmap_get(&bitcoind->pluginsmap, method); + fatal("%s error: bad response to %s (%s), response was %.*s", + p->cmd, method, reason, + toks->end - toks->start, buf + toks->start); +} + +/* `getrawblockbyheight` + * + * If no block were found at that height, will set each field to `null`. + * Plugin response: + * { + * "blockhash": "", + * "block": "rawblock" + * } + */ + +struct getrawblockbyheight_call { + struct bitcoind *bitcoind; void (*cb)(struct bitcoind *bitcoind, - u32 blockcount, - void *arg) = bcli->cb; + struct bitcoin_blkid *blkid, + struct bitcoin_block *block, + void *); + void *cb_arg; +}; + +static void +getrawblockbyheight_callback(const char *buf, const jsmntok_t *toks, + const jsmntok_t *idtok, + struct getrawblockbyheight_call *call) +{ + const jsmntok_t *resulttok, *blockhashtok, *blocktok; + const char *block_str, *blockhash_str; + struct bitcoin_blkid blkid; + struct bitcoin_block *blk; - p = tal_strndup(bcli, bcli->output, bcli->output_bytes); - blockcount = strtol(p, &end, 10); - if (end == p || *end != '\n') - fatal("%s: gave non-numeric blockcount %s", - bcli_args(tmpctx, bcli), p); + resulttok = json_get_member(buf, toks, "result"); + if (!resulttok) + bitcoin_plugin_error(call->bitcoind, buf, toks, + "getrawblockbyheight", + "bad 'result' field"); + + blockhashtok = json_get_member(buf, resulttok, "blockhash"); + if (!blockhashtok) + bitcoin_plugin_error(call->bitcoind, buf, toks, + "getrawblockbyheight", + "bad 'blockhash' field"); + /* If block hash is `null`, this means not found! Call the callback + * with NULL values. */ + if (json_tok_is_null(buf, blockhashtok)) { + db_begin_transaction(call->bitcoind->ld->wallet->db); + call->cb(call->bitcoind, NULL, NULL, call->cb_arg); + db_commit_transaction(call->bitcoind->ld->wallet->db); + goto clean; + } + blockhash_str = json_strdup(tmpctx, buf, blockhashtok); + if (!bitcoin_blkid_from_hex(blockhash_str, strlen(blockhash_str), &blkid)) + bitcoin_plugin_error(call->bitcoind, buf, toks, + "getrawblockbyheight", + "bad block hash"); + + blocktok = json_get_member(buf, resulttok, "block"); + if (!blocktok) + bitcoin_plugin_error(call->bitcoind, buf, toks, + "getrawblockbyheight", + "bad 'block' field"); + block_str = json_strdup(tmpctx, buf, blocktok); + blk = bitcoin_block_from_hex(tmpctx, chainparams, block_str, + strlen(block_str)); + if (!blk) + bitcoin_plugin_error(call->bitcoind, buf, toks, + "getrawblockbyheight", + "bad block"); - cb(bcli->bitcoind, blockcount, bcli->cb_arg); - return true; + db_begin_transaction(call->bitcoind->ld->wallet->db); + call->cb(call->bitcoind, &blkid, blk, call->cb_arg); + db_commit_transaction(call->bitcoind->ld->wallet->db); + +clean: + tal_free(call); } -void bitcoind_getblockcount_(struct bitcoind *bitcoind, - void (*cb)(struct bitcoind *bitcoind, - u32 blockcount, - void *arg), - void *arg) +void bitcoind_getrawblockbyheight_(struct bitcoind *bitcoind, + u32 height, + void (*cb)(struct bitcoind *bitcoind, + struct bitcoin_blkid *blkid, + struct bitcoin_block *blk, + void *arg), + void *cb_arg) { - start_bitcoin_cli(bitcoind, NULL, process_getblockcount, false, - BITCOIND_HIGH_PRIO, - cb, arg, - "getblockcount", NULL); + struct jsonrpc_request *req; + struct getrawblockbyheight_call *call = tal(NULL, + struct getrawblockbyheight_call); + + call->bitcoind = bitcoind; + call->cb = cb; + call->cb_arg = cb_arg; + + req = jsonrpc_request_start(bitcoind, "getrawblockbyheight", + bitcoind->log, getrawblockbyheight_callback, + /* Freed in cb. */ + notleak(call)); + json_add_num(req->stream, "height", height); + jsonrpc_request_end(req); + plugin_request_send(strmap_get(&bitcoind->pluginsmap, + "getrawblockbyheight"), req); +} + +/* `getchaininfo` + * + * Called at startup to check the network we are operating on, and to check + * if the Bitcoin backend is synced to the network tip. This also allows to + * get the current block count. + * { + * "chain": "", + * "headercount": , + * "blockcount": , + * "ibd": + * } + */ + +struct getchaininfo_call { + struct bitcoind *bitcoind; + /* Should we log verbosely? */ + bool first_call; + void (*cb)(struct bitcoind *bitcoind, + const char *chain, + u32 headercount, + u32 blockcount, + const bool ibd, + const bool first_call, + void *); + void *cb_arg; +}; + +static void getchaininfo_callback(const char *buf, const jsmntok_t *toks, + const jsmntok_t *idtok, + struct getchaininfo_call *call) +{ + const jsmntok_t *resulttok, *chaintok, *headerstok, *blktok, *ibdtok; + u32 headers = 0; + u32 blocks = 0; + bool ibd = false; + + resulttok = json_get_member(buf, toks, "result"); + if (!resulttok) + bitcoin_plugin_error(call->bitcoind, buf, toks, "getchaininfo", + "bad 'result' field"); + + chaintok = json_get_member(buf, resulttok, "chain"); + if (!chaintok) + bitcoin_plugin_error(call->bitcoind, buf, toks, "getchaininfo", + "bad 'chain' field"); + + headerstok = json_get_member(buf, resulttok, "headercount"); + if (!headerstok || !json_to_number(buf, headerstok, &headers)) + bitcoin_plugin_error(call->bitcoind, buf, toks, "getchaininfo", + "bad 'headercount' field"); + + blktok = json_get_member(buf, resulttok, "blockcount"); + if (!blktok || !json_to_number(buf, blktok, &blocks)) + bitcoin_plugin_error(call->bitcoind, buf, toks, "getchaininfo", + "bad 'blockcount' field"); + + ibdtok = json_get_member(buf, resulttok, "ibd"); + if (!ibdtok || !json_to_bool(buf, ibdtok, &ibd)) + bitcoin_plugin_error(call->bitcoind, buf, toks, "getchaininfo", + "bad 'ibd' field"); + + db_begin_transaction(call->bitcoind->ld->wallet->db); + call->cb(call->bitcoind, json_strdup(tmpctx, buf, chaintok), headers, + blocks, ibd, call->first_call, call->cb_arg); + db_commit_transaction(call->bitcoind->ld->wallet->db); + + tal_free(call); +} + +void bitcoind_getchaininfo_(struct bitcoind *bitcoind, + const bool first_call, + void (*cb)(struct bitcoind *bitcoind, + const char *chain, + u32 headercount, + u32 blockcount, + const bool ibd, + const bool first_call, + void *), + void *cb_arg) +{ + struct jsonrpc_request *req; + struct getchaininfo_call *call = tal(bitcoind, struct getchaininfo_call); + + call->bitcoind = bitcoind; + call->cb = cb; + call->cb_arg = cb_arg; + call->first_call = first_call; + + req = jsonrpc_request_start(bitcoind, "getchaininfo", bitcoind->log, + getchaininfo_callback, call); + jsonrpc_request_end(req); + plugin_request_send(strmap_get(&bitcoind->pluginsmap, "getchaininfo"), + req); } struct get_output { @@ -890,103 +1067,6 @@ void bitcoind_getfilteredblock_(struct bitcoind *bitcoind, u32 height, bitcoind_getblockhash(bitcoind, height, process_getfilteredblock_step1, call); } -/* Mutual recursion */ -static bool process_getblockchaininfo(struct bitcoin_cli *bcli); - -static void retry_getblockchaininfo(struct bitcoind *bitcoind) -{ - assert(!bitcoind->synced); - start_bitcoin_cli(bitcoind, NULL, - process_getblockchaininfo, - false, BITCOIND_LOW_PRIO, NULL, NULL, - "getblockchaininfo", NULL); -} - -/* Given JSON object from getblockchaininfo, are we synced? Poll if not. */ -static void is_bitcoind_synced_yet(struct bitcoind *bitcoind, - const char *output, size_t output_len, - const jsmntok_t *obj, - bool initial) -{ - const jsmntok_t *t; - unsigned int headers, blocks; - bool ibd; - - t = json_get_member(output, obj, "headers"); - if (!t || !json_to_number(output, t, &headers)) - fatal("Invalid 'headers' field in getblockchaininfo '%.*s'", - (int)output_len, output); - - t = json_get_member(output, obj, "blocks"); - if (!t || !json_to_number(output, t, &blocks)) - fatal("Invalid 'blocks' field in getblockchaininfo '%.*s'", - (int)output_len, output); - - t = json_get_member(output, obj, "initialblockdownload"); - if (!t || !json_to_bool(output, t, &ibd)) - fatal("Invalid 'initialblockdownload' field in getblockchaininfo '%.*s'", - (int)output_len, output); - - if (ibd) { - if (initial) - log_unusual(bitcoind->log, - "Waiting for initial block download" - " (this can take a while!)"); - else - log_debug(bitcoind->log, - "Still waiting for initial block download"); - } else if (headers != blocks) { - if (initial) - log_unusual(bitcoind->log, - "Waiting for bitcoind to catch up" - " (%u blocks of %u)", - blocks, headers); - else - log_debug(bitcoind->log, - "Waiting for bitcoind to catch up" - " (%u blocks of %u)", - blocks, headers); - } else { - if (!initial) - log_info(bitcoind->log, "Bitcoind now synced."); - bitcoind->synced = true; - return; - } - - bitcoind->synced = false; - notleak(new_reltimer(bitcoind->ld->timers, bitcoind, - /* Be 4x more aggressive in this case. */ - time_divide(time_from_sec(bitcoind->ld->topology - ->poll_seconds), 4), - retry_getblockchaininfo, bitcoind)); -} - -static bool process_getblockchaininfo(struct bitcoin_cli *bcli) -{ - const jsmntok_t *tokens; - bool valid; - - tokens = json_parse_input(bcli, bcli->output, bcli->output_bytes, - &valid); - if (!tokens) - fatal("%s: %s response (%.*s)", - bcli_args(tmpctx, bcli), - valid ? "partial" : "invalid", - (int)bcli->output_bytes, bcli->output); - - if (tokens[0].type != JSMN_OBJECT) { - log_unusual(bcli->bitcoind->log, - "%s: gave non-object (%.*s)?", - bcli_args(tmpctx, bcli), - (int)bcli->output_bytes, bcli->output); - return false; - } - - is_bitcoind_synced_yet(bcli->bitcoind, bcli->output, bcli->output_bytes, - tokens, false); - return true; -} - static void destroy_bitcoind(struct bitcoind *bitcoind) { strmap_clear(&bitcoind->pluginsmap); @@ -994,131 +1074,6 @@ static void destroy_bitcoind(struct bitcoind *bitcoind) bitcoind->shutdown = true; } -static const char **cmdarr(const tal_t *ctx, const struct bitcoind *bitcoind, - const char *cmd, ...) -{ - va_list ap; - const char **args; - - va_start(ap, cmd); - args = gather_args(bitcoind, ctx, cmd, ap); - va_end(ap); - return args; -} - -static void fatal_bitcoind_failure(struct bitcoind *bitcoind, const char *error_message) -{ - const char **cmd = cmdarr(bitcoind, bitcoind, "echo", NULL); - - fprintf(stderr, "%s\n\n", error_message); - fprintf(stderr, "Make sure you have bitcoind running and that bitcoin-cli is able to connect to bitcoind.\n\n"); - fprintf(stderr, "You can verify that your Bitcoin Core installation is ready for use by running:\n\n"); - fprintf(stderr, " $ %s 'hello world'\n", args_string(cmd, cmd)); - tal_free(cmd); - exit(1); -} - -/* This function is used to check "chain" field from - * bitcoin-cli "getblockchaininfo" API */ -static char* check_blockchain_from_bitcoincli(const tal_t *ctx, - struct bitcoind *bitcoind, - char* output, const char **cmd) -{ - size_t output_bytes; - const jsmntok_t *tokens, *valuetok; - bool valid; - - if (!output) - return tal_fmt(ctx, "Reading from %s failed: %s", - args_string(tmpctx, cmd), strerror(errno)); - - output_bytes = tal_count(output); - - tokens = json_parse_input(cmd, output, output_bytes, - &valid); - - if (!tokens) - return tal_fmt(ctx, "%s: %s response", - args_string(tmpctx, cmd), - valid ? "partial" : "invalid"); - - if (tokens[0].type != JSMN_OBJECT) - return tal_fmt(ctx, "%s: gave non-object (%.*s)?", - args_string(tmpctx, cmd), - (int)output_bytes, output); - - valuetok = json_get_member(output, tokens, "chain"); - if (!valuetok) - return tal_fmt(ctx, "%s: had no chain member (%.*s)?", - args_string(tmpctx, cmd), - (int)output_bytes, output); - - if (!json_tok_streq(output, valuetok, - chainparams->bip70_name)) - return tal_fmt(ctx, "Error blockchain for bitcoin-cli?" - " Should be: %s", - chainparams->bip70_name); - - is_bitcoind_synced_yet(bitcoind, output, output_bytes, tokens, true); - return NULL; -} - -void wait_for_bitcoind(struct bitcoind *bitcoind) -{ - int from, status, ret; - pid_t child; - const char **cmd = cmdarr(bitcoind, bitcoind, "getblockchaininfo", NULL); - bool printed = false; - char *errstr; - - for (;;) { - child = pipecmdarr(NULL, &from, &from, cast_const2(char **,cmd)); - if (child < 0) { - if (errno == ENOENT) { - fatal_bitcoind_failure(bitcoind, "bitcoin-cli not found. Is bitcoin-cli (part of Bitcoin Core) available in your PATH?"); - } - fatal("%s exec failed: %s", cmd[0], strerror(errno)); - } - - char *output = grab_fd(cmd, from); - - while ((ret = waitpid(child, &status, 0)) < 0 && errno == EINTR); - if (ret != child) - fatal("Waiting for %s: %s", cmd[0], strerror(errno)); - if (!WIFEXITED(status)) - fatal("Death of %s: signal %i", - cmd[0], WTERMSIG(status)); - - if (WEXITSTATUS(status) == 0) { - /* If succeeded, so check answer it gave. */ - errstr = check_blockchain_from_bitcoincli(tmpctx, bitcoind, output, cmd); - if (errstr) - fatal("%s", errstr); - - break; - } - - /* bitcoin/src/rpc/protocol.h: - * RPC_IN_WARMUP = -28, //!< Client still warming up - */ - if (WEXITSTATUS(status) != 28) { - if (WEXITSTATUS(status) == 1) { - fatal_bitcoind_failure(bitcoind, "Could not connect to bitcoind using bitcoin-cli. Is bitcoind running?"); - } - fatal("%s exited with code %i: %s", - cmd[0], WEXITSTATUS(status), output); - } - - if (!printed) { - log_unusual(bitcoind->log, - "Waiting for bitcoind to warm up..."); - printed = true; - } - sleep(1); - } - tal_free(cmd); -} - struct bitcoind *new_bitcoind(const tal_t *ctx, struct lightningd *ld, struct log *log) @@ -1143,6 +1098,7 @@ struct bitcoind *new_bitcoind(const tal_t *ctx, bitcoind->rpcconnect = NULL; bitcoind->rpcport = NULL; tal_add_destructor(bitcoind, destroy_bitcoind); + bitcoind->synced = false; return bitcoind; } diff --git a/lightningd/bitcoind.h b/lightningd/bitcoind.h index f39af0b7c9d4..7ccb8141be35 100644 --- a/lightningd/bitcoind.h +++ b/lightningd/bitcoind.h @@ -89,8 +89,6 @@ struct bitcoind *new_bitcoind(const tal_t *ctx, struct lightningd *ld, struct log *log); -void wait_for_bitcoind(struct bitcoind *bitcoind); - void bitcoind_estimate_fees_(struct bitcoind *bitcoind, const u32 blocks[], const char *estmode[], size_t num_estimates, @@ -120,20 +118,6 @@ void bitcoind_sendrawtx_(struct bitcoind *bitcoind, int, const char *), \ (arg)) -void bitcoind_getblockcount_(struct bitcoind *bitcoind, - void (*cb)(struct bitcoind *bitcoind, - u32 blockcount, - void *arg), - void *arg); - -#define bitcoind_getblockcount(bitcoind_, cb, arg) \ - bitcoind_getblockcount_((bitcoind_), \ - typesafe_cb_preargs(void, void *, \ - (cb), (arg), \ - struct bitcoind *, \ - u32 blockcount), \ - (arg)) - /* blkid is NULL if call fails. */ void bitcoind_getblockhash_(struct bitcoind *bitcoind, u32 height, @@ -178,6 +162,40 @@ void bitcoind_getrawblock_(struct bitcoind *bitcoind, struct bitcoin_block *), \ (arg)) +void bitcoind_getchaininfo_(struct bitcoind *bitcoind, + const bool first_call, + void (*cb)(struct bitcoind *bitcoind, + const char *chain, + u32 headercount, + u32 blockcount, + const bool ibd, + const bool first_call, void *), + void *cb_arg); +#define bitcoind_getchaininfo(bitcoind_, first_call_, cb, arg) \ + bitcoind_getchaininfo_((bitcoind_), (first_call_), \ + typesafe_cb_preargs(void, void *, \ + (cb), (arg), \ + struct bitcoind *, \ + const char *, u32, u32, \ + const bool, const bool), \ + (arg)) + +void bitcoind_getrawblockbyheight_(struct bitcoind *bitcoind, + u32 height, + void (*cb)(struct bitcoind *bitcoind, + struct bitcoin_blkid *blkid, + struct bitcoin_block *blk, + void *arg), + void *arg); +#define bitcoind_getrawblockbyheight(bitcoind_, height_, cb, arg) \ + bitcoind_getrawblockbyheight_((bitcoind_), (height_), \ + typesafe_cb_preargs(void, void *, \ + (cb), (arg), \ + struct bitcoind *, \ + struct bitcoin_blkid *, \ + struct bitcoin_block *),\ + (arg)) + void bitcoind_gettxout(struct bitcoind *bitcoind, const struct bitcoin_txid *txid, const u32 outnum, void (*cb)(struct bitcoind *bitcoind, diff --git a/lightningd/chaintopology.c b/lightningd/chaintopology.c index a9bdb48541ce..2ca6b1e65ecc 100644 --- a/lightningd/chaintopology.c +++ b/lightningd/chaintopology.c @@ -28,7 +28,7 @@ /* Mutual recursion via timer. */ static void try_extend_tip(struct chain_topology *topo); -/* get_init_blockhash sets topo->root, start_fee_estimate clears +/* init_topo sets topo->root, start_fee_estimate clears * feerate_uninitialized (even if unsuccessful) */ static void maybe_completed_init(struct chain_topology *topo) { @@ -780,6 +780,7 @@ static void try_extend_tip(struct chain_topology *topo) } static void init_topo(struct bitcoind *bitcoind UNUSED, + struct bitcoin_blkid *blkid UNUSED, struct bitcoin_block *blk, struct chain_topology *topo) { @@ -795,45 +796,6 @@ static void init_topo(struct bitcoind *bitcoind UNUSED, maybe_completed_init(topo); } -static void get_init_block(struct bitcoind *bitcoind, - const struct bitcoin_blkid *blkid, - struct chain_topology *topo) -{ - bitcoind_getrawblock(bitcoind, blkid, init_topo, topo); -} - -static void get_init_blockhash(struct bitcoind *bitcoind, u32 blockcount, - struct chain_topology *topo) -{ - /* If bitcoind's current blockheight is below the requested - * height, refuse. You can always explicitly request a reindex from - * that block number using --rescan=. */ - if (blockcount < topo->max_blockheight) { - /* UINT32_MAX == no blocks in database */ - if (topo->max_blockheight == UINT32_MAX) { - /* Relative rescan, but we didn't know the blockheight */ - /* Protect against underflow in subtraction. - * Possible in regtest mode. */ - if (blockcount < bitcoind->ld->config.rescan) - topo->max_blockheight = 0; - else - topo->max_blockheight = blockcount - bitcoind->ld->config.rescan; - } else - fatal("bitcoind has gone backwards from %u to %u blocks!", - topo->max_blockheight, blockcount); - } - - /* Rollback to the given blockheight, so we start track - * correctly again */ - wallet_blocks_rollback(topo->ld->wallet, topo->max_blockheight); - /* This may have unconfirmed txs: reconfirm as we add blocks. */ - watch_for_utxo_reconfirmation(topo, topo->ld->wallet); - - /* Get up to speed with topology. */ - bitcoind_getblockhash(bitcoind, topo->max_blockheight, - get_init_block, topo); -} - u32 get_block_height(const struct chain_topology *topo) { return topo->tip->height; @@ -942,9 +904,95 @@ struct chain_topology *new_topology(struct lightningd *ld, struct log *log) topo->root = NULL; topo->sync_waiters = tal(topo, struct list_head); list_head_init(topo->sync_waiters); + return topo; } +static void check_blockcount(struct chain_topology *topo, u32 blockcount) +{ + /* If bitcoind's current blockheight is below the requested + * height, refuse. You can always explicitly request a reindex from + * that block number using --rescan=. */ + if (blockcount < topo->max_blockheight) { + /* UINT32_MAX == no blocks in database */ + if (topo->max_blockheight == UINT32_MAX) { + /* Relative rescan, but we didn't know the blockheight */ + /* Protect against underflow in subtraction. + * Possible in regtest mode. */ + if (blockcount < topo->bitcoind->ld->config.rescan) + topo->max_blockheight = 0; + else + topo->max_blockheight = blockcount - topo->bitcoind->ld->config.rescan; + } else + fatal("bitcoind has gone backwards from %u to %u blocks!", + topo->max_blockheight, blockcount); + } + + /* Rollback to the given blockheight, so we start track + * correctly again */ + wallet_blocks_rollback(topo->ld->wallet, topo->max_blockheight); + /* This may have unconfirmed txs: reconfirm as we add blocks. */ + watch_for_utxo_reconfirmation(topo, topo->ld->wallet); +} + +static void retry_check_chain(struct chain_topology *topo); + +static void +check_chain(struct bitcoind *bitcoind, const char *chain, + const u32 headercount, const u32 blockcount, const bool ibd, + const bool first_call, struct chain_topology *topo) +{ + if (!streq(chain, chainparams->bip70_name)) + fatal("Wrong network! Our Bitcoin backend is running on '%s'," + " but we expect '%s'.", chain, chainparams->bip70_name); + + if (first_call) { + /* Has the Bitcoin backend gone backward ? */ + check_blockcount(topo, blockcount); + /* Get up to speed with topology. */ + bitcoind_getrawblockbyheight(topo->bitcoind, topo->max_blockheight, + init_topo, topo); + } + + if (ibd) { + if (first_call) + log_unusual(bitcoind->log, + "Waiting for initial block download (this can take" + " a while!)"); + else + log_debug(bitcoind->log, + "Still waiting for initial block download"); + } else if (headercount != blockcount) { + if (first_call) + log_unusual(bitcoind->log, + "Waiting for bitcoind to catch up" + " (%u blocks of %u)", + blockcount, headercount); + else + log_debug(bitcoind->log, + "Waiting for bitcoind to catch up" + " (%u blocks of %u)", + blockcount, headercount); + } else { + if (!first_call) + log_unusual(bitcoind->log, + "Bitcoin backend now synced."); + bitcoind->synced = true; + return; + } + + notleak(new_reltimer(bitcoind->ld->timers, bitcoind, + /* Be 4x more aggressive in this case. */ + time_divide(time_from_sec(bitcoind->ld->topology + ->poll_seconds), 4), + retry_check_chain, bitcoind->ld->topology)); + } + +static void retry_check_chain(struct chain_topology *topo) +{ + bitcoind_getchaininfo(topo->bitcoind, true, check_chain, topo); +} + void setup_topology(struct chain_topology *topo, struct timers *timers, u32 min_blockheight, u32 max_blockheight) @@ -958,10 +1006,8 @@ void setup_topology(struct chain_topology *topo, /* This waits for bitcoind. */ bitcoind_check_commands(topo->bitcoind); - /* Make sure bitcoind is started, and ready */ - wait_for_bitcoind(topo->bitcoind); - - bitcoind_getblockcount(topo->bitcoind, get_init_blockhash, topo); + /* Sanity checks, then topology initialization. */ + bitcoind_getchaininfo(topo->bitcoind, true, check_chain, topo); tal_add_destructor(topo, destroy_chain_topology); From 0ff12ce5cab37d6068aac13b63cb1520ca1fe1a3 Mon Sep 17 00:00:00 2001 From: darosior Date: Wed, 8 Jan 2020 18:47:35 +0100 Subject: [PATCH 12/21] lightningd/chaintopology: use plugin for tip polling --- lightningd/chaintopology.c | 33 ++++++++++++++------------------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/lightningd/chaintopology.c b/lightningd/chaintopology.c index 2ca6b1e65ecc..b9bfcc8593be 100644 --- a/lightningd/chaintopology.c +++ b/lightningd/chaintopology.c @@ -742,15 +742,22 @@ static void remove_tip(struct chain_topology *topo) tal_free(b); } -static void have_new_block(struct bitcoind *bitcoind UNUSED, - struct bitcoin_block *blk, - struct chain_topology *topo) +static void get_new_block(struct bitcoind *bitcoind, + struct bitcoin_blkid *blkid, + struct bitcoin_block *blk, + struct chain_topology *topo) { + if (!blkid && !blk) { + /* No such block, we're done. */ + updates_complete(topo); + return; + } + assert(blkid && blk); + /* Annotate all transactions with the chainparams */ - for (size_t i=0; itx); i++) + for (size_t i = 0; i < tal_count(blk->tx); i++) blk->tx[i]->chainparams = chainparams; - /* Unexpected predecessor? Free predecessor, refetch it. */ if (!bitcoin_blkid_eq(&topo->tip->blkid, &blk->hdr.prev_hash)) remove_tip(topo); @@ -761,22 +768,10 @@ static void have_new_block(struct bitcoind *bitcoind UNUSED, try_extend_tip(topo); } -static void get_new_block(struct bitcoind *bitcoind, - const struct bitcoin_blkid *blkid, - struct chain_topology *topo) -{ - if (!blkid) { - /* No such block, we're done. */ - updates_complete(topo); - return; - } - bitcoind_getrawblock(bitcoind, blkid, have_new_block, topo); -} - static void try_extend_tip(struct chain_topology *topo) { - bitcoind_getblockhash(topo->bitcoind, topo->tip->height + 1, - get_new_block, topo); + bitcoind_getrawblockbyheight(topo->bitcoind, topo->tip->height + 1, + get_new_block, topo); } static void init_topo(struct bitcoind *bitcoind UNUSED, From 6aa9233b89b0555955587e085b738f9de34c39cc Mon Sep 17 00:00:00 2001 From: darosior Date: Thu, 9 Jan 2020 12:25:45 +0100 Subject: [PATCH 13/21] lightningd/bitcoind: use the Bitcoin plugin to send transactions This restrains the informations we get about how the sending went to an errmsg as we cant rely on bitcoin-cli specific output nor its exit code. --- lightningd/bitcoind.c | 106 +++++++++++++++----- lightningd/bitcoind.h | 4 +- lightningd/chaintopology.c | 14 +-- lightningd/chaintopology.h | 4 +- lightningd/test/run-invoice-select-inchan.c | 2 +- wallet/test/run-wallet.c | 2 +- wallet/walletrpc.c | 6 +- 7 files changed, 95 insertions(+), 43 deletions(-) diff --git a/lightningd/bitcoind.c b/lightningd/bitcoind.c index f1bf7b7fec3a..662455f5b7f8 100644 --- a/lightningd/bitcoind.c +++ b/lightningd/bitcoind.c @@ -495,33 +495,6 @@ void bitcoind_estimate_fees_(struct bitcoind *bitcoind, do_one_estimatefee(bitcoind, efee); } -static bool process_sendrawtx(struct bitcoin_cli *bcli) -{ - void (*cb)(struct bitcoind *bitcoind, - int, const char *msg, void *) = bcli->cb; - const char *msg = tal_strndup(bcli, bcli->output, - bcli->output_bytes); - - log_debug(bcli->bitcoind->log, "sendrawtx exit %u, gave %s", - *bcli->exitstatus, msg); - - cb(bcli->bitcoind, *bcli->exitstatus, msg, bcli->cb_arg); - return true; -} - -void bitcoind_sendrawtx_(struct bitcoind *bitcoind, - const char *hextx, - void (*cb)(struct bitcoind *bitcoind, - int exitstatus, const char *msg, void *), - void *arg) -{ - log_debug(bitcoind->log, "sendrawtransaction: %s", hextx); - start_bitcoin_cli(bitcoind, NULL, process_sendrawtx, true, - BITCOIND_HIGH_PRIO, - cb, arg, - "sendrawtransaction", hextx, NULL); -} - static bool process_rawblock(struct bitcoin_cli *bcli) { struct bitcoin_block *blk; @@ -567,6 +540,85 @@ static void bitcoin_plugin_error(struct bitcoind *bitcoind, const char *buf, toks->end - toks->start, buf + toks->start); } +/* `sendrawtransaction` + * + * Send a transaction to the Bitcoin backend plugin. If the broadcast was + * not successful on its end, the plugin will populate the `errmsg` with + * the reason. + * + * Plugin response: + * { + * "success": , + * "errmsg": "" + * } + */ + +struct sendrawtx_call { + struct bitcoind *bitcoind; + void (*cb)(struct bitcoind *bitcoind, + bool success, + const char *err_msg, + void *); + void *cb_arg; +}; + +static void sendrawtx_callback(const char *buf, const jsmntok_t *toks, + const jsmntok_t *idtok, + struct sendrawtx_call *call) +{ + const jsmntok_t *resulttok, *successtok, *errtok; + bool success = false; + + resulttok = json_get_member(buf, toks, "result"); + if (!resulttok) + bitcoin_plugin_error(call->bitcoind, buf, toks, + "sendrawtransaction", + "bad 'result' field"); + + successtok = json_get_member(buf, resulttok, "success"); + if (!successtok || !json_to_bool(buf, successtok, &success)) + bitcoin_plugin_error(call->bitcoind, buf, toks, + "sendrawtransaction", + "bad 'success' field"); + + errtok = json_get_member(buf, resulttok, "errmsg"); + if (!success && !errtok) + bitcoin_plugin_error(call->bitcoind, buf, toks, + "sendrawtransaction", + "bad 'errmsg' field"); + + db_begin_transaction(call->bitcoind->ld->wallet->db); + call->cb(call->bitcoind, success, + errtok ? json_strdup(tmpctx, buf, errtok) : NULL, + call->cb_arg); + db_commit_transaction(call->bitcoind->ld->wallet->db); + + tal_free(call); +} + +void bitcoind_sendrawtx_(struct bitcoind *bitcoind, + const char *hextx, + void (*cb)(struct bitcoind *bitcoind, + bool success, const char *err_msg, void *), + void *cb_arg) +{ + struct jsonrpc_request *req; + struct sendrawtx_call *call = tal(bitcoind, struct sendrawtx_call); + + call->bitcoind = bitcoind; + call->cb = cb; + call->cb_arg = cb_arg; + log_debug(bitcoind->log, "sendrawtransaction: %s", hextx); + + req = jsonrpc_request_start(bitcoind, "sendrawtransaction", + bitcoind->log, sendrawtx_callback, + call); + json_add_string(req->stream, "tx", hextx); + jsonrpc_request_end(req); + plugin_request_send(strmap_get(&bitcoind->pluginsmap, + "sendrawtransaction"), req); +} + /* `getrawblockbyheight` * * If no block were found at that height, will set each field to `null`. diff --git a/lightningd/bitcoind.h b/lightningd/bitcoind.h index 7ccb8141be35..fc8719b4c9c1 100644 --- a/lightningd/bitcoind.h +++ b/lightningd/bitcoind.h @@ -107,7 +107,7 @@ void bitcoind_estimate_fees_(struct bitcoind *bitcoind, void bitcoind_sendrawtx_(struct bitcoind *bitcoind, const char *hextx, void (*cb)(struct bitcoind *bitcoind, - int exitstatus, const char *msg, void *), + bool success, const char *msg, void *), void *arg); #define bitcoind_sendrawtx(bitcoind_, hextx, cb, arg) \ @@ -115,7 +115,7 @@ void bitcoind_sendrawtx_(struct bitcoind *bitcoind, typesafe_cb_preargs(void, void *, \ (cb), (arg), \ struct bitcoind *, \ - int, const char *), \ + bool, const char *), \ (arg)) /* blkid is NULL if call fails. */ diff --git a/lightningd/chaintopology.c b/lightningd/chaintopology.c index b9bfcc8593be..1d5cd6a68196 100644 --- a/lightningd/chaintopology.c +++ b/lightningd/chaintopology.c @@ -127,13 +127,13 @@ struct txs_to_broadcast { /* We just sent the last entry in txs[]. Shrink and send the next last. */ static void broadcast_remainder(struct bitcoind *bitcoind, - int exitstatus, const char *msg, + bool success, const char *msg, struct txs_to_broadcast *txs) { /* These are expected. */ if (strstr(msg, "txn-mempool-conflict") || strstr(msg, "transaction already in block chain") - || exitstatus) + || !success) log_debug(bitcoind->log, "Expected error broadcasting tx %s: %s", txs->txs[txs->cursor], msg); @@ -174,7 +174,7 @@ static void rebroadcast_txs(struct chain_topology *topo, struct command *cmd) /* Let this do the dirty work. */ txs->cursor = (size_t)-1; - broadcast_remainder(topo->bitcoind, 0, "", txs); + broadcast_remainder(topo->bitcoind, true, "", txs); } static void destroy_outgoing_tx(struct outgoing_tx *otx) @@ -190,7 +190,7 @@ static void clear_otx_channel(struct channel *channel, struct outgoing_tx *otx) } static void broadcast_done(struct bitcoind *bitcoind, - int exitstatus, const char *msg, + bool success, const char *msg, struct outgoing_tx *otx) { /* Channel gone? Stop. */ @@ -203,7 +203,7 @@ static void broadcast_done(struct bitcoind *bitcoind, tal_del_destructor2(otx->channel, clear_otx_channel, otx); if (otx->failed_or_success) { - otx->failed_or_success(otx->channel, exitstatus, msg); + otx->failed_or_success(otx->channel, success, msg); tal_free(otx); } else { /* For continual rebroadcasting, until channel freed. */ @@ -216,7 +216,7 @@ static void broadcast_done(struct bitcoind *bitcoind, void broadcast_tx(struct chain_topology *topo, struct channel *channel, const struct bitcoin_tx *tx, void (*failed_or_success)(struct channel *channel, - int exitstatus, const char *err)) + bool success, const char *err)) { /* Channel might vanish: topo owns it to start with. */ struct outgoing_tx *otx = tal(topo, struct outgoing_tx); @@ -985,7 +985,7 @@ check_chain(struct bitcoind *bitcoind, const char *chain, static void retry_check_chain(struct chain_topology *topo) { - bitcoind_getchaininfo(topo->bitcoind, true, check_chain, topo); + bitcoind_getchaininfo(topo->bitcoind, false, check_chain, topo); } void setup_topology(struct chain_topology *topo, diff --git a/lightningd/chaintopology.h b/lightningd/chaintopology.h index 816c537f9e1b..03b25c0b4ae7 100644 --- a/lightningd/chaintopology.h +++ b/lightningd/chaintopology.h @@ -37,7 +37,7 @@ struct outgoing_tx { struct channel *channel; const char *hextx; struct bitcoin_txid txid; - void (*failed_or_success)(struct channel *channel, int exitstatus, const char *err); + void (*failed_or_success)(struct channel *channel, bool success, const char *err); }; struct block { @@ -166,7 +166,7 @@ struct command_result *param_feerate_estimate(struct command *cmd, void broadcast_tx(struct chain_topology *topo, struct channel *channel, const struct bitcoin_tx *tx, void (*failed)(struct channel *channel, - int exitstatus, + bool success, const char *err)); struct chain_topology *new_topology(struct lightningd *ld, struct log *log); diff --git a/lightningd/test/run-invoice-select-inchan.c b/lightningd/test/run-invoice-select-inchan.c index e227c282b92f..bd44eccef026 100644 --- a/lightningd/test/run-invoice-select-inchan.c +++ b/lightningd/test/run-invoice-select-inchan.c @@ -38,7 +38,7 @@ char *bolt11_encode_(const tal_t *ctx UNNEEDED, void broadcast_tx(struct chain_topology *topo UNNEEDED, struct channel *channel UNNEEDED, const struct bitcoin_tx *tx UNNEEDED, void (*failed)(struct channel *channel UNNEEDED, - int exitstatus UNNEEDED, + bool success UNNEEDED, const char *err)) { fprintf(stderr, "broadcast_tx called!\n"); abort(); } /* Generated stub for channel_tell_depth */ diff --git a/wallet/test/run-wallet.c b/wallet/test/run-wallet.c index 7b4932f67ab2..0f705cc38a41 100644 --- a/wallet/test/run-wallet.c +++ b/wallet/test/run-wallet.c @@ -49,7 +49,7 @@ void bitcoind_gettxout(struct bitcoind *bitcoind UNNEEDED, void broadcast_tx(struct chain_topology *topo UNNEEDED, struct channel *channel UNNEEDED, const struct bitcoin_tx *tx UNNEEDED, void (*failed)(struct channel *channel UNNEEDED, - int exitstatus UNNEEDED, + bool success UNNEEDED, const char *err)) { fprintf(stderr, "broadcast_tx called!\n"); abort(); } /* Generated stub for channel_tell_depth */ diff --git a/wallet/walletrpc.c b/wallet/walletrpc.c index dbef2a874dbb..311a6569ea18 100644 --- a/wallet/walletrpc.c +++ b/wallet/walletrpc.c @@ -43,7 +43,7 @@ * available outputs. */ static void wallet_withdrawal_broadcast(struct bitcoind *bitcoind UNUSED, - int exitstatus, const char *msg, + bool success, const char *msg, struct unreleased_tx *utx) { struct command *cmd = utx->wtx->cmd; @@ -53,7 +53,7 @@ static void wallet_withdrawal_broadcast(struct bitcoind *bitcoind UNUSED, /* FIXME: This won't be necessary once we use ccan/json_out! */ /* Massage output into shape so it doesn't kill the JSON serialization */ char *output = tal_strjoin(cmd, tal_strsplit(cmd, msg, "\n", STR_NO_EMPTY), " ", STR_NO_TRAIL); - if (exitstatus == 0) { + if (success) { /* Mark used outputs as spent */ wallet_confirm_utxos(ld->wallet, utx->wtx->utxos); @@ -66,7 +66,7 @@ static void wallet_withdrawal_broadcast(struct bitcoind *bitcoind UNUSED, struct json_stream *response = json_stream_success(cmd); json_add_tx(response, "tx", utx->tx); - json_add_string(response, "txid", output); + json_add_txid(response, "txid", &utx->txid); was_pending(command_success(cmd, response)); } else { was_pending(command_fail(cmd, LIGHTNINGD, From 900af4309f72689d9183a3ea108951b424e60e77 Mon Sep 17 00:00:00 2001 From: darosior Date: Thu, 9 Jan 2020 14:14:15 +0100 Subject: [PATCH 14/21] lightningd/bitcoind: use getrawblockatheight for getfilteredblock This avoids the getblockhash+getblock, and more importantly that was the last functionality making use of bitcoind_getrawblock() and bitcoin_getblockhash(), so we can also get rid of them. --- lightningd/bitcoind.c | 130 +++++++----------------------------------- lightningd/bitcoind.h | 30 ---------- 2 files changed, 22 insertions(+), 138 deletions(-) diff --git a/lightningd/bitcoind.c b/lightningd/bitcoind.c index 662455f5b7f8..d114c92fa5e7 100644 --- a/lightningd/bitcoind.c +++ b/lightningd/bitcoind.c @@ -495,40 +495,6 @@ void bitcoind_estimate_fees_(struct bitcoind *bitcoind, do_one_estimatefee(bitcoind, efee); } -static bool process_rawblock(struct bitcoin_cli *bcli) -{ - struct bitcoin_block *blk; - void (*cb)(struct bitcoind *bitcoind, - struct bitcoin_block *blk, - void *arg) = bcli->cb; - - blk = bitcoin_block_from_hex(bcli, chainparams, - bcli->output, bcli->output_bytes); - if (!blk) - fatal("%s: bad block '%.*s'?", - bcli_args(tmpctx, bcli), - (int)bcli->output_bytes, bcli->output); - - cb(bcli->bitcoind, blk, bcli->cb_arg); - return true; -} - -void bitcoind_getrawblock_(struct bitcoind *bitcoind, - const struct bitcoin_blkid *blockid, - void (*cb)(struct bitcoind *bitcoind, - struct bitcoin_block *blk, - void *arg), - void *arg) -{ - char hex[hex_str_size(sizeof(*blockid))]; - - bitcoin_blkid_to_hex(blockid, hex, sizeof(hex)); - start_bitcoin_cli(bitcoind, NULL, process_rawblock, false, - BITCOIND_HIGH_PRIO, - cb, arg, - "getblock", hex, "false", NULL); -} - /* Our Bitcoin backend plugin gave us a bad response. We can't recover. */ static void bitcoin_plugin_error(struct bitcoind *bitcoind, const char *buf, const jsmntok_t *toks, const char *method, @@ -885,50 +851,6 @@ static bool process_gettxout(struct bitcoin_cli *bcli) return true; } -static bool process_getblockhash(struct bitcoin_cli *bcli) -{ - struct bitcoin_blkid blkid; - void (*cb)(struct bitcoind *bitcoind, - const struct bitcoin_blkid *blkid, - void *arg) = bcli->cb; - - /* If it failed with error 8, call with NULL block. */ - if (*bcli->exitstatus != 0) { - /* Other error means we have to retry. */ - if (*bcli->exitstatus != 8) - return false; - cb(bcli->bitcoind, NULL, bcli->cb_arg); - return true; - } - - if (bcli->output_bytes == 0 - || !bitcoin_blkid_from_hex(bcli->output, bcli->output_bytes-1, - &blkid)) { - fatal("%s: bad blockid '%.*s'", - bcli_args(tmpctx, bcli), - (int)bcli->output_bytes, bcli->output); - } - - cb(bcli->bitcoind, &blkid, bcli->cb_arg); - return true; -} - -void bitcoind_getblockhash_(struct bitcoind *bitcoind, - u32 height, - void (*cb)(struct bitcoind *bitcoind, - const struct bitcoin_blkid *blkid, - void *arg), - void *arg) -{ - char str[STR_MAX_CHARS(height)]; - snprintf(str, sizeof(str), "%u", height); - - start_bitcoin_cli(bitcoind, NULL, process_getblockhash, true, - BITCOIND_HIGH_PRIO, - cb, arg, - "getblockhash", str, NULL); -} - void bitcoind_gettxout(struct bitcoind *bitcoind, const struct bitcoin_txid *txid, const u32 outnum, void (*cb)(struct bitcoind *bitcoind, @@ -965,7 +887,7 @@ process_getfiltered_block_final(struct bitcoind *bitcoind, const struct filteredblock_call *call); static void -process_getfilteredblock_step3(struct bitcoind *bitcoind, +process_getfilteredblock_step2(struct bitcoind *bitcoind, const struct bitcoin_tx_output *output, void *arg) { @@ -980,25 +902,36 @@ process_getfilteredblock_step3(struct bitcoind *bitcoind, if (call->current_outpoint < tal_count(call->outpoints)) { o = call->outpoints[call->current_outpoint]; bitcoind_gettxout(bitcoind, &o->txid, o->outnum, - process_getfilteredblock_step3, call); + process_getfilteredblock_step2, call); } else { /* If there were no more outpoints to check, we call the callback. */ process_getfiltered_block_final(bitcoind, call); } } -static void process_getfilteredblock_step2(struct bitcoind *bitcoind, +static void process_getfilteredblock_step1(struct bitcoind *bitcoind, + struct bitcoin_blkid *blkid, struct bitcoin_block *block, struct filteredblock_call *call) { struct filteredblock_outpoint *o; struct bitcoin_tx *tx; - /* If for some reason we couldn't get the block, just report a - * failure. */ - if (block == NULL) + /* If we were unable to fetch the block hash (bitcoind doesn't know + * about a block at that height), we can short-circuit and just call + * the callback. */ + if (!blkid) return process_getfiltered_block_final(bitcoind, call); + /* So we have the first piece of the puzzle, the block hash */ + call->result = tal(call, struct filteredblock); + call->result->height = call->height; + call->result->outpoints = tal_arr(call->result, struct filteredblock_outpoint *, 0); + call->result->id = *blkid; + + /* If the plugin gave us a block id, they MUST send us a block. */ + assert(block != NULL); + call->result->prev_hash = block->hdr.prev_hash; /* Allocate an array containing all the potentially interesting @@ -1037,31 +970,10 @@ static void process_getfilteredblock_step2(struct bitcoind *bitcoind, * call->result->outpoints. */ o = call->outpoints[call->current_outpoint]; bitcoind_gettxout(bitcoind, &o->txid, o->outnum, - process_getfilteredblock_step3, call); + process_getfilteredblock_step2, call); } } -static void process_getfilteredblock_step1(struct bitcoind *bitcoind, - const struct bitcoin_blkid *blkid, - struct filteredblock_call *call) -{ - /* If we were unable to fetch the block hash (bitcoind doesn't know - * about a block at that height), we can short-circuit and just call - * the callback. */ - if (!blkid) - return process_getfiltered_block_final(bitcoind, call); - - /* So we have the first piece of the puzzle, the block hash */ - call->result = tal(call, struct filteredblock); - call->result->height = call->height; - call->result->outpoints = tal_arr(call->result, struct filteredblock_outpoint *, 0); - call->result->id = *blkid; - - /* Now get the raw block to get all outpoints that were created in - * this block. */ - bitcoind_getrawblock(bitcoind, blkid, process_getfilteredblock_step2, call); -} - /* Takes a call, dispatches it to all queued requests that match the same * height, and then kicks off the next call. */ static void @@ -1091,7 +1003,8 @@ process_getfiltered_block_final(struct bitcoind *bitcoind, * pop here. */ if (!list_empty(&bitcoind->pending_getfilteredblock)) { c = list_top(&bitcoind->pending_getfilteredblock, struct filteredblock_call, list); - bitcoind_getblockhash(bitcoind, c->height, process_getfilteredblock_step1, c); + bitcoind_getrawblockbyheight(bitcoind, c->height, + process_getfilteredblock_step1, c); } } @@ -1116,7 +1029,8 @@ void bitcoind_getfilteredblock_(struct bitcoind *bitcoind, u32 height, list_add_tail(&bitcoind->pending_getfilteredblock, &call->list); if (start) - bitcoind_getblockhash(bitcoind, height, process_getfilteredblock_step1, call); + bitcoind_getrawblockbyheight(bitcoind, height, + process_getfilteredblock_step1, call); } static void destroy_bitcoind(struct bitcoind *bitcoind) diff --git a/lightningd/bitcoind.h b/lightningd/bitcoind.h index fc8719b4c9c1..c71eabf6e64c 100644 --- a/lightningd/bitcoind.h +++ b/lightningd/bitcoind.h @@ -118,22 +118,6 @@ void bitcoind_sendrawtx_(struct bitcoind *bitcoind, bool, const char *), \ (arg)) -/* blkid is NULL if call fails. */ -void bitcoind_getblockhash_(struct bitcoind *bitcoind, - u32 height, - void (*cb)(struct bitcoind *bitcoind, - const struct bitcoin_blkid *blkid, - void *arg), - void *arg); -#define bitcoind_getblockhash(bitcoind_, height, cb, arg) \ - bitcoind_getblockhash_((bitcoind_), \ - (height), \ - typesafe_cb_preargs(void, void *, \ - (cb), (arg), \ - struct bitcoind *, \ - const struct bitcoin_blkid *), \ - (arg)) - void bitcoind_getfilteredblock_(struct bitcoind *bitcoind, u32 height, void (*cb)(struct bitcoind *bitcoind, const struct filteredblock *fb, @@ -148,20 +132,6 @@ void bitcoind_getfilteredblock_(struct bitcoind *bitcoind, u32 height, const struct filteredblock *), \ (arg)) -void bitcoind_getrawblock_(struct bitcoind *bitcoind, - const struct bitcoin_blkid *blockid, - void (*cb)(struct bitcoind *bitcoind, - struct bitcoin_block *blk, - void *arg), - void *arg); -#define bitcoind_getrawblock(bitcoind_, blkid, cb, arg) \ - bitcoind_getrawblock_((bitcoind_), (blkid), \ - typesafe_cb_preargs(void, void *, \ - (cb), (arg), \ - struct bitcoind *, \ - struct bitcoin_block *), \ - (arg)) - void bitcoind_getchaininfo_(struct bitcoind *bitcoind, const bool first_call, void (*cb)(struct bitcoind *bitcoind, From fa7de2c798cd2a3a4dd1ec0659df098dc27bac6f Mon Sep 17 00:00:00 2001 From: darosior Date: Thu, 9 Jan 2020 16:38:12 +0100 Subject: [PATCH 15/21] lightningd/bitcoind: use the Bitcoin plugin for getutxout --- lightningd/bitcoind.c | 139 ++++++++++---------- lightningd/bitcoind.h | 19 ++- lightningd/channel_control.c | 10 +- lightningd/peer_control.c | 8 +- lightningd/test/run-invoice-select-inchan.c | 26 ++-- wallet/test/run-wallet.c | 54 ++++---- wallet/wallet.c | 8 +- wallet/walletrpc.c | 4 +- 8 files changed, 137 insertions(+), 131 deletions(-) diff --git a/lightningd/bitcoind.c b/lightningd/bitcoind.c index d114c92fa5e7..46f8c666cd08 100644 --- a/lightningd/bitcoind.c +++ b/lightningd/bitcoind.c @@ -781,89 +781,88 @@ void bitcoind_getchaininfo_(struct bitcoind *bitcoind, req); } -struct get_output { +/* `getutxout` + * + * Get informations about an UTXO. If the TXO is spent, the plugin will set + * all fields to `null`. + * { + * "amount": , + * "script": "The output's scriptPubKey", + * } + */ + +struct getutxout_call { + struct bitcoind *bitcoind; unsigned int blocknum, txnum, outnum; /* The real callback */ - void (*cb)(struct bitcoind *bitcoind, const struct bitcoin_tx_output *txout, void *arg); - + void (*cb)(struct bitcoind *bitcoind, + const struct bitcoin_tx_output *txout, void *arg); /* The real callback arg */ - void *cbarg; + void *cb_arg; }; -static bool process_gettxout(struct bitcoin_cli *bcli) +static void getutxout_callback(const char *buf, const jsmntok_t *toks, + const jsmntok_t *idtok, + struct getutxout_call *call) { - void (*cb)(struct bitcoind *bitcoind, - const struct bitcoin_tx_output *output, - void *arg) = bcli->cb; - const jsmntok_t *tokens, *valuetok, *scriptpubkeytok, *hextok; - struct bitcoin_tx_output out; - bool valid; + const jsmntok_t *resulttok, *amounttok, *scripttok; + struct bitcoin_tx_output txout; - /* As of at least v0.15.1.0, bitcoind returns "success" but an empty - string on a spent gettxout */ - if (*bcli->exitstatus != 0 || bcli->output_bytes == 0) { - cb(bcli->bitcoind, NULL, bcli->cb_arg); - return true; - } - - tokens = json_parse_input(bcli->output, bcli->output, bcli->output_bytes, - &valid); - if (!tokens) - fatal("%s: %s response", - bcli_args(tmpctx, bcli), valid ? "partial" : "invalid"); - - if (tokens[0].type != JSMN_OBJECT) - fatal("%s: gave non-object (%.*s)?", - bcli_args(tmpctx, bcli), - (int)bcli->output_bytes, bcli->output); - - valuetok = json_get_member(bcli->output, tokens, "value"); - if (!valuetok) - fatal("%s: had no value member (%.*s)?", - bcli_args(tmpctx, bcli), - (int)bcli->output_bytes, bcli->output); + resulttok = json_get_member(buf, toks, "result"); + if (!resulttok) + bitcoin_plugin_error(call->bitcoind, buf, toks, "getutxout", + "bad 'result' field"); - if (!json_to_bitcoin_amount(bcli->output, valuetok, &out.amount.satoshis)) /* Raw: talking to bitcoind */ - fatal("%s: had bad value (%.*s)?", - bcli_args(tmpctx, bcli), - (int)bcli->output_bytes, bcli->output); + scripttok = json_get_member(buf, resulttok, "script"); + if (!scripttok) + bitcoin_plugin_error(call->bitcoind, buf, toks, "getutxout", + "bad 'script' field"); + if (json_tok_is_null(buf, scripttok)) { + db_begin_transaction(call->bitcoind->ld->wallet->db); + call->cb(call->bitcoind, NULL, call->cb_arg); + db_commit_transaction(call->bitcoind->ld->wallet->db); + goto clean; + } + txout.script = json_tok_bin_from_hex(tmpctx, buf, scripttok); - scriptpubkeytok = json_get_member(bcli->output, tokens, "scriptPubKey"); - if (!scriptpubkeytok) - fatal("%s: had no scriptPubKey member (%.*s)?", - bcli_args(tmpctx, bcli), - (int)bcli->output_bytes, bcli->output); - hextok = json_get_member(bcli->output, scriptpubkeytok, "hex"); - if (!hextok) - fatal("%s: had no scriptPubKey->hex member (%.*s)?", - bcli_args(tmpctx, bcli), - (int)bcli->output_bytes, bcli->output); + amounttok = json_get_member(buf, resulttok, "amount"); + if (!amounttok) + bitcoin_plugin_error(call->bitcoind, buf, toks, "getutxout", + "bad 'amount' field"); + if (!json_to_sat(buf, amounttok, &txout.amount)) + bitcoin_plugin_error(call->bitcoind, buf, toks, "getutxout", + "bad sats amount"); - out.script = tal_hexdata(bcli, bcli->output + hextok->start, - hextok->end - hextok->start); - if (!out.script) - fatal("%s: scriptPubKey->hex invalid hex (%.*s)?", - bcli_args(tmpctx, bcli), - (int)bcli->output_bytes, bcli->output); + db_begin_transaction(call->bitcoind->ld->wallet->db); + call->cb(call->bitcoind, &txout, call->cb_arg); + db_commit_transaction(call->bitcoind->ld->wallet->db); - cb(bcli->bitcoind, &out, bcli->cb_arg); - return true; +clean: + tal_free(call); } -void bitcoind_gettxout(struct bitcoind *bitcoind, - const struct bitcoin_txid *txid, const u32 outnum, - void (*cb)(struct bitcoind *bitcoind, - const struct bitcoin_tx_output *txout, - void *arg), - void *arg) +void bitcoind_getutxout_(struct bitcoind *bitcoind, + const struct bitcoin_txid *txid, const u32 outnum, + void (*cb)(struct bitcoind *bitcoind, + const struct bitcoin_tx_output *txout, + void *arg), + void *cb_arg) { - start_bitcoin_cli(bitcoind, NULL, - process_gettxout, true, BITCOIND_LOW_PRIO, cb, arg, - "gettxout", - take(type_to_string(NULL, struct bitcoin_txid, txid)), - take(tal_fmt(NULL, "%u", outnum)), - NULL); + struct jsonrpc_request *req; + struct getutxout_call *call = tal(bitcoind, struct getutxout_call); + + call->bitcoind = bitcoind; + call->cb = cb; + call->cb_arg = cb_arg; + + req = jsonrpc_request_start(bitcoind, "getutxout", bitcoind->log, + getutxout_callback, call); + json_add_txid(req->stream, "txid", txid); + json_add_num(req->stream, "vout", outnum); + jsonrpc_request_end(req); + plugin_request_send(strmap_get(&bitcoind->pluginsmap, "getutxout"), + req); } /* Context for the getfilteredblock call. Wraps the actual arguments while we @@ -901,7 +900,7 @@ process_getfilteredblock_step2(struct bitcoind *bitcoind, call->current_outpoint++; if (call->current_outpoint < tal_count(call->outpoints)) { o = call->outpoints[call->current_outpoint]; - bitcoind_gettxout(bitcoind, &o->txid, o->outnum, + bitcoind_getutxout(bitcoind, &o->txid, o->outnum, process_getfilteredblock_step2, call); } else { /* If there were no more outpoints to check, we call the callback. */ @@ -969,7 +968,7 @@ static void process_getfilteredblock_step1(struct bitcoind *bitcoind, * store the one's that are unspent in * call->result->outpoints. */ o = call->outpoints[call->current_outpoint]; - bitcoind_gettxout(bitcoind, &o->txid, o->outnum, + bitcoind_getutxout(bitcoind, &o->txid, o->outnum, process_getfilteredblock_step2, call); } } diff --git a/lightningd/bitcoind.h b/lightningd/bitcoind.h index c71eabf6e64c..97f6f7bb2849 100644 --- a/lightningd/bitcoind.h +++ b/lightningd/bitcoind.h @@ -166,12 +166,19 @@ void bitcoind_getrawblockbyheight_(struct bitcoind *bitcoind, struct bitcoin_block *),\ (arg)) -void bitcoind_gettxout(struct bitcoind *bitcoind, - const struct bitcoin_txid *txid, const u32 outnum, - void (*cb)(struct bitcoind *bitcoind, - const struct bitcoin_tx_output *txout, - void *arg), - void *arg); +void bitcoind_getutxout_(struct bitcoind *bitcoind, + const struct bitcoin_txid *txid, const u32 outnum, + void (*cb)(struct bitcoind *bitcoind, + const struct bitcoin_tx_output *txout, + void *arg), + void *arg); +#define bitcoind_getutxout(bitcoind_, txid_, vout_, cb, arg) \ + bitcoind_getutxout_((bitcoind_), (txid_), (vout_), \ + typesafe_cb_preargs(void, void *, \ + (cb), (arg), \ + struct bitcoind *, \ + struct bitcoin_tx_output *),\ + (arg)) void bitcoind_getclientversion(struct bitcoind *bitcoind); diff --git a/lightningd/channel_control.c b/lightningd/channel_control.c index f779672b695e..7a3bd347912a 100644 --- a/lightningd/channel_control.c +++ b/lightningd/channel_control.c @@ -774,11 +774,11 @@ struct command_result *cancel_channel_before_broadcast(struct command *cmd, /* Note: The above check and this check can't completely ensure that * the funding transaction isn't broadcast. We can't know if the funding * is broadcast by external wallet and the transaction hasn't been onchain. */ - bitcoind_gettxout(cmd->ld->topology->bitcoind, - &cancel_channel->funding_txid, - cancel_channel->funding_outnum, - process_check_funding_broadcast, - notleak(cc)); + bitcoind_getutxout(cmd->ld->topology->bitcoind, + &cancel_channel->funding_txid, + cancel_channel->funding_outnum, + process_check_funding_broadcast, + notleak(cc)); return command_still_pending(cmd); } diff --git a/lightningd/peer_control.c b/lightningd/peer_control.c index 3ccc8dec505c..95bac4c0d729 100644 --- a/lightningd/peer_control.c +++ b/lightningd/peer_control.c @@ -2263,10 +2263,10 @@ static struct command_result *json_dev_forget_channel(struct command *cmd, "or `dev-fail` instead."); } - bitcoind_gettxout(cmd->ld->topology->bitcoind, - &forget->channel->funding_txid, - forget->channel->funding_outnum, - process_dev_forget_channel, forget); + bitcoind_getutxout(cmd->ld->topology->bitcoind, + &forget->channel->funding_txid, + forget->channel->funding_outnum, + process_dev_forget_channel, forget); return command_still_pending(cmd); } diff --git a/lightningd/test/run-invoice-select-inchan.c b/lightningd/test/run-invoice-select-inchan.c index bd44eccef026..496bb164fad2 100644 --- a/lightningd/test/run-invoice-select-inchan.c +++ b/lightningd/test/run-invoice-select-inchan.c @@ -13,14 +13,14 @@ size_t bigsize_get(const u8 *p UNNEEDED, size_t max UNNEEDED, bigsize_t *val UNN /* Generated stub for bigsize_put */ size_t bigsize_put(u8 buf[BIGSIZE_MAX_LEN] UNNEEDED, bigsize_t v UNNEEDED) { fprintf(stderr, "bigsize_put called!\n"); abort(); } -/* Generated stub for bitcoind_gettxout */ -void bitcoind_gettxout(struct bitcoind *bitcoind UNNEEDED, - const struct bitcoin_txid *txid UNNEEDED, const u32 outnum UNNEEDED, - void (*cb)(struct bitcoind *bitcoind UNNEEDED, - const struct bitcoin_tx_output *txout UNNEEDED, - void *arg) UNNEEDED, - void *arg UNNEEDED) -{ fprintf(stderr, "bitcoind_gettxout called!\n"); abort(); } +/* Generated stub for bitcoind_getutxout_ */ +void bitcoind_getutxout_(struct bitcoind *bitcoind UNNEEDED, + const struct bitcoin_txid *txid UNNEEDED, const u32 outnum UNNEEDED, + void (*cb)(struct bitcoind *bitcoind UNNEEDED, + const struct bitcoin_tx_output *txout UNNEEDED, + void *arg) UNNEEDED, + void *arg UNNEEDED) +{ fprintf(stderr, "bitcoind_getutxout_ called!\n"); abort(); } /* Generated stub for bolt11_decode */ struct bolt11 *bolt11_decode(const tal_t *ctx UNNEEDED, const char *str UNNEEDED, const char *description UNNEEDED, char **fail UNNEEDED) @@ -90,7 +90,7 @@ struct fee_states *dup_fee_states(const tal_t *ctx UNNEEDED, /* Generated stub for encode_scriptpubkey_to_addr */ char *encode_scriptpubkey_to_addr(const tal_t *ctx UNNEEDED, const struct chainparams *chainparams UNNEEDED, - const u8 *scriptPubkey UNNEEDED) + const u8 *scriptPubkey UNNEEDED) { fprintf(stderr, "encode_scriptpubkey_to_addr called!\n"); abort(); } /* Generated stub for fatal */ void fatal(const char *fmt UNNEEDED, ...) @@ -198,6 +198,10 @@ enum address_parse_result json_to_address_scriptpubkey(const tal_t *ctx UNNEEDED const char *buffer UNNEEDED, const jsmntok_t *tok UNNEEDED, const u8 **scriptpubkey UNNEEDED) { fprintf(stderr, "json_to_address_scriptpubkey called!\n"); abort(); } +/* Generated stub for json_tok_channel_id */ +bool json_tok_channel_id(const char *buffer UNNEEDED, const jsmntok_t *tok UNNEEDED, + struct channel_id *cid UNNEEDED) +{ fprintf(stderr, "json_tok_channel_id called!\n"); abort(); } /* Generated stub for json_to_node_id */ bool json_to_node_id(const char *buffer UNNEEDED, const jsmntok_t *tok UNNEEDED, struct node_id *id UNNEEDED) @@ -206,10 +210,6 @@ bool json_to_node_id(const char *buffer UNNEEDED, const jsmntok_t *tok UNNEEDED, bool json_to_short_channel_id(const char *buffer UNNEEDED, const jsmntok_t *tok UNNEEDED, struct short_channel_id *scid UNNEEDED) { fprintf(stderr, "json_to_short_channel_id called!\n"); abort(); } -/* Generated stub for json_tok_channel_id */ -bool json_tok_channel_id(const char *buffer UNNEEDED, const jsmntok_t *tok UNNEEDED, - struct channel_id *cid UNNEEDED) -{ fprintf(stderr, "json_tok_channel_id called!\n"); abort(); } /* Generated stub for kill_uncommitted_channel */ void kill_uncommitted_channel(struct uncommitted_channel *uc UNNEEDED, const char *why UNNEEDED) diff --git a/wallet/test/run-wallet.c b/wallet/test/run-wallet.c index 0f705cc38a41..d870e0b7470b 100644 --- a/wallet/test/run-wallet.c +++ b/wallet/test/run-wallet.c @@ -37,14 +37,14 @@ size_t bigsize_get(const u8 *p UNNEEDED, size_t max UNNEEDED, bigsize_t *val UNN /* Generated stub for bigsize_put */ size_t bigsize_put(u8 buf[BIGSIZE_MAX_LEN] UNNEEDED, bigsize_t v UNNEEDED) { fprintf(stderr, "bigsize_put called!\n"); abort(); } -/* Generated stub for bitcoind_gettxout */ -void bitcoind_gettxout(struct bitcoind *bitcoind UNNEEDED, - const struct bitcoin_txid *txid UNNEEDED, const u32 outnum UNNEEDED, - void (*cb)(struct bitcoind *bitcoind UNNEEDED, - const struct bitcoin_tx_output *txout UNNEEDED, - void *arg) UNNEEDED, - void *arg UNNEEDED) -{ fprintf(stderr, "bitcoind_gettxout called!\n"); abort(); } +/* Generated stub for bitcoind_getutxout_ */ +void bitcoind_getutxout_(struct bitcoind *bitcoind UNNEEDED, + const struct bitcoin_txid *txid UNNEEDED, const u32 outnum UNNEEDED, + void (*cb)(struct bitcoind *bitcoind UNNEEDED, + const struct bitcoin_tx_output *txout UNNEEDED, + void *arg) UNNEEDED, + void *arg UNNEEDED) +{ fprintf(stderr, "bitcoind_getutxout_ called!\n"); abort(); } /* Generated stub for broadcast_tx */ void broadcast_tx(struct chain_topology *topo UNNEEDED, struct channel *channel UNNEEDED, const struct bitcoin_tx *tx UNNEEDED, @@ -89,7 +89,7 @@ void delay_then_reconnect(struct channel *channel UNNEEDED, u32 seconds_delay UN /* Generated stub for encode_scriptpubkey_to_addr */ char *encode_scriptpubkey_to_addr(const tal_t *ctx UNNEEDED, const struct chainparams *chainparams UNNEEDED, - const u8 *scriptPubkey UNNEEDED) + const u8 *scriptPubkey UNNEEDED) { fprintf(stderr, "encode_scriptpubkey_to_addr called!\n"); abort(); } /* Generated stub for fatal */ void fatal(const char *fmt UNNEEDED, ...) @@ -343,21 +343,6 @@ enum address_parse_result json_to_address_scriptpubkey(const tal_t *ctx UNNEEDED /* Generated stub for json_to_bool */ bool json_to_bool(const char *buffer UNNEEDED, const jsmntok_t *tok UNNEEDED, bool *b UNNEEDED) { fprintf(stderr, "json_to_bool called!\n"); abort(); } -/* Generated stub for json_to_node_id */ -bool json_to_node_id(const char *buffer UNNEEDED, const jsmntok_t *tok UNNEEDED, - struct node_id *id UNNEEDED) -{ fprintf(stderr, "json_to_node_id called!\n"); abort(); } -/* Generated stub for json_to_number */ -bool json_to_number(const char *buffer UNNEEDED, const jsmntok_t *tok UNNEEDED, - unsigned int *num UNNEEDED) -{ fprintf(stderr, "json_to_number called!\n"); abort(); } -/* Generated stub for json_to_preimage */ -bool json_to_preimage(const char *buffer UNNEEDED, const jsmntok_t *tok UNNEEDED, struct preimage *preimage UNNEEDED) -{ fprintf(stderr, "json_to_preimage called!\n"); abort(); } -/* Generated stub for json_to_short_channel_id */ -bool json_to_short_channel_id(const char *buffer UNNEEDED, const jsmntok_t *tok UNNEEDED, - struct short_channel_id *scid UNNEEDED) -{ fprintf(stderr, "json_to_short_channel_id called!\n"); abort(); } /* Generated stub for json_tok_bin_from_hex */ u8 *json_tok_bin_from_hex(const tal_t *ctx UNNEEDED, const char *buffer UNNEEDED, const jsmntok_t *tok UNNEEDED) { fprintf(stderr, "json_tok_bin_from_hex called!\n"); abort(); } @@ -374,6 +359,21 @@ int json_tok_full_len(const jsmntok_t *t UNNEEDED) /* Generated stub for json_tok_streq */ bool json_tok_streq(const char *buffer UNNEEDED, const jsmntok_t *tok UNNEEDED, const char *str UNNEEDED) { fprintf(stderr, "json_tok_streq called!\n"); abort(); } +/* Generated stub for json_to_node_id */ +bool json_to_node_id(const char *buffer UNNEEDED, const jsmntok_t *tok UNNEEDED, + struct node_id *id UNNEEDED) +{ fprintf(stderr, "json_to_node_id called!\n"); abort(); } +/* Generated stub for json_to_number */ +bool json_to_number(const char *buffer UNNEEDED, const jsmntok_t *tok UNNEEDED, + unsigned int *num UNNEEDED) +{ fprintf(stderr, "json_to_number called!\n"); abort(); } +/* Generated stub for json_to_preimage */ +bool json_to_preimage(const char *buffer UNNEEDED, const jsmntok_t *tok UNNEEDED, struct preimage *preimage UNNEEDED) +{ fprintf(stderr, "json_to_preimage called!\n"); abort(); } +/* Generated stub for json_to_short_channel_id */ +bool json_to_short_channel_id(const char *buffer UNNEEDED, const jsmntok_t *tok UNNEEDED, + struct short_channel_id *scid UNNEEDED) +{ fprintf(stderr, "json_to_short_channel_id called!\n"); abort(); } /* Generated stub for kill_uncommitted_channel */ void kill_uncommitted_channel(struct uncommitted_channel *uc UNNEEDED, const char *why UNNEEDED) @@ -587,12 +587,12 @@ u8 *towire_channel_got_revoke_reply(const tal_t *ctx UNNEEDED) /* Generated stub for towire_channel_offer_htlc */ u8 *towire_channel_offer_htlc(const tal_t *ctx UNNEEDED, struct amount_msat amount_msat UNNEEDED, u32 cltv_expiry UNNEEDED, const struct sha256 *payment_hash UNNEEDED, const u8 onion_routing_packet[1366]) { fprintf(stderr, "towire_channel_offer_htlc called!\n"); abort(); } -/* Generated stub for towire_channel_send_shutdown */ -u8 *towire_channel_send_shutdown(const tal_t *ctx UNNEEDED, const u8 *shutdown_scriptpubkey UNNEEDED) -{ fprintf(stderr, "towire_channel_send_shutdown called!\n"); abort(); } /* Generated stub for towire_channel_sending_commitsig_reply */ u8 *towire_channel_sending_commitsig_reply(const tal_t *ctx UNNEEDED) { fprintf(stderr, "towire_channel_sending_commitsig_reply called!\n"); abort(); } +/* Generated stub for towire_channel_send_shutdown */ +u8 *towire_channel_send_shutdown(const tal_t *ctx UNNEEDED, const u8 *shutdown_scriptpubkey UNNEEDED) +{ fprintf(stderr, "towire_channel_send_shutdown called!\n"); abort(); } /* Generated stub for towire_channel_specific_feerates */ u8 *towire_channel_specific_feerates(const tal_t *ctx UNNEEDED, u32 feerate_base UNNEEDED, u32 feerate_ppm UNNEEDED) { fprintf(stderr, "towire_channel_specific_feerates called!\n"); abort(); } diff --git a/wallet/wallet.c b/wallet/wallet.c index 05ff3974a958..c83e0d663cc7 100644 --- a/wallet/wallet.c +++ b/wallet/wallet.c @@ -3599,8 +3599,8 @@ static void process_utxo_result(struct bitcoind *bitcoind, /* If we have more, resolve them too. */ tal_arr_remove(&utxos, 0); if (tal_count(utxos) != 0) { - bitcoind_gettxout(bitcoind, &utxos[0]->txid, utxos[0]->outnum, - process_utxo_result, utxos); + bitcoind_getutxout(bitcoind, &utxos[0]->txid, utxos[0]->outnum, + process_utxo_result, utxos); } else tal_free(utxos); } @@ -3610,8 +3610,8 @@ void wallet_clean_utxos(struct wallet *w, struct bitcoind *bitcoind) struct utxo **utxos = wallet_get_utxos(NULL, w, output_state_reserved); if (tal_count(utxos) != 0) { - bitcoind_gettxout(bitcoind, &utxos[0]->txid, utxos[0]->outnum, - process_utxo_result, notleak(utxos)); + bitcoind_getutxout(bitcoind, &utxos[0]->txid, utxos[0]->outnum, + process_utxo_result, notleak(utxos)); } else tal_free(utxos); } diff --git a/wallet/walletrpc.c b/wallet/walletrpc.c index 311a6569ea18..d5b75dcfbc75 100644 --- a/wallet/walletrpc.c +++ b/wallet/walletrpc.c @@ -930,7 +930,7 @@ static void process_utxo_result(struct bitcoind *bitcoind, json_array_end(rescan->response); was_pending(command_success(rescan->cmd, rescan->response)); } else { - bitcoind_gettxout( + bitcoind_getutxout( bitcoind->ld->topology->bitcoind, &rescan->utxos[0]->txid, rescan->utxos[0]->outnum, process_utxo_result, rescan); } @@ -956,7 +956,7 @@ static struct command_result *json_dev_rescan_outputs(struct command *cmd, json_array_end(rescan->response); return command_success(cmd, rescan->response); } - bitcoind_gettxout(cmd->ld->topology->bitcoind, &rescan->utxos[0]->txid, + bitcoind_getutxout(cmd->ld->topology->bitcoind, &rescan->utxos[0]->txid, rescan->utxos[0]->outnum, process_utxo_result, rescan); return command_still_pending(cmd); From a9239df8b29eed0573a05f60744c10ff996b8fb5 Mon Sep 17 00:00:00 2001 From: darosior Date: Thu, 9 Jan 2020 18:41:07 +0100 Subject: [PATCH 16/21] lightningd/bitcoind: use the Bitcoin plugin for fee estimates And remove bitcoin-cli interaction code, now unused. --- lightningd/bitcoind.c | 418 ++++++------------------------------------ 1 file changed, 61 insertions(+), 357 deletions(-) diff --git a/lightningd/bitcoind.c b/lightningd/bitcoind.c index 46f8c666cd08..6d6fde4d6433 100644 --- a/lightningd/bitcoind.c +++ b/lightningd/bitcoind.c @@ -26,13 +26,6 @@ #include #include -/* Bitcoind's web server has a default of 4 threads, with queue depth 16. - * It will *fail* rather than queue beyond that, so we must not stress it! - * - * This is how many request for each priority level we have. - */ -#define BITCOIND_MAX_PARALLEL 4 - /* The names of the request we can make to our Bitcoin backend. */ static const char *methods[] = {"getchaininfo", "getrawblockbyheight", "sendrawtransaction", "getutxout", @@ -91,324 +84,30 @@ void bitcoind_check_commands(struct bitcoind *bitcoind) } } -/* Add the n'th arg to *args, incrementing n and keeping args of size n+1 */ -static void add_arg(const char ***args, const char *arg) +/* Our Bitcoin backend plugin gave us a bad response. We can't recover. */ +static void bitcoin_plugin_error(struct bitcoind *bitcoind, const char *buf, + const jsmntok_t *toks, const char *method, + const char *reason) { - tal_arr_expand(args, arg); + struct plugin *p = strmap_get(&bitcoind->pluginsmap, method); + fatal("%s error: bad response to %s (%s), response was %.*s", + p->cmd, method, reason, + toks->end - toks->start, buf + toks->start); } -static const char **gather_args(const struct bitcoind *bitcoind, - const tal_t *ctx, const char *cmd, va_list ap) -{ - const char **args = tal_arr(ctx, const char *, 1); - const char *arg; - - args[0] = bitcoind->cli ? bitcoind->cli : chainparams->cli; - if (chainparams->cli_args) - add_arg(&args, chainparams->cli_args); - - if (bitcoind->datadir) - add_arg(&args, tal_fmt(args, "-datadir=%s", bitcoind->datadir)); - - - if (bitcoind->rpcconnect) - add_arg(&args, - tal_fmt(args, "-rpcconnect=%s", bitcoind->rpcconnect)); - - if (bitcoind->rpcport) - add_arg(&args, - tal_fmt(args, "-rpcport=%s", bitcoind->rpcport)); - - if (bitcoind->rpcuser) - add_arg(&args, tal_fmt(args, "-rpcuser=%s", bitcoind->rpcuser)); - - if (bitcoind->rpcpass) - add_arg(&args, - tal_fmt(args, "-rpcpassword=%s", bitcoind->rpcpass)); - - add_arg(&args, cmd); - - while ((arg = va_arg(ap, const char *)) != NULL) - add_arg(&args, tal_strdup(args, arg)); - - add_arg(&args, NULL); - return args; -} +/* `getfeerate` + * + * Gather feerate from our Bitcoin backend. Will set the feerate to `null` + * if estimation failed. + * + * Plugin response: + * { + * "feerate": , + * } + */ -struct bitcoin_cli { - struct list_node list; +struct estimatefee_call { struct bitcoind *bitcoind; - int fd; - int *exitstatus; - pid_t pid; - const char **args; - struct timeabs start; - enum bitcoind_prio prio; - char *output; - size_t output_bytes; - size_t new_output; - bool (*process)(struct bitcoin_cli *); - void *cb; - void *cb_arg; - struct bitcoin_cli **stopper; -}; - -static struct io_plan *read_more(struct io_conn *conn, struct bitcoin_cli *bcli) -{ - bcli->output_bytes += bcli->new_output; - if (bcli->output_bytes == tal_count(bcli->output)) - tal_resize(&bcli->output, bcli->output_bytes * 2); - return io_read_partial(conn, bcli->output + bcli->output_bytes, - tal_count(bcli->output) - bcli->output_bytes, - &bcli->new_output, read_more, bcli); -} - -static struct io_plan *output_init(struct io_conn *conn, struct bitcoin_cli *bcli) -{ - bcli->output_bytes = bcli->new_output = 0; - bcli->output = tal_arr(bcli, char, 100); - return read_more(conn, bcli); -} - -static void next_bcli(struct bitcoind *bitcoind, enum bitcoind_prio prio); - -/* For printing: simple string of args (no secrets!) */ -static char *args_string(const tal_t *ctx, const char **args) -{ - size_t i; - char *ret = tal_strdup(ctx, args[0]); - - for (i = 1; args[i]; i++) { - ret = tal_strcat(ctx, take(ret), " "); - if (strstarts(args[i], "-rpcpassword")) { - ret = tal_strcat(ctx, take(ret), "-rpcpassword=..."); - } else if (strstarts(args[i], "-rpcuser")) { - ret = tal_strcat(ctx, take(ret), "-rpcuser=..."); - } else { - ret = tal_strcat(ctx, take(ret), args[i]); - } - } - return ret; -} - -static char *bcli_args(const tal_t *ctx, struct bitcoin_cli *bcli) -{ - return args_string(ctx, bcli->args); -} - -static void retry_bcli(struct bitcoin_cli *bcli) -{ - list_add_tail(&bcli->bitcoind->pending[bcli->prio], &bcli->list); - next_bcli(bcli->bitcoind, bcli->prio); -} - -/* We allow 60 seconds of spurious errors, eg. reorg. */ -static void bcli_failure(struct bitcoind *bitcoind, - struct bitcoin_cli *bcli, - int exitstatus) -{ - struct timerel t; - - if (!bitcoind->error_count) - bitcoind->first_error_time = time_mono(); - - t = timemono_between(time_mono(), bitcoind->first_error_time); - if (time_greater(t, time_from_sec(bitcoind->retry_timeout))) - fatal("%s exited %u (after %u other errors) '%.*s'; " - "we have been retrying command for " - "--bitcoin-retry-timeout=%"PRIu64" seconds; " - "bitcoind setup or our --bitcoin-* configs broken?", - bcli_args(tmpctx, bcli), - exitstatus, - bitcoind->error_count, - (int)bcli->output_bytes, - bcli->output, - bitcoind->retry_timeout); - - log_unusual(bitcoind->log, - "%s exited with status %u", - bcli_args(tmpctx, bcli), exitstatus); - - bitcoind->error_count++; - - /* Retry in 1 second (not a leak!) */ - notleak(new_reltimer(bitcoind->ld->timers, notleak(bcli), - time_from_sec(1), - retry_bcli, bcli)); -} - -static void bcli_finished(struct io_conn *conn UNUSED, struct bitcoin_cli *bcli) -{ - int ret, status; - struct bitcoind *bitcoind = bcli->bitcoind; - enum bitcoind_prio prio = bcli->prio; - bool ok; - u64 msec = time_to_msec(time_between(time_now(), bcli->start)); - - /* If it took over 10 seconds, that's rather strange. */ - if (msec > 10000) - log_unusual(bitcoind->log, - "bitcoin-cli: finished %s (%"PRIu64" ms)", - bcli_args(tmpctx, bcli), msec); - - assert(bitcoind->num_requests[prio] > 0); - - /* FIXME: If we waited for SIGCHILD, this could never hang! */ - while ((ret = waitpid(bcli->pid, &status, 0)) < 0 && errno == EINTR); - if (ret != bcli->pid) - fatal("%s %s", bcli_args(tmpctx, bcli), - ret == 0 ? "not exited?" : strerror(errno)); - - if (!WIFEXITED(status)) - fatal("%s died with signal %i", - bcli_args(tmpctx, bcli), - WTERMSIG(status)); - - if (!bcli->exitstatus) { - if (WEXITSTATUS(status) != 0) { - bcli_failure(bitcoind, bcli, WEXITSTATUS(status)); - bitcoind->num_requests[prio]--; - goto done; - } - } else - *bcli->exitstatus = WEXITSTATUS(status); - - if (WEXITSTATUS(status) == 0) - bitcoind->error_count = 0; - - bitcoind->num_requests[bcli->prio]--; - - /* Don't continue if were only here because we were freed for shutdown */ - if (bitcoind->shutdown) - return; - - db_begin_transaction(bitcoind->ld->wallet->db); - ok = bcli->process(bcli); - db_commit_transaction(bitcoind->ld->wallet->db); - - if (!ok) - bcli_failure(bitcoind, bcli, WEXITSTATUS(status)); - else - tal_free(bcli); - -done: - next_bcli(bitcoind, prio); -} - -static void next_bcli(struct bitcoind *bitcoind, enum bitcoind_prio prio) -{ - struct bitcoin_cli *bcli; - struct io_conn *conn; - - if (bitcoind->num_requests[prio] >= BITCOIND_MAX_PARALLEL) - return; - - bcli = list_pop(&bitcoind->pending[prio], struct bitcoin_cli, list); - if (!bcli) - return; - - bcli->pid = pipecmdarr(NULL, &bcli->fd, &bcli->fd, - cast_const2(char **, bcli->args)); - if (bcli->pid < 0) - fatal("%s exec failed: %s", bcli->args[0], strerror(errno)); - - bcli->start = time_now(); - - bitcoind->num_requests[prio]++; - - /* This lifetime is attached to bitcoind command fd */ - conn = notleak(io_new_conn(bitcoind, bcli->fd, output_init, bcli)); - io_set_finish(conn, bcli_finished, bcli); -} - -static bool process_donothing(struct bitcoin_cli *bcli UNUSED) -{ - return true; -} - -/* If stopper gets freed first, set process() to a noop. */ -static void stop_process_bcli(struct bitcoin_cli **stopper) -{ - (*stopper)->process = process_donothing; - (*stopper)->stopper = NULL; -} - -/* It command finishes first, free stopper. */ -static void remove_stopper(struct bitcoin_cli *bcli) -{ - /* Calls stop_process_bcli, but we don't care. */ - tal_free(bcli->stopper); -} - -/* If ctx is non-NULL, and is freed before we return, we don't call process(). - * process returns false() if it's a spurious error, and we should retry. */ -static void -start_bitcoin_cli(struct bitcoind *bitcoind, - const tal_t *ctx, - bool (*process)(struct bitcoin_cli *), - bool nonzero_exit_ok, - enum bitcoind_prio prio, - void *cb, void *cb_arg, - char *cmd, ...) -{ - va_list ap; - struct bitcoin_cli *bcli = tal(bitcoind, struct bitcoin_cli); - - bcli->bitcoind = bitcoind; - bcli->process = process; - bcli->prio = prio; - bcli->cb = cb; - bcli->cb_arg = cb_arg; - if (ctx) { - /* Create child whose destructor will stop us calling */ - bcli->stopper = tal(ctx, struct bitcoin_cli *); - *bcli->stopper = bcli; - tal_add_destructor(bcli->stopper, stop_process_bcli); - tal_add_destructor(bcli, remove_stopper); - } else - bcli->stopper = NULL; - - if (nonzero_exit_ok) - bcli->exitstatus = tal(bcli, int); - else - bcli->exitstatus = NULL; - va_start(ap, cmd); - bcli->args = gather_args(bitcoind, bcli, cmd, ap); - va_end(ap); - - list_add_tail(&bitcoind->pending[bcli->prio], &bcli->list); - next_bcli(bitcoind, bcli->prio); -} - -static bool extract_feerate(struct bitcoin_cli *bcli, - const char *output, size_t output_bytes, - u64 *feerate) -{ - const jsmntok_t *tokens, *feeratetok; - bool valid; - - tokens = json_parse_input(output, output, output_bytes, &valid); - if (!tokens) - fatal("%s: %s response", - bcli_args(tmpctx, bcli), - valid ? "partial" : "invalid"); - - if (tokens[0].type != JSMN_OBJECT) { - log_unusual(bcli->bitcoind->log, - "%s: gave non-object (%.*s)?", - bcli_args(tmpctx, bcli), - (int)output_bytes, output); - return false; - } - - feeratetok = json_get_member(output, tokens, "feerate"); - if (!feeratetok) - return false; - - return json_to_bitcoin_amount(output, feeratetok, feerate); -} - -struct estimatefee { size_t i; const u32 *blocks; const char **estmode; @@ -420,59 +119,74 @@ struct estimatefee { }; static void do_one_estimatefee(struct bitcoind *bitcoind, - struct estimatefee *efee); + struct estimatefee_call *call); -static bool process_estimatefee(struct bitcoin_cli *bcli) +static void getfeerate_callback(const char *buf, const jsmntok_t *toks, + const jsmntok_t *idtok, + struct estimatefee_call *call) { + const jsmntok_t *resulttok, *feeratetok; u64 feerate; - struct estimatefee *efee = bcli->cb_arg; + + resulttok = json_get_member(buf, toks, "result"); + if (!resulttok) + bitcoin_plugin_error(call->bitcoind, buf, toks, + "getfeerate", + "bad 'result' field"); + + feeratetok = json_get_member(buf, resulttok, "feerate"); + if (!feeratetok) + bitcoin_plugin_error(call->bitcoind, buf, toks, + "getfeerate", + "bad 'feerate' field"); /* FIXME: We could trawl recent blocks for median fee... */ - if (!extract_feerate(bcli, bcli->output, bcli->output_bytes, &feerate)) { - log_unusual(bcli->bitcoind->log, "Unable to estimate %s/%u fee", - efee->estmode[efee->i], efee->blocks[efee->i]); + if (!json_to_u64(buf, feeratetok, &feerate)) { + log_unusual(call->bitcoind->log, "Unable to estimate %s/%u fee", + call->estmode[call->i], call->blocks[call->i]); #if DEVELOPER /* This is needed to test for failed feerate estimates * in DEVELOPER mode */ - efee->satoshi_per_kw[efee->i] = 0; + call->satoshi_per_kw[call->i] = 0; #else /* If we are in testnet mode we want to allow payments * with the minimal fee even if the estimate didn't * work out. This is less disruptive than erring out * all the time. */ if (chainparams->testnet) - efee->satoshi_per_kw[efee->i] = FEERATE_FLOOR; + call->satoshi_per_kw[call->i] = FEERATE_FLOOR; else - efee->satoshi_per_kw[efee->i] = 0; + call->satoshi_per_kw[call->i] = 0; #endif } else /* Rate in satoshi per kw. */ - efee->satoshi_per_kw[efee->i] + call->satoshi_per_kw[call->i] = feerate_from_style(feerate, FEERATE_PER_KBYTE); - efee->i++; - if (efee->i == tal_count(efee->satoshi_per_kw)) { - efee->cb(bcli->bitcoind, efee->satoshi_per_kw, efee->arg); - tal_free(efee); + call->i++; + if (call->i == tal_count(call->satoshi_per_kw)) { + call->cb(call->bitcoind, call->satoshi_per_kw, call->arg); + tal_free(call); } else { /* Next */ - do_one_estimatefee(bcli->bitcoind, efee); + do_one_estimatefee(call->bitcoind, call); } - return true; } static void do_one_estimatefee(struct bitcoind *bitcoind, - struct estimatefee *efee) + struct estimatefee_call *call) { - char blockstr[STR_MAX_CHARS(u32)]; - - snprintf(blockstr, sizeof(blockstr), "%u", efee->blocks[efee->i]); - start_bitcoin_cli(bitcoind, NULL, process_estimatefee, false, - BITCOIND_LOW_PRIO, - NULL, efee, - "estimatesmartfee", blockstr, efee->estmode[efee->i], - NULL); + struct jsonrpc_request *req; + + req = jsonrpc_request_start(bitcoind, "getfeerate", + bitcoind->log, getfeerate_callback, + call); + json_add_num(req->stream, "blocks", call->blocks[call->i]); + json_add_string(req->stream, "mode", call->estmode[call->i]); + jsonrpc_request_end(req); + plugin_request_send(strmap_get(&bitcoind->pluginsmap, + "getfeerate"), req); } void bitcoind_estimate_fees_(struct bitcoind *bitcoind, @@ -482,8 +196,9 @@ void bitcoind_estimate_fees_(struct bitcoind *bitcoind, const u32 satoshi_per_kw[], void *), void *arg) { - struct estimatefee *efee = tal(bitcoind, struct estimatefee); + struct estimatefee_call *efee = tal(bitcoind, struct estimatefee_call); + efee->bitcoind = bitcoind; efee->i = 0; efee->blocks = tal_dup_arr(efee, u32, blocks, num_estimates, 0); efee->estmode = tal_dup_arr(efee, const char *, estmode, num_estimates, @@ -495,17 +210,6 @@ void bitcoind_estimate_fees_(struct bitcoind *bitcoind, do_one_estimatefee(bitcoind, efee); } -/* Our Bitcoin backend plugin gave us a bad response. We can't recover. */ -static void bitcoin_plugin_error(struct bitcoind *bitcoind, const char *buf, - const jsmntok_t *toks, const char *method, - const char *reason) -{ - struct plugin *p = strmap_get(&bitcoind->pluginsmap, method); - fatal("%s error: bad response to %s (%s), response was %.*s", - p->cmd, method, reason, - toks->end - toks->start, buf + toks->start); -} - /* `sendrawtransaction` * * Send a transaction to the Bitcoin backend plugin. If the broadcast was From 6ee5fe482030985e6599bf6b069d011029fa9d61 Mon Sep 17 00:00:00 2001 From: darosior Date: Fri, 10 Jan 2020 12:12:20 +0100 Subject: [PATCH 17/21] lightningd/bitcoind: remove all bitcoin-cli specific code Changelog-Added: pluggable backends for Bitcoin data queries, default still bitcoind (using bitcoin-cli). --- lightningd/bitcoind.c | 21 +++++---------------- lightningd/bitcoind.h | 31 +------------------------------ lightningd/options.c | 24 ------------------------ 3 files changed, 6 insertions(+), 70 deletions(-) diff --git a/lightningd/bitcoind.c b/lightningd/bitcoind.c index 6d6fde4d6433..47c905e6549b 100644 --- a/lightningd/bitcoind.c +++ b/lightningd/bitcoind.c @@ -1,4 +1,8 @@ -/* Code for talking to bitcoind. We use bitcoin-cli. */ +/* Code for talking to bitcoind. We use a plugin as the Bitcoin backend. + * The default one shipped with C-lightning is a plugin which talks to bitcoind + * by using bitcoin-cli, but the interface we use to gather Bitcoin data is + * standardized and you can use another plugin as the Bitcoin backend, or + * even make your own! */ #include "bitcoin/base58.h" #include "bitcoin/block.h" #include "bitcoin/feerate.h" @@ -739,8 +743,6 @@ void bitcoind_getfilteredblock_(struct bitcoind *bitcoind, u32 height, static void destroy_bitcoind(struct bitcoind *bitcoind) { strmap_clear(&bitcoind->pluginsmap); - /* Suppresses the callbacks from bcli_finished as we free conns. */ - bitcoind->shutdown = true; } struct bitcoind *new_bitcoind(const tal_t *ctx, @@ -750,22 +752,9 @@ struct bitcoind *new_bitcoind(const tal_t *ctx, struct bitcoind *bitcoind = tal(ctx, struct bitcoind); strmap_init(&bitcoind->pluginsmap); - bitcoind->cli = NULL; - bitcoind->datadir = NULL; bitcoind->ld = ld; bitcoind->log = log; - for (size_t i = 0; i < BITCOIND_NUM_PRIO; i++) { - bitcoind->num_requests[i] = 0; - list_head_init(&bitcoind->pending[i]); - } list_head_init(&bitcoind->pending_getfilteredblock); - bitcoind->shutdown = false; - bitcoind->error_count = 0; - bitcoind->retry_timeout = 60; - bitcoind->rpcuser = NULL; - bitcoind->rpcpass = NULL; - bitcoind->rpcconnect = NULL; - bitcoind->rpcport = NULL; tal_add_destructor(bitcoind, destroy_bitcoind); bitcoind->synced = false; diff --git a/lightningd/bitcoind.h b/lightningd/bitcoind.h index 97f6f7bb2849..0afbbe110411 100644 --- a/lightningd/bitcoind.h +++ b/lightningd/bitcoind.h @@ -19,48 +19,19 @@ struct ripemd160; struct bitcoin_tx; struct bitcoin_block; -enum bitcoind_prio { - BITCOIND_LOW_PRIO, - BITCOIND_HIGH_PRIO -}; -#define BITCOIND_NUM_PRIO (BITCOIND_HIGH_PRIO+1) - struct bitcoind { - /* eg. "bitcoin-cli" */ - char *cli; - - /* -datadir arg for bitcoin-cli. */ - char *datadir; - /* Where to do logging. */ struct log *log; /* Main lightningd structure */ struct lightningd *ld; - /* Is bitcoind synced? If not, we retry. */ + /* Is our Bitcoin backend synced? If not, we retry. */ bool synced; - /* How many high/low prio requests are we running (it's ratelimited) */ - size_t num_requests[BITCOIND_NUM_PRIO]; - - /* Pending requests (high and low prio). */ - struct list_head pending[BITCOIND_NUM_PRIO]; - - /* If non-zero, time we first hit a bitcoind error. */ - unsigned int error_count; - struct timemono first_error_time; - /* Ignore results, we're shutting down. */ bool shutdown; - /* How long to keep trying to contact bitcoind - * before fatally exiting. */ - u64 retry_timeout; - - /* Passthrough parameters for bitcoin-cli */ - char *rpcuser, *rpcpass, *rpcconnect, *rpcport; - struct list_head pending_getfilteredblock; /* Map each method to a plugin, so we can have multiple plugins diff --git a/lightningd/options.c b/lightningd/options.c index e81256a81b7a..d76b30f6feb9 100644 --- a/lightningd/options.c +++ b/lightningd/options.c @@ -789,35 +789,11 @@ static void register_opts(struct lightningd *ld) opt_register_noarg("--help|-h", opt_lightningd_usage, ld, "Print this message."); - opt_register_arg("--bitcoin-datadir", opt_set_talstr, NULL, - &ld->topology->bitcoind->datadir, - "-datadir arg for bitcoin-cli"); opt_register_arg("--rgb", opt_set_rgb, NULL, ld, "RRGGBB hex color for node"); opt_register_arg("--alias", opt_set_alias, NULL, ld, "Up to 32-byte alias for node"); - opt_register_arg("--bitcoin-cli", opt_set_talstr, NULL, - &ld->topology->bitcoind->cli, - "bitcoin-cli pathname"); - opt_register_arg("--bitcoin-rpcuser", opt_set_talstr, NULL, - &ld->topology->bitcoind->rpcuser, - "bitcoind RPC username"); - opt_register_arg("--bitcoin-rpcpassword", opt_set_talstr, NULL, - &ld->topology->bitcoind->rpcpass, - "bitcoind RPC password"); - opt_register_arg("--bitcoin-rpcconnect", opt_set_talstr, NULL, - &ld->topology->bitcoind->rpcconnect, - "bitcoind RPC host to connect to"); - opt_register_arg("--bitcoin-rpcport", opt_set_talstr, NULL, - &ld->topology->bitcoind->rpcport, - "bitcoind RPC port"); - opt_register_arg("--bitcoin-retry-timeout", - opt_set_u64, opt_show_u64, - &ld->topology->bitcoind->retry_timeout, - "how long to keep trying to contact bitcoind " - "before fatally exiting"); - opt_register_arg("--pid-file=", opt_set_talstr, opt_show_charp, &ld->pidfile, "Specify pid file"); From c3b175d1c5314b8e237e22c42756bd85f04d959b Mon Sep 17 00:00:00 2001 From: darosior Date: Wed, 5 Feb 2020 10:27:03 +0100 Subject: [PATCH 18/21] lightningd: clean utxos after the topology is setup We need our Bitcoin backend to be ready to get infos about some utxos --- lightningd/lightningd.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lightningd/lightningd.c b/lightningd/lightningd.c index 3b2ac1520e55..2d0f56c98348 100644 --- a/lightningd/lightningd.c +++ b/lightningd/lightningd.c @@ -839,10 +839,6 @@ int main(int argc, char *argv[]) else if (max_blockheight != UINT32_MAX) max_blockheight -= ld->config.rescan; - /*~ Tell the wallet to start figuring out what to do for any reserved - * unspent outputs we may have crashed with. */ - wallet_clean_utxos(ld->wallet, ld->topology->bitcoind); - /*~ That's all of the wallet db operations for now. */ db_commit_transaction(ld->wallet->db); @@ -851,10 +847,14 @@ int main(int argc, char *argv[]) setup_topology(ld->topology, ld->timers, min_blockheight, max_blockheight); + db_begin_transaction(ld->wallet->db); + /*~ Tell the wallet to start figuring out what to do for any reserved + * unspent outputs we may have crashed with. */ + wallet_clean_utxos(ld->wallet, ld->topology->bitcoind); + /*~ Pull peers, channels and HTLCs from db. Needs to happen after the * topology is initialized since some decisions rely on being able to * know the blockheight. */ - db_begin_transaction(ld->wallet->db); unconnected_htlcs_in = load_channels_from_wallet(ld); db_commit_transaction(ld->wallet->db); From 08c0e08b23c3eec695c6916676f13c778eb82e58 Mon Sep 17 00:00:00 2001 From: darosior Date: Wed, 5 Feb 2020 16:59:00 +0100 Subject: [PATCH 19/21] pytest: adjust mocks and logs for bitcoind For bitcoind_fail_first: We only ever send `getblock` if we got a successful block hash from `getblockhash`, and if we can't get the block in that case it means our Bitcoin backend is faulty and we shouldnt continue. So, mock `getblockhash` instead, which is authorized to spuriously fail. For both bitcoind_fail_first and bitcoind_failure: Adapt the logs. --- tests/test_misc.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/test_misc.py b/tests/test_misc.py index fcd79e217b4a..4b3ce54e4783 100644 --- a/tests/test_misc.py +++ b/tests/test_misc.py @@ -105,11 +105,11 @@ def crash_bitcoincli(r): l1.daemon.rpcproxy.mock_rpc('getblockhash', crash_bitcoincli) # This should cause both estimatefee and getblockhash fail - l1.daemon.wait_for_logs(['estimatesmartfee .* exited with status 1', + l1.daemon.wait_for_logs(['Unable to estimate .* fee', 'getblockhash .* exited with status 1']) # And they should retry! - l1.daemon.wait_for_logs(['estimatesmartfee .* exited with status 1', + l1.daemon.wait_for_logs(['Unable to estimate .* fee', 'getblockhash .* exited with status 1']) # Restore, then it should recover and get blockheight. @@ -137,7 +137,7 @@ def test_bitcoin_ibd(node_factory, bitcoind): # "Finish" IDB. l1.daemon.rpcproxy.mock_rpc('getblockchaininfo', None) - l1.daemon.wait_for_log('Bitcoind now synced') + l1.daemon.wait_for_log('Bitcoin backend now synced') assert 'warning_bitcoind_sync' not in l1.rpc.getinfo() @@ -1648,7 +1648,7 @@ def test_bitcoind_fail_first(node_factory, bitcoind, executor): def mock_fail(*args): raise ValueError() - l1.daemon.rpcproxy.mock_rpc('getblock', mock_fail) + l1.daemon.rpcproxy.mock_rpc('getblockhash', mock_fail) l1.daemon.rpcproxy.mock_rpc('estimatesmartfee', mock_fail) f = executor.submit(l1.start) @@ -1657,12 +1657,12 @@ def mock_fail(*args): # Make sure it fails on the first `getblock` call (need to use `is_in_log` # since the `wait_for_log` in `start` sets the offset) wait_for(lambda: l1.daemon.is_in_log( - r'getblock [a-z0-9]* false exited with status 1')) + r'getblockhash [a-z0-9]* exited with status 1')) wait_for(lambda: l1.daemon.is_in_log( - r'estimatesmartfee 2 CONSERVATIVE exited with status 1')) + r'Unable to estimate CONSERVATIVE/2 fee')) # Now unset the mock, so calls go through again - l1.daemon.rpcproxy.mock_rpc('getblock', None) + l1.daemon.rpcproxy.mock_rpc('getblockhash', None) l1.daemon.rpcproxy.mock_rpc('estimatesmartfee', None) f.result() From 765033db5fe203a7cbf711f3f4f3f9526f9ce772 Mon Sep 17 00:00:00 2001 From: darosior Date: Wed, 5 Feb 2020 17:58:45 +0100 Subject: [PATCH 20/21] pytest: fixup flaky test_closing_specified_destination --- tests/test_closing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_closing.py b/tests/test_closing.py index 0e8700d500a6..88c7b9738c2c 100644 --- a/tests/test_closing.py +++ b/tests/test_closing.py @@ -328,7 +328,7 @@ def test_closing_specified_destination(node_factory, bitcoind, chainparams): # Both nodes should have disabled the channel in their view wait_for(lambda: len(l1.getactivechannels()) == 0) - assert bitcoind.rpc.getmempoolinfo()['size'] == 3 + wait_for(lambda: bitcoind.rpc.getmempoolinfo()['size'] == 3) # Now grab the close transaction closetxs = {} From f7b48ed0879c1943a7afd7076cd9e753472895e0 Mon Sep 17 00:00:00 2001 From: darosior Date: Thu, 6 Feb 2020 19:00:14 +0100 Subject: [PATCH 21/21] pytest: test Bitcoin plugin registration and the bcli plugin --- lightningd/bitcoind.c | 3 ++ lightningd/chaintopology.c | 3 ++ tests/plugins/bitcoin/part1.py | 37 ++++++++++++++++ tests/plugins/bitcoin/part2.py | 26 +++++++++++ tests/test_plugin.py | 81 ++++++++++++++++++++++++++++++++++ 5 files changed, 150 insertions(+) create mode 100755 tests/plugins/bitcoin/part1.py create mode 100755 tests/plugins/bitcoin/part2.py diff --git a/lightningd/bitcoind.c b/lightningd/bitcoind.c index 47c905e6549b..cb35f951f338 100644 --- a/lightningd/bitcoind.c +++ b/lightningd/bitcoind.c @@ -80,6 +80,9 @@ void bitcoind_check_commands(struct bitcoind *bitcoind) for (i = 0; i < ARRAY_SIZE(methods); i++) { p = find_plugin_for_command(bitcoind->ld, methods[i]); if (p == NULL) { + /* For testing .. */ + log_debug(bitcoind->ld->log, "Missing a Bitcoin plugin" + " command"); fatal("Could not access the plugin for %s, is a " "Bitcoin plugin (by default plugins/bcli) " "registered ?", methods[i]); diff --git a/lightningd/chaintopology.c b/lightningd/chaintopology.c index 1d5cd6a68196..9c86e9efd388 100644 --- a/lightningd/chaintopology.c +++ b/lightningd/chaintopology.c @@ -1001,6 +1001,9 @@ void setup_topology(struct chain_topology *topo, /* This waits for bitcoind. */ bitcoind_check_commands(topo->bitcoind); + /* For testing.. */ + log_debug(topo->ld->log, "All Bitcoin plugin commands registered"); + /* Sanity checks, then topology initialization. */ bitcoind_getchaininfo(topo->bitcoind, true, check_chain, topo); diff --git a/tests/plugins/bitcoin/part1.py b/tests/plugins/bitcoin/part1.py new file mode 100755 index 000000000000..930b786ca975 --- /dev/null +++ b/tests/plugins/bitcoin/part1.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +""" +This registers part of the Bitcoin backend methods. +We only use it for testing startup and we don't care about the actual values. +""" +import time + +from pyln.client import Plugin + + +plugin = Plugin() + + +@plugin.method("getfeerate") +def getfeerate(plugin, **kwargs): + time.sleep(1) + return {} + + +@plugin.method("getrawblockbyheight") +def getblock(plugin, **kwargs): + time.sleep(1) + return {} + + +@plugin.method("getchaininfo") +def getchaininfo(plugin, **kwargs): + time.sleep(1) + return {} + + +# We don't use these options, but it allows us to get to the expected failure. +plugin.add_option("bitcoin-rpcuser", "", "") +plugin.add_option("bitcoin-rpcpassword", "", "") +plugin.add_option("bitcoin-rpcport", "", "") + +plugin.run() diff --git a/tests/plugins/bitcoin/part2.py b/tests/plugins/bitcoin/part2.py new file mode 100755 index 000000000000..33c64e6ffc96 --- /dev/null +++ b/tests/plugins/bitcoin/part2.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +""" +This registers part of the Bitcoin backend methods. +We only use it for testing startup and we don't care about the actual values. +""" +import time + +from pyln.client import Plugin + + +plugin = Plugin() + + +@plugin.method("sendrawtransaction") +def sendtx(plugin, **kwargs): + time.sleep(1) + return {} + + +@plugin.method("getutxout") +def gettxout(plugin, **kwargs): + time.sleep(1) + return {} + + +plugin.run() diff --git a/tests/test_plugin.py b/tests/test_plugin.py index 180b2526168c..5d448def1750 100644 --- a/tests/test_plugin.py +++ b/tests/test_plugin.py @@ -932,3 +932,84 @@ def test_hook_chaining(node_factory): assert(l2.daemon.is_in_log( r'plugin-hook-chain-even.py: htlc_accepted called for payment_hash {}'.format(hash3) )) + + +def test_bitcoin_backend(node_factory, bitcoind): + """ + This tests interaction with the Bitcoin backend, but not specifically bcli + """ + l1 = node_factory.get_node(start=False, options={"disable-plugin": "bcli"}, + may_fail=True, allow_broken_log=True) + + # We don't start if we haven't all the required methods registered. + plugin = os.path.join(os.getcwd(), "tests/plugins/bitcoin/part1.py") + l1.daemon.opts["plugin"] = plugin + try: + l1.daemon.start() + except ValueError: + assert l1.daemon.is_in_log("Missing a Bitcoin plugin command") + # Now we should start if all the commands are registered, even if they + # are registered by two distincts plugins. + del l1.daemon.opts["plugin"] + l1.daemon.opts["plugin-dir"] = os.path.join(os.getcwd(), + "tests/plugins/bitcoin/") + try: + l1.daemon.start() + except ValueError: + msg = "All Bitcoin plugin commands registered" + assert l1.daemon.is_in_log(msg) + else: + raise Exception("We registered all commands but couldn't start!") + else: + raise Exception("We could start without all commands registered !!") + + # But restarting with just bcli is ok + del l1.daemon.opts["plugin-dir"] + del l1.daemon.opts["disable-plugin"] + l1.start() + assert l1.daemon.is_in_log("bitcoin-cli initialized and connected to" + " bitcoind") + + +def test_bcli(node_factory, bitcoind, chainparams): + """ + This tests the bcli plugin, used to gather Bitcoin data from a local + bitcoind. + Mostly sanity checks of the interface.. + """ + l1, l2 = node_factory.get_nodes(2) + + # We cant stop it dynamically + with pytest.raises(RpcError): + l1.rpc.plugin_stop("bcli") + + # Failure case of feerate is tested in test_misc.py + assert "feerate" in l1.rpc.call("getfeerate", {"blocks": 3, + "mode": "CONSERVATIVE"}) + + resp = l1.rpc.call("getchaininfo") + assert resp["chain"] == chainparams['name'] + for field in ["headercount", "blockcount", "ibd"]: + assert field in resp + + # We shouldn't get upset if we ask for an unknown-yet block + resp = l1.rpc.call("getrawblockbyheight", {"height": 500}) + assert resp["blockhash"] is resp["block"] is None + resp = l1.rpc.call("getrawblockbyheight", {"height": 50}) + assert resp["blockhash"] is not None and resp["blockhash"] is not None + # Some other bitcoind-failure cases for this call are covered in + # tests/test_misc.py + + l1.fundwallet(10**5) + l1.connect(l2) + txid = l1.rpc.fundchannel(l2.info["id"], 10**4)["txid"] + txo = l1.rpc.call("getutxout", {"txid": txid, "vout": 0}) + assert (Millisatoshi(txo["amount"]) == Millisatoshi(10**4 * 10**3) + and txo["script"].startswith("0020")) + l1.rpc.close(l2.info["id"]) + # When output is spent, it should give us null ! + txo = l1.rpc.call("getutxout", {"txid": txid, "vout": 0}) + assert txo["amount"] is txo["script"] is None + + resp = l1.rpc.call("sendrawtransaction", {"tx": "dummy"}) + assert not resp["success"] and "decode failed" in resp["errmsg"]