diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e35234c1341..db1a501cbebc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,25 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Added + +- JSON API: `listpeers` has new field `scratch_txid`: the latest tx in channel. + +### Changed + +### Deprecated + +Note: You should always set `allow-deprecated-apis=false` to test for +changes. + +### Removed + +### Fixed + +### Security + ## [0.6.1] - 2018-09-11: "Principled Opposition To Segwit" This release named by ZmnSCPxj. @@ -51,9 +70,6 @@ This release named by ZmnSCPxj. ### Deprecated -Note: You should always set `allow-deprecated-apis=false` to test for -changes. - ### Removed - JSON API: `listpeers` results no long have `alias` and `color` fields; diff --git a/Makefile b/Makefile index f2324f8d2eb2..6eaa29719847 100644 --- a/Makefile +++ b/Makefile @@ -38,7 +38,7 @@ ifeq ($(COMPAT),1) COMPAT_CFLAGS=-DCOMPAT_V052=1 -DCOMPAT_V060=1 endif -PYTEST_OPTS := -v -x +PYTEST_OPTS := -v # This is where we add new features as bitcoin adds them. FEATURES := @@ -186,7 +186,6 @@ include external/Makefile include bitcoin/Makefile include common/Makefile include wire/Makefile -include wallet/Makefile include hsmd/Makefile include gossipd/Makefile include openingd/Makefile @@ -206,8 +205,14 @@ ifneq ($(TEST_GROUP_COUNT),) PYTEST_OPTS += --test-group=$(TEST_GROUP) --test-group-count=$(TEST_GROUP_COUNT) endif +# If we run the tests in parallel we can speed testing up by a lot, however we +# then don't exit on the first error, since that'd kill the other tester +# processes and result in loads in loads of output. So we only tell py.test to +# abort early if we aren't running in parallel. ifneq ($(PYTEST_PAR),) PYTEST_OPTS += -n=$(PYTEST_PAR) +else +PYTEST_OPTS += -x endif check: diff --git a/README.md b/README.md index 12b7b548a1b3..2cbe86433a41 100644 --- a/README.md +++ b/README.md @@ -213,6 +213,7 @@ that `state` is `CHANNELD_NORMAL`; after 6 confirmations you can use * `FUNDING_SPEND_SEEN` means we've seen the funding transaction spent. * `ONCHAIN` means that the `lightning_onchaind` is tracking the onchain closing of the channel. +* `AWAITING_UNILATERAL` means that we're waiting for a unilateral close to hit the blockchain. All these states have more information about what's going on in the `status` field in `listpeers`. diff --git a/bitcoin/chainparams.c b/bitcoin/chainparams.c index fe4a1c64537e..ed0716b1cabc 100644 --- a/bitcoin/chainparams.c +++ b/bitcoin/chainparams.c @@ -4,54 +4,65 @@ #include const struct chainparams networks[] = { - {.index = 0, - .network_name = "bitcoin", + {.network_name = "bitcoin", .bip173_name = "bc", .genesis_blockhash = {{{.u.u8 = {0x6f, 0xe2, 0x8c, 0x0a, 0xb6, 0xf1, 0xb3, 0x72, 0xc1, 0xa6, 0xa2, 0x46, 0xae, 0x63, 0xf7, 0x4f, 0x93, 0x1e, 0x83, 0x65, 0xe1, 0x5a, 0x08, 0x9c, 0x68, 0xd6, 0x19, 0x00, 0x00, 0x00, 0x00, 0x00}}}}, .rpc_port = 8332, .cli = "bitcoin-cli", .cli_args = NULL, .dust_limit = 546, + /* BOLT #2: + * + * The sending node: + *... + * - MUST set `funding_satoshis` to less than 2^24 satoshi. + */ + .max_funding_satoshi = (1 << 24) - 1, + .max_payment_msat = 0xFFFFFFFFULL, /* "Lightning Charge Powers Developers & Blockstream Store" */ .when_lightning_became_cool = 504500, .testnet = false}, - {.index = 1, - .network_name = "regtest", + {.network_name = "regtest", .bip173_name = "bcrt", .genesis_blockhash = {{{.u.u8 = {0x06, 0x22, 0x6e, 0x46, 0x11, 0x1a, 0x0b, 0x59, 0xca, 0xaf, 0x12, 0x60, 0x43, 0xeb, 0x5b, 0xbf, 0x28, 0xc3, 0x4f, 0x3a, 0x5e, 0x33, 0x2a, 0x1f, 0xc7, 0xb2, 0xb7, 0x3c, 0xf1, 0x88, 0x91, 0x0f}}}}, .rpc_port = 18332, .cli = "bitcoin-cli", .cli_args = "-regtest", .dust_limit = 546, + .max_funding_satoshi = (1 << 24) - 1, + .max_payment_msat = 0xFFFFFFFFULL, .when_lightning_became_cool = 1, .testnet = true}, - {.index = 2, - .network_name = "testnet", + {.network_name = "testnet", .bip173_name = "tb", .genesis_blockhash = {{{.u.u8 = {0x43, 0x49, 0x7f, 0xd7, 0xf8, 0x26, 0x95, 0x71, 0x08, 0xf4, 0xa3, 0x0f, 0xd9, 0xce, 0xc3, 0xae, 0xba, 0x79, 0x97, 0x20, 0x84, 0xe9, 0x0e, 0xad, 0x01, 0xea, 0x33, 0x09, 0x00, 0x00, 0x00, 0x00}}}}, .rpc_port = 18332, .cli = "bitcoin-cli", .cli_args = "-testnet", .dust_limit = 546, + .max_funding_satoshi = (1 << 24) - 1, + .max_payment_msat = 0xFFFFFFFFULL, .testnet = true}, - {.index = 3, - .network_name = "litecoin", + {.network_name = "litecoin", .bip173_name = "ltc", .genesis_blockhash = {{{.u.u8 = {0xe2, 0xbf, 0x04, 0x7e, 0x7e, 0x5a, 0x19, 0x1a, 0xa4, 0xef, 0x34, 0xd3, 0x14, 0x97, 0x9d, 0xc9, 0x98, 0x6e, 0x0f, 0x19, 0x25, 0x1e, 0xda, 0xba, 0x59, 0x40, 0xfd, 0x1f, 0xe3, 0x65, 0xa7, 0x12 }}}}, .rpc_port = 9332, .cli = "litecoin-cli", .cli_args = NULL, .dust_limit = 100000, + .max_funding_satoshi = 60 * ((1 << 24) - 1), + .max_payment_msat = 60 * 0xFFFFFFFFULL, .when_lightning_became_cool = 1320000, .testnet = false}, - {.index = 4, - .network_name = "litecoin-testnet", + {.network_name = "litecoin-testnet", .bip173_name = "tltc", .genesis_blockhash = {{{.u.u8 = { 0xa0, 0x29, 0x3e, 0x4e, 0xeb, 0x3d, 0xa6, 0xe6, 0xf5, 0x6f, 0x81, 0xed, 0x59, 0x5f, 0x57, 0x88, 0x0d, 0x1a, 0x21, 0x56, 0x9e, 0x13, 0xee, 0xfd, 0xd9, 0x51, 0x28, 0x4b, 0x5a, 0x62, 0x66, 0x49 }}}}, .rpc_port = 19332, .cli = "litecoin-cli", .cli_args = "-testnet", .dust_limit = 100000, + .max_funding_satoshi = 60 * ((1 << 24) - 1), + .max_payment_msat = 60 * 0xFFFFFFFFULL, .when_lightning_became_cool = 1, .testnet = true} }; @@ -66,13 +77,14 @@ const struct chainparams *chainparams_for_network(const char *network_name) return NULL; } -const struct chainparams *chainparams_by_index(const int index) +const struct chainparams *chainparams_by_chainhash(const struct bitcoin_blkid *chain_hash) { - if (index >= ARRAY_SIZE(networks) || index < 0) { - return NULL; - } else { - return &networks[index]; + for (size_t i = 0; i < ARRAY_SIZE(networks); i++) { + if (bitcoin_blkid_eq(chain_hash, &networks[i].genesis_blockhash)) { + return &networks[i]; + } } + return NULL; } const struct chainparams *chainparams_by_bip173(const char *bip173_name) diff --git a/bitcoin/chainparams.h b/bitcoin/chainparams.h index 92a0d2d6dc32..44e762101747 100644 --- a/bitcoin/chainparams.h +++ b/bitcoin/chainparams.h @@ -7,7 +7,6 @@ #include struct chainparams { - const int index; const char *network_name; const char *bip173_name; const struct bitcoin_blkid genesis_blockhash; @@ -15,6 +14,8 @@ struct chainparams { const char *cli; const char *cli_args; const u64 dust_limit; + const u64 max_funding_satoshi; + const u64 max_payment_msat; const u32 when_lightning_became_cool; /* Whether this is a test network or not */ @@ -26,18 +27,16 @@ struct chainparams { */ const struct chainparams *chainparams_for_network(const char *network_name); -/** - * chainparams_by_index - Helper to get a network by its numeric index - * - * We may not want to pass the network name through to subdaemons, so - * we allows lookup by index. - */ -const struct chainparams *chainparams_by_index(const int index); - /** * chainparams_by_bip173 - Helper to get a network by its bip173 name * * This lets us decode BOLT11 addresses. */ const struct chainparams *chainparams_by_bip173(const char *bip173_name); + +/** + * chainparams_by_chainhash - Helper to get a network by its genesis blockhash + */ +const struct chainparams *chainparams_by_chainhash(const struct bitcoin_blkid *chain_hash); + #endif /* LIGHTNING_BITCOIN_CHAINPARAMS_H */ diff --git a/channeld/channeld.c b/channeld/channeld.c index 7441c06ccccd..028228ac4ba0 100644 --- a/channeld/channeld.c +++ b/channeld/channeld.c @@ -2614,7 +2614,9 @@ static void init_channel(struct peer *peer) /* channel_id is set from funding txout */ derive_channel_id(&peer->channel_id, &funding_txid, funding_txout); - peer->channel = new_full_channel(peer, &funding_txid, funding_txout, + peer->channel = new_full_channel(peer, + &peer->chain_hash, + &funding_txid, funding_txout, funding_satoshi, local_msatoshi, feerate_per_kw, diff --git a/channeld/full_channel.c b/channeld/full_channel.c index e8b395a41240..98a22b5f8e19 100644 --- a/channeld/full_channel.c +++ b/channeld/full_channel.c @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -22,6 +23,7 @@ #include "gen_full_channel_error_names.h" struct channel *new_full_channel(const tal_t *ctx, + const struct bitcoin_blkid *chain_hash, const struct bitcoin_txid *funding_txid, unsigned int funding_txout, u64 funding_satoshis, @@ -35,7 +37,9 @@ struct channel *new_full_channel(const tal_t *ctx, const struct pubkey *remote_funding_pubkey, enum side funder) { - struct channel *channel = new_initial_channel(ctx, funding_txid, + struct channel *channel = new_initial_channel(ctx, + chain_hash, + funding_txid, funding_txout, funding_satoshis, local_msatoshi, @@ -359,7 +363,7 @@ static enum channel_add_err add_htlc(struct channel *channel, * - for channels with `chain_hash` identifying the Bitcoin blockchain: * - MUST set the four most significant bytes of `amount_msat` to 0. */ - if (htlc->msatoshi & 0xFFFFFFFF00000000ULL) { + if (htlc->msatoshi > channel->chainparams->max_payment_msat) { return CHANNEL_ERR_MAX_HTLC_VALUE_EXCEEDED; } diff --git a/channeld/full_channel.h b/channeld/full_channel.h index b8ef156c8860..103de0a8f145 100644 --- a/channeld/full_channel.h +++ b/channeld/full_channel.h @@ -27,6 +27,7 @@ * Returns state, or NULL if malformed. */ struct channel *new_full_channel(const tal_t *ctx, + const struct bitcoin_blkid *chain_hash, const struct bitcoin_txid *funding_txid, unsigned int funding_txout, u64 funding_satoshis, diff --git a/channeld/test/run-full_channel.c b/channeld/test/run-full_channel.c index 58e8bb3914ba..f8098acf1e34 100644 --- a/channeld/test/run-full_channel.c +++ b/channeld/test/run-full_channel.c @@ -336,6 +336,7 @@ int main(void) const struct htlc **htlc_map, **htlcs; const u8 *funding_wscript, **wscripts; size_t i; + const struct chainparams *chainparams = chainparams_for_network("bitcoin"); secp256k1_ctx = wally_get_secp_context(); setup_tmpctx(); @@ -443,7 +444,9 @@ int main(void) to_local_msat = 7000000000; to_remote_msat = 3000000000; feerate_per_kw[LOCAL] = feerate_per_kw[REMOTE] = 15000; - lchannel = new_full_channel(tmpctx, &funding_txid, funding_output_index, + lchannel = new_full_channel(tmpctx, + &chainparams->genesis_blockhash, + &funding_txid, funding_output_index, funding_amount_satoshi, to_local_msat, feerate_per_kw, local_config, @@ -452,7 +455,9 @@ int main(void) &local_funding_pubkey, &remote_funding_pubkey, LOCAL); - rchannel = new_full_channel(tmpctx, &funding_txid, funding_output_index, + rchannel = new_full_channel(tmpctx, + &chainparams->genesis_blockhash, + &funding_txid, funding_output_index, funding_amount_satoshi, to_remote_msat, feerate_per_kw, remote_config, diff --git a/common/initial_channel.c b/common/initial_channel.c index f2ad81c18004..ad65ec591c9f 100644 --- a/common/initial_channel.c +++ b/common/initial_channel.c @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -8,6 +9,7 @@ #include struct channel *new_initial_channel(const tal_t *ctx, + const struct bitcoin_blkid *chain_hash, const struct bitcoin_txid *funding_txid, unsigned int funding_txout, u64 funding_satoshis, @@ -58,6 +60,9 @@ struct channel *new_initial_channel(const tal_t *ctx, channel->commitment_number_obscurer = commit_number_obscurer(&channel->basepoints[funder].payment, &channel->basepoints[!funder].payment); + channel->chainparams = chainparams_by_chainhash(chain_hash); + if (channel->chainparams == NULL) + return tal_free(channel); return channel; } diff --git a/common/initial_channel.h b/common/initial_channel.h index 9ea5df5732b1..a3ad474545c7 100644 --- a/common/initial_channel.h +++ b/common/initial_channel.h @@ -57,6 +57,9 @@ struct channel { /* What it looks like to each side. */ struct channel_view view[NUM_SIDES]; + + /* Chain params to check against */ + const struct chainparams *chainparams; }; /* Some requirements are self-specified (eg. my dust limit), others @@ -125,6 +128,7 @@ static inline u16 to_self_delay(const struct channel *channel, enum side side) /** * new_initial_channel: Given initial fees and funding, what is initial state? * @ctx: tal context to allocate return value from. + * @chain_hash: Which blockchain are we talking about? * @funding_txid: The commitment transaction id. * @funding_txout: The commitment transaction output number. * @funding_satoshis: The commitment transaction amount. @@ -142,6 +146,7 @@ static inline u16 to_self_delay(const struct channel *channel, enum side side) * Returns channel, or NULL if malformed. */ struct channel *new_initial_channel(const tal_t *ctx, + const struct bitcoin_blkid *chain_hash, const struct bitcoin_txid *funding_txid, unsigned int funding_txout, u64 funding_satoshis, diff --git a/contrib/Dockerfile.builder b/contrib/Dockerfile.builder index 737088071681..76827596671a 100644 --- a/contrib/Dockerfile.builder +++ b/contrib/Dockerfile.builder @@ -13,6 +13,7 @@ RUN apt-get -qq update && \ automake \ clang \ cppcheck \ + docbook-xml \ shellcheck \ eatmydata \ software-properties-common \ @@ -34,6 +35,7 @@ RUN apt-get -qq update && \ shellcheck \ libxml2-utils \ wget \ + xsltproc \ zlib1g-dev && \ rm -rf /var/lib/apt/lists/* @@ -49,4 +51,4 @@ RUN cd /tmp/ && \ rm -rf bitcoin.tar.gz /tmp/bitcoin-$BITCOIN_VERSION RUN pip3 install --upgrade pip && \ - python3 -m pip install python-bitcoinlib==0.7.0 pytest==3.0.5 setuptools==36.6.0 pytest-test-groups==1.0.3 flake8==3.5.0 pytest-rerunfailures==3.1 ephemeral-port-reserve==1.1.0 pytest-xdist==1.22.2 flaky==3.4.0 + python3 -m pip install python-bitcoinlib==0.7.0 pytest==3.0.5 setuptools==36.6.0 pytest-test-groups==1.0.3 flake8==3.5.0 pytest-rerunfailures==3.1 ephemeral-port-reserve==1.1.0 pytest-xdist==1.22.2 flaky==3.4.0 CherryPy==17.3.0 Flask==1.0.2 diff --git a/contrib/Dockerfile.builder.i386 b/contrib/Dockerfile.builder.i386 index 478c00259b3d..814c50a240e8 100644 --- a/contrib/Dockerfile.builder.i386 +++ b/contrib/Dockerfile.builder.i386 @@ -13,6 +13,7 @@ RUN apt-get -qq update && \ automake \ clang \ cppcheck \ + docbook-xml \ shellcheck \ eatmydata \ software-properties-common \ @@ -34,6 +35,7 @@ RUN apt-get -qq update && \ shellcheck \ libxml2-utils \ wget \ + xsltproc \ zlib1g-dev && \ rm -rf /var/lib/apt/lists/* @@ -49,4 +51,4 @@ RUN cd /tmp/ && \ rm -rf bitcoin.tar.gz /tmp/bitcoin-$BITCOIN_VERSION RUN pip3 install --upgrade pip && \ - python3 -m pip install python-bitcoinlib==0.7.0 pytest==3.0.5 setuptools==36.6.0 pytest-test-groups==1.0.3 flake8==3.5.0 pytest-rerunfailures==3.1 ephemeral-port-reserve==1.1.0 pytest-xdist==1.22.2 flaky==3.4.0 + python3 -m pip install python-bitcoinlib==0.7.0 pytest==3.0.5 setuptools==36.6.0 pytest-test-groups==1.0.3 flake8==3.5.0 pytest-rerunfailures==3.1 ephemeral-port-reserve==1.1.0 pytest-xdist==1.22.2 flaky==3.4.0 CherryPy==17.3.0 Flask==1.0.2 diff --git a/gossipd/gossip_wire.csv b/gossipd/gossip_wire.csv index 5e2abc5a43b8..45e11c494c4f 100644 --- a/gossipd/gossip_wire.csv +++ b/gossipd/gossip_wire.csv @@ -52,6 +52,7 @@ gossip_ping,,num_pong_bytes,u16 gossip_ping,,len,u16 gossip_ping_reply,3108 +gossip_ping_reply,,id,struct pubkey # False if id in gossip_ping was unknown. gossip_ping_reply,,sent,bool # 0 == no pong expected diff --git a/gossipd/gossipd.c b/gossipd/gossipd.c index e2dcfcad0a1b..45c00be6a4c9 100644 --- a/gossipd/gossipd.c +++ b/gossipd/gossipd.c @@ -648,7 +648,7 @@ static void handle_pong(struct peer *peer, const u8 *pong) } daemon_conn_send(&peer->daemon->master, - take(towire_gossip_ping_reply(NULL, true, + take(towire_gossip_ping_reply(NULL, &peer->id, true, tal_count(pong)))); } @@ -1557,7 +1557,8 @@ static struct io_plan *ping_req(struct io_conn *conn, struct daemon *daemon, peer = find_peer(daemon, &id); if (!peer) { daemon_conn_send(&daemon->master, - take(towire_gossip_ping_reply(NULL, false, 0))); + take(towire_gossip_ping_reply(NULL, &id, + false, 0))); goto out; } @@ -1581,7 +1582,8 @@ static struct io_plan *ping_req(struct io_conn *conn, struct daemon *daemon, */ if (num_pong_bytes >= 65532) daemon_conn_send(&daemon->master, - take(towire_gossip_ping_reply(NULL, true, 0))); + take(towire_gossip_ping_reply(NULL, &id, + true, 0))); else peer->num_pings_outstanding++; diff --git a/lightningd/Makefile b/lightningd/Makefile index 540c2cf3748c..682fe33a724f 100644 --- a/lightningd/Makefile +++ b/lightningd/Makefile @@ -75,12 +75,12 @@ LIGHTNINGD_SRC := \ lightningd/payalgo.c \ lightningd/peer_control.c \ lightningd/peer_htlcs.c \ + lightningd/ping.c \ lightningd/subd.c \ lightningd/watch.c # Source files without corresponding headers LIGHTNINGD_SRC_NOHDR := \ - lightningd/ping.c \ lightningd/memdump.c LIGHTNINGD_OBJS := $(LIGHTNINGD_SRC:.c=.o) $(LIGHTNINGD_SRC_NOHDR:.c=.o) @@ -100,8 +100,10 @@ LIGHTNINGD_HEADERS_GEN = \ ALL_GEN_HEADERS += $(LIGHTNINGD_HEADERS_GEN) +include wallet/Makefile + # All together in one convenient var -LIGHTNINGD_HEADERS = $(LIGHTNINGD_HEADERS_NOGEN) $(LIGHTNINGD_HEADERS_GEN) $(EXTERNAL_HEADERS) $(WIRE_HEADERS) $(BITCOIN_HEADERS) $(COMMON_HEADERS) $(WALLET_LIB_HEADERS) $(LIGHTNINGD_HSM_CLIENT_HEADERS) +LIGHTNINGD_HEADERS = $(LIGHTNINGD_HEADERS_NOGEN) $(LIGHTNINGD_HEADERS_GEN) $(WALLET_LIB_HEADERS) $(LIGHTNINGD_OBJS): $(LIGHTNINGD_HEADERS) diff --git a/lightningd/bitcoind.c b/lightningd/bitcoind.c index 744b579d92d2..20b434e24746 100644 --- a/lightningd/bitcoind.c +++ b/lightningd/bitcoind.c @@ -21,6 +21,13 @@ #include #include +/* Bitcoind's web server has a default of 4 threads, with queue depth 16. + * It will *fail* rather than queue beyond that, so we must not stress it! + * + * This is how many request for each priority level we have. + */ +#define BITCOIND_MAX_PARALLEL 4 + /* Add the n'th arg to *args, incrementing n and keeping args of size n+1 */ static void add_arg(const char ***args, const char *arg) { @@ -74,6 +81,8 @@ struct bitcoin_cli { int *exitstatus; pid_t pid; const char **args; + struct timeabs start; + enum bitcoind_prio prio; char *output; size_t output_bytes; size_t new_output; @@ -100,25 +109,25 @@ static struct io_plan *output_init(struct io_conn *conn, struct bitcoin_cli *bcl return read_more(conn, bcli); } -static void next_bcli(struct bitcoind *bitcoind); +static void next_bcli(struct bitcoind *bitcoind, enum bitcoind_prio prio); /* For printing: simple string of args. */ -static char *bcli_args(struct bitcoin_cli *bcli) +static char *bcli_args(const tal_t *ctx, struct bitcoin_cli *bcli) { size_t i; - char *ret = tal_strdup(bcli, bcli->args[0]); + char *ret = tal_strdup(ctx, bcli->args[0]); for (i = 1; bcli->args[i]; i++) { - ret = tal_strcat(bcli, take(ret), " "); - ret = tal_strcat(bcli, take(ret), bcli->args[i]); + ret = tal_strcat(ctx, take(ret), " "); + ret = tal_strcat(ctx, take(ret), bcli->args[i]); } return ret; } static void retry_bcli(struct bitcoin_cli *bcli) { - list_add_tail(&bcli->bitcoind->pending, &bcli->list); - next_bcli(bcli->bitcoind); + list_add_tail(&bcli->bitcoind->pending[bcli->prio], &bcli->list); + next_bcli(bcli->bitcoind, bcli->prio); } /* We allow 60 seconds of spurious errors, eg. reorg. */ @@ -134,14 +143,15 @@ static void bcli_failure(struct bitcoind *bitcoind, t = timemono_between(time_mono(), bitcoind->first_error_time); if (time_greater(t, time_from_sec(60))) fatal("%s exited %u (after %u other errors) '%.*s'", - bcli_args(bcli), + bcli_args(tmpctx, bcli), exitstatus, bitcoind->error_count, (int)bcli->output_bytes, bcli->output); log_unusual(bitcoind->log, - "%s exited with status %u", bcli_args(bcli), exitstatus); + "%s exited with status %u", + bcli_args(tmpctx, bcli), exitstatus); bitcoind->error_count++; @@ -154,23 +164,29 @@ static void bcli_finished(struct io_conn *conn UNUSED, struct bitcoin_cli *bcli) { int ret, status; struct bitcoind *bitcoind = bcli->bitcoind; + enum bitcoind_prio prio = bcli->prio; bool ok; + log_debug(bitcoind->log, "bitcoin-cli: finished %s (%"PRIu64" ms)", + bcli_args(tmpctx, bcli), + time_to_msec(time_between(time_now(), bcli->start))); + assert(bitcoind->num_requests[prio] > 0); + /* FIXME: If we waited for SIGCHILD, this could never hang! */ while ((ret = waitpid(bcli->pid, &status, 0)) < 0 && errno == EINTR); if (ret != bcli->pid) - fatal("%s %s", bcli_args(bcli), + fatal("%s %s", bcli_args(tmpctx, bcli), ret == 0 ? "not exited?" : strerror(errno)); if (!WIFEXITED(status)) fatal("%s died with signal %i", - bcli_args(bcli), + bcli_args(tmpctx, bcli), WTERMSIG(status)); if (!bcli->exitstatus) { if (WEXITSTATUS(status) != 0) { bcli_failure(bitcoind, bcli, WEXITSTATUS(status)); - bitcoind->current = NULL; + bitcoind->num_requests[prio]--; goto done; } } else @@ -179,7 +195,7 @@ static void bcli_finished(struct io_conn *conn UNUSED, struct bitcoin_cli *bcli) if (WEXITSTATUS(status) == 0) bitcoind->error_count = 0; - bitcoind->current = NULL; + bitcoind->num_requests[bcli->prio]--; /* Don't continue if were only here because we were freed for shutdown */ if (bitcoind->shutdown) @@ -195,27 +211,32 @@ static void bcli_finished(struct io_conn *conn UNUSED, struct bitcoin_cli *bcli) tal_free(bcli); done: - next_bcli(bitcoind); + next_bcli(bitcoind, prio); } -static void next_bcli(struct bitcoind *bitcoind) +static void next_bcli(struct bitcoind *bitcoind, enum bitcoind_prio prio) { struct bitcoin_cli *bcli; struct io_conn *conn; - if (bitcoind->current) + if (bitcoind->num_requests[prio] >= BITCOIND_MAX_PARALLEL) return; - bcli = list_pop(&bitcoind->pending, struct bitcoin_cli, list); + bcli = list_pop(&bitcoind->pending[prio], struct bitcoin_cli, list); if (!bcli) return; + log_debug(bitcoind->log, "bitcoin-cli: starting %s", + bcli_args(tmpctx, bcli)); bcli->pid = pipecmdarr(&bcli->fd, NULL, &bcli->fd, cast_const2(char **, bcli->args)); if (bcli->pid < 0) fatal("%s exec failed: %s", bcli->args[0], strerror(errno)); - bitcoind->current = bcli; + bcli->start = time_now(); + + bitcoind->num_requests[prio]++; + /* This lifetime is attached to bitcoind command fd */ conn = notleak(io_new_conn(bitcoind, bcli->fd, output_init, bcli)); io_set_finish(conn, bcli_finished, bcli); @@ -247,6 +268,7 @@ start_bitcoin_cli(struct bitcoind *bitcoind, const tal_t *ctx, bool (*process)(struct bitcoin_cli *), bool nonzero_exit_ok, + enum bitcoind_prio prio, void *cb, void *cb_arg, char *cmd, ...) { @@ -255,6 +277,7 @@ start_bitcoin_cli(struct bitcoind *bitcoind, bcli->bitcoind = bitcoind; bcli->process = process; + bcli->prio = prio; bcli->cb = cb; bcli->cb_arg = cb_arg; if (ctx) { @@ -274,8 +297,8 @@ start_bitcoin_cli(struct bitcoind *bitcoind, bcli->args = gather_args(bitcoind, bcli, cmd, ap); va_end(ap); - list_add_tail(&bitcoind->pending, &bcli->list); - next_bcli(bitcoind); + list_add_tail(&bitcoind->pending[bcli->prio], &bcli->list); + next_bcli(bitcoind, bcli->prio); } static bool extract_feerate(struct bitcoin_cli *bcli, @@ -288,13 +311,13 @@ static bool extract_feerate(struct bitcoin_cli *bcli, tokens = json_parse_input(output, output_bytes, &valid); if (!tokens) fatal("%s: %s response", - bcli_args(bcli), + bcli_args(tmpctx, bcli), valid ? "partial" : "invalid"); if (tokens[0].type != JSMN_OBJECT) { log_unusual(bcli->bitcoind->log, "%s: gave non-object (%.*s)?", - bcli_args(bcli), + bcli_args(tmpctx, bcli), (int)output_bytes, output); return false; } @@ -352,7 +375,9 @@ static void do_one_estimatefee(struct bitcoind *bitcoind, char blockstr[STR_MAX_CHARS(u32)]; snprintf(blockstr, sizeof(blockstr), "%u", efee->blocks[efee->i]); - start_bitcoin_cli(bitcoind, NULL, process_estimatefee, false, NULL, efee, + start_bitcoin_cli(bitcoind, NULL, process_estimatefee, false, + BITCOIND_LOW_PRIO, + NULL, efee, "estimatesmartfee", blockstr, efee->estmode[efee->i], NULL); } @@ -398,7 +423,9 @@ void bitcoind_sendrawtx_(struct bitcoind *bitcoind, void *arg) { log_debug(bitcoind->log, "sendrawtransaction: %s", hextx); - start_bitcoin_cli(bitcoind, NULL, process_sendrawtx, true, cb, arg, + start_bitcoin_cli(bitcoind, NULL, process_sendrawtx, true, + BITCOIND_HIGH_PRIO, + cb, arg, "sendrawtransaction", hextx, NULL); } @@ -412,7 +439,7 @@ static bool process_rawblock(struct bitcoin_cli *bcli) blk = bitcoin_block_from_hex(bcli, bcli->output, bcli->output_bytes); if (!blk) fatal("%s: bad block '%.*s'?", - bcli_args(bcli), + bcli_args(tmpctx, bcli), (int)bcli->output_bytes, bcli->output); cb(bcli->bitcoind, blk, bcli->cb_arg); @@ -429,7 +456,9 @@ void bitcoind_getrawblock_(struct bitcoind *bitcoind, char hex[hex_str_size(sizeof(*blockid))]; bitcoin_blkid_to_hex(blockid, hex, sizeof(hex)); - start_bitcoin_cli(bitcoind, NULL, process_rawblock, false, cb, arg, + start_bitcoin_cli(bitcoind, NULL, process_rawblock, false, + BITCOIND_HIGH_PRIO, + cb, arg, "getblock", hex, "false", NULL); } @@ -445,7 +474,7 @@ static bool process_getblockcount(struct bitcoin_cli *bcli) blockcount = strtol(p, &end, 10); if (end == p || *end != '\n') fatal("%s: gave non-numeric blockcount %s", - bcli_args(bcli), p); + bcli_args(tmpctx, bcli), p); cb(bcli->bitcoind, blockcount, bcli->cb_arg); return true; @@ -457,7 +486,9 @@ void bitcoind_getblockcount_(struct bitcoind *bitcoind, void *arg), void *arg) { - start_bitcoin_cli(bitcoind, NULL, process_getblockcount, false, cb, arg, + start_bitcoin_cli(bitcoind, NULL, process_getblockcount, false, + BITCOIND_HIGH_PRIO, + cb, arg, "getblockcount", NULL); } @@ -490,7 +521,7 @@ static bool process_gettxout(struct bitcoin_cli *bcli) string on a spent gettxout */ if (*bcli->exitstatus != 0 || bcli->output_bytes == 0) { log_debug(bcli->bitcoind->log, "%s: not unspent output?", - bcli_args(bcli)); + bcli_args(tmpctx, bcli)); cb(bcli->bitcoind, NULL, bcli->cb_arg); return true; } @@ -498,35 +529,41 @@ static bool process_gettxout(struct bitcoin_cli *bcli) tokens = json_parse_input(bcli->output, bcli->output_bytes, &valid); if (!tokens) fatal("%s: %s response", - bcli_args(bcli), valid ? "partial" : "invalid"); + bcli_args(tmpctx, bcli), valid ? "partial" : "invalid"); if (tokens[0].type != JSMN_OBJECT) fatal("%s: gave non-object (%.*s)?", - bcli_args(bcli), (int)bcli->output_bytes, bcli->output); + bcli_args(tmpctx, bcli), + (int)bcli->output_bytes, bcli->output); valuetok = json_get_member(bcli->output, tokens, "value"); if (!valuetok) fatal("%s: had no value member (%.*s)?", - bcli_args(bcli), (int)bcli->output_bytes, bcli->output); + bcli_args(tmpctx, bcli), + (int)bcli->output_bytes, bcli->output); if (!json_tok_bitcoin_amount(bcli->output, valuetok, &out.amount)) fatal("%s: had bad value (%.*s)?", - bcli_args(bcli), (int)bcli->output_bytes, bcli->output); + bcli_args(tmpctx, bcli), + (int)bcli->output_bytes, bcli->output); scriptpubkeytok = json_get_member(bcli->output, tokens, "scriptPubKey"); if (!scriptpubkeytok) fatal("%s: had no scriptPubKey member (%.*s)?", - bcli_args(bcli), (int)bcli->output_bytes, bcli->output); + bcli_args(tmpctx, bcli), + (int)bcli->output_bytes, bcli->output); hextok = json_get_member(bcli->output, scriptpubkeytok, "hex"); if (!hextok) fatal("%s: had no scriptPubKey->hex member (%.*s)?", - bcli_args(bcli), (int)bcli->output_bytes, bcli->output); + bcli_args(tmpctx, bcli), + (int)bcli->output_bytes, bcli->output); out.script = tal_hexdata(bcli, bcli->output + hextok->start, hextok->end - hextok->start); if (!out.script) fatal("%s: scriptPubKey->hex invalid hex (%.*s)?", - bcli_args(bcli), (int)bcli->output_bytes, bcli->output); + bcli_args(tmpctx, bcli), + (int)bcli->output_bytes, bcli->output); cb(bcli->bitcoind, &out, bcli->cb_arg); return true; @@ -555,7 +592,7 @@ static bool process_getblock(struct bitcoin_cli *bcli) * the callback with NULL to indicate failure */ log_debug(bcli->bitcoind->log, "%s: returned invalid block, is this a pruned node?", - bcli_args(bcli)); + bcli_args(tmpctx, bcli)); cb(bcli->bitcoind, NULL, cbarg); tal_free(go); return true; @@ -563,7 +600,8 @@ static bool process_getblock(struct bitcoin_cli *bcli) if (tokens[0].type != JSMN_OBJECT) fatal("%s: gave non-object (%.*s)?", - bcli_args(bcli), (int)bcli->output_bytes, bcli->output); + bcli_args(tmpctx, bcli), + (int)bcli->output_bytes, bcli->output); /* "tx": [ "1a7bb0f58a5d235d232deb61d9e2208dabe69848883677abe78e9291a00638e8", @@ -573,13 +611,14 @@ static bool process_getblock(struct bitcoin_cli *bcli) txstok = json_get_member(bcli->output, tokens, "tx"); if (!txstok) fatal("%s: had no tx member (%.*s)?", - bcli_args(bcli), (int)bcli->output_bytes, bcli->output); + bcli_args(tmpctx, bcli), + (int)bcli->output_bytes, bcli->output); /* Now, this can certainly happen, if txnum too large. */ txidtok = json_get_arr(txstok, go->txnum); if (!txidtok) { log_debug(bcli->bitcoind->log, "%s: no txnum %u", - bcli_args(bcli), go->txnum); + bcli_args(tmpctx, bcli), go->txnum); cb(bcli->bitcoind, NULL, cbarg); tal_free(go); return true; @@ -589,7 +628,7 @@ static bool process_getblock(struct bitcoin_cli *bcli) txidtok->end - txidtok->start, &txid)) fatal("%s: had bad txid (%.*s)?", - bcli_args(bcli), + bcli_args(tmpctx, bcli), txidtok->end - txidtok->start, bcli->output + txidtok->start); @@ -610,7 +649,7 @@ static bool process_getblockhash_for_txout(struct bitcoin_cli *bcli) if (*bcli->exitstatus != 0) { void *cbarg = go->cbarg; log_debug(bcli->bitcoind->log, "%s: invalid blocknum?", - bcli_args(bcli)); + bcli_args(tmpctx, bcli)); tal_free(go); cb(bcli->bitcoind, NULL, cbarg); return true; @@ -619,7 +658,9 @@ static bool process_getblockhash_for_txout(struct bitcoin_cli *bcli) /* Strip the newline at the end of the previous output */ blockhash = tal_strndup(NULL, bcli->output, bcli->output_bytes-1); - start_bitcoin_cli(bcli->bitcoind, NULL, process_getblock, false, cb, go, + start_bitcoin_cli(bcli->bitcoind, NULL, process_getblock, false, + BITCOIND_LOW_PRIO, + cb, go, "getblock", take(blockhash), NULL); return true; } @@ -640,7 +681,7 @@ void bitcoind_getoutput_(struct bitcoind *bitcoind, /* We may not have topology ourselves that far back, so ask bitcoind */ start_bitcoin_cli(bitcoind, NULL, process_getblockhash_for_txout, - true, cb, go, + true, BITCOIND_LOW_PRIO, cb, go, "getblockhash", take(tal_fmt(NULL, "%u", blocknum)), NULL); @@ -668,7 +709,8 @@ static bool process_getblockhash(struct bitcoin_cli *bcli) || !bitcoin_blkid_from_hex(bcli->output, bcli->output_bytes-1, &blkid)) { fatal("%s: bad blockid '%.*s'", - bcli_args(bcli), (int)bcli->output_bytes, bcli->output); + bcli_args(tmpctx, bcli), + (int)bcli->output_bytes, bcli->output); } cb(bcli->bitcoind, &blkid, bcli->cb_arg); @@ -685,7 +727,9 @@ void bitcoind_getblockhash_(struct bitcoind *bitcoind, char str[STR_MAX_CHARS(height)]; snprintf(str, sizeof(str), "%u", height); - start_bitcoin_cli(bitcoind, NULL, process_getblockhash, true, cb, arg, + start_bitcoin_cli(bitcoind, NULL, process_getblockhash, true, + BITCOIND_HIGH_PRIO, + cb, arg, "getblockhash", str, NULL); } @@ -697,7 +741,7 @@ void bitcoind_gettxout(struct bitcoind *bitcoind, void *arg) { start_bitcoin_cli(bitcoind, NULL, - process_gettxout, true, cb, arg, + process_gettxout, true, BITCOIND_LOW_PRIO, cb, arg, "gettxout", take(type_to_string(NULL, struct bitcoin_txid, txid)), take(tal_fmt(NULL, "%u", outnum)), @@ -803,14 +847,16 @@ struct bitcoind *new_bitcoind(const tal_t *ctx, bitcoind->datadir = NULL; bitcoind->ld = ld; bitcoind->log = log; - bitcoind->current = NULL; + for (size_t i = 0; i < BITCOIND_NUM_PRIO; i++) { + bitcoind->num_requests[i] = 0; + list_head_init(&bitcoind->pending[i]); + } bitcoind->shutdown = false; bitcoind->error_count = 0; bitcoind->rpcuser = NULL; bitcoind->rpcpass = NULL; bitcoind->rpcconnect = NULL; bitcoind->rpcport = NULL; - list_head_init(&bitcoind->pending); tal_add_destructor(bitcoind, destroy_bitcoind); return bitcoind; diff --git a/lightningd/bitcoind.h b/lightningd/bitcoind.h index 99fa8a0cc34d..379a56730ff9 100644 --- a/lightningd/bitcoind.h +++ b/lightningd/bitcoind.h @@ -24,6 +24,12 @@ enum bitcoind_mode { BITCOIND_REGTEST }; +enum bitcoind_prio { + BITCOIND_LOW_PRIO, + BITCOIND_HIGH_PRIO +}; +#define BITCOIND_NUM_PRIO (BITCOIND_HIGH_PRIO+1) + struct bitcoind { /* eg. "bitcoin-cli" */ char *cli; @@ -37,11 +43,11 @@ struct bitcoind { /* Main lightningd structure */ struct lightningd *ld; - /* Are we currently running a bitcoind request (it's ratelimited) */ - struct bitcoin_cli *current; + /* How many high/low prio requests are we running (it's ratelimited) */ + size_t num_requests[BITCOIND_NUM_PRIO]; - /* Pending requests. */ - struct list_head pending; + /* Pending requests (high and low prio). */ + struct list_head pending[BITCOIND_NUM_PRIO]; /* What network are we on? */ const struct chainparams *chainparams; diff --git a/lightningd/gossip_control.c b/lightningd/gossip_control.c index f75de7001807..83357b59e71a 100644 --- a/lightningd/gossip_control.c +++ b/lightningd/gossip_control.c @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -121,7 +122,6 @@ static unsigned gossip_msg(struct subd *gossip, const u8 *msg, const int *fds) case WIRE_GOSSIP_GETNODES_REPLY: case WIRE_GOSSIP_GETROUTE_REPLY: case WIRE_GOSSIP_GETCHANNELS_REPLY: - case WIRE_GOSSIP_PING_REPLY: case WIRE_GOSSIP_SCIDS_REPLY: case WIRE_GOSSIP_QUERY_CHANNEL_RANGE_REPLY: case WIRE_GOSSIP_RESOLVE_CHANNEL_REPLY: @@ -131,6 +131,10 @@ static unsigned gossip_msg(struct subd *gossip, const u8 *msg, const int *fds) case WIRE_GOSSIP_LOCAL_CHANNEL_CLOSE: break; + case WIRE_GOSSIP_PING_REPLY: + ping_reply(gossip, msg); + break; + case WIRE_GOSSIP_GET_TXOUT: get_txout(gossip, msg); break; diff --git a/lightningd/lightningd.c b/lightningd/lightningd.c index 6d09f1e97e4f..336ce6e77785 100644 --- a/lightningd/lightningd.c +++ b/lightningd/lightningd.c @@ -176,6 +176,7 @@ static struct lightningd *new_lightningd(const tal_t *ctx) list_head_init(&ld->waitsendpay_commands); list_head_init(&ld->sendpay_commands); list_head_init(&ld->close_commands); + list_head_init(&ld->ping_commands); /*~ Tal also explicitly supports arrays: it stores the number of * elements, which can be accessed with tal_count() (or tal_bytelen() @@ -493,9 +494,8 @@ static void daemonize_but_keep_dir(struct lightningd *ld) * file-lock a pidfile. This not only prevents accidentally running multiple * daemons on the same database at once, but lets nosy sysadmins see what pid * the currently-running daemon is supposed to be. */ -static void pidfile_create(const struct lightningd *ld) +static int pidfile_create(const struct lightningd *ld) { - char *pid; int pid_fd; /* Create PID file */ @@ -508,6 +508,18 @@ static void pidfile_create(const struct lightningd *ld) /* Problem locking file */ err(1, "lightningd already running? Error locking PID file"); + /*~ As closing the file will remove the lock, we need to keep it open; + * the OS will close it implicitly when we exit for any reason. */ + return pid_fd; +} + +/*~ Writing the pid into the lockfile provides a useful clue to users as to + * what created it; however, we can't do that until we've got a stable process + * id, and if --daemon is specified, that's quite late. */ +static void pidfile_write(const struct lightningd *ld, int pid_fd) +{ + char *pid; + /*~ Note that tal_fmt() is what asprintf() dreams of being. */ pid = tal_fmt(tmpctx, "%d\n", getpid()); /*~ CCAN's write_all writes to a file descriptor, looping if necessary @@ -516,9 +528,6 @@ static void pidfile_create(const struct lightningd *ld) * which write() is when FORTIFY_SOURCE is defined, so we're allowed * to ignore the result without jumping through hoops. */ write_all(pid_fd, pid, strlen(pid)); - - /*~ As closing the file will remove the lock, we need to keep it open; - * the OS will close it implicitly when we exit for any reason. */ } /*~ ccan/io allows overriding the poll() function that is the very core @@ -550,7 +559,7 @@ int main(int argc, char *argv[]) { struct lightningd *ld; u32 min_blockheight, max_blockheight; - int connectd_gossipd_fd; + int connectd_gossipd_fd, pid_fd; /*~ What happens in strange locales should stay there. */ setup_locale(); @@ -664,6 +673,10 @@ int main(int argc, char *argv[]) setup_topology(ld->topology, &ld->timers, min_blockheight, max_blockheight); + /*~ Now create the PID file: this errors out if there's already a + * daemon running, so we call before trying to create an RPC socket. */ + pid_fd = pidfile_create(ld); + /*~ Create RPC socket: now lightning-cli can send us JSON RPC commands * over a UNIX domain socket specified by `ld->rpc_filename`. */ setup_jsonrpc(ld, ld->rpc_filename); @@ -675,9 +688,8 @@ int main(int argc, char *argv[]) if (ld->daemon) daemonize_but_keep_dir(ld); - /*~ Now create the PID file: this has to be after daemonize, since that - * changes our pid! */ - pidfile_create(ld); + /*~ We have to do this after daemonize, since that changes our pid! */ + pidfile_write(ld, pid_fd); /*~ Activate connect daemon. Needs to be after the initialization of * chaintopology, otherwise peers may connect and ask for diff --git a/lightningd/lightningd.h b/lightningd/lightningd.h index cdd90b188562..bfc2b58a35b1 100644 --- a/lightningd/lightningd.h +++ b/lightningd/lightningd.h @@ -162,6 +162,8 @@ struct lightningd { struct list_head sendpay_commands; /* Outstanding close commands. */ struct list_head close_commands; + /* Outstanding ping commands. */ + struct list_head ping_commands; /* Maintained by invoices.c */ struct invoices *invoices; diff --git a/lightningd/log.c b/lightningd/log.c index 1175baa6ce38..ee89e9a4ad14 100644 --- a/lightningd/log.c +++ b/lightningd/log.c @@ -520,7 +520,7 @@ char *arg_log_to_file(const char *arg, struct lightningd *ld) void opt_register_logging(struct lightningd *ld) { opt_register_arg("--log-level", arg_log_level, show_log_level, ld->log, - "log level (debug, info, unusual, broken)"); + "log level (io, debug, info, unusual, broken)"); opt_register_arg("--log-prefix", arg_log_prefix, show_log_prefix, ld->log, "log prefix"); diff --git a/lightningd/opening_control.c b/lightningd/opening_control.c index b03b9120d683..8d43890843b8 100644 --- a/lightningd/opening_control.c +++ b/lightningd/opening_control.c @@ -730,7 +730,8 @@ void peer_start_openingd(struct peer *peer, */ uc->minimum_depth = peer->ld->config.anchor_confirms; - msg = towire_opening_init(NULL, get_chainparams(peer->ld)->index, + msg = towire_opening_init(NULL, + &get_chainparams(peer->ld)->genesis_blockhash, &uc->our_config, max_to_self_delay, min_effective_htlc_capacity_msat, @@ -766,6 +767,7 @@ static void json_fund_channel(struct command *cmd, struct channel *channel; u32 *feerate_per_kw; u8 *msg; + u64 max_funding_satoshi = get_chainparams(cmd->ld)->max_funding_satoshi; fc->cmd = cmd; fc->uc = NULL; @@ -777,7 +779,7 @@ static void json_fund_channel(struct command *cmd, NULL)) return; - if (!json_tok_wtx(&fc->wtx, buffer, sattok, MAX_FUNDING_SATOSHI)) + if (!json_tok_wtx(&fc->wtx, buffer, sattok, max_funding_satoshi)) return; if (!feerate_per_kw) { @@ -820,7 +822,7 @@ static void json_fund_channel(struct command *cmd, BITCOIN_SCRIPTPUBKEY_P2WSH_LEN)) return; - assert(fc->wtx.amount <= MAX_FUNDING_SATOSHI); + assert(fc->wtx.amount <= max_funding_satoshi); peer->uncommitted_channel->fc = tal_steal(peer->uncommitted_channel, fc); fc->uc = peer->uncommitted_channel; diff --git a/lightningd/peer_control.c b/lightningd/peer_control.c index cba4f5130ee5..4dc9fa945b97 100644 --- a/lightningd/peer_control.c +++ b/lightningd/peer_control.c @@ -642,6 +642,12 @@ static void json_add_peer(struct lightningd *ld, json_object_start(response, NULL); json_add_string(response, "state", channel_state_name(channel)); + if (channel->last_tx) { + struct bitcoin_txid txid; + bitcoin_txid(channel->last_tx, &txid); + + json_add_txid(response, "scratch_txid", &txid); + } if (channel->owner) json_add_string(response, "owner", channel->owner->name); diff --git a/lightningd/ping.c b/lightningd/ping.c index c533e96b096b..23cb4f251a87 100644 --- a/lightningd/ping.c +++ b/lightningd/ping.c @@ -10,28 +10,71 @@ #include #include #include +#include #include -static void ping_reply(struct subd *subd, const u8 *msg, const int *fds UNUSED, - struct command *cmd) +struct ping_command { + struct list_node list; + struct pubkey id; + struct command *cmd; +}; + +static struct ping_command *find_ping_cmd(struct lightningd *ld, + const struct pubkey *id) +{ + struct ping_command *i; + + list_for_each(&ld->ping_commands, i, list) { + if (pubkey_eq(id, &i->id)) + return i; + } + return NULL; +} + +static void destroy_ping_command(struct ping_command *pc) +{ + list_del(&pc->list); +} + +static struct ping_command *new_ping_command(const tal_t *ctx, + struct lightningd *ld, + const struct pubkey *peer_id, + struct command *cmd) +{ + struct ping_command *pc = tal(ctx, struct ping_command); + + pc->id = *peer_id; + pc->cmd = cmd; + list_add_tail(&ld->ping_commands, &pc->list); + tal_add_destructor(pc, destroy_ping_command); + + return pc; +} + +void ping_reply(struct subd *subd, const u8 *msg) { u16 totlen; bool ok, sent = true; + struct pubkey id; + struct ping_command *pc; log_debug(subd->ld->log, "Got ping reply!"); - ok = fromwire_gossip_ping_reply(msg, &sent, &totlen); + ok = fromwire_gossip_ping_reply(msg, &id, &sent, &totlen); + + pc = find_ping_cmd(subd->ld, &id); + assert(pc); if (!ok) - command_fail(cmd, LIGHTNINGD, "Bad reply message"); + command_fail(pc->cmd, LIGHTNINGD, "Bad reply message"); else if (!sent) - command_fail(cmd, LIGHTNINGD, "Unknown peer"); + command_fail(pc->cmd, LIGHTNINGD, "Unknown peer"); else { - struct json_result *response = new_json_result(cmd); + struct json_result *response = new_json_result(pc->cmd); json_object_start(response, NULL); json_add_num(response, "totlen", totlen); json_object_end(response); - command_success(cmd, response); + command_success(pc->cmd, response); } } @@ -76,10 +119,12 @@ static void json_ping(struct command *cmd, return; } + /* parent is cmd, so when we complete cmd, we free this. */ + new_ping_command(cmd, cmd->ld, id, cmd); + /* gossipd handles all pinging, even if it's in another daemon. */ msg = towire_gossip_ping(NULL, id, *pongbytes, *len); - subd_req(cmd->ld->gossip, cmd->ld->gossip, - take(msg), -1, 0, ping_reply, cmd); + subd_send_msg(cmd->ld->gossip, take(msg)); command_still_pending(cmd); } diff --git a/lightningd/ping.h b/lightningd/ping.h new file mode 100644 index 000000000000..fec5c90ee242 --- /dev/null +++ b/lightningd/ping.h @@ -0,0 +1,9 @@ +#ifndef LIGHTNING_LIGHTNINGD_PING_H +#define LIGHTNING_LIGHTNINGD_PING_H +#include "config.h" +#include + +struct subd; +void ping_reply(struct subd *subd, const u8 *msg); + +#endif /* LIGHTNING_LIGHTNINGD_PING_H */ diff --git a/openingd/opening_wire.csv b/openingd/opening_wire.csv index 51ae367777ba..5a25577c5b35 100644 --- a/openingd/opening_wire.csv +++ b/openingd/opening_wire.csv @@ -3,8 +3,8 @@ #include opening_init,6000 -# Which network are we configured for (as index into the chainparams)? -opening_init,,network_index,u32 +# Which network are we configured for? +opening_init,,chain_hash,struct bitcoin_blkid # Base configuration we'll offer (channel reserve will vary with amount) opening_init,,our_config,struct channel_config # Minimum/maximum configuration values we'll accept diff --git a/openingd/openingd.c b/openingd/openingd.c index 0aab661fbe52..b17e0eea3b89 100644 --- a/openingd/openingd.c +++ b/openingd/openingd.c @@ -353,9 +353,10 @@ static u8 *funder_channel(struct state *state, temporary_channel_id(&state->channel_id); - if (state->funding_satoshis > MAX_FUNDING_SATOSHI) + if (state->funding_satoshis > state->chainparams->max_funding_satoshi) status_failed(STATUS_FAIL_MASTER_IO, - "funding_satoshis must be < 2^24, not %"PRIu64, + "funding_satoshis must be < %"PRIu64", not %"PRIu64, + state->chainparams->max_funding_satoshi, state->funding_satoshis); /* BOLT #2: @@ -503,6 +504,7 @@ static u8 *funder_channel(struct state *state, bitcoin_txid(funding, &state->funding_txid); state->channel = new_initial_channel(state, + &state->chainparams->genesis_blockhash, &state->funding_txid, state->funding_txout, state->funding_satoshis, @@ -721,7 +723,7 @@ static u8 *fundee_channel(struct state *state, const u8 *open_channel_msg) * * The receiving node ... MUST fail the channel if `funding-satoshis` * is greater than or equal to 2^24 */ - if (state->funding_satoshis > MAX_FUNDING_SATOSHI) { + if (state->funding_satoshis > state->chainparams->max_funding_satoshi) { negotiation_failed(state, false, "funding_satoshis %"PRIu64" too large", state->funding_satoshis); @@ -842,6 +844,7 @@ static u8 *fundee_channel(struct state *state, const u8 *open_channel_msg) type_to_string(msg, struct channel_id, &id_in)); state->channel = new_initial_channel(state, + &chain_hash, &state->funding_txid, state->funding_txout, state->funding_satoshis, @@ -1092,7 +1095,7 @@ int main(int argc, char *argv[]) u8 *msg, *inner; struct pollfd pollfd[3]; struct state *state = tal(NULL, struct state); - u32 network_index; + struct bitcoin_blkid chain_hash; struct secret *none; subdaemon_setup(argc, argv); @@ -1101,7 +1104,7 @@ int main(int argc, char *argv[]) msg = wire_sync_read(tmpctx, REQ_FD); if (!fromwire_opening_init(tmpctx, msg, - &network_index, + &chain_hash, &state->localconf, &state->max_to_self_delay, &state->min_effective_htlc_capacity_msat, @@ -1122,7 +1125,7 @@ int main(int argc, char *argv[]) fail_if_all_error(inner); } - state->chainparams = chainparams_by_index(network_index); + state->chainparams = chainparams_by_chainhash(&chain_hash); /* Initially we're not associated with a channel, but * handle_peer_gossip_or_error wants this. */ memset(&state->channel_id, 0, sizeof(state->channel_id)); diff --git a/tests/btcproxy.py b/tests/btcproxy.py new file mode 100644 index 000000000000..9f7d3d348d58 --- /dev/null +++ b/tests/btcproxy.py @@ -0,0 +1,104 @@ +""" A bitcoind proxy that allows instrumentation and canned responses +""" +from flask import Flask, request +from bitcoin.rpc import JSONRPCError +from bitcoin.rpc import RawProxy as BitcoinProxy +from cheroot.wsgi import Server +from cheroot.wsgi import PathInfoDispatcher + +import decimal +import flask +import json +import logging +import os +import threading + + +class DecimalEncoder(json.JSONEncoder): + """By default json.dumps does not handle Decimals correctly, so we override it's handling + """ + def default(self, o): + if isinstance(o, decimal.Decimal): + return "{:.8f}".format(float(o)) + return super(DecimalEncoder, self).default(o) + + +class BitcoinRpcProxy(object): + def __init__(self, bitcoind, rpcport=0): + self.app = Flask("BitcoindProxy") + self.app.add_url_rule("/", "API entrypoint", self.proxy, methods=['POST']) + self.rpcport = rpcport + self.mocks = {} + self.bitcoind = bitcoind + self.request_count = 0 + + def _handle_request(self, r): + conf_file = os.path.join(self.bitcoind.bitcoin_dir, 'bitcoin.conf') + brpc = BitcoinProxy(btc_conf_file=conf_file) + method = r['method'] + + # If we have set a mock for this method reply with that instead of + # forwarding the request. + if method in self.mocks and type(method) == dict: + return self.mocks[method] + elif method in self.mocks and callable(self.mocks[method]): + return self.mocks[method](r) + + try: + reply = { + "result": brpc._call(r['method'], *r['params']), + "error": None, + "id": r['id'] + } + except JSONRPCError as e: + reply = { + "error": e.error, + "id": r['id'] + } + self.request_count += 1 + return reply + + def proxy(self): + r = json.loads(request.data.decode('ASCII')) + + if isinstance(r, list): + reply = [self._handle_request(subreq) for subreq in r] + else: + reply = self._handle_request(r) + + response = flask.Response(json.dumps(reply, cls=DecimalEncoder)) + response.headers['Content-Type'] = 'application/json' + return response + + def start(self): + d = PathInfoDispatcher({'/': self.app}) + self.server = Server(('0.0.0.0', self.rpcport), d) + self.proxy_thread = threading.Thread(target=self.server.start) + self.proxy_thread.daemon = True + self.proxy_thread.start() + + # Now that bitcoind is running on the real rpcport, let's tell all + # future callers to talk to the proxyport. We use the bind_addr as a + # signal that the port is bound and accepting connections. + while self.server.bind_addr[1] == 0: + pass + self.rpcport = self.server.bind_addr[1] + logging.debug("BitcoinRpcProxy proxying incoming port {} to {}".format(self.rpcport, self.bitcoind.rpcport)) + + def stop(self): + self.server.stop() + self.proxy_thread.join() + logging.debug("BitcoinRpcProxy shut down after processing {} requests".format(self.request_count)) + + def mock_rpc(self, method, response=None): + """Mock the response to a future RPC call of @method + + The response can either be a dict with the full JSON-RPC response, or a + function that returns such a response. If the response is None the mock + is removed and future calls will be passed through to bitcoind again. + + """ + if response is not None: + self.mocks[method] = response + elif method in self.mocks: + del self.mocks[method] diff --git a/tests/fixtures.py b/tests/fixtures.py index 85cfcb1d24dc..a097e249895d 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -1,5 +1,5 @@ from concurrent import futures -from utils import NodeFactory +from utils import NodeFactory, BitcoinD import logging import os @@ -8,7 +8,6 @@ import shutil import sys import tempfile -import utils with open('config.vars') as configfile: @@ -50,12 +49,13 @@ def directory(request, test_base_dir, test_name): # Auto set value if it isn't in the dict yet __attempts[test_name] = __attempts.get(test_name, 0) + 1 directory = os.path.join(test_base_dir, "{}_{}".format(test_name, __attempts[test_name])) + request.node.has_errors = False yield directory # This uses the status set in conftest.pytest_runtest_makereport to # determine whether we succeeded or failed. - if request.node.rep_call.outcome == 'passed': + if not request.node.has_errors and request.node.rep_call.outcome == 'passed': shutil.rmtree(directory) else: logging.debug("Test execution failed, leaving the test directory {} intact.".format(directory)) @@ -68,7 +68,7 @@ def test_name(request): @pytest.fixture def bitcoind(directory): - bitcoind = utils.BitcoinD(bitcoin_dir=directory, rpcport=None) + bitcoind = BitcoinD(bitcoin_dir=directory) try: bitcoind.start() except Exception: @@ -100,36 +100,42 @@ def bitcoind(directory): @pytest.fixture -def node_factory(directory, test_name, bitcoind, executor): +def node_factory(request, directory, test_name, bitcoind, executor): nf = NodeFactory(test_name, bitcoind, executor, directory=directory) yield nf err_count = 0 ok = nf.killall([not n.may_fail for n in nf.nodes]) + + def check_errors(request, err_count, msg): + """Just a simple helper to format a message, set flags on request and then raise + """ + if err_count: + request.node.has_errors = True + raise ValueError(msg.format(err_count)) + if VALGRIND: for node in nf.nodes: err_count += printValgrindErrors(node) - if err_count: - raise ValueError("{} nodes reported valgrind errors".format(err_count)) + check_errors(request, err_count, "{} nodes reported valgrind errors") for node in nf.nodes: err_count += printCrashLog(node) - if err_count: - raise ValueError("{} nodes had crash.log files".format(err_count)) + check_errors(request, err_count, "{} nodes had crash.log files") + for node in nf.nodes: err_count += checkReconnect(node) - if err_count: - raise ValueError("{} nodes had unexpected reconnections".format(err_count)) + check_errors(request, err_count, "{} nodes had unexpected reconnections") + for node in nf.nodes: err_count += checkBadGossipOrder(node) - if err_count: - raise ValueError("{} nodes had bad gossip order".format(err_count)) + check_errors(request, err_count, "{} nodes had bad gossip order") for node in nf.nodes: err_count += checkBadReestablish(node) - if err_count: - raise ValueError("{} nodes had bad reestablish".format(err_count)) + check_errors(request, err_count, "{} nodes had bad reestablish") if not ok: + request.node.has_errors = True raise Exception("At least one lightning exited with unexpected non-zero return code") diff --git a/tests/requirements.txt b/tests/requirements.txt index b0193f03a282..818937667dfe 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -3,3 +3,5 @@ ephemeral-port-reserve==1.1.0 pytest-forked==0.2 pytest-xdist==1.22.2 flaky==3.4.0 +CherryPy==17.3.0 +Flask==1.0.2 diff --git a/tests/test_closing.py b/tests/test_closing.py index 9eacc727dc30..271b998f68da 100644 --- a/tests/test_closing.py +++ b/tests/test_closing.py @@ -475,7 +475,7 @@ def test_onchain_unwatch(node_factory, bitcoind): l1.rpc.dev_fail(l2.info['id']) l1.daemon.wait_for_log('Failing due to dev-fail command') - l1.daemon.wait_for_log('sendrawtx exit 0') + l1.wait_for_channel_onchain(l2.info['id']) l1.bitcoin.generate_block(1) l1.daemon.wait_for_log(' to ONCHAIN') @@ -594,7 +594,7 @@ def test_onchain_dust_out(node_factory, bitcoind, executor): # l1 will drop to chain. l1.daemon.wait_for_log('permfail') - l1.daemon.wait_for_log('sendrawtx exit 0') + l1.wait_for_channel_onchain(l2.info['id']) l1.bitcoin.generate_block(1) l1.daemon.wait_for_log(' to ONCHAIN') l2.daemon.wait_for_log(' to ONCHAIN') @@ -662,7 +662,7 @@ def test_onchain_timeout(node_factory, bitcoind, executor): # l1 will drop to chain. l1.daemon.wait_for_log('permfail') - l1.daemon.wait_for_log('sendrawtx exit 0') + l1.wait_for_channel_onchain(l2.info['id']) l1.bitcoin.generate_block(1) l1.daemon.wait_for_log(' to ONCHAIN') l2.daemon.wait_for_log(' to ONCHAIN') @@ -814,7 +814,7 @@ def test_onchain_feechange(node_factory, bitcoind, executor): # l2 will drop to chain. l2.daemon.wait_for_log('permfail') - l2.daemon.wait_for_log('sendrawtx exit 0') + l2.wait_for_channel_onchain(l1.info['id']) bitcoind.generate_block(1) l1.daemon.wait_for_log(' to ONCHAIN') l2.daemon.wait_for_log(' to ONCHAIN') @@ -890,7 +890,7 @@ def test_onchain_all_dust(node_factory, bitcoind, executor): # l2 will drop to chain. l2.daemon.wait_for_log('permfail') - l2.daemon.wait_for_log('sendrawtx exit 0') + l2.wait_for_channel_onchain(l1.info['id']) # Make l1's fees really high (and wait for it to exceed 50000) l1.set_feerates((100000, 100000, 100000)) @@ -944,7 +944,7 @@ def test_onchain_different_fees(node_factory, bitcoind, executor): # Drop to chain l1.rpc.dev_fail(l2.info['id']) - l1.daemon.wait_for_log('sendrawtx exit 0') + l1.wait_for_channel_onchain(l2.info['id']) bitcoind.generate_block(1) l1.daemon.wait_for_log(' to ONCHAIN') @@ -999,7 +999,7 @@ def test_permfail_new_commit(node_factory, bitcoind, executor): t = executor.submit(l1.pay, l2, 200000000) l2.daemon.wait_for_log('dev_disconnect permfail') - l2.daemon.wait_for_log('sendrawtx exit 0') + l2.wait_for_channel_onchain(l1.info['id']) bitcoind.generate_block(1) l1.daemon.wait_for_log('Their unilateral tx, new commit point') l1.daemon.wait_for_log(' to ONCHAIN') @@ -1037,7 +1037,7 @@ def test_permfail_htlc_in(node_factory, bitcoind, executor): t = executor.submit(l1.pay, l2, 200000000) l2.daemon.wait_for_log('dev_disconnect permfail') - l2.daemon.wait_for_log('sendrawtx exit 0') + l2.wait_for_channel_onchain(l1.info['id']) bitcoind.generate_block(1) l1.daemon.wait_for_log('Their unilateral tx, old commit point') l1.daemon.wait_for_log(' to ONCHAIN') @@ -1082,7 +1082,7 @@ def test_permfail_htlc_out(node_factory, bitcoind, executor): t = executor.submit(l2.pay, l1, 200000000) l2.daemon.wait_for_log('dev_disconnect permfail') - l2.daemon.wait_for_log('sendrawtx exit 0') + l2.wait_for_channel_onchain(l1.info['id']) bitcoind.generate_block(1) l1.daemon.wait_for_log('Their unilateral tx, old commit point') l1.daemon.wait_for_log(' to ONCHAIN') @@ -1144,7 +1144,7 @@ def test_permfail(node_factory, bitcoind): # We fail l2, so l1 will reconnect to it. l2.rpc.dev_fail(l1.info['id']) l2.daemon.wait_for_log('Failing due to dev-fail command') - l2.daemon.wait_for_log('sendrawtx exit 0') + l2.wait_for_channel_onchain(l1.info['id']) assert l1.bitcoin.rpc.getmempoolinfo()['size'] == 1 diff --git a/tests/test_connection.py b/tests/test_connection.py index 5b555866ae9a..5c30f8b56c1b 100644 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -783,7 +783,7 @@ def test_channel_persistence(node_factory, bitcoind, executor): # Now make sure l1 is watching for unilateral closes l2.rpc.dev_fail(l1.info['id']) l2.daemon.wait_for_log('Failing due to dev-fail command') - l2.daemon.wait_for_log('sendrawtx exit 0') + l2.wait_for_channel_onchain(l1.info['id']) bitcoind.generate_block(1) # L1 must notice. @@ -836,8 +836,8 @@ def test_update_fee(node_factory, bitcoind): l2.daemon.wait_for_log(' to CLOSINGD_COMPLETE') # And should put closing into mempool. - l1.daemon.wait_for_log('sendrawtx exit 0') - l2.daemon.wait_for_log('sendrawtx exit 0') + l1.wait_for_channel_onchain(l2.info['id']) + l2.wait_for_channel_onchain(l1.info['id']) bitcoind.generate_block(1) l1.daemon.wait_for_log(' to ONCHAIN') @@ -925,8 +925,8 @@ def test_update_fee_reconnect(node_factory, bitcoind): l1.rpc.close(chan) # And should put closing into mempool. - l1.daemon.wait_for_log('sendrawtx exit 0') - l2.daemon.wait_for_log('sendrawtx exit 0') + l1.wait_for_channel_onchain(l2.info['id']) + l2.wait_for_channel_onchain(l1.info['id']) bitcoind.generate_block(1) l1.daemon.wait_for_log(' to ONCHAIN') @@ -1088,19 +1088,23 @@ def test_fundee_forget_funding_tx_unconfirmed(node_factory, bitcoind): # Let blocks settle. time.sleep(1) - # Prevent funder from broadcasting funding tx. - l1.bitcoind_cmd_override('exit 1') + def mock_sendrawtransaction(r): + return {'error': 'sendrawtransaction disabled'} + + # Prevent funder from broadcasting funding tx (any tx really). + l1.daemon.rpcproxy.mock_rpc('sendrawtransaction', mock_sendrawtransaction) + # Fund the channel. # The process will complete, but funder will be unable # to broadcast and confirm funding tx. l1.rpc.fundchannel(l2.info['id'], 10**6) - # Prevent l1 from timing out bitcoin-cli. - l1.bitcoind_cmd_remove_override() + # Generate blocks until unconfirmed. bitcoind.generate_block(blocks) # fundee will forget channel! l2.daemon.wait_for_log('Forgetting channel: It has been {} blocks'.format(blocks)) + # fundee will also forget and disconnect from peer. assert len(l2.rpc.listpeers(l1.info['id'])['peers']) == 0 @@ -1108,8 +1112,11 @@ def test_fundee_forget_funding_tx_unconfirmed(node_factory, bitcoind): @unittest.skipIf(not DEVELOPER, "needs dev_fail") def test_no_fee_estimate(node_factory, bitcoind, executor): l1 = node_factory.get_node(start=False) - l1.bitcoind_cmd_override(cmd='estimatesmartfee', - failscript="""echo '{ "errors": [ "Insufficient data or no feerate found" ], "blocks": 0 }'; exit 0""") + + # Fail any fee estimation requests until we allow them further down + l1.daemon.rpcproxy.mock_rpc('estimatesmartfee', { + 'error': {"errors": ["Insufficient data or no feerate found"], "blocks": 0} + }) l1.start() l2 = node_factory.get_node() @@ -1143,7 +1150,7 @@ def test_no_fee_estimate(node_factory, bitcoind, executor): l1.daemon.wait_for_log('sendrawtx exit 0') l1.rpc.dev_fail(l2.info['id']) l1.daemon.wait_for_log('Failing due to dev-fail command') - l1.daemon.wait_for_log('sendrawtx exit 0') + l1.wait_for_channel_onchain(l2.info['id']) bitcoind.generate_block(6) wait_for(lambda: only_one(l1.rpc.getpeer(l2.info['id'])['channels'])['state'] == 'ONCHAIN') wait_for(lambda: only_one(l2.rpc.getpeer(l1.info['id'])['channels'])['state'] == 'ONCHAIN') @@ -1165,9 +1172,9 @@ def test_no_fee_estimate(node_factory, bitcoind, executor): l2.pay(l1, 10**9 // 2) l1.rpc.dev_fail(l2.info['id']) l1.daemon.wait_for_log('Failing due to dev-fail command') - l1.daemon.wait_for_log('sendrawtx exit 0') + l1.wait_for_channel_onchain(l2.info['id']) bitcoind.generate_block(5) - l1.daemon.wait_for_log('sendrawtx exit 0') + wait_for(lambda: len(bitcoind.rpc.getrawmempool()) > 0) bitcoind.generate_block(100) # Start estimatesmartfee. @@ -1273,7 +1280,7 @@ def test_dataloss_protection(node_factory, bitcoind): l2.daemon.wait_for_log("Peer permanent failure in CHANNELD_NORMAL: Awaiting unilateral close") # l1 should drop to chain. - l1.daemon.wait_for_log('sendrawtx exit 0') + l1.wait_for_channel_onchain(l2.info['id']) # l2 must NOT drop to chain. l2.daemon.wait_for_log("Cannot broadcast our commitment tx: they have a future one") diff --git a/tests/test_misc.py b/tests/test_misc.py index ed377055ba8c..c57f6a44ed43 100644 --- a/tests/test_misc.py +++ b/tests/test_misc.py @@ -89,7 +89,12 @@ def test_bitcoin_failure(node_factory, bitcoind): # Make sure we're not failing it between getblockhash and getblock. sync_blockheight(bitcoind, [l1]) - l1.bitcoind_cmd_override('exit 1') + def crash_bitcoincli(r): + return {'error': 'go away'} + + # This is not a JSON-RPC response by purpose + l1.daemon.rpcproxy.mock_rpc('estimatesmartfee', crash_bitcoincli) + l1.daemon.rpcproxy.mock_rpc('getblockhash', crash_bitcoincli) # This should cause both estimatefee and getblockhash fail l1.daemon.wait_for_logs(['estimatesmartfee .* exited with status 1', @@ -100,7 +105,9 @@ def test_bitcoin_failure(node_factory, bitcoind): 'getblockhash .* exited with status 1']) # Restore, then it should recover and get blockheight. - l1.bitcoind_cmd_remove_override() + l1.daemon.rpcproxy.mock_rpc('estimatesmartfee', None) + l1.daemon.rpcproxy.mock_rpc('getblockhash', None) + bitcoind.generate_block(5) sync_blockheight(bitcoind, [l1]) @@ -175,7 +182,7 @@ def test_htlc_sig_persistence(node_factory, executor): # This should reload the htlc_sig l2.rpc.dev_fail(l1.info['id']) # Make sure it broadcasts to chain. - l2.daemon.wait_for_log('sendrawtx exit 0') + l2.wait_for_channel_onchain(l1.info['id']) l2.stop() l1.bitcoin.rpc.generate(1) l1.start() @@ -570,8 +577,6 @@ def test_listconfigs(node_factory, bitcoind): configs = l1.rpc.listconfigs() # See utils.py - assert configs['bitcoin-datadir'] == bitcoind.bitcoin_dir - assert configs['lightning-dir'] == l1.daemon.lightning_dir assert configs['allow-deprecated-apis'] is False assert configs['network'] == 'regtest' assert configs['ignore-fee-limits'] is False @@ -860,8 +865,9 @@ def test_ipv4_and_ipv6(node_factory): def test_feerates(node_factory): l1 = node_factory.get_node(options={'log-level': 'io'}, start=False) - l1.bitcoind_cmd_override(cmd='estimatesmartfee', - failscript="""echo '{ "errors": [ "Insufficient data or no feerate found" ], "blocks": 0 }'; exit 0""") + l1.daemon.rpcproxy.mock_rpc('estimatesmartfee', { + 'error': {"errors": ["Insufficient data or no feerate found"], "blocks": 0} + }) l1.start() # Query feerates (shouldn't give any!) @@ -922,6 +928,8 @@ def test_logging(node_factory): logpath = os.path.join(l1.daemon.lightning_dir, 'logfile') logpath_moved = os.path.join(l1.daemon.lightning_dir, 'logfile_moved') + l1.daemon.rpcproxy.start() + l1.daemon.opts['bitcoin-rpcport'] = l1.daemon.rpcproxy.rpcport TailableProc.start(l1.daemon) wait_for(lambda: os.path.exists(logpath)) @@ -931,10 +939,12 @@ def test_logging(node_factory): wait_for(lambda: os.path.exists(logpath)) log1 = open(logpath_moved).readlines() - log2 = open(logpath).readlines() - assert log1[-1].endswith("Ending log due to SIGHUP\n") - assert log2[0].endswith("Started log due to SIGHUP\n") + + def check_new_log(): + log2 = open(logpath).readlines() + return len(log2) > 1 and log2[0].endswith("Started log due to SIGHUP\n") + wait_for(check_new_log) @unittest.skipIf(VALGRIND and not DEVELOPER, diff --git a/tests/utils.py b/tests/utils.py index cc2fe9373798..565d306e6c72 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -4,18 +4,17 @@ import re import shutil import sqlite3 -import stat import string import subprocess import threading import time +from btcproxy import BitcoinRpcProxy from bitcoin.rpc import RawProxy as BitcoinProxy from decimal import Decimal from ephemeral_port_reserve import reserve from lightning import LightningRpc - BITCOIND_CONFIG = { "regtest": 1, "rpcuser": "rpcuser", @@ -233,7 +232,8 @@ def __getattr__(self, name): # Create a callable to do the actual call proxy = BitcoinProxy(btc_conf_file=self.__btc_conf_file__) - f = lambda *args: proxy._call(name, *args) + def f(*args): + return proxy._call(name, *args) # Make debuggers show rather than > @@ -286,21 +286,24 @@ def generate_block(self, numblocks=1): class LightningD(TailableProc): - def __init__(self, lightning_dir, bitcoin_dir, port=9735, random_hsm=False, node_id=0): + def __init__(self, lightning_dir, bitcoind, port=9735, random_hsm=False, node_id=0): TailableProc.__init__(self, lightning_dir) self.lightning_dir = lightning_dir self.port = port self.cmd_prefix = [] self.disconnect_file = None + self.rpcproxy = BitcoinRpcProxy(bitcoind) + self.opts = LIGHTNINGD_CONFIG.copy() opts = { - 'bitcoin-datadir': bitcoin_dir, 'lightning-dir': lightning_dir, 'addr': '127.0.0.1:{}'.format(port), 'allow-deprecated-apis': 'false', 'network': 'regtest', 'ignore-fee-limits': 'false', + 'bitcoin-rpcuser': BITCOIND_CONFIG['rpcuser'], + 'bitcoin-rpcpassword': BITCOIND_CONFIG['rpcpassword'], } for k, v in opts.items(): @@ -341,6 +344,9 @@ def cmd_line(self): return self.cmd_prefix + ['lightningd/lightningd'] + opts def start(self): + self.rpcproxy.start() + + self.opts['bitcoin-rpcport'] = self.rpcproxy.rpcport TailableProc.start(self) self.wait_for_log("Server started with public key") logging.info("LightningD started") @@ -352,6 +358,7 @@ def wait(self, timeout=10): not return before the timeout triggers. """ self.proc.wait(timeout) + self.rpcproxy.stop() return self.proc.returncode @@ -553,6 +560,10 @@ def is_channel_active(self, chanid): active = [(c['short_channel_id'], c['flags']) for c in channels if c['active']] return (chanid, 0) in active and (chanid, 1) in active + def wait_for_channel_onchain(self, peerid): + txid = only_one(only_one(self.rpc.listpeers(peerid)['peers'])['channels'])['scratch_txid'] + wait_for(lambda: txid in self.bitcoin.rpc.getrawmempool()) + def wait_channel_active(self, chanid): wait_for(lambda: self.is_channel_active(chanid), interval=1) @@ -592,31 +603,30 @@ def wait_pay(): # wait for sendpay to comply self.rpc.waitsendpay(rhash) - def bitcoind_cmd_override(self, failscript='exit 1', cmd=None): - # Create and rename, for atomicity. - f = os.path.join(self.daemon.lightning_dir, "bitcoin-cli-fail.tmp") - with open(f, "w") as text_file: - text_file.write(failscript) - os.chmod(f, os.stat(f).st_mode | stat.S_IEXEC) - if cmd: - failfile = "bitcoin-cli-fail-{}".format(cmd) - else: - failfile = "bitcoin-cli-fail" - os.rename(f, os.path.join(self.daemon.lightning_dir, failfile)) - - def bitcoind_cmd_remove_override(self, cmd=None): - if cmd: - failfile = "bitcoin-cli-fail-{}".format(cmd) - else: - failfile = "bitcoin-cli-fail" - os.remove(os.path.join(self.daemon.lightning_dir, failfile)) - # Note: this feeds through the smoother in update_feerate, so changing # it on a running daemon may not give expected result! def set_feerates(self, feerates, wait_for_effect=True): # (bitcoind returns bitcoin per kb, so these are * 4) - self.bitcoind_cmd_override("""case "$*" in *2\ CONSERVATIVE*) FEERATE={};; *4\ ECONOMICAL*) FEERATE={};; *100\ ECONOMICAL*) FEERATE={};; *) exit 98;; esac; echo '{{ "feerate": '$(printf 0.%08u $FEERATE)' }}'; exit 0""".format(feerates[0] * 4, feerates[1] * 4, feerates[2] * 4), - 'estimatesmartfee') + + def mock_estimatesmartfee(r): + params = r['params'] + if params == [2, 'CONSERVATIVE']: + feerate = feerates[0] * 4 + elif params == [4, 'ECONOMICAL']: + feerate = feerates[1] * 4 + elif params == [100, 'ECONOMICAL']: + feerate = feerates[2] * 4 + else: + raise ValueError() + return { + 'id': r['id'], + 'error': None, + 'result': { + 'feerate': Decimal(feerate) / 10**8 + }, + } + self.daemon.rpcproxy.mock_rpc('estimatesmartfee', mock_estimatesmartfee) + if wait_for_effect: self.daemon.wait_for_log('Feerate estimate for .* set to') @@ -673,7 +683,9 @@ def get_nodes(self, num_nodes, opts=None): return [j.result() for j in jobs] - def get_node(self, disconnect=None, options=None, may_fail=False, may_reconnect=False, random_hsm=False, feerates=(15000, 7500, 3750), start=True, fake_bitcoin_cli=False, log_all_io=False): + def get_node(self, disconnect=None, options=None, may_fail=False, + may_reconnect=False, random_hsm=False, + feerates=(15000, 7500, 3750), start=True, log_all_io=False): with self.lock: node_id = self.next_id self.next_id += 1 @@ -687,7 +699,7 @@ def get_node(self, disconnect=None, options=None, may_fail=False, may_reconnect= socket_path = os.path.join(lightning_dir, "lightning-rpc").format(node_id) daemon = LightningD( - lightning_dir, self.bitcoind.bitcoin_dir, + lightning_dir, self.bitcoind, port=port, random_hsm=random_hsm, node_id=node_id ) # If we have a disconnect string, dump it to a file for daemon. @@ -710,17 +722,6 @@ def get_node(self, disconnect=None, options=None, may_fail=False, may_reconnect= if not may_reconnect: daemon.opts["dev-no-reconnect"] = None - cli = os.path.join(lightning_dir, "fake-bitcoin-cli") - with open(cli, "w") as text_file: - text_file.write('#! /bin/sh\n' - '! [ -x bitcoin-cli-fail ] || exec ./bitcoin-cli-fail "$@"\n' - 'for a in "$@"; do\n' - '\t! [ -x bitcoin-cli-fail-"$a" ] || exec ./bitcoin-cli-fail-"$a" "$@"\n' - 'done\n' - 'exec bitcoin-cli "$@"\n') - os.chmod(cli, os.stat(cli).st_mode | stat.S_IEXEC) - daemon.opts['bitcoin-cli'] = cli - if options is not None: daemon.opts.update(options) diff --git a/wallet/Makefile b/wallet/Makefile index 4c82a78db2e4..6f5c9e28e9e0 100644 --- a/wallet/Makefile +++ b/wallet/Makefile @@ -15,7 +15,9 @@ WALLET_LIB_OBJS := $(WALLET_LIB_SRC:.c=.o) WALLET_LIB_HEADERS := $(WALLET_LIB_SRC:.c=.h) # Make sure these depend on everything. -ALL_OBJS += $(LIGHTNINGD_OBJS) +ALL_OBJS += $(WALLET_LIB_OBJS) + +$(WALLET_LIB_OBJS): $(LIGHTNINGD_HEADERS_NOGEN) $(WALLET_LIB_HEADERS) check-whitespace: $(WALLET_LIB_SRC:%=check-whitespace/%) $(WALLET_LIB_HEADERS:%=check-whitespace/%) check-source: $(WALLET_LIB_SRC:%=check-src-include-order/%) diff --git a/wire/peer_wire.h b/wire/peer_wire.h index e1d68029d256..28dd4cec43cf 100644 --- a/wire/peer_wire.h +++ b/wire/peer_wire.h @@ -29,12 +29,4 @@ bool extract_channel_id(const u8 *in_pkt, struct channel_id *channel_id); * the network, as detailed within [BOLT #7] */ #define CHANNEL_FLAGS_ANNOUNCE_CHANNEL 1 - -/* BOLT #2: - * - * The sending node: - *... - * - MUST set `funding_satoshis` to less than 2^24 satoshi. - */ -#define MAX_FUNDING_SATOSHI ((1 << 24) - 1) #endif /* LIGHTNING_WIRE_PEER_WIRE_H */