Merge with master

CHANGELOG.md

@@ -4,6 +4,25 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [Unreleased]
+
+### Added
+
+- JSON API: `listpeers` has new field `scratch_txid`: the latest tx in channel.
+
+### Changed
+
+### Deprecated
+
+Note: You should always set `allow-deprecated-apis=false` to test for
+changes.
+
+### Removed
+
+### Fixed
+
+### Security
+
 ## [0.6.1] - 2018-09-11: "Principled Opposition To Segwit"
 
 This release named by ZmnSCPxj.
@@ -51,9 +70,6 @@ This release named by ZmnSCPxj.
 
 ### Deprecated
 
-Note: You should always set `allow-deprecated-apis=false` to test for
-changes.
-
 ### Removed
 
 - JSON API: `listpeers` results no long have `alias` and `color` fields;

Makefile

@@ -38,7 +38,7 @@ ifeq ($(COMPAT),1)
 COMPAT_CFLAGS=-DCOMPAT_V052=1 -DCOMPAT_V060=1
 endif
 
-PYTEST_OPTS := -v -x
+PYTEST_OPTS := -v
 
 # This is where we add new features as bitcoin adds them.
 FEATURES :=
@@ -186,7 +186,6 @@ include external/Makefile
 include bitcoin/Makefile
 include common/Makefile
 include wire/Makefile
-include wallet/Makefile
 include hsmd/Makefile
 include gossipd/Makefile
 include openingd/Makefile
@@ -206,8 +205,14 @@ ifneq ($(TEST_GROUP_COUNT),)
 PYTEST_OPTS += --test-group=$(TEST_GROUP) --test-group-count=$(TEST_GROUP_COUNT)
 endif
 
+# If we run the tests in parallel we can speed testing up by a lot, however we
+# then don't exit on the first error, since that'd kill the other tester
+# processes and result in loads in loads of output. So we only tell py.test to
+# abort early if we aren't running in parallel.
 ifneq ($(PYTEST_PAR),)
 PYTEST_OPTS += -n=$(PYTEST_PAR)
+else
+PYTEST_OPTS += -x
 endif
 
 check:

README.md

@@ -213,6 +213,7 @@ that `state` is `CHANNELD_NORMAL`; after 6 confirmations you can use
 * `FUNDING_SPEND_SEEN` means we've seen the funding transaction spent.
 * `ONCHAIN` means that the `lightning_onchaind` is tracking the onchain
   closing of the channel.
+* `AWAITING_UNILATERAL` means that we're waiting for a unilateral close to hit the blockchain.
 
 All these states have more information about what's going on in the
 `status` field in `listpeers`.

bitcoin/chainparams.c

@@ -4,54 +4,65 @@
 #include <string.h>
 
 const struct chainparams networks[] = {
-    {.index = 0,
-     .network_name = "bitcoin",
+    {.network_name = "bitcoin",
      .bip173_name = "bc",
      .genesis_blockhash = {{{.u.u8 = {0x6f, 0xe2, 0x8c, 0x0a, 0xb6, 0xf1, 0xb3, 0x72, 0xc1, 0xa6, 0xa2, 0x46, 0xae, 0x63, 0xf7, 0x4f, 0x93, 0x1e, 0x83, 0x65, 0xe1, 0x5a, 0x08, 0x9c, 0x68, 0xd6, 0x19, 0x00, 0x00, 0x00, 0x00, 0x00}}}},
      .rpc_port = 8332,
      .cli = "bitcoin-cli",
      .cli_args = NULL,
      .dust_limit = 546,
+     /* BOLT #2:
+      *
+      * The sending node:
+      *...
+      *   - MUST set `funding_satoshis` to less than 2^24 satoshi.
+      */
+     .max_funding_satoshi = (1 << 24) - 1,
+     .max_payment_msat = 0xFFFFFFFFULL,
      /* "Lightning Charge Powers Developers & Blockstream Store" */
      .when_lightning_became_cool = 504500,
      .testnet = false},
-    {.index = 1,
-     .network_name = "regtest",
+    {.network_name = "regtest",
      .bip173_name = "bcrt",
      .genesis_blockhash = {{{.u.u8 = {0x06, 0x22, 0x6e, 0x46, 0x11, 0x1a, 0x0b, 0x59, 0xca, 0xaf, 0x12, 0x60, 0x43, 0xeb, 0x5b, 0xbf, 0x28, 0xc3, 0x4f, 0x3a, 0x5e, 0x33, 0x2a, 0x1f, 0xc7, 0xb2, 0xb7, 0x3c, 0xf1, 0x88, 0x91, 0x0f}}}},
      .rpc_port = 18332,
      .cli = "bitcoin-cli",
      .cli_args = "-regtest",
      .dust_limit = 546,
+     .max_funding_satoshi = (1 << 24) - 1,
+     .max_payment_msat = 0xFFFFFFFFULL,
      .when_lightning_became_cool = 1,
      .testnet = true},
-    {.index = 2,
-     .network_name = "testnet",
+    {.network_name = "testnet",
      .bip173_name = "tb",
      .genesis_blockhash = {{{.u.u8 = {0x43, 0x49, 0x7f, 0xd7, 0xf8, 0x26, 0x95, 0x71, 0x08, 0xf4, 0xa3, 0x0f, 0xd9, 0xce, 0xc3, 0xae, 0xba, 0x79, 0x97, 0x20, 0x84, 0xe9, 0x0e, 0xad, 0x01, 0xea, 0x33, 0x09, 0x00, 0x00, 0x00, 0x00}}}},
      .rpc_port = 18332,
      .cli = "bitcoin-cli",
      .cli_args = "-testnet",
      .dust_limit = 546,
+     .max_funding_satoshi = (1 << 24) - 1,
+     .max_payment_msat = 0xFFFFFFFFULL,
      .testnet = true},
-    {.index = 3,
-     .network_name = "litecoin",
+    {.network_name = "litecoin",
      .bip173_name = "ltc",
      .genesis_blockhash = {{{.u.u8 = {0xe2, 0xbf, 0x04, 0x7e, 0x7e, 0x5a, 0x19, 0x1a, 0xa4, 0xef, 0x34, 0xd3, 0x14, 0x97, 0x9d, 0xc9, 0x98, 0x6e, 0x0f, 0x19, 0x25, 0x1e, 0xda, 0xba, 0x59, 0x40, 0xfd, 0x1f, 0xe3, 0x65, 0xa7, 0x12 }}}},
      .rpc_port = 9332,
      .cli = "litecoin-cli",
      .cli_args = NULL,
      .dust_limit = 100000,
+     .max_funding_satoshi = 60 * ((1 << 24) - 1),
+     .max_payment_msat = 60 * 0xFFFFFFFFULL,
      .when_lightning_became_cool = 1320000,
      .testnet = false},
-    {.index = 4,
-     .network_name = "litecoin-testnet",
+    {.network_name = "litecoin-testnet",
      .bip173_name = "tltc",
      .genesis_blockhash = {{{.u.u8 = { 0xa0, 0x29, 0x3e, 0x4e, 0xeb, 0x3d, 0xa6, 0xe6, 0xf5, 0x6f, 0x81, 0xed, 0x59, 0x5f, 0x57, 0x88, 0x0d, 0x1a, 0x21, 0x56, 0x9e, 0x13, 0xee, 0xfd, 0xd9, 0x51, 0x28, 0x4b, 0x5a, 0x62, 0x66, 0x49 }}}},
      .rpc_port = 19332,
      .cli = "litecoin-cli",
      .cli_args = "-testnet",
      .dust_limit = 100000,
+     .max_funding_satoshi = 60 * ((1 << 24) - 1),
+     .max_payment_msat = 60 * 0xFFFFFFFFULL,
      .when_lightning_became_cool = 1,
      .testnet = true}
 };
@@ -66,13 +77,14 @@ const struct chainparams *chainparams_for_network(const char *network_name)
 	return NULL;
 }
 
-const struct chainparams *chainparams_by_index(const int index)
+const struct chainparams *chainparams_by_chainhash(const struct bitcoin_blkid *chain_hash)
 {
-	if (index >= ARRAY_SIZE(networks) || index < 0) {
-		return NULL;
-	} else {
-		return &networks[index];
+	for (size_t i = 0; i < ARRAY_SIZE(networks); i++) {
+		if (bitcoin_blkid_eq(chain_hash, &networks[i].genesis_blockhash)) {
+			return &networks[i];
+		}
 	}
+	return NULL;
 }
 
 const struct chainparams *chainparams_by_bip173(const char *bip173_name)

bitcoin/chainparams.h

@@ -7,14 +7,15 @@
 #include <stdbool.h>
 
 struct chainparams {
-	const int index;
 	const char *network_name;
 	const char *bip173_name;
 	const struct bitcoin_blkid genesis_blockhash;
 	const int rpc_port;
 	const char *cli;
 	const char *cli_args;
 	const u64 dust_limit;
+	const u64 max_funding_satoshi;
+	const u64 max_payment_msat;
 	const u32 when_lightning_became_cool;
 
 	/* Whether this is a test network or not */
@@ -26,18 +27,16 @@ struct chainparams {
  */
 const struct chainparams *chainparams_for_network(const char *network_name);
 
-/**
- * chainparams_by_index - Helper to get a network by its numeric index
- *
- * We may not want to pass the network name through to subdaemons, so
- * we allows lookup by index.
- */
-const struct chainparams *chainparams_by_index(const int index);
-
 /**
  * chainparams_by_bip173 - Helper to get a network by its bip173 name
  *
  * This lets us decode BOLT11 addresses.
  */
 const struct chainparams *chainparams_by_bip173(const char *bip173_name);
+
+/**
+ * chainparams_by_chainhash - Helper to get a network by its genesis blockhash
+ */
+const struct chainparams *chainparams_by_chainhash(const struct bitcoin_blkid *chain_hash);
+
 #endif /* LIGHTNING_BITCOIN_CHAINPARAMS_H */

channeld/channeld.c

@@ -2614,7 +2614,9 @@ static void init_channel(struct peer *peer)
 	/* channel_id is set from funding txout */
 	derive_channel_id(&peer->channel_id, &funding_txid, funding_txout);
 
-	peer->channel = new_full_channel(peer, &funding_txid, funding_txout,
+	peer->channel = new_full_channel(peer,
+					 &peer->chain_hash,
+					 &funding_txid, funding_txout,
 					 funding_satoshi,
 					 local_msatoshi,
 					 feerate_per_kw,

channeld/full_channel.c

@@ -1,4 +1,5 @@
 #include <assert.h>
+#include <bitcoin/chainparams.h>
 #include <bitcoin/preimage.h>
 #include <bitcoin/script.h>
 #include <bitcoin/tx.h>
@@ -22,6 +23,7 @@
   #include "gen_full_channel_error_names.h"
 
 struct channel *new_full_channel(const tal_t *ctx,
+				 const struct bitcoin_blkid *chain_hash,
 				 const struct bitcoin_txid *funding_txid,
 				 unsigned int funding_txout,
 				 u64 funding_satoshis,
@@ -35,7 +37,9 @@ struct channel *new_full_channel(const tal_t *ctx,
 				 const struct pubkey *remote_funding_pubkey,
 				 enum side funder)
 {
-	struct channel *channel = new_initial_channel(ctx, funding_txid,
+	struct channel *channel = new_initial_channel(ctx,
+						      chain_hash,
+						      funding_txid,
 						      funding_txout,
 						      funding_satoshis,
 						      local_msatoshi,
@@ -359,7 +363,7 @@ static enum channel_add_err add_htlc(struct channel *channel,
 	 * - for channels with `chain_hash` identifying the Bitcoin blockchain:
 	 *    - MUST set the four most significant bytes of `amount_msat` to 0.
 	 */
-	if (htlc->msatoshi & 0xFFFFFFFF00000000ULL) {
+	if (htlc->msatoshi > channel->chainparams->max_payment_msat) {
 		return CHANNEL_ERR_MAX_HTLC_VALUE_EXCEEDED;
 	}
 

channeld/full_channel.h

@@ -27,6 +27,7 @@
  * Returns state, or NULL if malformed.
  */
 struct channel *new_full_channel(const tal_t *ctx,
+				 const struct bitcoin_blkid *chain_hash,
 				 const struct bitcoin_txid *funding_txid,
 				 unsigned int funding_txout,
 				 u64 funding_satoshis,

channeld/test/run-full_channel.c

@@ -336,6 +336,7 @@ int main(void)
 	const struct htlc **htlc_map, **htlcs;
 	const u8 *funding_wscript, **wscripts;
 	size_t i;
+	const struct chainparams *chainparams = chainparams_for_network("bitcoin");
 
 	secp256k1_ctx = wally_get_secp_context();
 	setup_tmpctx();
@@ -443,7 +444,9 @@ int main(void)
 	to_local_msat = 7000000000;
 	to_remote_msat = 3000000000;
 	feerate_per_kw[LOCAL] = feerate_per_kw[REMOTE] = 15000;
-	lchannel = new_full_channel(tmpctx, &funding_txid, funding_output_index,
+	lchannel = new_full_channel(tmpctx,
+				    &chainparams->genesis_blockhash,
+				    &funding_txid, funding_output_index,
 				    funding_amount_satoshi, to_local_msat,
 				    feerate_per_kw,
 				    local_config,
@@ -452,7 +455,9 @@ int main(void)
 				    &local_funding_pubkey,
 				    &remote_funding_pubkey,
 				    LOCAL);
-	rchannel = new_full_channel(tmpctx, &funding_txid, funding_output_index,
+	rchannel = new_full_channel(tmpctx,
+				    &chainparams->genesis_blockhash,
+				    &funding_txid, funding_output_index,
 				    funding_amount_satoshi, to_remote_msat,
 				    feerate_per_kw,
 				    remote_config,

common/initial_channel.c

@@ -1,4 +1,5 @@
 #include <assert.h>
+#include <bitcoin/chainparams.h>
 #include <bitcoin/script.h>
 #include <ccan/tal/str/str.h>
 #include <common/initial_channel.h>
@@ -8,6 +9,7 @@
 #include <inttypes.h>
 
 struct channel *new_initial_channel(const tal_t *ctx,
+				    const struct bitcoin_blkid *chain_hash,
 				    const struct bitcoin_txid *funding_txid,
 				    unsigned int funding_txout,
 				    u64 funding_satoshis,
@@ -58,6 +60,9 @@ struct channel *new_initial_channel(const tal_t *ctx,
 	channel->commitment_number_obscurer
 		= commit_number_obscurer(&channel->basepoints[funder].payment,
 					 &channel->basepoints[!funder].payment);
+	channel->chainparams = chainparams_by_chainhash(chain_hash);
+	if (channel->chainparams == NULL)
+		return tal_free(channel);
 
 	return channel;
 }

common/initial_channel.h

@@ -57,6 +57,9 @@ struct channel {
 
 	/* What it looks like to each side. */
 	struct channel_view view[NUM_SIDES];
+
+	/* Chain params to check against */
+	const struct chainparams *chainparams;
 };
 
 /* Some requirements are self-specified (eg. my dust limit), others
@@ -125,6 +128,7 @@ static inline u16 to_self_delay(const struct channel *channel, enum side side)
 /**
  * new_initial_channel: Given initial fees and funding, what is initial state?
  * @ctx: tal context to allocate return value from.
+ * @chain_hash: Which blockchain are we talking about?
  * @funding_txid: The commitment transaction id.
  * @funding_txout: The commitment transaction output number.
  * @funding_satoshis: The commitment transaction amount.
@@ -142,6 +146,7 @@ static inline u16 to_self_delay(const struct channel *channel, enum side side)
  * Returns channel, or NULL if malformed.
  */
 struct channel *new_initial_channel(const tal_t *ctx,
+				    const struct bitcoin_blkid *chain_hash,
 				    const struct bitcoin_txid *funding_txid,
 				    unsigned int funding_txout,
 				    u64 funding_satoshis,

contrib/Dockerfile.builder

@@ -13,6 +13,7 @@ RUN apt-get -qq update && \
 	automake \
 	clang \
 	cppcheck \
+	docbook-xml \
 	shellcheck \
 	eatmydata \
 	software-properties-common \
@@ -34,6 +35,7 @@ RUN apt-get -qq update && \
 	shellcheck \
 	libxml2-utils \
 	wget \
+	xsltproc \
 	zlib1g-dev && \
 	rm -rf /var/lib/apt/lists/*
 
@@ -49,4 +51,4 @@ RUN cd /tmp/ && \
     rm -rf bitcoin.tar.gz /tmp/bitcoin-$BITCOIN_VERSION
 
 RUN pip3 install --upgrade pip && \
-    python3 -m pip install python-bitcoinlib==0.7.0 pytest==3.0.5 setuptools==36.6.0 pytest-test-groups==1.0.3 flake8==3.5.0 pytest-rerunfailures==3.1 ephemeral-port-reserve==1.1.0 pytest-xdist==1.22.2 flaky==3.4.0
+    python3 -m pip install python-bitcoinlib==0.7.0 pytest==3.0.5 setuptools==36.6.0 pytest-test-groups==1.0.3 flake8==3.5.0 pytest-rerunfailures==3.1 ephemeral-port-reserve==1.1.0 pytest-xdist==1.22.2 flaky==3.4.0 CherryPy==17.3.0 Flask==1.0.2

contrib/Dockerfile.builder.i386

@@ -13,6 +13,7 @@ RUN apt-get -qq update && \
 	automake \
 	clang \
 	cppcheck \
+	docbook-xml \
 	shellcheck \
 	eatmydata \
 	software-properties-common \
@@ -34,6 +35,7 @@ RUN apt-get -qq update && \
 	shellcheck \
 	libxml2-utils \
 	wget \
+	xsltproc \
 	zlib1g-dev && \
 	rm -rf /var/lib/apt/lists/*
 
@@ -49,4 +51,4 @@ RUN cd /tmp/ && \
     rm -rf bitcoin.tar.gz /tmp/bitcoin-$BITCOIN_VERSION
 
 RUN pip3 install --upgrade pip && \
-    python3 -m pip install python-bitcoinlib==0.7.0 pytest==3.0.5 setuptools==36.6.0 pytest-test-groups==1.0.3 flake8==3.5.0 pytest-rerunfailures==3.1 ephemeral-port-reserve==1.1.0 pytest-xdist==1.22.2 flaky==3.4.0
+    python3 -m pip install python-bitcoinlib==0.7.0 pytest==3.0.5 setuptools==36.6.0 pytest-test-groups==1.0.3 flake8==3.5.0 pytest-rerunfailures==3.1 ephemeral-port-reserve==1.1.0 pytest-xdist==1.22.2 flaky==3.4.0 CherryPy==17.3.0 Flask==1.0.2

gossipd/gossip_wire.csv

@@ -52,6 +52,7 @@ gossip_ping,,num_pong_bytes,u16
 gossip_ping,,len,u16
 
 gossip_ping_reply,3108
+gossip_ping_reply,,id,struct pubkey
 # False if id in gossip_ping was unknown.
 gossip_ping_reply,,sent,bool
 # 0 == no pong expected