From c31f7f12d5cf1f0e35eb54b0d1d84a25f0a02ca2 Mon Sep 17 00:00:00 2001 From: niftynei Date: Wed, 22 Jul 2020 21:03:59 -0500 Subject: [PATCH 01/14] psbt: add helpers for creating + adding input/outputs --- bitcoin/psbt.c | 91 ++++++++++++++++++++++++++++++++++++++++++++++++++ bitcoin/psbt.h | 58 ++++++++++++++++++++++++++++++-- 2 files changed, 147 insertions(+), 2 deletions(-) diff --git a/bitcoin/psbt.c b/bitcoin/psbt.c index 5c8a7b4eb80b..c23641645e04 100644 --- a/bitcoin/psbt.c +++ b/bitcoin/psbt.c @@ -21,6 +21,36 @@ void psbt_destroy(struct wally_psbt *psbt) wally_psbt_free(psbt); } +static struct wally_psbt *init_psbt(const tal_t *ctx, size_t num_inputs, size_t num_outputs) +{ + int wally_err; + struct wally_psbt *psbt; + + if (is_elements(chainparams)) + wally_err = wally_psbt_elements_init_alloc(0, num_inputs, num_outputs, 0, &psbt); + else + wally_err = wally_psbt_init_alloc(0, num_inputs, num_outputs, 0, &psbt); + assert(wally_err == WALLY_OK); + tal_add_destructor(psbt, psbt_destroy); + return tal_steal(ctx, psbt); +} + +struct wally_psbt *create_psbt(const tal_t *ctx, size_t num_inputs, size_t num_outputs) +{ + int wally_err; + struct wally_tx *wtx; + struct wally_psbt *psbt; + + if (wally_tx_init_alloc(WALLY_TX_VERSION_2, 0, num_inputs, num_outputs, &wtx) != WALLY_OK) + abort(); + + psbt = init_psbt(ctx, num_inputs, num_outputs); + + wally_err = wally_psbt_set_global_tx(psbt, wtx); + assert(wally_err == WALLY_OK); + return psbt; +} + struct wally_psbt *new_psbt(const tal_t *ctx, const struct wally_tx *wtx) { struct wally_psbt *psbt; @@ -80,6 +110,28 @@ struct wally_psbt_input *psbt_add_input(struct wally_psbt *psbt, return &psbt->inputs[insert_at]; } +struct wally_psbt_input *psbt_append_input(struct wally_psbt *psbt, + const struct bitcoin_txid *txid, + u32 outnum, u32 sequence) +{ + struct wally_tx_input *tx_in; + struct wally_psbt_input *input; + size_t insert_at; + + insert_at = psbt->num_inputs; + + if (wally_tx_input_init_alloc(txid->shad.sha.u.u8, + sizeof(struct bitcoin_txid), + outnum, sequence, NULL, 0, NULL, + &tx_in) != WALLY_OK) + abort(); + + tx_in->features = chainparams->is_elements ? WALLY_TX_IS_ELEMENTS : 0; + input = psbt_add_input(psbt, tx_in, insert_at); + wally_tx_input_free(tx_in); + return input; +} + void psbt_rm_input(struct wally_psbt *psbt, size_t remove_at) { @@ -96,6 +148,18 @@ struct wally_psbt_output *psbt_add_output(struct wally_psbt *psbt, return &psbt->outputs[insert_at]; } +struct wally_psbt_output *psbt_append_out(struct wally_psbt *psbt, + const u8 *script, + struct amount_sat amount) +{ + struct wally_psbt_output *out; + struct wally_tx_output *tx_out = wally_tx_output(script, amount); + + out = psbt_add_output(psbt, tx_out, psbt->tx->num_outputs); + wally_tx_output_free(tx_out); + return out; +} + void psbt_rm_output(struct wally_psbt *psbt, size_t remove_at) { @@ -298,6 +362,24 @@ void psbt_elements_input_init(struct wally_psbt *psbt, size_t in, abort(); } +bool psbt_has_input(struct wally_psbt *psbt, + struct bitcoin_txid *txid, + u32 outnum) +{ + for (size_t i = 0; i < psbt->num_inputs; i++) { + struct bitcoin_txid in_txid; + struct wally_tx_input *in = &psbt->tx->inputs[i]; + + if (outnum != in->index) + continue; + + wally_tx_input_get_txid(in, &in_txid); + if (bitcoin_txid_eq(txid, &in_txid)) + return true; + } + return false; +} + bool psbt_input_set_redeemscript(struct wally_psbt *psbt, size_t in, const u8 *redeemscript) { @@ -329,6 +411,15 @@ struct amount_sat psbt_input_get_amount(struct wally_psbt *psbt, return val; } +struct amount_sat psbt_output_get_amount(struct wally_psbt *psbt, + size_t out) +{ + struct amount_asset asset; + assert(out < psbt->num_outputs); + asset = wally_tx_output_get_amount(&psbt->tx->outputs[out]); + assert(amount_asset_is_main(&asset)); + return amount_asset_to_sat(&asset); +} struct wally_tx *psbt_finalize(struct wally_psbt *psbt, bool finalize_in_place) { struct wally_psbt *tmppsbt; diff --git a/bitcoin/psbt.h b/bitcoin/psbt.h index ff154d985dc6..40e57229677d 100644 --- a/bitcoin/psbt.h +++ b/bitcoin/psbt.h @@ -5,11 +5,11 @@ #include #include -struct wally_tx_input; -struct wally_tx_output; struct wally_psbt; struct wally_psbt_input; struct wally_tx; +struct wally_tx_input; +struct wally_tx_output; struct amount_asset; struct amount_sat; struct bitcoin_signature; @@ -18,6 +18,22 @@ struct pubkey; void psbt_destroy(struct wally_psbt *psbt); +/** + * create_psbt - Create a new psbt object + * + * @ctx - allocation context + * @num_inputs - number of inputs to allocate + * @num_outputs - number of outputs to allocate + */ +struct wally_psbt *create_psbt(const tal_t *ctx, size_t num_inputs, size_t num_outputs); + +/* + * new_psbt - Create a PSBT, using the passed in tx + * as the global_tx + * + * @ctx - allocation context + * @wtx - global_tx starter kit + */ struct wally_psbt *new_psbt(const tal_t *ctx, const struct wally_tx *wtx); @@ -47,6 +63,10 @@ struct wally_psbt_input *psbt_add_input(struct wally_psbt *psbt, struct wally_tx_input *input, size_t insert_at); +struct wally_psbt_input *psbt_append_input(struct wally_psbt *psbt, + const struct bitcoin_txid *txid, + u32 outnum, u32 sequence); + void psbt_rm_input(struct wally_psbt *psbt, size_t remove_at); @@ -54,6 +74,17 @@ struct wally_psbt_output *psbt_add_output(struct wally_psbt *psbt, struct wally_tx_output *output, size_t insert_at); +/** + * wally_psbt_output - Append a new output to the PSBT + * + * @psbt - PSBT to append output to + * @script - scriptPubKey of the output + * @amount - value of the output + */ +struct wally_psbt_output *psbt_append_out(struct wally_psbt *psbt, + const u8 *script, + struct amount_sat amount); + void psbt_rm_output(struct wally_psbt *psbt, size_t remove_at); @@ -81,9 +112,32 @@ void psbt_elements_input_init_witness(struct wally_psbt *psbt, size_t in, const u8 *nonce); bool psbt_input_set_redeemscript(struct wally_psbt *psbt, size_t in, const u8 *redeemscript); +/* psbt_input_get_amount - Returns the value of this input + * + * @psbt - psbt + * @in - index of input whose value you're returning + * */ struct amount_sat psbt_input_get_amount(struct wally_psbt *psbt, size_t in); +/* psbt_output_get_amount - Returns the value of this output + * + * @psbt - psbt + * @out -index of output whose value you're returning + */ +struct amount_sat psbt_output_get_amount(struct wally_psbt *psbt, + size_t out); + +/* psbt_has_input - Is this input present on this psbt + * + * @psbt - psbt + * @txid - txid of input + * @outnum - output index of input + */ +bool psbt_has_input(struct wally_psbt *psbt, + struct bitcoin_txid *txid, + u32 outnum); + struct wally_psbt *psbt_from_b64(const tal_t *ctx, const char *b64, size_t b64len); From 4f449d09050b4fc19677c9c5abbeb434f86a447d Mon Sep 17 00:00:00 2001 From: niftynei Date: Wed, 22 Jul 2020 21:08:12 -0500 Subject: [PATCH 02/14] psbt: utilities for adding unknown info / proprietary c-lightning keydata --- bitcoin/psbt.c | 102 +++++++++++++++++++++++++++++++++++++++++++++++++ bitcoin/psbt.h | 54 ++++++++++++++++++++++++++ 2 files changed, 156 insertions(+) diff --git a/bitcoin/psbt.c b/bitcoin/psbt.c index c23641645e04..94d878dbea52 100644 --- a/bitcoin/psbt.c +++ b/bitcoin/psbt.c @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -420,6 +421,105 @@ struct amount_sat psbt_output_get_amount(struct wally_psbt *psbt, assert(amount_asset_is_main(&asset)); return amount_asset_to_sat(&asset); } + +static void add(u8 **key, const void *mem, size_t len) +{ + size_t oldlen = tal_count(*key); + tal_resize(key, oldlen + len); + memcpy(*key + oldlen, memcheck(mem, len), len); +} + +static void add_type(u8 **key, const u8 num) +{ + add(key, &num, 1); +} + +static void add_varint(u8 **key, size_t val) +{ + u8 vt[VARINT_MAX_LEN]; + size_t vtlen; + vtlen = varint_put(vt, val); + add(key, vt, vtlen); +} + +#define LIGHTNING_PROPRIETARY_PREFIX "lightning" + +u8 *psbt_make_key(const tal_t *ctx, u8 key_subtype, const u8 *key_data) +{ + /** + * BIP174: + * Type: Proprietary Use Type PSBT_GLOBAL_PROPRIETARY = 0xFC + ** Key: Variable length identifier prefix, followed + * by a subtype, followed by the key data itself. + *** {0xFC}||{subtype}|{key data} + ** Value: Any value data as defined by the proprietary type user. + *** + */ + u8 *key = tal_arr(ctx, u8, 0); + add_type(&key, PSBT_PROPRIETARY_TYPE); + add_varint(&key, strlen(LIGHTNING_PROPRIETARY_PREFIX)); + add(&key, LIGHTNING_PROPRIETARY_PREFIX, + strlen(LIGHTNING_PROPRIETARY_PREFIX)); + add_type(&key, key_subtype); + if (key_data) + add(&key, key_data, tal_bytelen(key_data)); + return key; +} + +void psbt_input_add_unknown(struct wally_psbt_input *in, + const u8 *key, + const void *value, + size_t value_len) +{ + if (wally_map_add(&in->unknowns, + cast_const(unsigned char *, key), tal_bytelen(key), + (unsigned char *) memcheck(value, value_len), value_len) + != WALLY_OK) + abort(); +} + +void *psbt_get_unknown(const struct wally_map *map, + const u8 *key, + size_t *val_len) +{ + size_t index; + + if (wally_map_find(map, key, tal_bytelen(key), &index) != WALLY_OK) + return NULL; + + /* Zero: item not found. */ + if (index == 0) + return NULL; + + /* ++: item is at this index minus 1 */ + *val_len = map->items[index - 1].value_len; + return map->items[index - 1].value; +} + +void *psbt_get_lightning(const struct wally_map *map, + const u8 proprietary_type, + size_t *val_len) +{ + void *res; + u8 *key = psbt_make_key(NULL, proprietary_type, NULL); + res = psbt_get_unknown(map, key, val_len); + tal_free(key); + return res; +} + + +void psbt_output_add_unknown(struct wally_psbt_output *out, + const u8 *key, + const void *value, + size_t value_len) +{ + if (wally_map_add(&out->unknowns, + cast_const(unsigned char *, key), tal_bytelen(key), + (unsigned char *) memcheck(value, value_len), value_len) + != WALLY_OK) + abort(); +} + struct wally_tx *psbt_finalize(struct wally_psbt *psbt, bool finalize_in_place) { struct wally_psbt *tmppsbt; @@ -480,6 +580,8 @@ char *psbt_to_b64(const tal_t *ctx, const struct wally_psbt *psbt) wally_free_string(serialized_psbt); return ret_val; } + +/* Do not remove this line, it is magic */ REGISTER_TYPE_TO_STRING(wally_psbt, psbt_to_b64); const u8 *psbt_get_bytes(const tal_t *ctx, const struct wally_psbt *psbt, diff --git a/bitcoin/psbt.h b/bitcoin/psbt.h index 40e57229677d..26a3b29ebe2d 100644 --- a/bitcoin/psbt.h +++ b/bitcoin/psbt.h @@ -10,6 +10,7 @@ struct wally_psbt_input; struct wally_tx; struct wally_tx_input; struct wally_tx_output; +struct wally_map; struct amount_asset; struct amount_sat; struct bitcoin_signature; @@ -59,6 +60,16 @@ void psbt_txid(const struct wally_psbt *psbt, struct bitcoin_txid *txid, struct wally_tx *psbt_finalize(struct wally_psbt *psbt, bool finalize_in_place); +/* psbt_make_key - Create a new, proprietary c-lightning key + * + * @ctx - allocation context + * @key_subtype - type for this key + * @key_data - any extra data to append to the key + * + * Returns a proprietary-prefixed key. + */ +u8 *psbt_make_key(const tal_t *ctx, u8 key_subtype, const u8 *key_data); + struct wally_psbt_input *psbt_add_input(struct wally_psbt *psbt, struct wally_tx_input *input, size_t insert_at); @@ -112,6 +123,49 @@ void psbt_elements_input_init_witness(struct wally_psbt *psbt, size_t in, const u8 *nonce); bool psbt_input_set_redeemscript(struct wally_psbt *psbt, size_t in, const u8 *redeemscript); +/* psbt_input_add_unknown - Add the given Key-Value to the psbt's input keymap + * @in - psbt input to add key-value to + * @key - key for key-value pair + * @value - value to add + * @value_len - length of {@value} + */ +void psbt_input_add_unknown(struct wally_psbt_input *in, + const u8 *key, + const void *value, + size_t value_len); +/* psbt_get_unknown - Fetch the value from the given map at key + * + * @map - map of unknowns to search for key + * @key - key of key-value pair to return value for + * @value_len - (out) length of value (if found) + * + * Returns: value at @key, or NULL if not found */ +void *psbt_get_unknown(const struct wally_map *map, + const u8 *key, + size_t *val_len); + +/* psbt_get_lightning - Fetch a proprietary lightning value from the given map + * + * @map - map of unknowns to search for key + * @proprietary_type - type no. to look for + * @val_len - (out) length of value (if found) + * + * Returns: value of type {proprietary_type}, or NULL if not found */ +void *psbt_get_lightning(const struct wally_map *map, + const u8 proprietary_type, + size_t *val_len); + +/* psbt_output_add_unknown - Add the given Key-Value to the psbt's output keymap + * + * @out - psbt output to add key-value to + * @key - key for key-value pair + * @value - value to add + * @value_len - length of {@value} + */ +void psbt_output_add_unknown(struct wally_psbt_output *out, + const u8 *key, const void *value, + size_t value_len); + /* psbt_input_get_amount - Returns the value of this input * * @psbt - psbt From 764e5126e63c4e3c19a4e1558603337ea4ead48d Mon Sep 17 00:00:00 2001 From: niftynei Date: Wed, 22 Jul 2020 21:08:58 -0500 Subject: [PATCH 03/14] pubkey: update comment to reflect location --- bitcoin/pubkey.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bitcoin/pubkey.h b/bitcoin/pubkey.h index 7477d9f56da3..725fde369eb4 100644 --- a/bitcoin/pubkey.h +++ b/bitcoin/pubkey.h @@ -45,7 +45,7 @@ void pubkey_to_der(u8 der[PUBKEY_CMPR_LEN], const struct pubkey *key); /* Compare the keys `a` and `b`. Return <0 if `a`<`b`, 0 if equal and >0 otherwise */ int pubkey_cmp(const struct pubkey *a, const struct pubkey *b); -/* If the two nodes[] are id1 and id2, which index would id1 be? */ +/* If the two pubkeys[] are id1 and id2, which index would id1 be? */ static inline int pubkey_idx(const struct pubkey *id1, const struct pubkey *id2) { return pubkey_cmp(id1, id2) > 0; From 6971ad42d0ee300d1aed8f3ff489eef240b0d4d0 Mon Sep 17 00:00:00 2001 From: niftynei Date: Wed, 22 Jul 2020 21:15:38 -0500 Subject: [PATCH 04/14] psbt-common: shared psbt utilities includes facilities for - sorting psbt inputs by serial_id - sorting psbt outputs by serial_id - adding a serial_id - getting a serial_id - finding the diffset between two psbts - adding a max_len to a psbt input - getting a max_len from a psbt input --- common/Makefile | 1 + common/psbt_open.c | 389 +++++++++++++++++++++++++++++++++++++++++++++ common/psbt_open.h | 145 +++++++++++++++++ 3 files changed, 535 insertions(+) create mode 100644 common/psbt_open.c create mode 100644 common/psbt_open.h diff --git a/common/Makefile b/common/Makefile index c56be7466152..bcc433906692 100644 --- a/common/Makefile +++ b/common/Makefile @@ -55,6 +55,7 @@ COMMON_SRC_NOGEN := \ common/peer_failed.c \ common/permute_tx.c \ common/ping.c \ + common/psbt_open.c \ common/pseudorand.c \ common/read_peer_msg.c \ common/setup.c \ diff --git a/common/psbt_open.c b/common/psbt_open.c new file mode 100644 index 000000000000..eb9201b423c3 --- /dev/null +++ b/common/psbt_open.c @@ -0,0 +1,389 @@ +#include "common/psbt_open.h" +#include +#include +#include +#include +#include +#include +#include + +bool psbt_get_serial_id(const struct wally_map *map, u16 *serial_id) +{ + size_t value_len; + void *result = psbt_get_lightning(map, PSBT_TYPE_SERIAL_ID, &value_len); + if (!result) + return false; + + if (value_len != sizeof(*serial_id)) + return false; + + memcpy(serial_id, result, value_len); + return true; +} + +static int compare_serials(const struct wally_map *map_a, + const struct wally_map *map_b) +{ + u16 serial_left, serial_right; + bool ok; + + ok = psbt_get_serial_id(map_a, &serial_left); + assert(ok); + ok = psbt_get_serial_id(map_b, &serial_right); + assert(ok); + if (serial_left > serial_right) + return 1; + if (serial_left < serial_right) + return -1; + return 0; +} + +static int compare_inputs_at(const struct input_set *a, + const struct input_set *b, + void *unused UNUSED) +{ + return compare_serials(&a->input.unknowns, + &b->input.unknowns); +} + +static int compare_outputs_at(const struct output_set *a, + const struct output_set *b, + void *unused UNUSED) +{ + return compare_serials(&a->output.unknowns, + &b->output.unknowns); +} + +static const u8 *linearize_input(const tal_t *ctx, + const struct wally_psbt_input *in, + const struct wally_tx_input *tx_in) +{ + struct wally_psbt *psbt = create_psbt(NULL, 1, 0); + size_t byte_len; + + + if (wally_tx_add_input(psbt->tx, tx_in) != WALLY_OK) + abort(); + psbt->inputs[0] = *in; + psbt->num_inputs++; + + const u8 *bytes = psbt_get_bytes(ctx, psbt, &byte_len); + + /* Hide the inputs we added, so it doesn't get freed */ + psbt->num_inputs--; + tal_free(psbt); + return bytes; +} + +static const u8 *linearize_output(const tal_t *ctx, + const struct wally_psbt_output *out, + const struct wally_tx_output *tx_out) +{ + struct wally_psbt *psbt = create_psbt(NULL, 1, 1); + size_t byte_len; + struct bitcoin_txid txid; + + /* Add a 'fake' input so this will linearize the tx */ + memset(&txid, 0, sizeof(txid)); + psbt_append_input(psbt, &txid, 0, 0); + + if (wally_tx_add_output(psbt->tx, tx_out) != WALLY_OK) + abort(); + psbt->outputs[0] = *out; + psbt->num_outputs++; + + const u8 *bytes = psbt_get_bytes(ctx, psbt, &byte_len); + + /* Hide the outputs we added, so it doesn't get freed */ + psbt->num_outputs--; + tal_free(psbt); + return bytes; +} + +static bool input_identical(const struct wally_psbt *a, + size_t a_index, + const struct wally_psbt *b, + size_t b_index) +{ + const u8 *a_in = linearize_input(tmpctx, + &a->inputs[a_index], + &a->tx->inputs[a_index]); + const u8 *b_in = linearize_input(tmpctx, + &b->inputs[b_index], + &b->tx->inputs[b_index]); + + return memeq(a_in, tal_bytelen(a_in), + b_in, tal_bytelen(b_in)); +} + +static bool output_identical(const struct wally_psbt *a, + size_t a_index, + const struct wally_psbt *b, + size_t b_index) +{ + const u8 *a_out = linearize_output(tmpctx, + &a->outputs[a_index], + &a->tx->outputs[a_index]); + const u8 *b_out = linearize_output(tmpctx, + &b->outputs[b_index], + &b->tx->outputs[b_index]); + + return memeq(a_out, tal_bytelen(a_out), + b_out, tal_bytelen(b_out)); +} + +static void sort_inputs(struct wally_psbt *psbt) +{ + /* Build an input map */ + struct input_set *set = tal_arr(NULL, + struct input_set, + psbt->num_inputs); + + for (size_t i = 0; i < tal_count(set); i++) { + set[i].tx_input = psbt->tx->inputs[i]; + set[i].input = psbt->inputs[i]; + } + + asort(set, tal_count(set), + compare_inputs_at, NULL); + + /* Put PSBT parts into place */ + for (size_t i = 0; i < tal_count(set); i++) { + psbt->inputs[i] = set[i].input; + psbt->tx->inputs[i] = set[i].tx_input; + } + + tal_free(set); +} + +static void sort_outputs(struct wally_psbt *psbt) +{ + /* Build an output map */ + struct output_set *set = tal_arr(NULL, + struct output_set, + psbt->num_outputs); + for (size_t i = 0; i < tal_count(set); i++) { + set[i].tx_output = psbt->tx->outputs[i]; + set[i].output = psbt->outputs[i]; + } + + asort(set, tal_count(set), + compare_outputs_at, NULL); + + /* Put PSBT parts into place */ + for (size_t i = 0; i < tal_count(set); i++) { + psbt->outputs[i] = set[i].output; + psbt->tx->outputs[i] = set[i].tx_output; + } + + tal_free(set); +} + +void psbt_sort_by_serial_id(struct wally_psbt *psbt) +{ + sort_inputs(psbt); + sort_outputs(psbt); +} + +#define ADD(type, add_to, from, index) \ + do { \ + struct type##_set a; \ + a.type = from->type##s[index]; \ + a.tx_##type = from->tx->type##s[index]; \ + tal_arr_expand(add_to, a); \ + } while (0) + +/* this requires having a serial_id entry on everything */ +/* YOU MUST KEEP orig + new AROUND TO USE THE RESULTING SETS */ +bool psbt_has_diff(const tal_t *ctx, + struct wally_psbt *orig, + struct wally_psbt *new, + struct input_set **added_ins, + struct input_set **rm_ins, + struct output_set **added_outs, + struct output_set **rm_outs) +{ + int result; + size_t i = 0, j = 0; + + psbt_sort_by_serial_id(orig); + psbt_sort_by_serial_id(new); + + *added_ins = tal_arr(ctx, struct input_set, 0); + *rm_ins = tal_arr(ctx, struct input_set, 0); + *added_outs = tal_arr(ctx, struct output_set, 0); + *rm_outs = tal_arr(ctx, struct output_set, 0); + + /* Find the input diff */ + while (i < orig->num_inputs || j < new->num_inputs) { + if (i >= orig->num_inputs) { + ADD(input, added_ins, new, j); + j++; + continue; + } + if (j >= new->num_inputs) { + ADD(input, rm_ins, orig, i); + i++; + continue; + } + + result = compare_serials(&orig->inputs[i].unknowns, + &new->inputs[j].unknowns); + if (result == -1) { + ADD(input, rm_ins, orig, i); + i++; + continue; + } + if (result == 1) { + ADD(input, added_ins, new, j); + j++; + continue; + } + + if (!input_identical(orig, i, new, j)) { + ADD(input, rm_ins, orig, i); + ADD(input, added_ins, new, j); + } + i++; + j++; + } + /* Find the output diff */ + i = 0; + j = 0; + while (i < orig->num_outputs || j < new->num_outputs) { + if (i >= orig->num_outputs) { + ADD(output, added_outs, new, j); + j++; + continue; + } + if (j >= new->num_outputs) { + ADD(output, rm_outs, orig, i); + i++; + continue; + } + + result = compare_serials(&orig->outputs[i].unknowns, + &new->outputs[j].unknowns); + if (result == -1) { + ADD(output, rm_outs, orig, i); + i++; + continue; + } + if (result == 1) { + ADD(output, added_outs, new, j); + j++; + continue; + } + if (!output_identical(orig, i, new, j)) { + ADD(output, rm_outs, orig, i); + ADD(output, added_outs, new, j); + } + i++; + j++; + } + + return tal_count(*added_ins) != 0 || + tal_count(*rm_ins) != 0 || + tal_count(*added_outs) != 0 || + tal_count(*rm_outs) != 0; +} + +void psbt_input_add_serial_id(struct wally_psbt_input *input, + u16 serial_id) +{ + u8 *key = psbt_make_key(tmpctx, PSBT_TYPE_SERIAL_ID, NULL); + psbt_input_add_unknown(input, key, &serial_id, sizeof(serial_id)); +} + + +void psbt_output_add_serial_id(struct wally_psbt_output *output, + u16 serial_id) +{ + u8 *key = psbt_make_key(tmpctx, PSBT_TYPE_SERIAL_ID, NULL); + psbt_output_add_unknown(output, key, &serial_id, sizeof(serial_id)); +} + +bool psbt_has_serial_input(struct wally_psbt *psbt, u16 serial_id) +{ + for (size_t i = 0; i < psbt->num_inputs; i++) { + u16 in_serial; + if (!psbt_get_serial_id(&psbt->inputs[i].unknowns, &in_serial)) + continue; + if (in_serial == serial_id) + return true; + } + return false; +} + +bool psbt_has_serial_output(struct wally_psbt *psbt, u16 serial_id) +{ + for (size_t i = 0; i < psbt->num_outputs; i++) { + u16 out_serial; + if (!psbt_get_serial_id(&psbt->outputs[i].unknowns, &out_serial)) + continue; + if (out_serial == serial_id) + return true; + } + return false; +} + +void psbt_input_add_max_witness_len(struct wally_psbt_input *input, + u16 max_witness_len) +{ + u8 *key = psbt_make_key(NULL, PSBT_TYPE_MAX_WITNESS_LEN, NULL); + psbt_input_add_unknown(input, key, &max_witness_len, sizeof(max_witness_len)); + tal_free(key); +} + + +bool psbt_input_get_max_witness_len(struct wally_psbt_input *input, + u16 *max_witness_len) +{ + size_t value_len; + void *result = psbt_get_lightning(&input->unknowns, + PSBT_TYPE_MAX_WITNESS_LEN, + &value_len); + if (!result) + return false; + + if (value_len != sizeof(*max_witness_len)) + return false; + + memcpy(max_witness_len, result, value_len); + return true; +} + +bool psbt_has_required_fields(struct wally_psbt *psbt) +{ + u16 max_witness, serial_id; + for (size_t i = 0; i < psbt->num_inputs; i++) { + struct wally_psbt_input *input = &psbt->inputs[i]; + + if (!psbt_get_serial_id(&input->unknowns, &serial_id)) + return false; + + /* Inputs had also better have their max_witness_lens + * filled in! */ + if (!psbt_input_get_max_witness_len(input, &max_witness)) + return false; + + /* Required because we send the full tx over the wire now */ + if (!input->utxo) + return false; + + /* If is P2SH, redeemscript must be present */ + const u8 *outscript = + wally_tx_output_get_script(tmpctx, + &input->utxo->outputs[psbt->tx->inputs[i].index]); + if (is_p2sh(outscript, NULL) && input->redeem_script_len == 0) + return false; + + } + + for (size_t i = 0; i < psbt->num_outputs; i++) { + if (!psbt_get_serial_id(&psbt->outputs[i].unknowns, &serial_id)) + return false; + } + + return true; +} diff --git a/common/psbt_open.h b/common/psbt_open.h new file mode 100644 index 000000000000..776811e13df4 --- /dev/null +++ b/common/psbt_open.h @@ -0,0 +1,145 @@ +#ifndef LIGHTNING_COMMON_PSBT_OPEN_H +#define LIGHTNING_COMMON_PSBT_OPEN_H +#include "config.h" +#include +#include +#include +#include +#include + +struct wally_tx_input; +struct wally_tx_output; +struct wally_psbt; +struct wally_psbt_input; +struct wally_psbt_output; +struct wally_map; + +struct input_set { + struct wally_tx_input tx_input; + struct wally_psbt_input input; +}; + +struct output_set { + struct wally_tx_output tx_output; + struct wally_psbt_output output; +}; + +#define PSBT_TYPE_SERIAL_ID 0x01 +#define PSBT_TYPE_MAX_WITNESS_LEN 0x02 + +/* psbt_get_serial_id - Returns the serial_id from an unknowns map + * + * @map - the map to find the serial id entry within + * @serial_id - found serial_id + * + * Returns false if serial_id is not present + */ +WARN_UNUSED_RESULT bool psbt_get_serial_id(const struct wally_map *map, + u16 *serial_id); + +/* psbt_sort_by_serial_id - Sort PSBT by serial_ids + * + * MUST have a serial_id on every input/output. + * + * @psbt - psbt to sort + */ +void psbt_sort_by_serial_id(struct wally_psbt *psbt); + +/* psbt_has_diff - Returns set of diffs btw orig + new psbt + * + * All inputs+outputs MUST have a serial_id field present before + * calling this. + * + * @ctx - allocation context for returned diffsets + * @orig - original psbt + * @new - updated psbt + * @added_ins - inputs added {new} + * @rm_ins - inputs removed {orig} + * @added_outs - outputs added {new} + * @rm_outs - outputs removed {orig} + * + * Note that the input + output data returned in the diff sets + * contain references to the originating PSBT; they are not copies. + * + * Returns true if changes are found. + */ +bool psbt_has_diff(const tal_t *ctx, + struct wally_psbt *orig, + struct wally_psbt *new, + struct input_set **added_ins, + struct input_set **rm_ins, + struct output_set **added_outs, + struct output_set **rm_outs); + +/* psbt_input_add_serial_id - Adds a serial id to given input + * + * @input - to add serial_id to + * @serial_id - to add + */ +void psbt_input_add_serial_id(struct wally_psbt_input *input, + u16 serial_id); +/* psbt_output_add_serial_id - Adds a serial id to given output + * + * @output - to add serial_id to + * @serial_id - to add + */ +void psbt_output_add_serial_id(struct wally_psbt_output *output, + u16 serial_id); + +/* psbt_sort_by_serial_id - Sorts the inputs + outputs by serial_id + * + * Requires every input/output to have a serial_id entry. + * + * @psbt - psbt to sort inputs/outputs + */ +void psbt_sort_by_serial_id(struct wally_psbt *psbt); + +/* psbt_has_serial_input - Checks inputs for provided serial_id + * + * @psbt - psbt's inputs to check + * @serial_id - id to look for + * Returns true if serial_id found. + */ +WARN_UNUSED_RESULT bool +psbt_has_serial_input(struct wally_psbt *psbt, u16 serial_id); + +/* psbt_has_serial_output - Checks outputs for provided serial_id + * + * @psbt - psbt's outputs to check + * @serial_id - id to look for + * Returns true if serial_id found. + */ +WARN_UNUSED_RESULT bool +psbt_has_serial_output(struct wally_psbt *psbt, u16 serial_id); + +/* psbt_input_add_max_witness_len - Put a max witness len on a thing + * + * @input - input to add max-witness-len to + * @max_witness_len - value + */ +void psbt_input_add_max_witness_len(struct wally_psbt_input *input, + u16 max_witness_len); + +/* psbt_input_get_max_witness_len - Get the max_witness_len + * + * @input - psbt input to look for max witness len on + * @max_witness_len - found length + * + * Returns false if key not present */ +WARN_UNUSED_RESULT bool +psbt_input_get_max_witness_len(struct wally_psbt_input *input, + u16 *max_witness_len); + +/* psbt_has_required_fields - Validates psbt field completion + * + * Required fields are: + * - a serial_id; input+output + * - a prev_tx; input,non_witness_utxo + * - redeemscript; input,iff is P2SH-P2W* + * @psbt - psbt to validate + * + * Returns true if all required fields are present + */ +bool psbt_has_required_fields(struct wally_psbt *psbt); + +#endif /* LIGHTNING_COMMON_PSBT_OPEN_H */ From 67662d2101f6325f36d26dff219e94e6e613dda9 Mon Sep 17 00:00:00 2001 From: niftynei Date: Fri, 31 Jul 2020 12:35:24 -0500 Subject: [PATCH 05/14] psbt: re-use init function for create_psbt Less code is more code! --- bitcoin/psbt.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/bitcoin/psbt.c b/bitcoin/psbt.c index 94d878dbea52..0432eb72a511 100644 --- a/bitcoin/psbt.c +++ b/bitcoin/psbt.c @@ -57,12 +57,7 @@ struct wally_psbt *new_psbt(const tal_t *ctx, const struct wally_tx *wtx) struct wally_psbt *psbt; int wally_err; - if (is_elements(chainparams)) - wally_err = wally_psbt_elements_init_alloc(0, wtx->num_inputs, wtx->num_outputs, 0, &psbt); - else - wally_err = wally_psbt_init_alloc(0, wtx->num_inputs, wtx->num_outputs, 0, &psbt); - assert(wally_err == WALLY_OK); - tal_add_destructor(psbt, psbt_destroy); + psbt = init_psbt(ctx, wtx->num_inputs, wtx->num_outputs); /* Set directly: avoids psbt checks for non-NULL scripts/witnesses */ wally_err = wally_tx_clone_alloc(wtx, 0, &psbt->tx); From 1a4038dade221398ce5ce2b9d829c6edd72b1b19 Mon Sep 17 00:00:00 2001 From: niftynei Date: Tue, 4 Aug 2020 12:34:21 -0500 Subject: [PATCH 06/14] psbt: rename append_out -> append_output --- bitcoin/psbt.c | 6 +++--- bitcoin/psbt.h | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/bitcoin/psbt.c b/bitcoin/psbt.c index 0432eb72a511..95c65b43c2ae 100644 --- a/bitcoin/psbt.c +++ b/bitcoin/psbt.c @@ -144,9 +144,9 @@ struct wally_psbt_output *psbt_add_output(struct wally_psbt *psbt, return &psbt->outputs[insert_at]; } -struct wally_psbt_output *psbt_append_out(struct wally_psbt *psbt, - const u8 *script, - struct amount_sat amount) +struct wally_psbt_output *psbt_append_output(struct wally_psbt *psbt, + const u8 *script, + struct amount_sat amount) { struct wally_psbt_output *out; struct wally_tx_output *tx_out = wally_tx_output(script, amount); diff --git a/bitcoin/psbt.h b/bitcoin/psbt.h index 26a3b29ebe2d..effd9fac05ef 100644 --- a/bitcoin/psbt.h +++ b/bitcoin/psbt.h @@ -92,9 +92,9 @@ struct wally_psbt_output *psbt_add_output(struct wally_psbt *psbt, * @script - scriptPubKey of the output * @amount - value of the output */ -struct wally_psbt_output *psbt_append_out(struct wally_psbt *psbt, - const u8 *script, - struct amount_sat amount); +struct wally_psbt_output *psbt_append_output(struct wally_psbt *psbt, + const u8 *script, + struct amount_sat amount); void psbt_rm_output(struct wally_psbt *psbt, size_t remove_at); From abc2e28ee3148b170bf298dda0eb3a3df09efeac Mon Sep 17 00:00:00 2001 From: niftynei Date: Tue, 4 Aug 2020 12:56:09 -0500 Subject: [PATCH 07/14] psbt-common: unit test for diff/sort/add serial etc --- common/test/run-psbt_diff.c | 177 ++++++++++++++++++++++++++++++++++++ 1 file changed, 177 insertions(+) create mode 100644 common/test/run-psbt_diff.c diff --git a/common/test/run-psbt_diff.c b/common/test/run-psbt_diff.c new file mode 100644 index 000000000000..543c4289e80e --- /dev/null +++ b/common/test/run-psbt_diff.c @@ -0,0 +1,177 @@ +#include "config.h" +#include +#include +#include +#include +#include +#include +#include +#include "../amount.c" +#include "../psbt_open.c" + +/* AUTOGENERATED MOCKS START */ +/* Generated stub for fromwire */ +const u8 *fromwire(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, void *copy UNNEEDED, size_t n UNNEEDED) +{ fprintf(stderr, "fromwire called!\n"); abort(); } +/* Generated stub for fromwire_bool */ +bool fromwire_bool(const u8 **cursor UNNEEDED, size_t *max UNNEEDED) +{ fprintf(stderr, "fromwire_bool called!\n"); abort(); } +/* Generated stub for fromwire_fail */ +void *fromwire_fail(const u8 **cursor UNNEEDED, size_t *max UNNEEDED) +{ fprintf(stderr, "fromwire_fail called!\n"); abort(); } +/* Generated stub for fromwire_secp256k1_ecdsa_signature */ +void fromwire_secp256k1_ecdsa_signature(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, + secp256k1_ecdsa_signature *signature UNNEEDED) +{ fprintf(stderr, "fromwire_secp256k1_ecdsa_signature called!\n"); abort(); } +/* Generated stub for fromwire_sha256 */ +void fromwire_sha256(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, struct sha256 *sha256 UNNEEDED) +{ fprintf(stderr, "fromwire_sha256 called!\n"); abort(); } +/* Generated stub for fromwire_tal_arrn */ +u8 *fromwire_tal_arrn(const tal_t *ctx UNNEEDED, + const u8 **cursor UNNEEDED, size_t *max UNNEEDED, size_t num UNNEEDED) +{ fprintf(stderr, "fromwire_tal_arrn called!\n"); abort(); } +/* Generated stub for fromwire_u16 */ +u16 fromwire_u16(const u8 **cursor UNNEEDED, size_t *max UNNEEDED) +{ fprintf(stderr, "fromwire_u16 called!\n"); abort(); } +/* Generated stub for fromwire_u32 */ +u32 fromwire_u32(const u8 **cursor UNNEEDED, size_t *max UNNEEDED) +{ fprintf(stderr, "fromwire_u32 called!\n"); abort(); } +/* Generated stub for fromwire_u64 */ +u64 fromwire_u64(const u8 **cursor UNNEEDED, size_t *max UNNEEDED) +{ fprintf(stderr, "fromwire_u64 called!\n"); abort(); } +/* Generated stub for fromwire_u8 */ +u8 fromwire_u8(const u8 **cursor UNNEEDED, size_t *max UNNEEDED) +{ fprintf(stderr, "fromwire_u8 called!\n"); abort(); } +/* Generated stub for towire */ +void towire(u8 **pptr UNNEEDED, const void *data UNNEEDED, size_t len UNNEEDED) +{ fprintf(stderr, "towire called!\n"); abort(); } +/* Generated stub for towire_bool */ +void towire_bool(u8 **pptr UNNEEDED, bool v UNNEEDED) +{ fprintf(stderr, "towire_bool called!\n"); abort(); } +/* Generated stub for towire_secp256k1_ecdsa_signature */ +void towire_secp256k1_ecdsa_signature(u8 **pptr UNNEEDED, + const secp256k1_ecdsa_signature *signature UNNEEDED) +{ fprintf(stderr, "towire_secp256k1_ecdsa_signature called!\n"); abort(); } +/* Generated stub for towire_sha256 */ +void towire_sha256(u8 **pptr UNNEEDED, const struct sha256 *sha256 UNNEEDED) +{ fprintf(stderr, "towire_sha256 called!\n"); abort(); } +/* Generated stub for towire_u16 */ +void towire_u16(u8 **pptr UNNEEDED, u16 v UNNEEDED) +{ fprintf(stderr, "towire_u16 called!\n"); abort(); } +/* Generated stub for towire_u32 */ +void towire_u32(u8 **pptr UNNEEDED, u32 v UNNEEDED) +{ fprintf(stderr, "towire_u32 called!\n"); abort(); } +/* Generated stub for towire_u64 */ +void towire_u64(u8 **pptr UNNEEDED, u64 v UNNEEDED) +{ fprintf(stderr, "towire_u64 called!\n"); abort(); } +/* Generated stub for towire_u8 */ +void towire_u8(u8 **pptr UNNEEDED, u8 v UNNEEDED) +{ fprintf(stderr, "towire_u8 called!\n"); abort(); } +/* Generated stub for towire_u8_array */ +void towire_u8_array(u8 **pptr UNNEEDED, const u8 *arr UNNEEDED, size_t num UNNEEDED) +{ fprintf(stderr, "towire_u8_array called!\n"); abort(); } +/* AUTOGENERATED MOCKS END */ + +static void diff_count(struct wally_psbt *a, + struct wally_psbt *b, + size_t diff_added, + size_t diff_rm) +{ + bool has_diff; + struct input_set *added_in, *rm_in; + struct output_set *added_out, *rm_out; + + has_diff = psbt_has_diff(tmpctx, a, b, + &added_in, &rm_in, + &added_out, &rm_out); + + assert(has_diff == (diff_added + diff_rm != 0)); + assert(tal_count(added_in) == diff_added); + assert(tal_count(added_out) == diff_added); + assert(tal_count(rm_in) == diff_rm); + assert(tal_count(rm_out) == diff_rm); +} + +static void add_in_out_with_serial(struct wally_psbt *psbt, + size_t serial_id, + size_t default_value) +{ + struct bitcoin_txid txid; + u8 *script; + struct amount_sat sat; + struct wally_psbt_input *in; + struct wally_psbt_output *out; + + memset(&txid, default_value, sizeof(txid)); + in = psbt_append_input(psbt, &txid, default_value, default_value); + if (!in) + abort(); + psbt_input_add_serial_id(in, serial_id); + + script = tal_arr(tmpctx, u8, 20); + memset(script, default_value, 20); + sat = amount_sat(default_value); + out = psbt_append_output(psbt, script, sat); + if (!out) + abort(); + psbt_output_add_serial_id(out, serial_id); +} + +int main(int argc, const char *argv[]) +{ + common_setup(argv[0]); + + struct wally_psbt *start, *end; + u32 flags = 0; + + chainparams = chainparams_for_network("bitcoin"); + + /* Create two psbts! */ + end = create_psbt(tmpctx, 1, 1); + if (wally_psbt_clone_alloc(end, flags, &start) != WALLY_OK) + abort(); + diff_count(start, end, 0, 0); + diff_count(end, start, 0, 0); + + /* New input/output added */ + add_in_out_with_serial(end, 10, 1); + diff_count(start, end, 1, 0); + diff_count(end, start, 0, 1); + + /* Add another one, before previous */ + if (wally_psbt_clone_alloc(end, flags, &start) != WALLY_OK) + abort(); + add_in_out_with_serial(end, 5, 2); + diff_count(start, end, 1, 0); + diff_count(end, start, 0, 1); + + /* Add another, at end */ + if (wally_psbt_clone_alloc(end, flags, &start) != WALLY_OK) + abort(); + add_in_out_with_serial(end, 15, 3); + diff_count(start, end, 1, 0); + diff_count(end, start, 0, 1); + + /* Add another, in middle */ + if (wally_psbt_clone_alloc(end, flags, &start) != WALLY_OK) + abort(); + add_in_out_with_serial(end, 11, 4); + diff_count(start, end, 1, 0); + diff_count(end, start, 0, 1); + + /* Change existing input/output info + * (we accomplish this by removing and then + * readding an input/output with the same serial_id + * but different value) */ + if (wally_psbt_clone_alloc(end, flags, &start) != WALLY_OK) + abort(); + psbt_rm_output(end, 0); + psbt_rm_input(end, 0); + add_in_out_with_serial(end, 5, 5); + diff_count(start, end, 1, 1); + diff_count(end, start, 1, 1); + + /* No memory leaks please */ + common_shutdown(); + return 0; +} From 4c370efe4b950fe61d9af7635352f0cb3c236d65 Mon Sep 17 00:00:00 2001 From: niftynei Date: Fri, 7 Aug 2020 11:35:32 -0500 Subject: [PATCH 08/14] psbt-utils: ignore the 'unknown' map for input + output comparison There's no stable ordering on unknown serialization, so linearizing identical but mis-ordered unknown data will lead to 'wrong' results. Instead, we just ignore any data that's in the psbt unknown struct. There's probably also problems here with other PSBT maps. Really, this needs a finer grained comparison function .... fuck --- common/psbt_open.c | 9 +++++++-- common/test/run-psbt_diff.c | 15 +++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/common/psbt_open.c b/common/psbt_open.c index eb9201b423c3..2061a17968ca 100644 --- a/common/psbt_open.c +++ b/common/psbt_open.c @@ -61,11 +61,13 @@ static const u8 *linearize_input(const tal_t *ctx, struct wally_psbt *psbt = create_psbt(NULL, 1, 0); size_t byte_len; - if (wally_tx_add_input(psbt->tx, tx_in) != WALLY_OK) abort(); psbt->inputs[0] = *in; psbt->num_inputs++; + /* Blank out unknowns. These are unordered and serializing + * them might have different outputs for identical data */ + psbt->inputs[0].unknowns.num_items = 0; const u8 *bytes = psbt_get_bytes(ctx, psbt, &byte_len); @@ -89,8 +91,12 @@ static const u8 *linearize_output(const tal_t *ctx, if (wally_tx_add_output(psbt->tx, tx_out) != WALLY_OK) abort(); + psbt->outputs[0] = *out; psbt->num_outputs++; + /* Blank out unknowns. These are unordered and serializing + * them might have different outputs for identical data */ + psbt->outputs[0].unknowns.num_items = 0; const u8 *bytes = psbt_get_bytes(ctx, psbt, &byte_len); @@ -127,7 +133,6 @@ static bool output_identical(const struct wally_psbt *a, const u8 *b_out = linearize_output(tmpctx, &b->outputs[b_index], &b->tx->outputs[b_index]); - return memeq(a_out, tal_bytelen(a_out), b_out, tal_bytelen(b_out)); } diff --git a/common/test/run-psbt_diff.c b/common/test/run-psbt_diff.c index 543c4289e80e..de5be99b86a3 100644 --- a/common/test/run-psbt_diff.c +++ b/common/test/run-psbt_diff.c @@ -171,6 +171,21 @@ int main(int argc, const char *argv[]) diff_count(start, end, 1, 1); diff_count(end, start, 1, 1); + /* Add some extra unknown info to a PSBT */ + psbt_input_add_max_witness_len(&end->inputs[1], 100); + psbt_input_add_max_witness_len(&start->inputs[1], 100); + + /* Swap locations */ + struct wally_map_item tmp; + tmp = end->inputs[1].unknowns.items[0]; + end->inputs[1].unknowns.items[0] = end->inputs[1].unknowns.items[1]; + end->inputs[1].unknowns.items[1] = tmp; + + /* We expect nothing to change ? */ + /* FIXME: stable ordering of unknowns ? */ + diff_count(start, end, 1, 1); + diff_count(end, start, 1, 1); + /* No memory leaks please */ common_shutdown(); return 0; From 3e7f3e1a46feca0df732b5286272325e34091d39 Mon Sep 17 00:00:00 2001 From: niftynei Date: Thu, 13 Aug 2020 11:38:03 -0500 Subject: [PATCH 09/14] psbt: use elements input init alloc --- bitcoin/psbt.c | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/bitcoin/psbt.c b/bitcoin/psbt.c index 95c65b43c2ae..39b7de9bebf8 100644 --- a/bitcoin/psbt.c +++ b/bitcoin/psbt.c @@ -112,18 +112,27 @@ struct wally_psbt_input *psbt_append_input(struct wally_psbt *psbt, { struct wally_tx_input *tx_in; struct wally_psbt_input *input; - size_t insert_at; - insert_at = psbt->num_inputs; - - if (wally_tx_input_init_alloc(txid->shad.sha.u.u8, - sizeof(struct bitcoin_txid), - outnum, sequence, NULL, 0, NULL, - &tx_in) != WALLY_OK) - abort(); + if (chainparams->is_elements) { + if (wally_tx_elements_input_init_alloc(txid->shad.sha.u.u8, + sizeof(txid->shad.sha.u.u8), + outnum, sequence, NULL, 0, + NULL, + NULL, 0, + NULL, 0, NULL, 0, + NULL, 0, NULL, 0, + NULL, 0, NULL, + &tx_in) != WALLY_OK) + abort(); + } else { + if (wally_tx_input_init_alloc(txid->shad.sha.u.u8, + sizeof(txid->shad.sha.u.u8), + outnum, sequence, NULL, 0, NULL, + &tx_in) != WALLY_OK) + abort(); + } - tx_in->features = chainparams->is_elements ? WALLY_TX_IS_ELEMENTS : 0; - input = psbt_add_input(psbt, tx_in, insert_at); + input = psbt_add_input(psbt, tx_in, psbt->num_inputs); wally_tx_input_free(tx_in); return input; } From 7ca23a515a955bf85d509c4ee673fe713e5fea6c Mon Sep 17 00:00:00 2001 From: niftynei Date: Thu, 13 Aug 2020 11:38:23 -0500 Subject: [PATCH 10/14] tx: use elements_input_init_alloc --- bitcoin/tx.c | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/bitcoin/tx.c b/bitcoin/tx.c index fb47428e42b3..946ab574e8a1 100644 --- a/bitcoin/tx.c +++ b/bitcoin/tx.c @@ -193,14 +193,26 @@ int bitcoin_tx_add_input(struct bitcoin_tx *tx, const struct bitcoin_txid *txid, assert(tx->wtx != NULL); i = tx->wtx->num_inputs; - wally_err = wally_tx_input_init_alloc(txid->shad.sha.u.u8, - sizeof(struct bitcoin_txid), - outnum, sequence, - scriptSig, tal_bytelen(scriptSig), - NULL /* Empty witness stack */, - &input); + if (chainparams->is_elements) + wally_err = + wally_tx_elements_input_init_alloc(txid->shad.sha.u.u8, + sizeof(struct bitcoin_txid), + outnum, sequence, + scriptSig, tal_bytelen(scriptSig), + NULL /* Empty witness stack */, + NULL, 0, NULL, 0, + NULL, 0, NULL, 0, + NULL, 0, NULL, 0, + NULL, &input); + else + wally_err = wally_tx_input_init_alloc(txid->shad.sha.u.u8, + sizeof(struct bitcoin_txid), + outnum, sequence, + scriptSig, tal_bytelen(scriptSig), + NULL /* Empty witness stack */, + &input); + assert(wally_err == WALLY_OK); - input->features = chainparams->is_elements ? WALLY_TX_IS_ELEMENTS : 0; wally_tx_add_input(tx->wtx, input); psbt_add_input(tx->psbt, input, i); From 17f6dd3d0bfdef7e0f3090d4b824d3f36cd7da7c Mon Sep 17 00:00:00 2001 From: niftynei Date: Thu, 13 Aug 2020 11:38:46 -0500 Subject: [PATCH 11/14] psbt: assertion for index in bounds --- common/psbt_open.c | 1 + 1 file changed, 1 insertion(+) diff --git a/common/psbt_open.c b/common/psbt_open.c index 2061a17968ca..70a027b8ff49 100644 --- a/common/psbt_open.c +++ b/common/psbt_open.c @@ -377,6 +377,7 @@ bool psbt_has_required_fields(struct wally_psbt *psbt) return false; /* If is P2SH, redeemscript must be present */ + assert(psbt->tx->inputs[i].index < input->utxo->num_outputs); const u8 *outscript = wally_tx_output_get_script(tmpctx, &input->utxo->outputs[psbt->tx->inputs[i].index]); From 9090f785fe7e23c08a645d5f5582654b6c17ea21 Mon Sep 17 00:00:00 2001 From: niftynei Date: Thu, 13 Aug 2020 12:14:17 -0500 Subject: [PATCH 12/14] psbt: use platform-independent serialization for psbt fields portability for the win Suggested-By: @jgriffiths --- common/psbt_open.c | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/common/psbt_open.c b/common/psbt_open.c index 70a027b8ff49..e168ada8228a 100644 --- a/common/psbt_open.c +++ b/common/psbt_open.c @@ -4,20 +4,23 @@ #include #include #include +#include #include #include bool psbt_get_serial_id(const struct wally_map *map, u16 *serial_id) { size_t value_len; + beint16_t bev; void *result = psbt_get_lightning(map, PSBT_TYPE_SERIAL_ID, &value_len); if (!result) return false; - if (value_len != sizeof(*serial_id)) + if (value_len != sizeof(bev)) return false; - memcpy(serial_id, result, value_len); + memcpy(&bev, result, value_len); + *serial_id = be16_to_cpu(bev); return true; } @@ -297,7 +300,9 @@ void psbt_input_add_serial_id(struct wally_psbt_input *input, u16 serial_id) { u8 *key = psbt_make_key(tmpctx, PSBT_TYPE_SERIAL_ID, NULL); - psbt_input_add_unknown(input, key, &serial_id, sizeof(serial_id)); + beint16_t bev = cpu_to_be16(serial_id); + + psbt_input_add_unknown(input, key, &bev, sizeof(bev)); } @@ -305,7 +310,8 @@ void psbt_output_add_serial_id(struct wally_psbt_output *output, u16 serial_id) { u8 *key = psbt_make_key(tmpctx, PSBT_TYPE_SERIAL_ID, NULL); - psbt_output_add_unknown(output, key, &serial_id, sizeof(serial_id)); + beint16_t bev = cpu_to_be16(serial_id); + psbt_output_add_unknown(output, key, &bev, sizeof(bev)); } bool psbt_has_serial_input(struct wally_psbt *psbt, u16 serial_id) @@ -345,16 +351,18 @@ bool psbt_input_get_max_witness_len(struct wally_psbt_input *input, u16 *max_witness_len) { size_t value_len; + beint16_t bev; void *result = psbt_get_lightning(&input->unknowns, PSBT_TYPE_MAX_WITNESS_LEN, &value_len); if (!result) return false; - if (value_len != sizeof(*max_witness_len)) + if (value_len != sizeof(bev)) return false; - memcpy(max_witness_len, result, value_len); + memcpy(&bev, result, value_len); + *max_witness_len = be16_to_cpu(bev); return true; } From ba040cd70b63eb2014db2ae4e5a76edee6c2858d Mon Sep 17 00:00:00 2001 From: niftynei Date: Thu, 13 Aug 2020 18:19:26 -0500 Subject: [PATCH 13/14] libwally: update to latest master, includes map sort --- external/libwally-core | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/libwally-core b/external/libwally-core index 058aad016040..b8d7ea91049c 160000 --- a/external/libwally-core +++ b/external/libwally-core @@ -1 +1 @@ -Subproject commit 058aad016040c493f076215397ad76784144ce4e +Subproject commit b8d7ea91049c3d5522768c77c8bfe4936cbabbd7 From f99dbfa87191cdd92d95e20553e7870effd9ea05 Mon Sep 17 00:00:00 2001 From: niftynei Date: Thu, 13 Aug 2020 18:30:46 -0500 Subject: [PATCH 14/14] psbt: sort maps before serializing, to stabilize comparison Our psbt input/output comparison functions use serialization to compare the things, but if there's a map with things in it and the map isn't sorted exactly the same, it's highly likely you'll mark an identical inputs as different. To fix this, we sort all the input/output maps before linearizing them. --- common/psbt_open.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/common/psbt_open.c b/common/psbt_open.c index e168ada8228a..bfc74bba1add 100644 --- a/common/psbt_open.c +++ b/common/psbt_open.c @@ -68,9 +68,12 @@ static const u8 *linearize_input(const tal_t *ctx, abort(); psbt->inputs[0] = *in; psbt->num_inputs++; - /* Blank out unknowns. These are unordered and serializing - * them might have different outputs for identical data */ - psbt->inputs[0].unknowns.num_items = 0; + + + /* Sort the inputs, so serializing them is ok */ + wally_map_sort(&psbt->inputs[0].unknowns, 0); + wally_map_sort(&psbt->inputs[0].keypaths, 0); + wally_map_sort(&psbt->inputs[0].signatures, 0); const u8 *bytes = psbt_get_bytes(ctx, psbt, &byte_len); @@ -97,9 +100,9 @@ static const u8 *linearize_output(const tal_t *ctx, psbt->outputs[0] = *out; psbt->num_outputs++; - /* Blank out unknowns. These are unordered and serializing - * them might have different outputs for identical data */ - psbt->outputs[0].unknowns.num_items = 0; + /* Sort the outputs, so serializing them is ok */ + wally_map_sort(&psbt->outputs[0].unknowns, 0); + wally_map_sort(&psbt->outputs[0].keypaths, 0); const u8 *bytes = psbt_get_bytes(ctx, psbt, &byte_len);