ElementsProject · rustyrussell · Jun 26, 2021 · Jun 17, 2021 · Jun 17, 2021 · Jun 17, 2021
diff --git a/channeld/channeld_wiregen.c b/channeld/channeld_wiregen.c
diff --git a/channeld/channeld_wiregen.h b/channeld/channeld_wiregen.h
diff --git a/closingd/closingd_wiregen.c b/closingd/closingd_wiregen.c
diff --git a/closingd/closingd_wiregen.h b/closingd/closingd_wiregen.h
diff --git a/common/json_tok.c b/common/json_tok.c
@@ -543,3 +543,44 @@ struct command_result *param_outpoint_arr(struct command *cmd,
 	}
 	return NULL;
 }
+
+struct command_result *param_extra_tlvs(struct command *cmd, const char *name,
+					const char *buffer,
+					const jsmntok_t *tok,
+					struct tlv_field **fields)
+{
+	size_t i;
+	const jsmntok_t *curr;
+	struct tlv_field *f, *temp;
+
+	if (tok->type != JSMN_OBJECT) {
+		return command_fail(
+		    cmd, JSONRPC2_INVALID_PARAMS,
+		    "Could not decode the TLV object from %s: "
+		    "\"%s\" is not a valid JSON object.",
+		    name, json_strdup(tmpctx, buffer, tok));
+	}
+
+	temp = tal_arr(cmd, struct tlv_field, tok->size);
+	json_for_each_obj(i, curr, tok) {
+		f = &temp[i];
+		if (!json_to_u64(buffer, curr, &f->numtype)) {
+			return command_fail(
+			    cmd, JSONRPC2_INVALID_PARAMS,
+			    "\"%s\" is not a valid numeric TLV type.",
+			    json_strdup(tmpctx, buffer, curr));
+		}
+		f->value = json_tok_bin_from_hex(temp, buffer, curr + 1);
+
+		if (f->value == NULL) {
+			return command_fail(
+			    cmd, JSONRPC2_INVALID_PARAMS,
+			    "\"%s\" is not a valid hex encoded TLV value.",
+			    json_strdup(tmpctx, buffer, curr));
+		}
+		f->length = tal_bytelen(f->value);
+		f->meta = NULL;
+	}
+	*fields = temp;
+	return NULL;
+}
diff --git a/common/json_tok.h b/common/json_tok.h
@@ -189,4 +189,9 @@ struct command_result *param_outpoint_arr(struct command *cmd,
 					  const char *buffer,
 					  const jsmntok_t *tok,
 					  struct bitcoin_outpoint **outpoints);
+
+struct command_result *param_extra_tlvs(struct command *cmd, const char *name,
+					const char *buffer,
+					const jsmntok_t *tok,
+					struct tlv_field **fields);
 #endif /* LIGHTNING_COMMON_JSON_TOK_H */
diff --git a/common/onion.c b/common/onion.c
@@ -267,6 +267,7 @@ struct onion_payload *onion_decode(const tal_t *ctx,
 				   const struct route_step *rs,
 				   const struct pubkey *blinding,
 				   const struct secret *blinding_ss,
+				   u64 *accepted_extra_tlvs,
 				   u64 *failtlvtype,
 				   size_t *failtlvpos)
 {
@@ -305,14 +306,15 @@ struct onion_payload *onion_decode(const tal_t *ctx,
 		/* If they somehow got an invalid onion this far, fail. */
 		if (!cursor)
 			return tal_free(p);
+		p->tlv = NULL;
 		return p;
 
 	case ONION_TLV_PAYLOAD:
 		tlv = tlv_tlv_payload_new(p);
 		if (!fromwire_tlv_payload(&cursor, &max, tlv))
 			goto fail;
 
-		if (!tlv_payload_is_valid(tlv, failtlvpos)) {
+		if (!tlv_fields_valid(tlv->fields, accepted_extra_tlvs, failtlvpos)) {
 			*failtlvtype = tlv->fields[*failtlvpos].numtype;
 			goto fail;
 		}
@@ -403,7 +405,7 @@ struct onion_payload *onion_decode(const tal_t *ctx,
 			*p->total_msat
 				= amount_msat(tlv->payment_data->total_msat);
 		}
-		tal_free(tlv);
+		p->tlv = tal_steal(p, tlv);
 		return p;
 	}
 

diff --git a/common/onion.h b/common/onion.h
@@ -24,6 +24,9 @@ struct onion_payload {
 	/* If blinding is set, blinding_ss is the shared secret.*/
 	struct pubkey *blinding;
 	struct secret blinding_ss;
+
+	/* The raw TLVs contained in the payload. */
+	struct tlv_tlv_payload *tlv;
 };
 
 u8 *onion_nonfinal_hop(const tal_t *ctx,
@@ -68,6 +71,7 @@ size_t onion_payload_length(const u8 *raw_payload, size_t len,
  *       onion_payload_length().
  * @blinding: the optional incoming blinding point.
  * @blinding_ss: the shared secret derived from @blinding (iff that's non-NULL)
+ * @accepted_extra_tlvs: Allow these types to be in the TLV without failing
  * @failtlvtype: (out) the tlv type which failed to parse.
  * @failtlvpos: (out) the offset in the tlv which failed to parse.
  *
@@ -77,6 +81,7 @@ struct onion_payload *onion_decode(const tal_t *ctx,
 				   const struct route_step *rs,
 				   const struct pubkey *blinding,
 				   const struct secret *blinding_ss,
+				   u64 *accepted_extra_tlvs,
 				   u64 *failtlvtype,
 				   size_t *failtlvpos);
 

diff --git a/common/peer_status_wiregen.c b/common/peer_status_wiregen.c
diff --git a/common/peer_status_wiregen.h b/common/peer_status_wiregen.h
diff --git a/common/status_wiregen.c b/common/status_wiregen.c
diff --git a/common/status_wiregen.h b/common/status_wiregen.h
diff --git a/common/test/run-gossmap_local.c b/common/test/run-gossmap_local.c
@@ -24,7 +24,8 @@ bool fromwire_tlv(const u8 **cursor UNNEEDED, size_t *max UNNEEDED,
 		  void *record UNNEEDED, struct tlv_field **fields UNNEEDED)
 { fprintf(stderr, "fromwire_tlv called!\n"); abort(); }
 /* Generated stub for tlv_fields_valid */
-bool tlv_fields_valid(const struct tlv_field *fields UNNEEDED, size_t *err_index UNNEEDED)
+bool tlv_fields_valid(const struct tlv_field *fields UNNEEDED, u64 *allow_extra UNNEEDED,
+		      size_t *err_index UNNEEDED)
 { fprintf(stderr, "tlv_fields_valid called!\n"); abort(); }
 /* Generated stub for towire_bigsize */
 void towire_bigsize(u8 **pptr UNNEEDED, const bigsize_t val UNNEEDED)

diff --git a/common/test/run-json.c b/common/test/run-json.c
@@ -19,7 +19,8 @@ bool fromwire_tlv(const u8 **cursor UNNEEDED, size_t *max UNNEEDED,
 bool node_id_from_hexstr(const char *str UNNEEDED, size_t slen UNNEEDED, struct node_id *id UNNEEDED)
 { fprintf(stderr, "node_id_from_hexstr called!\n"); abort(); }
 /* Generated stub for tlv_fields_valid */
-bool tlv_fields_valid(const struct tlv_field *fields UNNEEDED, size_t *err_index UNNEEDED)
+bool tlv_fields_valid(const struct tlv_field *fields UNNEEDED, u64 *allow_extra UNNEEDED,
+		      size_t *err_index UNNEEDED)
 { fprintf(stderr, "tlv_fields_valid called!\n"); abort(); }
 /* Generated stub for towire_tlv */
 void towire_tlv(u8 **pptr UNNEEDED,

diff --git a/common/test/run-param.c b/common/test/run-param.c
@@ -79,7 +79,8 @@ int segwit_addr_decode(
 )
 { fprintf(stderr, "segwit_addr_decode called!\n"); abort(); }
 /* Generated stub for tlv_fields_valid */
-bool tlv_fields_valid(const struct tlv_field *fields UNNEEDED, size_t *err_index UNNEEDED)
+bool tlv_fields_valid(const struct tlv_field *fields UNNEEDED, u64 *allow_extra UNNEEDED,
+		      size_t *err_index UNNEEDED)
 { fprintf(stderr, "tlv_fields_valid called!\n"); abort(); }
 /* Generated stub for towire_tlv */
 void towire_tlv(u8 **pptr UNNEEDED,

diff --git a/common/test/run-route-specific.c b/common/test/run-route-specific.c
@@ -30,7 +30,8 @@ bool fromwire_tlv(const u8 **cursor UNNEEDED, size_t *max UNNEEDED,
 		  void *record UNNEEDED, struct tlv_field **fields UNNEEDED)
 { fprintf(stderr, "fromwire_tlv called!\n"); abort(); }
 /* Generated stub for tlv_fields_valid */
-bool tlv_fields_valid(const struct tlv_field *fields UNNEEDED, size_t *err_index UNNEEDED)
+bool tlv_fields_valid(const struct tlv_field *fields UNNEEDED, u64 *allow_extra UNNEEDED,
+		      size_t *err_index UNNEEDED)
 { fprintf(stderr, "tlv_fields_valid called!\n"); abort(); }
 /* Generated stub for towire_bigsize */
 void towire_bigsize(u8 **pptr UNNEEDED, const bigsize_t val UNNEEDED)

diff --git a/common/test/run-route.c b/common/test/run-route.c
@@ -23,7 +23,8 @@ bool fromwire_tlv(const u8 **cursor UNNEEDED, size_t *max UNNEEDED,
 		  void *record UNNEEDED, struct tlv_field **fields UNNEEDED)
 { fprintf(stderr, "fromwire_tlv called!\n"); abort(); }
 /* Generated stub for tlv_fields_valid */
-bool tlv_fields_valid(const struct tlv_field *fields UNNEEDED, size_t *err_index UNNEEDED)
+bool tlv_fields_valid(const struct tlv_field *fields UNNEEDED, u64 *allow_extra UNNEEDED,
+		      size_t *err_index UNNEEDED)
 { fprintf(stderr, "tlv_fields_valid called!\n"); abort(); }
 /* Generated stub for towire_bigsize */
 void towire_bigsize(u8 **pptr UNNEEDED, const bigsize_t val UNNEEDED)

diff --git a/common/test/run-sphinx.c b/common/test/run-sphinx.c
@@ -74,7 +74,8 @@ bool fromwire_tlv(const u8 **cursor UNNEEDED, size_t *max UNNEEDED,
 bool pubkey_from_node_id(struct pubkey *key UNNEEDED, const struct node_id *id UNNEEDED)
 { fprintf(stderr, "pubkey_from_node_id called!\n"); abort(); }
 /* Generated stub for tlv_fields_valid */
-bool tlv_fields_valid(const struct tlv_field *fields UNNEEDED, size_t *err_index UNNEEDED)
+bool tlv_fields_valid(const struct tlv_field *fields UNNEEDED, u64 *allow_extra UNNEEDED,
+		      size_t *err_index UNNEEDED)
 { fprintf(stderr, "tlv_fields_valid called!\n"); abort(); }
 /* Generated stub for towire_amount_msat */
 void towire_amount_msat(u8 **pptr UNNEEDED, const struct amount_msat msat UNNEEDED)

diff --git a/connectd/connectd_gossipd_wiregen.c b/connectd/connectd_gossipd_wiregen.c
diff --git a/connectd/connectd_gossipd_wiregen.h b/connectd/connectd_gossipd_wiregen.h
diff --git a/connectd/connectd_wiregen.c b/connectd/connectd_wiregen.c
diff --git a/connectd/connectd_wiregen.h b/connectd/connectd_wiregen.h
@@ -1353,3 +1353,26 @@ def getsharedsecret(self, point, **kwargs):
         }
         payload.update({k: v for k, v in kwargs.items()})
         return self.call("getsharedsecret", payload)
+
+    def keysend(self, destination, msatoshi, label=None, maxfeepercent=None,
+                retry_for=None, maxdelay=None, exemptfee=None,
+                extratlvs=None):
+        """
+        """
+
+        if extratlvs is not None and not isinstance(extratlvs, dict):
+            raise ValueError(
+                "extratlvs is not a dictionary with integer keys and hexadecimal values"
+            )
+
+        payload = {
+            "destination": destination,
+            "msatoshi": msatoshi,
+            "label": label,
+            "maxfeepercent": maxfeepercent,
+            "retry_for": retry_for,
+            "maxdelay": maxdelay,
+            "exemptfee": exemptfee,
+            "extratlvs": extratlvs,
+        }
+        return self.call("keysend", payload)
diff --git a/gossipd/gossip_store_wiregen.c b/gossipd/gossip_store_wiregen.c
diff --git a/gossipd/gossip_store_wiregen.h b/gossipd/gossip_store_wiregen.h
diff --git a/gossipd/gossipd_peerd_wiregen.c b/gossipd/gossipd_peerd_wiregen.c
diff --git a/gossipd/gossipd_peerd_wiregen.h b/gossipd/gossipd_peerd_wiregen.h
diff --git a/gossipd/gossipd_wiregen.c b/gossipd/gossipd_wiregen.c
diff --git a/gossipd/gossipd_wiregen.h b/gossipd/gossipd_wiregen.h
diff --git a/hsmd/hsmd_wiregen.c b/hsmd/hsmd_wiregen.c
diff --git a/hsmd/hsmd_wiregen.h b/hsmd/hsmd_wiregen.h
diff --git a/lightningd/htlc_end.h b/lightningd/htlc_end.h
@@ -58,6 +58,9 @@ struct htlc_in {
 
 	/* A simple text annotation shown in `listpeers` */
 	char *status;
+
+	/* The decoded onion payload after hooks processed it. */
+	struct onion_payload *payload;
 };
 
 struct htlc_out {

diff --git a/lightningd/invoice.c b/lightningd/invoice.c
@@ -15,6 +15,7 @@
 #include <common/json_command.h>
 #include <common/json_helpers.h>
 #include <common/jsonrpc_errors.h>
+#include <common/onion.h>
 #include <common/overflows.h>
 #include <common/param.h>
 #include <common/random_select.h>
@@ -167,6 +168,38 @@ struct invoice_payment_hook_payload {
 	/* FIXME: Include raw payload! */
 };
 
+#ifdef DEVELOPER
+static void invoice_payment_add_tlvs(struct json_stream *stream,
+				     struct htlc_set *hset)
+{
+	struct htlc_in *hin;
+	struct tlv_tlv_payload *tlvs;
+	assert(tal_count(hset->htlcs) > 0);
+
+	/* Pick the first HTLC as representative for the entire set. */
+	hin = hset->htlcs[0];
+
+	if (hin->payload->type != ONION_TLV_PAYLOAD)
+		return;
+	tlvs = hin->payload->tlv;
+
+	json_array_start(stream, "extratlvs");
+
+	for (size_t i = 0; i < tal_count(tlvs->fields); i++) {
+		struct tlv_field *field = &tlvs->fields[i];
+		/* If we have metadata attached it is not an extra TLV field. */
+		if (field->meta == NULL) {
+			json_object_start(stream, NULL);
+			json_add_u64(stream, "type", field->numtype);
+			json_add_num(stream, "length", field->length);
+			json_add_hex_talarr(stream, "value", field->value);
+			json_object_end(stream);
+		}
+	}
+	json_array_end(stream);
+}
+#endif
+
 static void
 invoice_payment_serialize(struct invoice_payment_hook_payload *payload,
 			  struct json_stream *stream,
@@ -178,6 +211,9 @@ invoice_payment_serialize(struct invoice_payment_hook_payload *payload,
 	json_add_string(stream, "msat",
 			type_to_string(tmpctx, struct amount_msat,
 				       &payload->msat));
+#ifdef DEVELOPER
+	invoice_payment_add_tlvs(stream, payload->set);
+#endif
 	json_object_end(stream); /* .payment */
 }
 

diff --git a/lightningd/lightningd.c b/lightningd/lightningd.c
@@ -233,6 +233,9 @@ static struct lightningd *new_lightningd(const tal_t *ctx)
 	 * so set it to NULL explicitly now. */
 	ld->wallet = NULL;
 
+	/*~ Behavioral options */
+	ld->accept_extra_tlv_types = tal_arr(ld, u64, 0);
+
 	/*~ In the next step we will initialize the plugins. This will
 	 *  also populate the JSON-RPC with passthrough methods, hence
 	 *  lightningd needs to have something to put those in. This