New pyln modules.

contrib/pyln-proto/pyln/proto/message/bolts/Makefile

@@ -0,0 +1,14 @@
+#! /usr/bin/make
+
+SPECDIR := ../../../../../../../lightning-rfc
+
+# Only consider specs which have types in them.
+SPECS := $(shell fgrep -l '1. type' $(SPECDIR)/[0-9]*.md)
+
+bolts.py: $(SPECS) Makefile
+	for f in $(SPECS); do SPECNUM=`basename $$f | sed 's/-.*//'`; echo bolt_$${SPECNUM}_csv = '['; python3 $(SPECDIR)/tools/extract-formats.py $$f | sed 's/\(.*\)/    "\1",/'; echo ']'; done > $@
+	chmod a+x $@
+
+CSVFILES = $(SPECS:%.md=bolt%.py)
+
+refresh: $(CSVFILES)

contrib/pyln-proto/pyln/proto/message/bolts/__init__.py

@@ -0,0 +1,10 @@
+from .bolts import bolt_01_csv, bolt_02_csv, bolt_04_csv, bolt_07_csv
+
+__version__ = '0.0.1'
+
+__all__ = [
+    "bolt_01_csv",
+    "bolt_02_csv",
+    "bolt_04_csv",
+    "bolt_07_csv",
+]

contrib/pyln-proto/pyln/proto/message/bolts/bolts.py

@@ -0,0 +1,273 @@
+bolt_01_csv = [
+    "msgtype,init,16",
+    "msgdata,init,gflen,u16,",
+    "msgdata,init,globalfeatures,byte,gflen",
+    "msgdata,init,flen,u16,",
+    "msgdata,init,features,byte,flen",
+    "msgdata,init,tlvs,init_tlvs,",
+    "tlvtype,init_tlvs,networks,1",
+    "tlvdata,init_tlvs,networks,chains,chain_hash,...",
+    "msgtype,error,17",
+    "msgdata,error,channel_id,channel_id,",
+    "msgdata,error,len,u16,",
+    "msgdata,error,data,byte,len",
+    "msgtype,ping,18",
+    "msgdata,ping,num_pong_bytes,u16,",
+    "msgdata,ping,byteslen,u16,",
+    "msgdata,ping,ignored,byte,byteslen",
+    "msgtype,pong,19",
+    "msgdata,pong,byteslen,u16,",
+    "msgdata,pong,ignored,byte,byteslen",
+    "tlvtype,n1,tlv1,1",
+    "tlvdata,n1,tlv1,amount_msat,tu64,",
+    "tlvtype,n1,tlv2,2",
+    "tlvdata,n1,tlv2,scid,short_channel_id,",
+    "tlvtype,n1,tlv3,3",
+    "tlvdata,n1,tlv3,node_id,point,",
+    "tlvdata,n1,tlv3,amount_msat_1,u64,",
+    "tlvdata,n1,tlv3,amount_msat_2,u64,",
+    "tlvtype,n1,tlv4,254",
+    "tlvdata,n1,tlv4,cltv_delta,u16,",
+    "tlvtype,n2,tlv1,0",
+    "tlvdata,n2,tlv1,amount_msat,tu64,",
+    "tlvtype,n2,tlv2,11",
+    "tlvdata,n2,tlv2,cltv_expiry,tu32,",
+]
+bolt_02_csv = [
+    "msgtype,open_channel,32",
+    "msgdata,open_channel,chain_hash,chain_hash,",
+    "msgdata,open_channel,temporary_channel_id,byte,32",
+    "msgdata,open_channel,funding_satoshis,u64,",
+    "msgdata,open_channel,push_msat,u64,",
+    "msgdata,open_channel,dust_limit_satoshis,u64,",
+    "msgdata,open_channel,max_htlc_value_in_flight_msat,u64,",
+    "msgdata,open_channel,channel_reserve_satoshis,u64,",
+    "msgdata,open_channel,htlc_minimum_msat,u64,",
+    "msgdata,open_channel,feerate_per_kw,u32,",
+    "msgdata,open_channel,to_self_delay,u16,",
+    "msgdata,open_channel,max_accepted_htlcs,u16,",
+    "msgdata,open_channel,funding_pubkey,point,",
+    "msgdata,open_channel,revocation_basepoint,point,",
+    "msgdata,open_channel,payment_basepoint,point,",
+    "msgdata,open_channel,delayed_payment_basepoint,point,",
+    "msgdata,open_channel,htlc_basepoint,point,",
+    "msgdata,open_channel,first_per_commitment_point,point,",
+    "msgdata,open_channel,channel_flags,byte,",
+    "msgdata,open_channel,tlvs,open_channel_tlvs,",
+    "tlvtype,open_channel_tlvs,upfront_shutdown_script,0",
+    "tlvdata,open_channel_tlvs,upfront_shutdown_script,shutdown_scriptpubkey,byte,...",
+    "msgtype,accept_channel,33",
+    "msgdata,accept_channel,temporary_channel_id,byte,32",
+    "msgdata,accept_channel,dust_limit_satoshis,u64,",
+    "msgdata,accept_channel,max_htlc_value_in_flight_msat,u64,",
+    "msgdata,accept_channel,channel_reserve_satoshis,u64,",
+    "msgdata,accept_channel,htlc_minimum_msat,u64,",
+    "msgdata,accept_channel,minimum_depth,u32,",
+    "msgdata,accept_channel,to_self_delay,u16,",
+    "msgdata,accept_channel,max_accepted_htlcs,u16,",
+    "msgdata,accept_channel,funding_pubkey,point,",
+    "msgdata,accept_channel,revocation_basepoint,point,",
+    "msgdata,accept_channel,payment_basepoint,point,",
+    "msgdata,accept_channel,delayed_payment_basepoint,point,",
+    "msgdata,accept_channel,htlc_basepoint,point,",
+    "msgdata,accept_channel,first_per_commitment_point,point,",
+    "msgdata,accept_channel,tlvs,accept_channel_tlvs,",
+    "tlvtype,accept_channel_tlvs,upfront_shutdown_script,0",
+    "tlvdata,accept_channel_tlvs,upfront_shutdown_script,shutdown_scriptpubkey,byte,...",
+    "msgtype,funding_created,34",
+    "msgdata,funding_created,temporary_channel_id,byte,32",
+    "msgdata,funding_created,funding_txid,sha256,",
+    "msgdata,funding_created,funding_output_index,u16,",
+    "msgdata,funding_created,signature,signature,",
+    "msgtype,funding_signed,35",
+    "msgdata,funding_signed,channel_id,channel_id,",
+    "msgdata,funding_signed,signature,signature,",
+    "msgtype,funding_locked,36",
+    "msgdata,funding_locked,channel_id,channel_id,",
+    "msgdata,funding_locked,next_per_commitment_point,point,",
+    "msgtype,shutdown,38",
+    "msgdata,shutdown,channel_id,channel_id,",
+    "msgdata,shutdown,len,u16,",
+    "msgdata,shutdown,scriptpubkey,byte,len",
+    "msgtype,closing_signed,39",
+    "msgdata,closing_signed,channel_id,channel_id,",
+    "msgdata,closing_signed,fee_satoshis,u64,",
+    "msgdata,closing_signed,signature,signature,",
+    "msgtype,update_add_htlc,128",
+    "msgdata,update_add_htlc,channel_id,channel_id,",
+    "msgdata,update_add_htlc,id,u64,",
+    "msgdata,update_add_htlc,amount_msat,u64,",
+    "msgdata,update_add_htlc,payment_hash,sha256,",
+    "msgdata,update_add_htlc,cltv_expiry,u32,",
+    "msgdata,update_add_htlc,onion_routing_packet,byte,1366",
+    "msgtype,update_fulfill_htlc,130",
+    "msgdata,update_fulfill_htlc,channel_id,channel_id,",
+    "msgdata,update_fulfill_htlc,id,u64,",
+    "msgdata,update_fulfill_htlc,payment_preimage,byte,32",
+    "msgtype,update_fail_htlc,131",
+    "msgdata,update_fail_htlc,channel_id,channel_id,",
+    "msgdata,update_fail_htlc,id,u64,",
+    "msgdata,update_fail_htlc,len,u16,",
+    "msgdata,update_fail_htlc,reason,byte,len",
+    "msgtype,update_fail_malformed_htlc,135",
+    "msgdata,update_fail_malformed_htlc,channel_id,channel_id,",
+    "msgdata,update_fail_malformed_htlc,id,u64,",
+    "msgdata,update_fail_malformed_htlc,sha256_of_onion,sha256,",
+    "msgdata,update_fail_malformed_htlc,failure_code,u16,",
+    "msgtype,commitment_signed,132",
+    "msgdata,commitment_signed,channel_id,channel_id,",
+    "msgdata,commitment_signed,signature,signature,",
+    "msgdata,commitment_signed,num_htlcs,u16,",
+    "msgdata,commitment_signed,htlc_signature,signature,num_htlcs",
+    "msgtype,revoke_and_ack,133",
+    "msgdata,revoke_and_ack,channel_id,channel_id,",
+    "msgdata,revoke_and_ack,per_commitment_secret,byte,32",
+    "msgdata,revoke_and_ack,next_per_commitment_point,point,",
+    "msgtype,update_fee,134",
+    "msgdata,update_fee,channel_id,channel_id,",
+    "msgdata,update_fee,feerate_per_kw,u32,",
+    "msgtype,channel_reestablish,136",
+    "msgdata,channel_reestablish,channel_id,channel_id,",
+    "msgdata,channel_reestablish,next_commitment_number,u64,",
+    "msgdata,channel_reestablish,next_revocation_number,u64,",
+    "msgdata,channel_reestablish,your_last_per_commitment_secret,byte,32",
+    "msgdata,channel_reestablish,my_current_per_commitment_point,point,",
+]
+bolt_04_csv = [
+    "tlvtype,tlv_payload,amt_to_forward,2",
+    "tlvdata,tlv_payload,amt_to_forward,amt_to_forward,tu64,",
+    "tlvtype,tlv_payload,outgoing_cltv_value,4",
+    "tlvdata,tlv_payload,outgoing_cltv_value,outgoing_cltv_value,tu32,",
+    "tlvtype,tlv_payload,short_channel_id,6",
+    "tlvdata,tlv_payload,short_channel_id,short_channel_id,short_channel_id,",
+    "tlvtype,tlv_payload,payment_data,8",
+    "tlvdata,tlv_payload,payment_data,payment_secret,byte,32",
+    "tlvdata,tlv_payload,payment_data,total_msat,tu64,",
+    "msgtype,invalid_realm,PERM|1",
+    "msgtype,temporary_node_failure,NODE|2",
+    "msgtype,permanent_node_failure,PERM|NODE|2",
+    "msgtype,required_node_feature_missing,PERM|NODE|3",
+    "msgtype,invalid_onion_version,BADONION|PERM|4",
+    "msgdata,invalid_onion_version,sha256_of_onion,sha256,",
+    "msgtype,invalid_onion_hmac,BADONION|PERM|5",
+    "msgdata,invalid_onion_hmac,sha256_of_onion,sha256,",
+    "msgtype,invalid_onion_key,BADONION|PERM|6",
+    "msgdata,invalid_onion_key,sha256_of_onion,sha256,",
+    "msgtype,temporary_channel_failure,UPDATE|7",
+    "msgdata,temporary_channel_failure,len,u16,",
+    "msgdata,temporary_channel_failure,channel_update,byte,len",
+    "msgtype,permanent_channel_failure,PERM|8",
+    "msgtype,required_channel_feature_missing,PERM|9",
+    "msgtype,unknown_next_peer,PERM|10",
+    "msgtype,amount_below_minimum,UPDATE|11",
+    "msgdata,amount_below_minimum,htlc_msat,u64,",
+    "msgdata,amount_below_minimum,len,u16,",
+    "msgdata,amount_below_minimum,channel_update,byte,len",
+    "msgtype,fee_insufficient,UPDATE|12",
+    "msgdata,fee_insufficient,htlc_msat,u64,",
+    "msgdata,fee_insufficient,len,u16,",
+    "msgdata,fee_insufficient,channel_update,byte,len",
+    "msgtype,incorrect_cltv_expiry,UPDATE|13",
+    "msgdata,incorrect_cltv_expiry,cltv_expiry,u32,",
+    "msgdata,incorrect_cltv_expiry,len,u16,",
+    "msgdata,incorrect_cltv_expiry,channel_update,byte,len",
+    "msgtype,expiry_too_soon,UPDATE|14",
+    "msgdata,expiry_too_soon,len,u16,",
+    "msgdata,expiry_too_soon,channel_update,byte,len",
+    "msgtype,incorrect_or_unknown_payment_details,PERM|15",
+    "msgdata,incorrect_or_unknown_payment_details,htlc_msat,u64,",
+    "msgdata,incorrect_or_unknown_payment_details,height,u32,",
+    "msgtype,final_incorrect_cltv_expiry,18",
+    "msgdata,final_incorrect_cltv_expiry,cltv_expiry,u32,",
+    "msgtype,final_incorrect_htlc_amount,19",
+    "msgdata,final_incorrect_htlc_amount,incoming_htlc_amt,u64,",
+    "msgtype,channel_disabled,UPDATE|20",
+    "msgtype,expiry_too_far,21",
+    "msgtype,invalid_onion_payload,PERM|22",
+    "msgdata,invalid_onion_payload,type,varint,",
+    "msgdata,invalid_onion_payload,offset,u16,",
+    "msgtype,mpp_timeout,23",
+]
+bolt_07_csv = [
+    "msgtype,announcement_signatures,259",
+    "msgdata,announcement_signatures,channel_id,channel_id,",
+    "msgdata,announcement_signatures,short_channel_id,short_channel_id,",
+    "msgdata,announcement_signatures,node_signature,signature,",
+    "msgdata,announcement_signatures,bitcoin_signature,signature,",
+    "msgtype,channel_announcement,256",
+    "msgdata,channel_announcement,node_signature_1,signature,",
+    "msgdata,channel_announcement,node_signature_2,signature,",
+    "msgdata,channel_announcement,bitcoin_signature_1,signature,",
+    "msgdata,channel_announcement,bitcoin_signature_2,signature,",
+    "msgdata,channel_announcement,len,u16,",
+    "msgdata,channel_announcement,features,byte,len",
+    "msgdata,channel_announcement,chain_hash,chain_hash,",
+    "msgdata,channel_announcement,short_channel_id,short_channel_id,",
+    "msgdata,channel_announcement,node_id_1,point,",
+    "msgdata,channel_announcement,node_id_2,point,",
+    "msgdata,channel_announcement,bitcoin_key_1,point,",
+    "msgdata,channel_announcement,bitcoin_key_2,point,",
+    "msgtype,node_announcement,257",
+    "msgdata,node_announcement,signature,signature,",
+    "msgdata,node_announcement,flen,u16,",
+    "msgdata,node_announcement,features,byte,flen",
+    "msgdata,node_announcement,timestamp,u32,",
+    "msgdata,node_announcement,node_id,point,",
+    "msgdata,node_announcement,rgb_color,byte,3",
+    "msgdata,node_announcement,alias,byte,32",
+    "msgdata,node_announcement,addrlen,u16,",
+    "msgdata,node_announcement,addresses,byte,addrlen",
+    "msgtype,channel_update,258",
+    "msgdata,channel_update,signature,signature,",
+    "msgdata,channel_update,chain_hash,chain_hash,",
+    "msgdata,channel_update,short_channel_id,short_channel_id,",
+    "msgdata,channel_update,timestamp,u32,",
+    "msgdata,channel_update,message_flags,byte,",
+    "msgdata,channel_update,channel_flags,byte,",
+    "msgdata,channel_update,cltv_expiry_delta,u16,",
+    "msgdata,channel_update,htlc_minimum_msat,u64,",
+    "msgdata,channel_update,fee_base_msat,u32,",
+    "msgdata,channel_update,fee_proportional_millionths,u32,",
+    "msgdata,channel_update,htlc_maximum_msat,u64,,option_channel_htlc_max",
+    "msgtype,query_short_channel_ids,261,gossip_queries",
+    "msgdata,query_short_channel_ids,chain_hash,chain_hash,",
+    "msgdata,query_short_channel_ids,len,u16,",
+    "msgdata,query_short_channel_ids,encoded_short_ids,byte,len",
+    "msgdata,query_short_channel_ids,tlvs,query_short_channel_ids_tlvs,",
+    "tlvtype,query_short_channel_ids_tlvs,query_flags,1",
+    "tlvdata,query_short_channel_ids_tlvs,query_flags,encoding_type,u8,",
+    "tlvdata,query_short_channel_ids_tlvs,query_flags,encoded_query_flags,byte,...",
+    "msgtype,reply_short_channel_ids_end,262,gossip_queries",
+    "msgdata,reply_short_channel_ids_end,chain_hash,chain_hash,",
+    "msgdata,reply_short_channel_ids_end,full_information,byte,",
+    "msgtype,query_channel_range,263,gossip_queries",
+    "msgdata,query_channel_range,chain_hash,chain_hash,",
+    "msgdata,query_channel_range,first_blocknum,u32,",
+    "msgdata,query_channel_range,number_of_blocks,u32,",
+    "msgdata,query_channel_range,tlvs,query_channel_range_tlvs,",
+    "tlvtype,query_channel_range_tlvs,query_option,1",
+    "tlvdata,query_channel_range_tlvs,query_option,query_option_flags,varint,",
+    "msgtype,reply_channel_range,264,gossip_queries",
+    "msgdata,reply_channel_range,chain_hash,chain_hash,",
+    "msgdata,reply_channel_range,first_blocknum,u32,",
+    "msgdata,reply_channel_range,number_of_blocks,u32,",
+    "msgdata,reply_channel_range,full_information,byte,",
+    "msgdata,reply_channel_range,len,u16,",
+    "msgdata,reply_channel_range,encoded_short_ids,byte,len",
+    "msgdata,reply_channel_range,tlvs,reply_channel_range_tlvs,",
+    "tlvtype,reply_channel_range_tlvs,timestamps_tlv,1",
+    "tlvdata,reply_channel_range_tlvs,timestamps_tlv,encoding_type,u8,",
+    "tlvdata,reply_channel_range_tlvs,timestamps_tlv,encoded_timestamps,byte,...",
+    "tlvtype,reply_channel_range_tlvs,checksums_tlv,3",
+    "tlvdata,reply_channel_range_tlvs,checksums_tlv,checksums,channel_update_checksums,...",
+    "subtype,channel_update_timestamps",
+    "subtypedata,channel_update_timestamps,timestamp_node_id_1,u32,",
+    "subtypedata,channel_update_timestamps,timestamp_node_id_2,u32,",
+    "subtype,channel_update_checksums",
+    "subtypedata,channel_update_checksums,checksum_node_id_1,u32,",
+    "subtypedata,channel_update_checksums,checksum_node_id_2,u32,",
+    "msgtype,gossip_timestamp_filter,265,gossip_queries",
+    "msgdata,gossip_timestamp_filter,chain_hash,chain_hash,",
+    "msgdata,gossip_timestamp_filter,first_timestamp,u32,",
+    "msgdata,gossip_timestamp_filter,timestamp_range,u32,",
+]

contrib/pyln-proto/setup.py

@@ -17,7 +17,7 @@
       author='Christian Decker',
       author_email='decker.christian@gmail.com',
       license='MIT',
-      packages=['pyln.proto', 'pyln.proto.message'],
+      packages=['pyln.proto', 'pyln.proto.message', 'pyln.proto.message.bolts'],
       scripts=[],
       zip_safe=True,
       install_requires=requirements)

contrib/pyln-proto/tests/test_bolts.py

@@ -0,0 +1,69 @@
+#! /usr/bin/python3
+from pyln.proto.message import Message, MessageNamespace
+from pyln.proto.message.bolts import bolt_01_csv, bolt_02_csv, bolt_04_csv, bolt_07_csv
+
+
+def test_bolt_01_csv_tlv():
+    ns = MessageNamespace(bolt_01_csv)
+
+    n1 = ns.get_tlvtype('n1')
+
+    # FIXME: Test failure cases too!
+    for t in [['0x', ''],
+              ['0x21 00', '33='],
+              ['0xfd0201 00', '513='],
+              ['0xfd00fd 00', '253='],
+              ['0xfd00ff 00', '255='],
+              ['0xfe02000001 00', '33554433='],
+              ['0xff0200000000000001 00', '144115188075855873='],
+              ['0x01 00', 'tlv1={amount_msat=0}'],
+              ['0x01 01 01', 'tlv1={amount_msat=1}'],
+              ['0x01 02 0100', 'tlv1={amount_msat=256}'],
+              ['0x01 03 010000', 'tlv1={amount_msat=65536}'],
+              ['0x01 04 01000000', 'tlv1={amount_msat=16777216}'],
+              ['0x01 05 0100000000', 'tlv1={amount_msat=4294967296}'],
+              ['0x01 06 010000000000', 'tlv1={amount_msat=1099511627776}'],
+              ['0x01 07 01000000000000', 'tlv1={amount_msat=281474976710656}'],
+              ['0x01 08 0100000000000000', 'tlv1={amount_msat=72057594037927936}'],
+              ['0x02 08 0000000000000226', 'tlv2={scid=0x0x550}'],
+              ['0x03 31 023da092f6980e58d2c037173180e9a465476026ee50f96695963e8efe436f54eb00000000000000010000000000000002', 'tlv3={node_id=023da092f6980e58d2c037173180e9a465476026ee50f96695963e8efe436f54eb,amount_msat_1=1,amount_msat_2=2}'],
+              ['0xfd00fe 02 0226', 'tlv4={cltv_delta=550}']]:
+        msg = bytes.fromhex(t[0][2:].replace(' ', ''))
+
+        val, size = n1.val_from_bin(msg, None)
+        assert size == len(msg)
+        assert n1.val_to_str(val, None) == '{' + t[1] + '}'
+
+
+def test_bolt_01_csv():
+    ns = MessageNamespace(bolt_01_csv)
+    # string [expected string]
+    for t in [['init globalfeatures= features=80',
+               'init globalfeatures= features=80 tlvs={}'],
+              ['init globalfeatures= features=80 tlvs={}'],
+              ['init globalfeatures= features=80 tlvs={networks={chains=[6fe28c0ab6f1b372c1a6a246ae63f74f931e8365e15a089c68d6190000000000]}}'],
+              ['init globalfeatures= features=80 tlvs={networks={chains=[6fe28c0ab6f1b372c1a6a246ae63f74f931e8365e15a089c68d6190000000000,1fe28c0ab6f1b372c1a6a246ae63f74f931e8365e15a089c68d6190000000000]}}'],
+              ['error channel_id=0000000000000000000000000000000000000000000000000000000000000000 data=00'],
+              ['ping num_pong_bytes=0 ignored='],
+              ['ping num_pong_bytes=3 ignored=0000'],
+              ['pong ignored='],
+              ['pong ignored=000000']]:
+        m = Message.from_str(ns, t[0])
+        b = m.to_bin()
+        m2 = Message.from_bin(ns, b)
+        assert m2.to_str() == t[-1]
+
+
+def test_bolt_02_csv():
+    MessageNamespace(bolt_02_csv)
+    # FIXME: Add tests.
+
+
+def test_bolt_04_csv():
+    MessageNamespace(bolt_04_csv)
+    # FIXME: Add tests.
+
+
+def test_bolt_07_csv():
+    MessageNamespace(bolt_07_csv)
+    # FIXME: Add tests.