pyln minor fixes, move bolts to pyln.spec.bolt*

Makefile

@@ -340,7 +340,7 @@ check-markdown:
 check-spelling:
 	@tools/check-spelling.sh
 
-PYSRC=$(shell git ls-files "*.py") contrib/pylightning/lightning-pay
+PYSRC=$(shell git ls-files "*.py" | grep -v /text.py) contrib/pylightning/lightning-pay
 
 check-python:
 	@# E501 line too long (N > 79 characters)

contrib/pyln-proto/Makefile

@@ -17,21 +17,21 @@ check-flake8:
 
 # mypy . does not recurse.  I have no idea why...
 check-mypy:
-	mypy --ignore-missing-imports `find * -name '*.py'`
+	mypy --ignore-missing-imports `find pyln/proto/message/ -name '*.py'`
 
 $(SDIST_FILE):
 	python3 setup.py sdist
 
 $(BDIST_FILE):
-	python3 setup.py bdist
+	python3 setup.py bdist_wheel
 
 test-release: check $(ARTEFACTS)
 	python3 -m twine upload --repository testpypi --skip-existing $(ARTEFACTS)
 
 	# Create a test virtualenv, install from the testpypi and run the
 	# tests against it (make sure not to use any virtualenv that may have
 	# pyln-proto already installed).
-	virtualenv --no-site-packages testpypi --python=/usr/bin/python3 --download --always-copy --clear
+	virtualenv testpypi --python=/usr/bin/python3 --download --always-copy --clear
 	# Install the requirements from the prod repo, they are not being kept up to date on the test repo
 	testpypi/bin/python3 -m pip install -r requirements.txt pytest flaky pytest-timeout
 	testpypi/bin/python3 -m pip install -I --index-url https://test.pypi.org/simple/ --no-deps pyln-proto

contrib/pyln-proto/pyln/proto/__init__.py

@@ -2,7 +2,7 @@
 from .onion import OnionPayload, TlvPayload, LegacyOnionPayload
 from .wire import LightningConnection, LightningServerSocket
 
-__version__ = '0.8.2'
+__version__ = '0.8.3'
 
 __all__ = [
     "Invoice",

contrib/pyln-proto/pyln/proto/message/array_types.py

@@ -1,8 +1,8 @@
 from .fundamental_types import FieldType, IntegerType, split_field
-from typing import List, Optional, Dict, Tuple, TYPE_CHECKING, Any, Union
+from typing import List, Optional, Dict, Tuple, TYPE_CHECKING, Any, Union, cast
 from io import BufferedIOBase
 if TYPE_CHECKING:
-    from .message import SubtypeType, TlvStreamType
+    from .message import SubtypeType, TlvMessageType, MessageTypeField
 
 
 class ArrayType(FieldType):
@@ -98,7 +98,7 @@ def read(self, io_in: BufferedIOBase, otherfields: Dict[str, Any]) -> List[Any]:
 class EllipsisArrayType(ArrayType):
     """This is used for ... fields at the end of a tlv: the array ends
 when the tlv ends"""
-    def __init__(self, tlv: 'TlvStreamType', name: str, elemtype: FieldType):
+    def __init__(self, tlv: 'TlvMessageType', name: str, elemtype: FieldType):
         super().__init__(tlv, name, elemtype)
 
     def read(self, io_in: BufferedIOBase, otherfields: Dict[str, Any]) -> List[Any]:
@@ -119,13 +119,13 @@ def __init__(self, inttype: IntegerType):
         super().__init__(inttype.name)
         self.underlying_type = inttype
         # You can be length for more than one field!
-        self.len_for: List[DynamicArrayType] = []
+        self.len_for: List['MessageTypeField'] = []
 
     def is_optional(self) -> bool:
         """This field value is always implies, never specified directly"""
         return True
 
-    def add_length_for(self, field: 'DynamicArrayType') -> None:
+    def add_length_for(self, field: 'MessageTypeField') -> None:
         assert isinstance(field.fieldtype, DynamicArrayType)
         self.len_for.append(field)
 
@@ -160,7 +160,7 @@ def name_and_val(self, name: str, v: int) -> str:
 they're implied by the length of other fields"""
         return ''
 
-    def read(self, io_in: BufferedIOBase, otherfields: Dict[str, Any]) -> None:
+    def read(self, io_in: BufferedIOBase, otherfields: Dict[str, Any]) -> Optional[int]:
         """We store this, but it'll be removed from the fields as soon as it's used (i.e. by DynamicArrayType's val_from_bin)"""
         return self.underlying_type.read(io_in, otherfields)
 
@@ -186,11 +186,11 @@ def len_fields_bad(self, fieldname: str, otherfields: Dict[str, Any]) -> List[st
 
 class DynamicArrayType(ArrayType):
     """This is used for arrays where another field controls the size"""
-    def __init__(self, outer: 'SubtypeType', name: str, elemtype: FieldType, lenfield: LengthFieldType):
+    def __init__(self, outer: 'SubtypeType', name: str, elemtype: FieldType, lenfield: 'MessageTypeField'):
         super().__init__(outer, name, elemtype)
         assert type(lenfield.fieldtype) is LengthFieldType
         self.lenfield = lenfield
 
     def read(self, io_in: BufferedIOBase, otherfields: Dict[str, Any]) -> List[Any]:
         return super().read_arr(io_in, otherfields,
-                                self.lenfield.fieldtype._maybe_calc_value(self.lenfield.name, otherfields))
+                                cast(LengthFieldType, self.lenfield.fieldtype)._maybe_calc_value(self.lenfield.name, otherfields))

contrib/pyln-proto/pyln/proto/message/fundamental_types.py

@@ -59,6 +59,15 @@ def len_fields_bad(self, fieldname: str, fieldvals: Dict[str, Any]) -> List[str]
     def val_to_str(self, v: Any, otherfields: Dict[str, Any]) -> str:
         raise NotImplementedError()
 
+    def val_from_str(self, s: str) -> Tuple[Any, str]:
+        raise NotImplementedError()
+
+    def write(self, io_out: BufferedIOBase, v: Any, otherfields: Dict[str, Any]) -> None:
+        raise NotImplementedError()
+
+    def read(self, io_in: BufferedIOBase, otherfields: Dict[str, Any]) -> Any:
+        raise NotImplementedError()
+
     def val_to_py(self, v: Any, otherfields: Dict[str, Any]) -> Any:
         """Convert to a python object: for simple fields, this means a string"""
         return self.val_to_str(v, otherfields)
@@ -83,7 +92,7 @@ def val_from_str(self, s: str) -> Tuple[int, str]:
         a, b = split_field(s)
         return int(a), b
 
-    def val_to_py(self, v: Any, otherfields: Dict[str, Any]) -> int:
+    def val_to_py(self, v: Any, otherfields: Dict[str, Any]) -> Any:
         """Convert to a python object: for integer fields, this means an int"""
         return int(v)
 
@@ -240,8 +249,37 @@ def val_to_py(self, v: Any, otherfields: Dict[str, Any]) -> int:
         return int(v)
 
 
-def fundamental_types():
-    # From 01-messaging.md#fundamental-types:
+def fundamental_types() -> List[FieldType]:
+    # BOLT #1:
+    # Various fundamental types are referred to in the message specifications:
+    #
+    # * `byte`: an 8-bit byte
+    # * `u16`: a 2 byte unsigned integer
+    # * `u32`: a 4 byte unsigned integer
+    # * `u64`: an 8 byte unsigned integer
+    #
+    # Inside TLV records which contain a single value, leading zeros in
+    # integers can be omitted:
+    #
+    # * `tu16`: a 0 to 2 byte unsigned integer
+    # * `tu32`: a 0 to 4 byte unsigned integer
+    # * `tu64`: a 0 to 8 byte unsigned integer
+    #
+    # The following convenience types are also defined:
+    #
+    # * `chain_hash`: a 32-byte chain identifier (see [BOLT
+    #   #0](00-introduction.md#glossary-and-terminology-guide))
+    # * `channel_id`: a 32-byte channel_id (see [BOLT
+    #   #2](02-peer-protocol.md#definition-of-channel-id))
+    # * `sha256`: a 32-byte SHA2-256 hash
+    # * `signature`: a 64-byte bitcoin Elliptic Curve signature
+    # * `point`: a 33-byte Elliptic Curve point (compressed encoding as per
+    #   [SEC 1 standard](http://www.secg.org/sec1-v2.pdf#subsubsection.2.3.3))
+    # * `short_channel_id`: an 8 byte value identifying a channel (see [BOLT
+    #   #7](07-routing-gossip.md#definition-of-short-channel-id))
+    # * `bigsize`: a variable-length, unsigned integer similar to Bitcoin's
+    #   CompactSize encoding, but big-endian.  Described in
+    #   [BigSize](#appendix-a-bigsize-test-vectors).
     return [IntegerType('byte', 1, 'B'),
             IntegerType('u16', 2, '>H'),
             IntegerType('u32', 4, '>I'),
@@ -256,10 +294,6 @@ def fundamental_types():
             ShortChannelIDType('short_channel_id'),
             FundamentalHexType('signature', 64),
             BigSizeType('bigsize'),
-            # FIXME: See https://github.com/lightningnetwork/lightning-rfc/pull/778
-            BigSizeType('varint'),
-            # FIXME
-            IntegerType('u8', 1, 'B'),
             ]