crypto: speed up secp256k1/field.py (#160)

* Speed up crypto/secp256k1/field.py by 100x (!) using Cython.

* Fill in the build.py docstring, adjust data types in field.pxd per review.

* Update a secp256k1 mention in the decred package's README.md .

* Stop the build from failing when a C compiler cannot be found.

.gitignore

@@ -1,13 +1,16 @@
 decred/examples/wallets
-decred/dist
-tinywallet/dist
+decred/dist/
 decred/poetry.lock
+tinywallet/dist/
 htmlcov/
 prof/
+.venv/
 __pycache__/
 .pytest_cache
 .coverage
-.venv
+*.so
+*.dll
+*.html
 *.egg-info
 *.sublime-project
 *.sublime-workspace

decred/README.md

@@ -5,7 +5,7 @@ The `decred` package contains everything needed to create
 
 ## Features
 
-1. Pure-Python secp256k1 elliptic curve.
+1. A secp256k1 elliptic curve written in Python and accelerated by Cython.
 
 1. Serializable and de-serializable python versions of important types
 from the `dcrd/wire` package: `MsgTx`, `BlockHeader`, `OutPoint`, etc.

decred/build.py

@@ -0,0 +1,42 @@
+"""
+Copyright (c) 2020, The Decred developers
+See LICENSE for details
+
+This file implements Cython integration (see https://cython.org/) to generate
+a C extension that speeds up the low-level secp256k1 crypto code. This is used
+by the Poetry tool when generating the wheel archive via its `build` command.
+
+It uses a currently undocumented Poetry feature, see:
+https://github.com/python-poetry/poetry/issues/11#issuecomment-379484540
+
+If Cython or a C compiler cannot be found, we skip the compilation
+of the C extension, and the Python code will be used.
+
+The shared library can also be built manually using the command:
+
+$ cythonize -X language_level=3 -a -i ./decred/crypto/secp256k1/field.py
+"""
+
+from distutils.command.build_ext import build_ext
+
+
+class BuildExt(build_ext):
+    def build_extensions(self):
+        try:
+            super().build_extensions()
+        except Exception:
+            pass
+
+
+def build(setup_kwargs):
+    try:
+        from Cython.Build import cythonize
+
+        setup_kwargs.update(
+            dict(
+                ext_modules=cythonize(["decred/crypto/secp256k1/field.py"]),
+                cmdclass=dict(build_ext=BuildExt),
+            )
+        )
+    except Exception:
+        pass

decred/decred/crypto/secp256k1/field.pxd

@@ -0,0 +1,120 @@
+"""
+Copyright (c) 2020, The Decred developers
+See LICENSE for details
+
+Definitions allowing Cython to generate optimized C code that builds a
+dynamic library speeding up the field.py code.
+"""
+
+import cython
+
+
+cdef unsigned long twoBitsMask = 0x03
+cdef unsigned long fourBitsMask = 0x0F
+cdef unsigned long sixBitsMask = 0x3F
+cdef unsigned long eightBitsMask = 0xFF
+cdef unsigned long fieldWords = 10
+cdef unsigned long fieldBase = 26
+cdef unsigned long fieldBaseMask = (1 << fieldBase) - 1
+cdef unsigned long fieldMSBBits = 256 - (fieldBase * (fieldWords - 1))
+cdef unsigned long fieldMSBMask = (1 << fieldMSBBits) - 1
+cdef unsigned long fieldPrimeWordZero = 0x3FFFC2F
+cdef unsigned long fieldPrimeWordOne = 0x3FFFFBF
+cdef unsigned long primePartBy16 = 68719492368
+
+
+cdef class FieldVal:
+    cdef public unsigned long n[10]
+
+    @cython.locals(
+        m=cython.ulong,
+        t0=cython.ulong,
+        t1=cython.ulong,
+        t2=cython.ulong,
+        t3=cython.ulong,
+        t4=cython.ulong,
+        t5=cython.ulong,
+        t6=cython.ulong,
+        t7=cython.ulong,
+        t8=cython.ulong,
+        t9=cython.ulong,
+    )
+    cpdef normalize(self)
+
+    cpdef negateVal(self, FieldVal val, unsigned long magnitude)
+
+    cpdef add(self, FieldVal val)
+
+    @cython.locals(
+        m=cython.ulong,
+        n=cython.ulong,
+        t0=cython.ulong,
+        t1=cython.ulong,
+        t2=cython.ulong,
+        t3=cython.ulong,
+        t4=cython.ulong,
+        t5=cython.ulong,
+        t6=cython.ulong,
+        t7=cython.ulong,
+        t8=cython.ulong,
+        t9=cython.ulong,
+        t10=cython.ulong,
+        t11=cython.ulong,
+        t12=cython.ulong,
+        t13=cython.ulong,
+        t14=cython.ulong,
+        t15=cython.ulong,
+        t16=cython.ulong,
+        t17=cython.ulong,
+        t18=cython.ulong,
+        t19=cython.ulong,
+    )
+    cpdef squareVal(self, FieldVal val)
+
+    cpdef mulInt(self, long val)
+
+    @cython.locals(
+        d=cython.ulong,
+        m=cython.ulong,
+        t0=cython.ulong,
+        t1=cython.ulong,
+        t2=cython.ulong,
+        t3=cython.ulong,
+        t4=cython.ulong,
+        t5=cython.ulong,
+        t6=cython.ulong,
+        t7=cython.ulong,
+        t8=cython.ulong,
+        t9=cython.ulong,
+        t10=cython.ulong,
+        t11=cython.ulong,
+        t12=cython.ulong,
+        t13=cython.ulong,
+        t14=cython.ulong,
+        t15=cython.ulong,
+        t16=cython.ulong,
+        t17=cython.ulong,
+        t18=cython.ulong,
+        t19=cython.ulong,
+    )
+    cpdef mul2(self, FieldVal val, FieldVal val2)
+
+    cpdef add2(self, FieldVal val, FieldVal val2)
+
+    cpdef putBytes(self, char b[32])
+
+    # inverse relies heavily on FieldVal methods, preventing optimization.
+    # @cython.locals(
+    #     a2=FieldVal,
+    #     a3=FieldVal,
+    #     a4=FieldVal,
+    #     a10=FieldVal,
+    #     a11=FieldVal,
+    #     a21=FieldVal,
+    #     a42=FieldVal,
+    #     a45=FieldVal,
+    #     a63=FieldVal,
+    #     a1019=FieldVal,
+    #     a1023=FieldVal,
+    # )
+    # cpdef inverse(self)

decred/decred/crypto/secp256k1/field.py

@@ -49,6 +49,21 @@
 # internal field representation.  It is used during negation.
 fieldPrimeWordOne = 0x3FFFFBF
 
+# The secp256k1 prime is equivalent to 2^256 - 4294968273.
+# 4294968273 in field representation (base 2^26) is:
+# n[0] = 977
+# n[1] = 64
+# That is to say (2^26 * 64) + 977 = 4294968273
+# Since each word is in base 26, the upper terms (t10 and up) start
+# at 260 bits (versus the final desired range of 256 bits), so the
+# field representation of 'c' from above needs to be adjusted for the
+# extra 4 bits by multiplying it by 2^4 = 16.  4294968273 * 16 =
+# 68719492368.  Thus, the adjusted field representation of 'c' is:
+# n[0] = 977 * 16 = 15632
+# n[1] = 64 * 16 = 1024
+# That is to say (2^26 * 1024) + 15632 = 68719492368
+primePartBy16 = 68719492368
+
 
 class FieldVal:
     """
@@ -144,9 +159,9 @@ def fromHex(hexString):
         field value representation.  Only the first 32-bytes are used.
 
         The field value is returned to support chaining, enabling syntax like:
-            f = FieldVal.fromHex("0abc").add(1)
+            f = FieldVal.fromHex("abc").add(1)
         so that:
-            f = 0x0abc + 1
+            f = 0xabc + 1
 
         Args:
             hexString (str): the hex string to be used as field value.
@@ -221,16 +236,11 @@ def equals(self, f):
 
     def setBytes(self, b):
         """
-        SetBytes packs the passed 32-byte big-endian value into the internal
+        setBytes packs the passed 32-byte big-endian value into the internal
         field value representation.
 
-        The field value is returned to support chaining, enabling syntax like:
-            f = FieldVal().setBytes(b).mul(f2)
-        so that:
-            f = b * f2
-
         Preconditions: None
-        Output Normalized: Yes
+        Output Normalized: Yes if no overflow, no otherwise
         Output Max Magnitude: 1
 
         Args:
@@ -763,7 +773,7 @@ def squareVal(self, val):
         t7 = m & fieldBaseMask
         m = (m >> fieldBase) + t8 + t17 * 1024 + t18 * 15632
         t8 = m & fieldBaseMask
-        m = (m >> fieldBase) + t9 + t18 * 1024 + t19 * 68719492368
+        m = (m >> fieldBase) + t9 + t18 * 1024 + t19 * primePartBy16
         t9 = m & fieldMSBMask
         m = m >> fieldMSBBits
 
@@ -1144,7 +1154,7 @@ def mul2(self, val, val2):
         t7 = m & fieldBaseMask
         m = (m >> fieldBase) + t8 + t17 * 1024 + t18 * 15632
         t8 = m & fieldBaseMask
-        m = (m >> fieldBase) + t9 + t18 * 1024 + t19 * 68719492368
+        m = (m >> fieldBase) + t9 + t18 * 1024 + t19 * primePartBy16
         t9 = m & fieldMSBMask
         m = m >> fieldMSBBits
 
@@ -1268,7 +1278,7 @@ def putBytes(self, b):
 
     def bytes(self):
         """
-        bytes unpacks the field value to a 32-byte big-endian ByteArray.  See
+        bytes unpacks the field value to a 32-byte big-endian bytearray.  See
         putBytes for a variant that allows a bytearray to be passed, which can
         be useful to cut down on the number of allocations by allowing the
         caller to reuse a bytearray.
@@ -1277,9 +1287,9 @@ def bytes(self):
           - The field value MUST be normalized.
 
         Return:
-            ByteArray: the field value converted to a 32-byte ByteArray.
+            bytearray: the field value converted to a 32-byte bytearray.
         """
-        b = ByteArray(0, length=32)
+        b = bytearray(32)
         self.putBytes(b)
         return b
 

decred/pyproject.toml

@@ -20,6 +20,7 @@ classifiers = [
     "Programming Language :: Python :: 3.8",
     "Topic :: Office/Business :: Financial"
 ]
+build = "build.py"
 
 [tool.poetry.dependencies]
 python = "^3.6"
@@ -39,6 +40,7 @@ isort = "^4.3"
 websocket-server = "^0.4"
 pytest-benchmark = "^3.2.3"
 pytest-profiling = "^1.7.0"
+Cython = "^0.29.16"
 
 [tool.isort]
 atomic = "true"

decred/tests/unit/crypto/secp256k1/test_field.py

@@ -7,7 +7,7 @@
 
 
 class Test_FieldVal:
-    def test_set_int(self):
+    def test_setInt(self):
         """
         test_set_int ensures that setting a field value to various native
         integers works as expected.
@@ -36,7 +36,7 @@ def test_zero(self):
         f.zero()
         assert all((x == 0 for x in f.n))
 
-    def test_is_zero(self):
+    def test_isZero(self):
         """
         test_is_zero ensures that checking if a field is zero works as expected.
         """