pytest: add test for bug found by Travis

We fail to restore HTLCs on reconnect sometimes, depending on traverse order:

	2019-10-30T18:39:40.012Z **BROKEN** lightningd(7652): lightning_channeld-0266e4598d1d3c415f572a8488830b60f7e744ed9235eb0b1ba93283b315c03518 chan #1: Cannot add htlc #0 10000msat to LOCAL
	2019-10-30T18:39:40.024Z **BROKEN** lightningd(7652): lightning_channeld-0266e4598d1d3c415f572a8488830b60f7e744ed9235eb0b1ba93283b315c03518 chan #1: Could not restore HTLCs (version v0.7.3-12-ga0a271a)

Or, alternatively:

lightning_channeld: Could not restore HTLCs (version v0.7.3-11-gd7838db-modded)
0x564d1c1b53bd send_backtrace
	common/daemon.c:41
0x564d1c1c23c9 status_failed
	common/status.c:199
0x564d1c1a7509 init_channel
	channeld/channeld.c:3073
0x564d1c1a7959 main
	channeld/channeld.c:3165
0x7fdc73be01e2 ???
	???:0
0x564d1c19ee5d ???
	???:0
0xffffffffffffffff ???
	???:0

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>

tests/test_connection.py

@@ -2196,3 +2196,35 @@ def test_feerate_stress(node_factory, executor):
     l1.rpc.call('dev-feerate', [l2.info['id'], rate - 5])
     assert not l1.daemon.is_in_log('Bad.*signature')
     assert not l2.daemon.is_in_log('Bad.*signature')
+
+
+@pytest.mark.xfail(strict=True)
+@unittest.skipIf(not DEVELOPER, "need dev_disconnect")
+def test_pay_disconnect_stress(node_factory, executor):
+    """Expose race in htlc restoration in channeld: 50% chance of failure"""
+    for i in range(5):
+        l1, l2 = node_factory.line_graph(2, opts=[{'may_reconnect': True},
+                                                  {'may_reconnect': True,
+                                                   'disconnect': ['=WIRE_UPDATE_ADD_HTLC',
+                                                                  '-WIRE_COMMITMENT_SIGNED']}])
+
+        scid12 = l1.get_channel_scid(l2)
+        routel2l1 = [{'msatoshi': '10000msat', 'id': l1.info['id'], 'delay': 5, 'channel': scid12}]
+
+        # Get invoice from l1 to pay.
+        payhash1 = l1.rpc.invoice(10000, "invoice", "invoice")['payment_hash']
+
+        # Start balancing payment.
+        fut = executor.submit(l1.pay, l2, 10**9 // 2)
+
+        # As soon as reverse payment is accepted, reconnect.
+        while True:
+            l2.rpc.sendpay(routel2l1, payhash1)
+            try:
+                # This will usually fail with Capacity exceeded
+                l2.rpc.waitsendpay(payhash1, timeout=TIMEOUT)
+                break
+            except RpcError:
+                pass
+
+        fut.result()