pyln-testing: get unused port for grpc by default to avoid collissions

contrib/pyln-testing/pyln/testing/utils.py

@@ -587,14 +587,13 @@ def __init__(
             port=9735,
             random_hsm=False,
             node_id=0,
-            grpc_port="9736"
+            grpc_port=None
     ):
         # We handle our own version of verbose, below.
         TailableProc.__init__(self, lightning_dir, verbose=False)
         self.executable = 'lightningd'
         self.lightning_dir = lightning_dir
         self.port = port
-        self.grpc_port = grpc_port
         self.cmd_prefix = []
 
         self.rpcproxy = bitcoindproxy
@@ -606,7 +605,6 @@ def __init__(
             'addr': '127.0.0.1:{}'.format(port),
             'allow-deprecated-apis': '{}'.format("true" if DEPRECATED_APIS
                                                  else "false"),
-            'grpc-port': grpc_port,
 
             'network': TEST_NETWORK,
             'ignore-fee-limits': 'false',
@@ -617,6 +615,11 @@ def __init__(
             'bitcoin-datadir': lightning_dir,
         }
 
+        if grpc_port is None:
+            opts['disable-plugin'] = 'cln-grpc'
+        else:
+            opts['grpc-port'] = grpc_port
+
         for k, v in opts.items():
             self.opts[k] = v
 
@@ -750,7 +753,7 @@ def __init__(self, node_id, lightning_dir, bitcoind, executor, valgrind, may_fai
                  broken_log=None,
                  allow_warning=False,
                  allow_bad_gossip=False,
-                 db=None, port=None, disconnect=None, random_hsm=None, options=None,
+                 db=None, port=None, grpc_port=None, disconnect=None, random_hsm=None, options=None,
                  jsonschemas={},
                  valgrind_plugins=True,
                  **kwargs):
@@ -763,7 +766,7 @@ def __init__(self, node_id, lightning_dir, bitcoind, executor, valgrind, may_fai
         self.allow_warning = allow_warning
         self.db = db
         self.lightning_dir = Path(lightning_dir)
-        self.grpc_port = options.get("grpc-port", "9736") if options else "9736"
+        self.grpc_port = grpc_port
 
         # Assume successful exit
         self.rc = 0
@@ -873,12 +876,7 @@ def grpc(self):
         """Tiny helper to return a grpc stub if grpc was configured.
         """
         # Before doing anything let's see if we have a grpc-port at all
-        try:
-            grpc_port = int(filter(
-                lambda v: v[0] == 'grpc-port',
-                self.daemon.opts.items()
-            ).__next__()[1])
-        except Exception:
+        if not self.grpc_port:
             raise ValueError("grpc-port is not specified, can't connect over grpc")
 
         import grpc
@@ -894,7 +892,7 @@ def grpc(self):
         )
 
         channel = grpc.secure_channel(
-            f"localhost:{grpc_port}",
+            f"localhost:{self.grpc_port}",
             creds,
             options=(('grpc.ssl_target_name_override', 'cln'),)
         )
@@ -1582,9 +1580,10 @@ def get_nodes(self, num_nodes, opts=None):
     def get_node(self, node_id=None, options=None, dbfile=None,
                  bkpr_dbfile=None, feerates=(15000, 11000, 7500, 3750),
                  start=True, wait_for_bitcoind_sync=True, may_fail=False,
-                 expect_fail=False, cleandir=True, **kwargs):
+                 expect_fail=False, cleandir=True, grpc=True, **kwargs):
         node_id = self.get_node_id() if not node_id else node_id
         port = reserve_unused_port()
+        grpc_port = self.get_unused_port() if grpc else None
 
         lightning_dir = os.path.join(
             self.directory, "lightning-{}/".format(node_id))
@@ -1598,7 +1597,7 @@ def get_node(self, node_id=None, options=None, dbfile=None,
         db.provider = self.db_provider
         node = self.node_cls(
             node_id, lightning_dir, self.bitcoind, self.executor, self.valgrind, db=db,
-            port=port, options=options, may_fail=may_fail or expect_fail,
+            port=port, grpc_port=grpc_port, options=options, may_fail=may_fail or expect_fail,
             jsonschemas=self.jsonschemas,
             **kwargs
         )

tests/test_cln_rs.py

@@ -20,7 +20,7 @@
 
 def wait_for_grpc_start(node):
     """This can happen before "public key" which start() swallows"""
-    wait_for(lambda: node.daemon.is_in_log(r'serving grpc on 0.0.0.0:'))
+    wait_for(lambda: node.daemon.is_in_log(r'serving grpc'))
 
 
 def test_rpc_client(node_factory):
@@ -36,8 +36,8 @@ def test_plugin_start(node_factory):
     """
     bin_path = Path.cwd() / "target" / RUST_PROFILE / "examples" / "cln-plugin-startup"
     l1, l2 = node_factory.get_nodes(2, opts=[
-        {"grpc-port": str(node_factory.get_unused_port()), "plugin": str(bin_path), 'test-option': 31337},
-        {"grpc-port": str(node_factory.get_unused_port())}])
+        {"plugin": str(bin_path), 'test-option': 31337}, {}
+    ])
 
     # The plugin should be in the list of active plugins
     plugins = l1.rpc.plugin('list')['plugins']
@@ -77,7 +77,6 @@ def test_plugin_options_handle_defaults(node_factory):
     bin_path = Path.cwd() / "target" / RUST_PROFILE / "examples" / "cln-plugin-startup"
     l1 = node_factory.get_node(
         options={
-            "grpc-port": str(node_factory.get_unused_port()),
             "plugin": str(bin_path),
             "opt-option": 31337,
             "test-option": 31338,
@@ -96,7 +95,7 @@ def test_plugin_options_handle_defaults(node_factory):
     assert opts["multi-i64-option-default"] == [5, 6]
 
     # Do not set any value, should be None now
-    l1 = node_factory.get_node(options={"grpc-port": str(node_factory.get_unused_port()), "plugin": str(bin_path)})
+    l1 = node_factory.get_node(options={"plugin": str(bin_path)})
     opts = l1.rpc.testoptions()
     assert opts["opt-option"] is None, "opt-option has no default"
     assert opts["test-option"] == 42, "test-option has a default of 42"
@@ -109,8 +108,7 @@ def test_plugin_options_handle_defaults(node_factory):
 def test_grpc_connect(node_factory):
     """Attempts to connect to the grpc interface and call getinfo"""
     # These only exist if we have rust!
-    grpc_port = node_factory.get_unused_port()
-    l1 = node_factory.get_node(options={"grpc-port": grpc_port})
+    l1 = node_factory.get_node()
 
     p = Path(l1.daemon.lightning_dir) / TEST_NETWORK
     cert_path = p / "client.pem"
@@ -124,7 +122,7 @@ def test_grpc_connect(node_factory):
 
     wait_for_grpc_start(l1)
     channel = grpc.secure_channel(
-        f"localhost:{grpc_port}",
+        f"localhost:{l1.grpc_port}",
         creds,
         options=(('grpc.ssl_target_name_override', 'cln'),)
     )
@@ -165,9 +163,7 @@ def test_grpc_generate_certificate(node_factory):
      - If we have certs, we they should just get loaded
      - If we delete one cert or its key it should get regenerated.
     """
-    l1 = node_factory.get_node(options={
-        "grpc-port": str(node_factory.get_unused_port()),
-    }, start=False)
+    l1 = node_factory.get_node(start=False)
 
     p = Path(l1.daemon.lightning_dir) / TEST_NETWORK
     files = [p / f for f in [
@@ -222,13 +218,11 @@ def test_grpc_wrong_auth(node_factory):
     and then we try to cross the wires.
     """
     # These only exist if we have rust!
-    grpc_port_1 = node_factory.get_unused_port()
-    grpc_port_2 = node_factory.get_unused_port()
-    l1, l2 = node_factory.get_nodes(2, opts=[{"start": False, "grpc-port": str(grpc_port_1)}, {"start": False, "grpc-port": str(grpc_port_2)}])
+    l1, l2 = node_factory.get_nodes(2, opts=[{"start": False}, {"start": False}])
     l1.start()
     wait_for_grpc_start(l1)
 
-    def connect(node, grpcport):
+    def connect(node):
         p = Path(node.daemon.lightning_dir) / TEST_NETWORK
         cert, key, ca = [f.open('rb').read() for f in [
             p / 'client.pem',
@@ -242,13 +236,13 @@ def connect(node, grpcport):
         )
 
         channel = grpc.secure_channel(
-            f"localhost:{grpcport}",
+            f"localhost:{node.grpc_port}",
             creds,
             options=(('grpc.ssl_target_name_override', 'cln'),)
         )
         return clnpb.NodeStub(channel)
 
-    stub = connect(l1, grpc_port_1)
+    stub = connect(l1)
     # This should work, it's the correct node
     stub.Getinfo(clnpb.GetinfoRequest())
 
@@ -261,7 +255,7 @@ def connect(node, grpcport):
         stub.Getinfo(clnpb.GetinfoRequest())
 
     # Now load the correct ones and we should be good to go
-    stub = connect(l2, grpc_port_2)
+    stub = connect(l2)
     stub.Getinfo(clnpb.GetinfoRequest())
 
 
@@ -278,7 +272,7 @@ def test_cln_plugin_reentrant(node_factory, executor):
 
     """
     bin_path = Path.cwd() / "target" / RUST_PROFILE / "examples" / "cln-plugin-reentrant"
-    l1, l2 = node_factory.get_nodes(2, opts=[{"grpc-port": str(node_factory.get_unused_port()), "plugin": str(bin_path)}, {"grpc-port": str(node_factory.get_unused_port())}])
+    l1, l2 = node_factory.get_nodes(2, opts=[{"plugin": str(bin_path)}, {}])
     l2.connect(l1)
     l2.fundchannel(l1)
 
@@ -308,7 +302,6 @@ def test_grpc_keysend_routehint(bitcoind, node_factory):
     """
     l1, l2, l3 = node_factory.line_graph(
         3,
-        opts=[{"grpc-port": str(node_factory.get_unused_port())} for _ in range(3)],
         announce_channels=True,  # Do not enforce scid-alias
     )
     bitcoind.generate_block(3)
@@ -348,7 +341,6 @@ def test_grpc_listpeerchannels(bitcoind, node_factory):
     """
     l1, l2 = node_factory.line_graph(
         2,
-        opts=[{"grpc-port": str(node_factory.get_unused_port())} for _ in range(2)],
         announce_channels=True,  # Do not enforce scid-alias
     )
 
@@ -373,7 +365,7 @@ def test_grpc_listpeerchannels(bitcoind, node_factory):
 
 
 def test_grpc_decode(node_factory):
-    l1 = node_factory.get_node(options={'grpc-port': str(node_factory.get_unused_port())})
+    l1 = node_factory.get_node()
     inv = l1.grpc.Invoice(clnpb.InvoiceRequest(
         amount_msat=clnpb.AmountOrAny(any=True),
         description="desc",
@@ -397,15 +389,15 @@ def test_rust_plugin_subscribe_wildcard(node_factory):
     """
     bin_path = Path.cwd() / "target" / RUST_PROFILE / "examples" / "cln-subscribe-wildcard"
     l1 = node_factory.get_node(options={"plugin": bin_path})
-    l2 = node_factory.get_node(options={"grpc-port": str(node_factory.get_unused_port())})
+    l2 = node_factory.get_node()
 
     l2.connect(l1)
 
     l1.daemon.wait_for_log("Received notification connect")
 
 
 def test_grpc_block_added_notifications(node_factory, bitcoind):
-    l1 = node_factory.get_node(options={"grpc-port": str(node_factory.get_unused_port())})
+    l1 = node_factory.get_node()
 
     # Test the block_added notification
     # Start listening to block added events over grpc
@@ -421,7 +413,7 @@ def test_grpc_block_added_notifications(node_factory, bitcoind):
 
 
 def test_grpc_connect_notification(node_factory):
-    l1, l2 = node_factory.get_nodes(2, opts=[{"grpc-port": str(node_factory.get_unused_port())} for _ in range(2)])
+    l1, l2 = node_factory.get_nodes(2)
 
     # Test the connect notification
     connect_stream = l1.grpc.SubscribeConnect(clnpb.StreamConnectRequest())
@@ -433,7 +425,7 @@ def test_grpc_connect_notification(node_factory):
 
 
 def test_grpc_custommsg_notification(node_factory):
-    l1, l2 = node_factory.get_nodes(2, opts=[{"grpc-port": str(node_factory.get_unused_port())} for _ in range(2)])
+    l1, l2 = node_factory.get_nodes(2)
 
     # Test the connect notification
     custommsg_stream = l1.grpc.SubscribeCustomMsg(clnpb.StreamCustomMsgRequest())

tests/test_clnrest.py

@@ -37,9 +37,8 @@ def test_clnrest_self_signed_certificates(node_factory):
     rest_port = str(node_factory.get_unused_port())
     rest_host = '127.0.0.1'
     base_url = f'https://{rest_host}:{rest_port}'
-    l1 = node_factory.get_node(options={'disable-plugin': 'cln-grpc',
-                                        'clnrest-port': rest_port,
-                                        'clnrest-host': rest_host})
+    l1 = node_factory.get_node(options={'clnrest-port': rest_port,
+                                        'clnrest-host': rest_host}, grpc=False)
     # This might happen really early!
     l1.daemon.logsearch_start = 0
     l1.daemon.wait_for_log(r'plugin-clnrest: REST server running at ' + base_url)
@@ -57,13 +56,12 @@ def test_clnrest_uses_grpc_plugin_certificates(node_factory):
     - clnrest-protocol: https
     """
     rest_host = 'localhost'
-    grpc_port = str(node_factory.get_unused_port())
     rest_port = str(node_factory.get_unused_port())
-    l1 = node_factory.get_node(options={'grpc-port': grpc_port, 'clnrest-host': rest_host, 'clnrest-port': rest_port})
+    l1 = node_factory.get_node(options={'clnrest-host': rest_host, 'clnrest-port': rest_port})
     base_url = f'https://{rest_host}:{rest_port}'
     # This might happen really early!
     l1.daemon.logsearch_start = 0
-    l1.daemon.wait_for_logs([r'serving grpc on 0.0.0.0:',
+    l1.daemon.wait_for_logs([r'serving grpc',
                              r'plugin-clnrest: REST server running at ' + base_url])
     ca_cert = Path(l1.daemon.lightning_dir) / TEST_NETWORK / 'ca.pem'
     http_session = http_session_with_retry()