tests: adjust cln-grpc tests for new default start

contrib/pyln-testing/pyln/testing/utils.py

@@ -595,7 +595,6 @@ def __init__(
             port=9735,
             random_hsm=False,
             node_id=0,
-            grpc_port=None
     ):
         # We handle our own version of verbose, below.
         TailableProc.__init__(self, lightning_dir, verbose=False)
@@ -613,6 +612,7 @@ def __init__(
             'addr': '127.0.0.1:{}'.format(port),
             'allow-deprecated-apis': '{}'.format("true" if DEPRECATED_APIS
                                                  else "false"),
+
             'network': TEST_NETWORK,
             'ignore-fee-limits': 'false',
             'bitcoin-rpcuser': BITCOIND_CONFIG['rpcuser'],
@@ -622,9 +622,6 @@ def __init__(
             'bitcoin-datadir': lightning_dir,
         }
 
-        if grpc_port is not None:
-            opts['grpc-port'] = grpc_port
-
         for k, v in opts.items():
             self.opts[k] = v
 
@@ -758,7 +755,7 @@ def __init__(self, node_id, lightning_dir, bitcoind, executor, valgrind, may_fai
                  broken_log=None,
                  allow_warning=False,
                  allow_bad_gossip=False,
-                 db=None, port=None, disconnect=None, random_hsm=None, options=None,
+                 db=None, port=None, grpc_port=None, disconnect=None, random_hsm=None, options=None,
                  jsonschemas={},
                  valgrind_plugins=True,
                  **kwargs):
@@ -783,7 +780,6 @@ def __init__(self, node_id, lightning_dir, bitcoind, executor, valgrind, may_fai
         self.daemon = LightningD(
             lightning_dir, bitcoindproxy=bitcoind.get_proxy(),
             port=port, random_hsm=random_hsm, node_id=node_id,
-            grpc_port=self.grpc_port,
         )
 
         self.disconnect = disconnect
@@ -834,6 +830,13 @@ def __init__(self, node_id, lightning_dir, bitcoind, executor, valgrind, may_fai
             if SLOW_MACHINE:
                 self.daemon.cmd_prefix += ['--read-inline-info=no']
 
+        if self.daemon.opts.get('disable-plugin') == 'cln-grpc':
+            self.grpc_port = None
+        else:
+            if grpc_port:
+                self.daemon.opts['grpc-port'] = grpc_port
+            self.grpc_port = grpc_port or 9736
+
     def _create_rpc(self, jsonschemas):
         """Prepares anything related to the RPC.
         """
@@ -845,7 +848,7 @@ def _create_rpc(self, jsonschemas):
 
     def _create_grpc_rpc(self):
         from pyln.testing import grpc
-        self.grpc_port = reserve_unused_port()
+        self.grpc_port = self.grpc_port or reserve_unused_port()
         d = self.lightning_dir / TEST_NETWORK
         d.mkdir(parents=True, exist_ok=True)
 
@@ -868,7 +871,6 @@ def _create_grpc_rpc(self):
 
     def _create_jsonrpc_rpc(self, jsonschemas):
         socket_path = self.lightning_dir / TEST_NETWORK / "lightning-rpc"
-        self.grpc_port = None
 
         self.rpc = PrettyPrintingLightningRpc(
             str(socket_path),
@@ -881,12 +883,7 @@ def grpc(self):
         """Tiny helper to return a grpc stub if grpc was configured.
         """
         # Before doing anything let's see if we have a grpc-port at all
-        try:
-            grpc_port = int(filter(
-                lambda v: v[0] == 'grpc-port',
-                self.daemon.opts.items()
-            ).__next__()[1])
-        except Exception:
+        if not self.grpc_port:
             raise ValueError("grpc-port is not specified, can't connect over grpc")
 
         import grpc
@@ -902,7 +899,7 @@ def grpc(self):
         )
 
         channel = grpc.secure_channel(
-            f"localhost:{grpc_port}",
+            f"localhost:{self.grpc_port}",
             creds,
             options=(('grpc.ssl_target_name_override', 'cln'),)
         )
@@ -1590,9 +1587,10 @@ def get_nodes(self, num_nodes, opts=None):
     def get_node(self, node_id=None, options=None, dbfile=None,
                  bkpr_dbfile=None, feerates=(15000, 11000, 7500, 3750),
                  start=True, wait_for_bitcoind_sync=True, may_fail=False,
-                 expect_fail=False, cleandir=True, gossip_store_file=None, **kwargs):
+                 expect_fail=False, cleandir=True, gossip_store_file=None, unused_grpc_port=True, **kwargs):
         node_id = self.get_node_id() if not node_id else node_id
         port = reserve_unused_port()
+        grpc_port = self.get_unused_port() if unused_grpc_port else None
 
         lightning_dir = os.path.join(
             self.directory, "lightning-{}/".format(node_id))
@@ -1606,7 +1604,7 @@ def get_node(self, node_id=None, options=None, dbfile=None,
         db.provider = self.db_provider
         node = self.node_cls(
             node_id, lightning_dir, self.bitcoind, self.executor, self.valgrind, db=db,
-            port=port, options=options, may_fail=may_fail or expect_fail,
+            port=port, grpc_port=grpc_port, options=options, may_fail=may_fail or expect_fail,
             jsonschemas=self.jsonschemas,
             **kwargs
         )

tests/test_cln_rs.py

@@ -20,7 +20,7 @@
 
 def wait_for_grpc_start(node):
     """This can happen before "public key" which start() swallows"""
-    wait_for(lambda: node.daemon.is_in_log(r'serving grpc on 0.0.0.0:'))
+    wait_for(lambda: node.daemon.is_in_log(r'serving grpc'))
 
 
 def test_rpc_client(node_factory):
@@ -35,8 +35,9 @@ def test_plugin_start(node_factory):
     """Start a minimal plugin and ensure it is well-behaved
     """
     bin_path = Path.cwd() / "target" / RUST_PROFILE / "examples" / "cln-plugin-startup"
-    l1 = node_factory.get_node(options={"plugin": str(bin_path), 'test-option': 31337})
-    l2 = node_factory.get_node()
+    l1, l2 = node_factory.get_nodes(2, opts=[
+        {"plugin": str(bin_path), 'test-option': 31337}, {}
+    ])
 
     # The plugin should be in the list of active plugins
     plugins = l1.rpc.plugin('list')['plugins']
@@ -107,9 +108,7 @@ def test_plugin_options_handle_defaults(node_factory):
 def test_grpc_connect(node_factory):
     """Attempts to connect to the grpc interface and call getinfo"""
     # These only exist if we have rust!
-
-    grpc_port = node_factory.get_unused_port()
-    l1 = node_factory.get_node(options={"grpc-port": str(grpc_port)})
+    l1 = node_factory.get_node()
 
     p = Path(l1.daemon.lightning_dir) / TEST_NETWORK
     cert_path = p / "client.pem"
@@ -123,7 +122,7 @@ def test_grpc_connect(node_factory):
 
     wait_for_grpc_start(l1)
     channel = grpc.secure_channel(
-        f"localhost:{grpc_port}",
+        f"localhost:{l1.grpc_port}",
         creds,
         options=(('grpc.ssl_target_name_override', 'cln'),)
     )
@@ -164,10 +163,7 @@ def test_grpc_generate_certificate(node_factory):
      - If we have certs, we they should just get loaded
      - If we delete one cert or its key it should get regenerated.
     """
-    grpc_port = node_factory.get_unused_port()
-    l1 = node_factory.get_node(options={
-        "grpc-port": str(grpc_port),
-    }, start=False)
+    l1 = node_factory.get_node(start=False)
 
     p = Path(l1.daemon.lightning_dir) / TEST_NETWORK
     files = [p / f for f in [
@@ -202,18 +198,20 @@ def test_grpc_generate_certificate(node_factory):
     assert all(private)
 
 
-def test_grpc_no_auto_start(node_factory):
-    """Ensure that we do not start cln-grpc unless a port is configured.
-    Also check that we do not generate certificates.
-    """
-    l1 = node_factory.get_node()
+def test_grpc_default_port_auto_starts(node_factory):
+    """Ensure that we start cln-grpc on default port. Also check that certificates are generated."""
+    l1 = node_factory.get_node(unused_grpc_port=False)
 
-    wait_for(lambda: [p for p in l1.rpc.plugin('list')['plugins'] if 'cln-grpc' in p['name']] == [])
-    assert l1.daemon.is_in_log(r'plugin-cln-grpc: Killing plugin: disabled itself at init')
-    p = Path(l1.daemon.lightning_dir) / TEST_NETWORK
-    files = os.listdir(p)
-    pem_files = [f for f in files if re.match(r".*\.pem$", f)]
-    assert pem_files == []
+    grpcplugin = next((p for p in l1.rpc.plugin('list')['plugins'] if 'cln-grpc' in p['name'] and p['active']), None)
+    # Check that the plugin is active
+    assert grpcplugin is not None
+    # Check that the plugin is listening on the default port
+    assert l1.daemon.is_in_log(f'plugin-cln-grpc: Plugin logging initialized')
+    # Check that the certificates are generated
+    assert len([f for f in os.listdir(Path(l1.daemon.lightning_dir) / TEST_NETWORK) if re.match(r".*\.pem$", f)]) >= 6
+
+    # Check server connection
+    l1.grpc.Getinfo(clnpb.GetinfoRequest())
 
 
 def test_grpc_wrong_auth(node_factory):
@@ -223,12 +221,7 @@ def test_grpc_wrong_auth(node_factory):
     and then we try to cross the wires.
     """
     # These only exist if we have rust!
-
-    grpc_port = node_factory.get_unused_port()
-    l1, l2 = node_factory.get_nodes(2, opts={
-        "start": False,
-        "grpc-port": str(grpc_port),
-    })
+    l1, l2 = node_factory.get_nodes(2, opts=[{"start": False}, {"start": False}])
     l1.start()
     wait_for_grpc_start(l1)
 
@@ -246,7 +239,7 @@ def connect(node):
         )
 
         channel = grpc.secure_channel(
-            f"localhost:{grpc_port}",
+            f"localhost:{node.grpc_port}",
             creds,
             options=(('grpc.ssl_target_name_override', 'cln'),)
         )
@@ -282,8 +275,7 @@ def test_cln_plugin_reentrant(node_factory, executor):
 
     """
     bin_path = Path.cwd() / "target" / RUST_PROFILE / "examples" / "cln-plugin-reentrant"
-    l1 = node_factory.get_node(options={"plugin": str(bin_path)})
-    l2 = node_factory.get_node()
+    l1, l2 = node_factory.get_nodes(2, opts=[{"plugin": str(bin_path)}, {}])
     l2.connect(l1)
     l2.fundchannel(l1)
 
@@ -311,18 +303,13 @@ def test_grpc_keysend_routehint(bitcoind, node_factory):
     recipient.
 
     """
-    grpc_port = node_factory.get_unused_port()
     l1, l2, l3 = node_factory.line_graph(
         3,
-        opts=[
-            {"grpc-port": str(grpc_port)}, {}, {}
-        ],
         announce_channels=True,  # Do not enforce scid-alias
     )
     bitcoind.generate_block(3)
     sync_blockheight(bitcoind, [l1, l2, l3])
 
-    stub = l1.grpc
     chan = l2.rpc.listpeerchannels(l3.info['id'])
 
     routehint = clnpb.RoutehintList(hints=[
@@ -348,19 +335,15 @@ def test_grpc_keysend_routehint(bitcoind, node_factory):
         routehints=routehint,
     )
 
-    res = stub.KeySend(call)
+    res = l1.grpc.KeySend(call)
     print(res)
 
 
 def test_grpc_listpeerchannels(bitcoind, node_factory):
     """ Check that conversions of this rather complex type work.
     """
-    grpc_port = node_factory.get_unused_port()
     l1, l2 = node_factory.line_graph(
         2,
-        opts=[
-            {"grpc-port": str(grpc_port)}, {}
-        ],
         announce_channels=True,  # Do not enforce scid-alias
     )
 
@@ -385,8 +368,7 @@ def test_grpc_listpeerchannels(bitcoind, node_factory):
 
 
 def test_grpc_decode(node_factory):
-    grpc_port = node_factory.get_unused_port()
-    l1 = node_factory.get_node(options={'grpc-port': str(grpc_port)})
+    l1 = node_factory.get_node()
     inv = l1.grpc.Invoice(clnpb.InvoiceRequest(
         amount_msat=clnpb.AmountOrAny(any=True),
         description="desc",
@@ -418,9 +400,7 @@ def test_rust_plugin_subscribe_wildcard(node_factory):
 
 
 def test_grpc_block_added_notifications(node_factory, bitcoind):
-    grpc_port = node_factory.get_unused_port()
-
-    l1 = node_factory.get_node(options={"grpc-port": str(grpc_port)})
+    l1 = node_factory.get_node()
 
     # Test the block_added notification
     # Start listening to block added events over grpc
@@ -436,10 +416,7 @@ def test_grpc_block_added_notifications(node_factory, bitcoind):
 
 
 def test_grpc_connect_notification(node_factory):
-    grpc_port = node_factory.get_unused_port()
-
-    l1 = node_factory.get_node(options={"grpc-port": str(grpc_port)})
-    l2 = node_factory.get_node()
+    l1, l2 = node_factory.get_nodes(2)
 
     # Test the connect notification
     connect_stream = l1.grpc.SubscribeConnect(clnpb.StreamConnectRequest())
@@ -451,10 +428,7 @@ def test_grpc_connect_notification(node_factory):
 
 
 def test_grpc_custommsg_notification(node_factory):
-    grpc_port = node_factory.get_unused_port()
-
-    l1 = node_factory.get_node(options={"grpc-port": str(grpc_port)})
-    l2 = node_factory.get_node()
+    l1, l2 = node_factory.get_nodes(2)
 
     # Test the connect notification
     custommsg_stream = l1.grpc.SubscribeCustomMsg(clnpb.StreamCustomMsgRequest())

tests/test_clnrest.py

@@ -63,7 +63,7 @@ def test_clnrest_uses_grpc_plugin_certificates(node_factory):
     base_url = f'https://{rest_host}:{rest_port}'
     # This might happen really early!
     l1.daemon.logsearch_start = 0
-    l1.daemon.wait_for_logs([r'serving grpc on 0.0.0.0:',
+    l1.daemon.wait_for_logs([r'serving grpc on 127.0.0.1:',
                              r'plugin-clnrest: REST server running at ' + base_url])
     ca_cert = Path(l1.daemon.lightning_dir) / TEST_NETWORK / 'ca.pem'
     http_session = http_session_with_retry()

tests/test_plugin.py

@@ -799,7 +799,7 @@ def test_channel_state_changed_bilateral(node_factory, bitcoind):
 
     # a helper that gives us the next channel_state_changed log entry
     def wait_for_event(node):
-        msg = node.daemon.wait_for_log("channel_state_changed.*new_state.*")
+        msg = node.daemon.wait_for_log("plugin-misc_notifications.py: channel_state_changed.*new_state.*")
         event = ast.literal_eval(re.findall(".*({.*}).*", msg)[0])
         return event
 
@@ -967,7 +967,7 @@ def test_channel_state_changed_unilateral(node_factory, bitcoind):
 
     # a helper that gives us the next channel_state_changed log entry
     def wait_for_event(node):
-        msg = node.daemon.wait_for_log("channel_state_changed.*new_state.*")
+        msg = node.daemon.wait_for_log("plugin-misc_notifications.py: channel_state_changed.*new_state.*")
         event = ast.literal_eval(re.findall(".*({.*}).*", msg)[0])
         return event