Cap the number of probes to send

This commit does two things: It caps the number of probes
to send per prefix and per TTL to 4095 after the first 256
probes.  It also pins the ClickHouse server to version 22.8
because the latest ClickHouse version does not work with
this version of diamond-miner

.github/workflows/tests.yml

@@ -7,7 +7,7 @@ jobs:
     runs-on: ubuntu-latest
     services:
       clickhouse:
-        image: clickhouse/clickhouse-server
+        image: clickhouse/clickhouse-server:22.8
         ports: ["8123:8123"]
     steps:
       - uses: actions/checkout@v3

diamond_miner/generators/database.py

@@ -16,6 +16,7 @@
 from diamond_miner.queries import GetProbesDiff
 from diamond_miner.typing import FlowMapper, IPNetwork, Probe
 
+max_probes = 0
 
 def probe_generator_from_database(
     client: ClickHouseClient,
@@ -45,6 +46,11 @@ def probe_generator_from_database(
         >>> (str(ip_address(probes[0][0])), *probes[0][1:])
         ('::ffff:808:100', 24000, 33434, 1, 'icmp')
     """
+    global max_probes
+    if max_probes == 0:
+        max_probes = 4095 # XXX make this a parameter
+        logger.info("capping the number of probes to send at %d", max_probes)
+
     rows = GetProbesDiff(
         round_eq=round_, probe_ttl_geq=probe_ttl_geq, probe_ttl_leq=probe_ttl_leq
     ).execute_iter(client, measurement_id, subsets=subsets)
@@ -60,8 +66,8 @@ def probe_generator_from_database(
                 addr_offset, port_offset = mapper.offset(flow_id, dst_prefix_int)
                 dst_addr = dst_prefix_int + addr_offset
                 src_port = probe_src_port + port_offset
-                if src_port > (2**16 - 1):
-                    # TEMP: Log prefixes that overflows the port number and skip prefix.
-                    logger.warning("Port overflow for %s", row)
+                # Note that port_offset is actually the number of probes sent after having already sent 256 probes.
+                if port_offset > max_probes:
+                    logger.warning("not probing %s after having already sent %d probes", row, max_probes+256)
                     break
                 yield dst_addr, src_port, probe_dst_port, ttl, protocol_str  # type: ignore

docs/dev.md

@@ -31,7 +31,7 @@ poetry run bumpversion patch # or minor/major
 Most tests require a running instance of ClickHouse with pre-populated tables.
 To start a ClickHouse server and insert the test data:
 ```bash
-docker run --rm -d -p 8123:8123 clickhouse/clickhouse-server:22.6
+docker run --rm -d -p 8123:8123 clickhouse/clickhouse-server:22.8
 poetry run python tests/data/insert.py
 ```
 

docs/usage.md

@@ -12,7 +12,7 @@ These components can be pieced together to conduct various kind of topology meas
 
 To run the examples below, you need a running [ClickHouse](https://clickhouse.com) server:
 ```bash
-docker run --rm -d -p 8123:8123 clickhouse/clickhouse-server:22.6
+docker run --rm -d -p 8123:8123 clickhouse/clickhouse-server:22.8
 ```
 
 You also need [`pycaracal`](https://github.com/dioptra-io/caracal) and [`pych-client`](https://github.com/dioptra-io/pych-client).