Docs: Add --without-daemon flag to benchmark script (#5839)

The benchmark script sends a number of `ArithmeticAddCalculation`s to
the daemon and reports the time taken per process at the end. This is
intended for users to run to get an idea of the performance of their
setup. However, running over the daemon is not necessary the most
straightforward when it comes to analysing the results and the timings
can differ quite a bit. It makes more sense to simply run the N jobs
serially, which should give a better more consistent picture of the
average overhead per process.

The option `--without-daemon/--with-daemon` is added to the command
to allow switching the behavior. By default the jobs are now run
sequentially, not over the daemon.

docs/source/howto/include/scripts/performance_benchmark_base.py

@@ -10,8 +10,9 @@
 @click.command()
 @options.CODE(required=False, help='A code that can run the ``ArithmeticAddCalculation``, for example bash.')
 @click.option('-n', 'number', type=int, default=100, show_default=True, help='The number of processes to submit.')
+@click.option('--daemon/--without-daemon', default=False, is_flag=True, help='Submit to daemon or run synchronously.')
 @decorators.with_dbenv()
-def main(code, number):
+def main(code, number, daemon):
     """Submit a number of ``ArithmeticAddCalculation`` to the daemon and record time to completion.
 
     This command requires the daemon to be running.
@@ -28,14 +29,14 @@ def main(code, number):
 
     from aiida import orm
     from aiida.common import exceptions
-    from aiida.engine import submit
+    from aiida.engine import run_get_node, submit
     from aiida.engine.daemon.client import get_daemon_client
     from aiida.plugins import CalculationFactory
     from aiida.tools.graph.deletions import delete_nodes
 
     client = get_daemon_client()
 
-    if not client.is_daemon_running:
+    if daemon and not client.is_daemon_running:
         echo.echo_critical('The daemon is not running.')
 
     computer_created = False
@@ -72,34 +73,42 @@ def main(code, number):
     time_start = time.time()
     nodes = []
 
-    with click.progressbar(range(number), label=f'Submitting {number} calculations.') as bar:
-        for iteration in bar:
-            node = submit(builder)
-            nodes.append(node)
+    if daemon:
+        with click.progressbar(range(number), label=f'Submitting {number} calculations.') as bar:
+            for iteration in bar:
+                node = submit(builder)
+                nodes.append(node)
 
-    time_end = time.time()
-    echo.echo(f'Submission completed in {(time_end - time_start):.2f} seconds.')
+        time_end = time.time()
+        echo.echo(f'Submission completed in {(time_end - time_start):.2f} seconds.')
+
+        completed = 0
 
-    completed = 0
+        with click.progressbar(length=number, label='Waiting for calculations to complete') as bar:
+            while True:
+                time.sleep(0.2)
 
-    with click.progressbar(length=number, label='Waiting for calculations to complete') as bar:
-        while True:
-            time.sleep(0.2)
+                terminated = [node.is_terminated for node in nodes]
+                newly_completed = terminated.count(True) - completed
+                completed = terminated.count(True)
 
-            terminated = [node.is_terminated for node in nodes]
-            newly_completed = terminated.count(True) - completed
-            completed = terminated.count(True)
+                bar.update(newly_completed)
 
-            bar.update(newly_completed)
+                if all(terminated):
+                    break
 
-            if all(terminated):
-                break
+    else:
+        with click.progressbar(range(number), label=f'Running {number} calculations.') as bar:
+            for iteration in bar:
+                _, node = run_get_node(builder)
+                nodes.append(node)
 
     if any(node.is_excepted or node.is_killed for node in nodes):
         echo.echo_warning('At least one submitted calculation excepted or was killed.')
     else:
         echo.echo_success('All calculations finished successfully.')
 
+
     time_end = time.time()
     echo.echo(f'Elapsed time: {(time_end - time_start):.2f} seconds.')
 

docs/source/howto/installation.rst

@@ -347,26 +347,27 @@ Further in-depth information is available in the dedicated :ref:`topic on perfor
 
 .. dropdown:: Benchmark workflow engine performance
 
-    Start the AiiDA daemon with a single worker, download the :download:`benchmark script <include/scripts/performance_benchmark_base.py>` :fa:`download`, and run it in your AiiDA environment.
+    Download the :download:`benchmark script <include/scripts/performance_benchmark_base.py>` :fa:`download`, and run it in your AiiDA environment.
 
     .. code:: console
 
         sph@citadel:~/$ python performance_benchmark_base.py -n 100
-            Success: Created and configured temporary `Computer` benchmark-5fa8c67f for localhost.
+            Success: Created and configured temporary `Computer` benchmark-e73b8647 for localhost.
             Success: Created temporary `Code` bash for localhost.
-            Submitting 100 calculations.  [####################################]  100%
-            Submission completed in 9.36 seconds.
-            Waiting for calculations to complete  [####################################]  100%
+            Running 100 calculations.  [####################################]  100%
             Success: All calculations finished successfully.
-            Elapsed time: 46.55 seconds.
+            Elapsed time: 24.90 seconds.
             Cleaning up...
+            12/19/2022 10:57:43 AM <12625> aiida.delete: [REPORT] 400 Node(s) marked for deletion
+            12/19/2022 10:57:43 AM <12625> aiida.delete: [REPORT] Starting node deletion...
+            12/19/2022 10:57:43 AM <12625> aiida.delete: [REPORT] Deletion of nodes completed.
             Success: Deleted all calculations.
-            Success: Deleted the created code bash@benchmark-5fa8c67f.
-            Success: Deleted the created computer benchmark-5fa8c67f.
-            Performance: 0.47 s / process
+            Success: Deleted the created code bash@benchmark-e73b8647.
+            Success: Deleted the created computer benchmark-e73b8647.
+            Performance: 0.25 s / process
 
-    The output above was generated with a *single* daemon worker on one core of an AMD Ryzen 5 3600 6-Core processor (3.6 GHz, 4.2 GHz turbo boost) using AiiDA v1.6.9, and RabbitMQ and PostgreSQL running on the same machine.
-    Here, 100 ``ArithmeticAddCalculation`` processes completed in ~47s (including the time needed to submit them), corresponding to an average of half a second per process.
+    The output above was generated on an AMD Ryzen 5 3600 6-Core processor (3.6 GHz, 4.2 GHz turbo boost) using AiiDA v2.2.0, and RabbitMQ and PostgreSQL running on the same machine.
+    Here, 100 ``ArithmeticAddCalculation`` processes completed in ~25s, corresponding to 0.25 seconds per process.
 
     If you observe a significantly higher runtime, you may want to check whether any relevant component (CPU, disk, postgresql, rabbitmq) is congested.