DECTRIS support; use PipelinedExecutor for live processing

.gitignore

@@ -28,3 +28,4 @@ data
 .mypy_cache/
 .benchmarks
 junit.xml
+profiles/

README.rst

@@ -56,9 +56,11 @@ Detectors
 ---------
 
 * `Quantum Detectors Merlin
-  <https://libertem.github.io/LiberTEM-live/reference.html#quantum-detectors-merlin>`_
+  <https://libertem.github.io/LiberTEM-live/detectors.html#merlin-detector>`_
+* `DECTRIS EIGER2-based
+  <https://libertem.github.io/LiberTEM-live/detectors.html#dectris-detectors>`_
 
-Support for the Gatan K2 IS is currently under development.
+Support for the Gatan K2 IS and X-Spectrum cameras is currently under development.
 
 License
 -------

azure-pipelines-data.yml

@@ -71,7 +71,6 @@ stages:
           Python36:
             python.version: '3.6'
             TOXENV: 'py36-data'
-          # 3.7: default in current Debian stable
           Python37:
             python.version: '3.7'
             TOXENV: 'py37-data'
@@ -82,6 +81,9 @@ stages:
           Python39:
             python.version: '3.9'
             TOXENV: 'py39-data'
+          Python310:
+            python.version: '3.10'
+            TOXENV: 'py310-data'
       variables:
         TESTDATA_BASE_PATH: '/data/'
       steps:
@@ -133,6 +135,9 @@ stages:
           Python39:
             python.version: '3.9'
             TOXENV: 'py39'
+          Python310:
+            python.version: '3.10'
+            TOXENV: 'py310'
       variables:
         TOXENV: '$(TOXENV)'
       steps:
@@ -144,8 +149,13 @@ stages:
       - bash: pip install -U tox
         displayName: 'install requirements'
 
-      - bash: tox
-        displayName: 'Run tox tests $(TOXENV) $(Agent.OS)'
+      - ${{ if eq(parameters.recreate_tox, true) }}:
+        - bash: tox -r
+          displayName: 'Run tox tests $(TOXENV) $(Agent.OS) (recreating)'
+
+      - ${{ if eq(parameters.recreate_tox, false) }}:
+        - bash: tox
+          displayName: 'Run tox tests $(TOXENV) $(Agent.OS)'
 
       - bash: bash <(curl -s https://codecov.io/bash) -f coverage.xml
         displayName: 'Submit coverage to codecov.io'
@@ -169,8 +179,13 @@ stages:
       - bash: pip install -U tox
         displayName: 'install requirements'
 
-      - bash: tox
-        displayName: 'Run tox tests $(TOXENV) $(Agent.OS)'
+      - ${{ if eq(parameters.recreate_tox, true) }}:
+        - bash: tox -r
+          displayName: 'Run tox tests $(TOXENV) $(Agent.OS) (recreating)'
+
+      - ${{ if eq(parameters.recreate_tox, false) }}:
+        - bash: tox
+          displayName: 'Run tox tests $(TOXENV) $(Agent.OS)'
 
       - bash: bash <(curl -s https://codecov.io/bash) -f coverage.xml
         displayName: 'Submit coverage to codecov.io'
@@ -189,6 +204,7 @@ stages:
             TOXENV: 'docs-check'
       variables:
         TOXENV: '$(TOXENV)'
+        TESTDATA_BASE_PATH: '/data/'
       steps:
       - task: UsePythonVersion@0
         displayName: 'Use Python 3.9'

ci/run-with-sim.sh

@@ -1,20 +1,28 @@
 #!/bin/bash
 # run both nbval and libertem-live-mib-sim
+# assumes default ports are usable (i.e. isolated environment)
 
 .tox/notebooks/bin/libertem-live-mib-sim --host 127.0.0.1 "$TESTDATA_BASE_PATH/20200518 165148/default.hdr" --cached=MEMFD --wait-trigger&
-SIM_PID=$!
+MERLIN_SIM_PID=$!
+
+.tox/notebooks/bin/libertem-live-dectris-sim "$TESTDATA_BASE_PATH/dectris/zmqdump.dat.128x128-id34-exte-bslz4" --port 8910 --zmqport 9999&
+DECTRIS_SIM_PID=$!
 
 cleanup() {
-    kill "$SIM_PID"
+    kill "$MERLIN_SIM_PID"
+    kill "$DECTRIS_SIM_PID"
 }
 
 trap cleanup EXIT
 
-echo "started simulator in background, pid=$SIM_PID"
-echo "waiting for data socket to be ready..."
+echo "started merlin simulator in background, pid=$MERLIN_SIM_PID"
+echo "started dectris simulator in background, pid=$DECTRIS_SIM_PID"
+echo "waiting for data sockets to be ready..."
 
-while ! (echo "" > /dev/tcp/127.0.0.1/6342) 2>/dev/null; do echo "Waiting..."; sleep 1; done
+while ! (echo "" > /dev/tcp/127.0.0.1/6342) 2>/dev/null; do echo "Waiting for 6342..."; sleep 1; done
+while ! (echo "" > /dev/tcp/127.0.0.1/8910) 2>/dev/null; do echo "Waiting for 8910..."; sleep 1; done
+while ! (echo "" > /dev/tcp/127.0.0.1/9999) 2>/dev/null; do echo "Waiting for 9999..."; sleep 1; done
 
-echo "done waiting, data socket ready!"
+echo "done waiting, sockets ready!"
 
 "$@"

conftest.py

@@ -3,6 +3,7 @@
 in both doctests and regular tests.
 """
 from libertem.executor.inline import InlineJobExecutor
+from libertem.executor.pipelined import PipelinedExecutor
 import pytest
 
 from libertem.viz.base import Dummy2DPlot
@@ -23,6 +24,17 @@ def ltl_ctx_fast():
     return ltl.LiveContext(executor=inline_executor, plot_class=Dummy2DPlot)
 
 
+@pytest.fixture(scope="session")
+def ctx_pipelined():
+    executor = None
+    try:
+        executor = PipelinedExecutor()
+        yield ltl.LiveContext(executor=executor, plot_class=Dummy2DPlot)
+    finally:
+        if executor is not None:
+            executor.close()
+
+
 @pytest.fixture()
 def default_aq(ltl_ctx):
     return ltl_ctx.prepare_acquisition('memory', trigger=None, datashape=[16, 16, 32, 32])

docs/source/changelog/features/dectris.rst

@@ -0,0 +1,3 @@
+[Feature] Support for DECTRIS detectors
+=======================================
+* Initial support for DECTRIS detectors, including QUADRO and ARINA (:pr:`51`).

docs/source/changelog/features/pipelined.rst

@@ -0,0 +1,4 @@
+[Feature] Parallel pipelined processing
+=======================================
+* Scalability improvements - now uses the pipelined executor added in LiberTEM
+  0.10 for parallel processing (:pr:`51`).

docs/source/changelog/features/record.rst

@@ -0,0 +1,4 @@
+[Feature] Data recording
+========================
+
+* :ref:`recording` with :class:`~libertem_live.udf.record.RecordUDF` (:pr:`51`).

docs/source/conf.py

@@ -66,6 +66,7 @@
     'nbsphinx_link',
     'IPython.sphinxext.ipython_console_highlighting',
     'sphinx_issues',
+    'sphinx_rtd_theme',
 ]
 
 bibtex_bibfiles = ['references-libertem_live.bib']
@@ -87,7 +88,7 @@
 #
 # This is also used if you do content translation via gettext catalogs.
 # Usually you set "language" from the command line for these cases.
-language = 'en'
+language = "en"
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
@@ -103,7 +104,9 @@
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
 #
-html_theme = 'alabaster'
+html_theme = 'sphinx_rtd_theme'
+
+html_logo = '_static/logo.png'
 
 html_favicon = '../../corporatedesign/logo/favicon.ico'
 
@@ -112,7 +115,6 @@
 # documentation.
 #
 html_theme_options = {
-    'logo': 'logo.png'
 }
 
 # Add any paths that contain custom static files (such as style sheets) here,
@@ -207,7 +209,7 @@
 
 intersphinx_mapping = {
     'numpy': ('http://docs.scipy.org/doc/numpy/', None),
-    'python': ('https://docs.python.org/3.7', None),
+    'python': ('https://docs.python.org/3.10', None),
     'libertem': ('https://libertem.github.io/LiberTEM/', None),
 }
 
@@ -220,3 +222,19 @@
     # Freezes the link checker for unknown reasons within CI, hard to reproduce
     r'http://quantumdetectors.com/wp-content/uploads/2017/01/1532-Merlin-for-EM-Technical-Datasheet-v2.pdf',  # NOQA:E501
 ]
+
+doctest_global_setup = '''
+import os
+def get_testdata_path():
+    return os.environ.get(
+        'TESTDATA_BASE_PATH',
+        os.path.normpath(
+            os.path.join(os.getcwd(), 'data')
+        )
+    )
+DECTRIS_TESTDATA_PATH = os.path.join(
+    get_testdata_path(),
+    'dectris', 'zmqdump.dat.128x128-id34-exte-bslz4'
+)
+HAVE_DECTRIS_TESTDATA = os.path.exists(DECTRIS_TESTDATA_PATH)
+'''

docs/source/detectors.rst

@@ -0,0 +1,149 @@
+Supported Detectors
+===================
+
+.. _`dectris detectors`:
+
+DECTRIS EIGER2-based
+--------------------
+
+.. versionadded:: 0.2
+
+LiberTEM-live has support for all DECTRIS detectors that support
+the `SIMPLON API <https://media.dectris.com/210607-DECTRIS-SIMPLON-API-Manual_EIGER2-chip-based_detectros.pdf>`_,
+including QUADRO and ARINA.
+
+.. note::
+    Python 3.7+ is required to use the DECTRIS-related features of
+    LiberTEM-live, and we recommend to use Python 3.10. Currently, Windows and
+    Linux on x86_64 are supported.
+
+Installation
+............
+
+There are a few extra dependencies needed for DECTRIS support. You can easily
+install them using:
+
+.. code-block:: shell
+
+    (libertem) $ python -m pip install "libertem-live[dectris]"
+
+Currently, at least LiberTEM 0.10 is required. Using LiberTEM master is
+recommended until a number of stability fixes for the pipelined executor are
+released in LiberTEM.
+
+Usage example
+.............
+
+.. testsetup::
+    :skipif: not HAVE_DECTRIS_TESTDATA
+
+    from libertem_live.detectors.dectris.sim import DectrisSim
+    server = DectrisSim(path=DECTRIS_TESTDATA_PATH, port=0, zmqport=0, verbose=False)
+    server.start()
+    server.wait_for_listen()
+    DCU_API_PORT = server.port
+    DCU_DATA_PORT = server.zmqport
+
+.. testoutput::
+    :hide:
+
+    RustedReplay listening on tcp://127.0.0.1:...
+    Waiting for arm
+
+.. testcode::
+    :skipif: not HAVE_DECTRIS_TESTDATA
+
+    from libertem.executor.pipelined import PipelinedExecutor
+    from libertem.viz.bqp import BQLive2DPlot
+    from libertem_live.api import LiveContext
+    from libertem.udf.sum import SumUDF
+
+    executor = PipelinedExecutor(spec=PipelinedExecutor.make_spec(cpus=range(10), cudas=[]))
+    ctx = LiveContext(executor=executor, plot_class=BQLive2DPlot)
+
+    def trigger(aq):
+        """
+        You can trigger the scan here, if you have a microscope control API
+        """
+        # you can imagine something like:
+        # trigger_scan(shape=tuple(aq.shape.nav), dwelltime=67e-6)
+        pass
+
+    # prepare for acquisition, setting up scan parameters etc.
+    aq = ctx.prepare_acquisition(
+        'dectris',
+        api_host="127.0.0.1",
+        api_port=DCU_API_PORT,
+        data_host="127.0.0.1",
+        data_port=DCU_DATA_PORT,
+        nav_shape=(128, 128),
+        trigger_mode="exte",
+        trigger=trigger,
+        frames_per_partition=512,
+    )
+
+    # run one or more UDFs on the live data stream:
+    # (this can be run multiple times on the same `aq` object)
+    ctx.run_udf(dataset=aq, udf=SumUDF(), plots=True)
+
+.. testoutput::
+    :hide:
+
+    ...
+
+.. testcleanup::
+    :skipif: not HAVE_DECTRIS_TESTDATA
+
+    ctx.close()
+    server.stop()
+    server.maybe_raise()
+
+See the :ref:`DECTRIS reference section <dectris reference>` for a description of
+the acquisition parameters.
+
+.. _`merlin detector`:
+
+Quantum Detectors Merlin
+------------------------
+
+No extra dependencies are needed for using Merlin detectors.
+
+Supported are currently 1 bit, 6 bit and 12 bit :code:`COUNTERDEPTH` for both the "binary"
+(:code:`FILEFORMAT 0`) and the "raw binary" format (:code:`FILEFORMAT 2`).
+
+For testing, an acquisition with soft trigger (:code:`TRIGGERSTART 5`) is
+recommended since internal trigger (:code:`TRIGGERSTART 0`) may cause issues
+with finding the beginning of the data stream. For a real STEM acquisition a
+hardware trigger setup that matches the given instrument is required. See the
+MerlinEM User Manual from Quantum Detectors for details!
+
+A simple simulator for testing live acquisition without the actual hardware is
+included in LiberTEM-live. It replays an MIB dataset and accepts the following
+parameters:
+
+.. code-block:: shell
+
+    (libertem) $ libertem-live-mib-sim --help
+    Usage: libertem-live-mib-sim [OPTIONS] PATH
+
+    Options:
+    --nav-shape <INTEGER INTEGER>...
+    --continuous
+    --cached [NONE|MEM|MEMFD]
+    --host TEXT                     Address to listen on (data, control, and
+                                    trigger sockets)
+    --data-port INTEGER
+    --control-port INTEGER
+    --wait-trigger                  Wait for a SOFTTRIGGER command on the
+                                    control port, or a trigger signal on the
+                                    trigger socket
+    --garbage                       Send garbage before trigger. Implies --wait-
+                                    trigger
+    --max-runs INTEGER
+    --help                          Show this message and exit.
+
+A suitable MIB dataset for testing can be downloaded at
+https://zenodo.org/record/5113449.
+
+See the :ref:`Merlin reference section <merlin reference>` for a description of
+the acquisition parameters.

docs/source/index.rst

@@ -19,10 +19,10 @@ Documentation
    :caption: Contents:
 
    usage
+   detectors
+   reference
    changelog
    acknowledgments
-   reference
-
 
 Indices and tables
 ------------------