diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..8d7e16e --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,44 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at: +// https://github.com/microsoft/vscode-dev-containers/tree/v0.194.0/containers/ubuntu +{ + "name": "xspress-detector devcontainer", + "build":{ + "dockerfile": "../Dockerfile", + "context": "..", + "target": "build" + }, + "remoteUser": "${localEnv:VSCODE_REMOTE_USER}", + // Add the IDs of extensions you want installed when the container is created. + "customizations": { + "vscode": { + "extensions": [ + "ms-python.vscode-pylance", + "tamasfe.even-better-toml", + "redhat.vscode-yaml", + "ryanluker.vscode-coverage-gutters", + "epicsdeb.vscode-epics", + "charliermarsh.ruff", + "ms-vscode.cmake-tools", + "ms-vscode.cpptools", + "ms-azuretools.vscode-docker" + ] + } + }, + + "features": { + // Some default things like git config + "ghcr.io/devcontainers/features/common-utils:2": { + "upgradePackages": false + } + }, + + "runArgs": [ + "--net=host", + "--security-opt=label=disable" + ], + // Mount the parent of the project folder so we can access peer projects + "workspaceMount": "source=${localWorkspaceFolder}/..,target=/workspaces,type=bind", + "mounts": [ + "source=/dev/shm,target=/dev/shm,type=bind" + ] +} diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..a71039e --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,29 @@ +{ + "cmake.generator": "Unix Makefiles", + "cmake.sourceDirectory": "${workspaceFolder}/cpp", + "cmake.buildDirectory": "${workspaceFolder}/vscode_build", + "cmake.installPrefix": "/odin", + "cmake.configureArgs": [ + "-DODINDATA_ROOT_DIR=/odin", + "-DLIBXSPRESS_ROOT_DIR=/libxspress/prefix" + ], + "C_Cpp.default.configurationProvider": "ms-vscode.cmake-tools", + "editor.formatOnSave": false, + "[python]": { + "editor.codeActionsOnSave": { + "source.organizeImports": "never" + }, + "editor.defaultFormatter": "ms-python.python", + "editor.rulers": [ + 88 + ] + }, + "python.linting.flake8Args": [ + "--max-line-length=88" + ], + "python.formatting.provider": "black", + "isort.args": [ + "--profile", + "black" + ], +} \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index c878d1b..98afda0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,13 +1,19 @@ FROM ghcr.io/odin-detector/odin-data-build:latest AS build +# Install numactl for the MetaWriter process +RUN apt-get update +RUN apt-get install numactl + # Copy xspress-detector source in for build COPY . /tmp/xspress-detector + +# In here you should copy your version of libxspress inside the container # COPY ./libxspress /libxspress # C++ WORKDIR /tmp/xspress-detector RUN mkdir -p build && cd build && \ - cmake -DCMAKE_INSTALL_PREFIX=/odin -DODINDATA_ROOT_DIR=/odin -DLIBXSPRESS_ROOT_DIR=/libxspress ../cpp && \ + cmake -DCMAKE_INSTALL_PREFIX=/odin -DODINDATA_ROOT_DIR=/odin ../cpp && \ make -j8 VERBOSE=1 && \ make install @@ -20,4 +26,4 @@ FROM ghcr.io/odin-detector/odin-data-runtime:latest COPY --from=build /odin /odin # COPY --from=build /libxspress /libxspress ENV PATH=/odin/bin:/odin/venv/bin:$PATH -WORKDIR /odin +WORKDIR /odin \ No newline at end of file diff --git a/deploy/fp1.json b/deploy/fp1.json new file mode 100644 index 0000000..dc40db3 --- /dev/null +++ b/deploy/fp1.json @@ -0,0 +1,416 @@ +[ + { + "fr_setup": { + "fr_ready_cnxn": "tcp://127.0.0.1:10001", + "fr_release_cnxn": "tcp://127.0.0.1:10002" + }, + "meta_endpoint": "tcp://*:10008" + }, + { + "plugin": { + "load": { + "index": "xspress", + "name": "XspressProcessPlugin", + "library": "/odin/lib/libXspressProcessPlugin.so" + } + } + }, + { + "plugin": { + "load": { + "index": "xspresslist", + "name": "XspressListModeProcessPlugin", + "library": "/odin/lib/libXspressListModeProcessPlugin.so" + } + } + }, + { + "plugin": { + "load": { + "index": "view", + "name": "LiveViewPlugin", + "library": "/odin/lib/libLiveViewPlugin.so" + } + } + }, + { + "plugin": { + "load": { + "index": "blosc", + "name": "BloscPlugin", + "library": "/odin/lib/libBloscPlugin.so" + } + } + }, + { + "plugin": { + "load": { + "index": "hdf", + "name": "FileWriterPlugin", + "library": "/odin/lib/libHdf5Plugin.so" + } + } + }, + { + "plugin": { + "connect": { + "index": "xspress", + "connection": "frame_receiver" + } + } + }, + { + "plugin": { + "connect": { + "index": "view", + "connection": "xspress" + } + } + }, + { + "plugin": { + "connect": { + "index": "blosc", + "connection": "xspress" + } + } + }, + { + "plugin": { + "connect": { + "index": "hdf", + "connection": "blosc" + } + } + }, + { + "hdf": { + "process": { + "number": 1, + "rank": 0 + }, + "file": { + "postfix": "_A" + } + } + }, + { + "hdf": { + "dataset": { + "mca_0": { + "datatype": "uint32", + "chunks": [ + 256, + 1, + 4096 + ], + "dims": [ + 1, + 4096 + ], + "compression": "blosc", + "indexes": true + } + } + } + }, + { + "hdf": { + "dataset": { + "mca_1": { + "datatype": "uint32", + "chunks": [ + 256, + 1, + 4096 + ], + "dims": [ + 1, + 4096 + ], + "compression": "blosc", + "indexes": true + } + } + } + }, + { + "hdf": { + "dataset": { + "mca_2": { + "datatype": "uint32", + "chunks": [ + 256, + 1, + 4096 + ], + "dims": [ + 1, + 4096 + ], + "compression": "blosc", + "indexes": true + } + } + } + }, + { + "hdf": { + "dataset": { + "mca_3": { + "datatype": "uint32", + "chunks": [ + 256, + 1, + 4096 + ], + "dims": [ + 1, + 4096 + ], + "compression": "blosc", + "indexes": true + } + } + } + }, + { + "hdf": { + "master": "mca_3" + } + }, + { + "xspress": { + "live_view": "view" + } + }, + { + "view": { + "live_view_socket_addr": "tcp://0.0.0.0:15500", + "dataset_name": "live", + "frame_frequency": 0, + "per_second": 20 + } + }, + { + "store": { + "index": "mca", + "value": [ + { + "plugin": { + "disconnect": "all" + } + }, + { + "plugin": { + "disconnect": { + "index": "xspresslist", + "connection": "frame_receiver" + } + } + }, + { + "plugin": { + "connect": { + "index": "xspress", + "connection": "frame_receiver" + } + } + }, + { + "plugin": { + "connect": { + "index": "view", + "connection": "xspress" + } + } + }, + { + "plugin": { + "connect": { + "index": "blosc", + "connection": "xspress" + } + } + }, + { + "plugin": { + "connect": { + "index": "hdf", + "connection": "blosc" + } + } + }, + { + "hdf": { + "delete_datasets": true + } + }, + { + "hdf": { + "dataset": { + "mca_0": { + "datatype": "uint32", + "chunks": [ + 256, + 1, + 4096 + ], + "dims": [ + 1, + 4096 + ], + "compression": "blosc", + "indexes": true + } + } + } + }, + { + "hdf": { + "dataset": { + "mca_1": { + "datatype": "uint32", + "chunks": [ + 256, + 1, + 4096 + ], + "dims": [ + 1, + 4096 + ], + "compression": "blosc", + "indexes": true + } + } + } + }, + { + "hdf": { + "dataset": { + "mca_2": { + "datatype": "uint32", + "chunks": [ + 256, + 1, + 4096 + ], + "dims": [ + 1, + 4096 + ], + "compression": "blosc", + "indexes": true + } + } + } + }, + { + "hdf": { + "dataset": { + "mca_3": { + "datatype": "uint32", + "chunks": [ + 256, + 1, + 4096 + ], + "dims": [ + 1, + 4096 + ], + "compression": "blosc", + "indexes": true + } + } + } + } + ] + } + }, + { + "store": { + "index": "list", + "value": [ + { + "plugin": { + "disconnect": "all" + } + }, + { + "plugin": { + "disconnect": { + "index": "xspress", + "connection": "frame_receiver" + } + } + }, + { + "plugin": { + "connect": { + "index": "xspresslist", + "connection": "frame_receiver" + } + } + }, + { + "plugin": { + "connect": { + "index": "hdf", + "connection": "xspresslist" + } + } + }, + { + "xspresslist": { + "channels": [0, 1, 2, 3, 4 + ], + "frame_size": 4194304 + } + }, + { + "hdf": { + "delete_datasets": true + } + }, + { + "hdf": { + "dataset": { + "raw_0": { + "datatype": "uint64", + "chunks": [ + 524288 + ] + }, + "raw_1": { + "datatype": "uint64", + "chunks": [ + 524288 + ] + }, + "raw_2": { + "datatype": "uint64", + "chunks": [ + 524288 + ] + }, + "raw_3": { + "datatype": "uint64", + "chunks": [ + 524288 + ] + }, + "raw_4": { + "datatype": "uint64", + "chunks": [ + 524288 + ] + } + } + } + } + ] + } + } +] \ No newline at end of file diff --git a/deploy/fp2.json b/deploy/fp2.json new file mode 100644 index 0000000..a0c35c5 --- /dev/null +++ b/deploy/fp2.json @@ -0,0 +1,416 @@ +[ + { + "fr_setup": { + "fr_ready_cnxn": "tcp://127.0.0.1:10011", + "fr_release_cnxn": "tcp://127.0.0.1:10012" + }, + "meta_endpoint": "tcp://*:10018" + }, + { + "plugin": { + "load": { + "index": "xspress", + "name": "XspressProcessPlugin", + "library": "/odin/lib/libXspressProcessPlugin.so" + } + } + }, + { + "plugin": { + "load": { + "index": "xspresslist", + "name": "XspressListModeProcessPlugin", + "library": "/odin/lib/libXspressListModeProcessPlugin.so" + } + } + }, + { + "plugin": { + "load": { + "index": "view", + "name": "LiveViewPlugin", + "library": "/odin/lib/libLiveViewPlugin.so" + } + } + }, + { + "plugin": { + "load": { + "index": "blosc", + "name": "BloscPlugin", + "library": "/odin/lib/libBloscPlugin.so" + } + } + }, + { + "plugin": { + "load": { + "index": "hdf", + "name": "FileWriterPlugin", + "library": "/odin/lib/libHdf5Plugin.so" + } + } + }, + { + "plugin": { + "connect": { + "index": "xspress", + "connection": "frame_receiver" + } + } + }, + { + "plugin": { + "connect": { + "index": "view", + "connection": "xspress" + } + } + }, + { + "plugin": { + "connect": { + "index": "blosc", + "connection": "xspress" + } + } + }, + { + "plugin": { + "connect": { + "index": "hdf", + "connection": "blosc" + } + } + }, + { + "hdf": { + "process": { + "number": 1, + "rank": 0 + }, + "file": { + "postfix": "_B" + } + } + }, + { + "hdf": { + "dataset": { + "mca_4": { + "datatype": "uint32", + "chunks": [ + 256, + 1, + 4096 + ], + "dims": [ + 1, + 4096 + ], + "compression": "blosc", + "indexes": true + } + } + } + }, + { + "hdf": { + "dataset": { + "mca_5": { + "datatype": "uint32", + "chunks": [ + 256, + 1, + 4096 + ], + "dims": [ + 1, + 4096 + ], + "compression": "blosc", + "indexes": true + } + } + } + }, + { + "hdf": { + "dataset": { + "mca_6": { + "datatype": "uint32", + "chunks": [ + 256, + 1, + 4096 + ], + "dims": [ + 1, + 4096 + ], + "compression": "blosc", + "indexes": true + } + } + } + }, + { + "hdf": { + "dataset": { + "mca_7": { + "datatype": "uint32", + "chunks": [ + 256, + 1, + 4096 + ], + "dims": [ + 1, + 4096 + ], + "compression": "blosc", + "indexes": true + } + } + } + }, + { + "hdf": { + "master": "mca_7" + } + }, + { + "xspress": { + "live_view": "view" + } + }, + { + "view": { + "live_view_socket_addr": "tcp://0.0.0.0:15501", + "dataset_name": "live", + "frame_frequency": 0, + "per_second": 20 + } + }, + { + "store": { + "index": "mca", + "value": [ + { + "plugin": { + "disconnect": "all" + } + }, + { + "plugin": { + "disconnect": { + "index": "xspresslist", + "connection": "frame_receiver" + } + } + }, + { + "plugin": { + "connect": { + "index": "xspress", + "connection": "frame_receiver" + } + } + }, + { + "plugin": { + "connect": { + "index": "view", + "connection": "xspress" + } + } + }, + { + "plugin": { + "connect": { + "index": "blosc", + "connection": "xspress" + } + } + }, + { + "plugin": { + "connect": { + "index": "hdf", + "connection": "blosc" + } + } + }, + { + "hdf": { + "delete_datasets": true + } + }, + { + "hdf": { + "dataset": { + "mca_4": { + "datatype": "uint32", + "chunks": [ + 256, + 1, + 4096 + ], + "dims": [ + 1, + 4096 + ], + "compression": "blosc", + "indexes": true + } + } + } + }, + { + "hdf": { + "dataset": { + "mca_5": { + "datatype": "uint32", + "chunks": [ + 256, + 1, + 4096 + ], + "dims": [ + 1, + 4096 + ], + "compression": "blosc", + "indexes": true + } + } + } + }, + { + "hdf": { + "dataset": { + "mca_6": { + "datatype": "uint32", + "chunks": [ + 256, + 1, + 4096 + ], + "dims": [ + 1, + 4096 + ], + "compression": "blosc", + "indexes": true + } + } + } + }, + { + "hdf": { + "dataset": { + "mca_7": { + "datatype": "uint32", + "chunks": [ + 256, + 1, + 4096 + ], + "dims": [ + 1, + 4096 + ], + "compression": "blosc", + "indexes": true + } + } + } + } + ] + } + }, + { + "store": { + "index": "list", + "value": [ + { + "plugin": { + "disconnect": "all" + } + }, + { + "plugin": { + "disconnect": { + "index": "xspress", + "connection": "frame_receiver" + } + } + }, + { + "plugin": { + "connect": { + "index": "xspresslist", + "connection": "frame_receiver" + } + } + }, + { + "plugin": { + "connect": { + "index": "hdf", + "connection": "xspresslist" + } + } + }, + { + "xspresslist": { + "channels": [5, 6, 7, 8, 9 + ], + "frame_size": 4194304 + } + }, + { + "hdf": { + "delete_datasets": true + } + }, + { + "hdf": { + "dataset": { + "raw_5": { + "datatype": "uint64", + "chunks": [ + 524288 + ] + }, + "raw_6": { + "datatype": "uint64", + "chunks": [ + 524288 + ] + }, + "raw_7": { + "datatype": "uint64", + "chunks": [ + 524288 + ] + }, + "raw_8": { + "datatype": "uint64", + "chunks": [ + 524288 + ] + }, + "raw_9": { + "datatype": "uint64", + "chunks": [ + 524288 + ] + } + } + } + } + ] + } + } +] \ No newline at end of file diff --git a/deploy/fr1.json b/deploy/fr1.json new file mode 100644 index 0000000..e514543 --- /dev/null +++ b/deploy/fr1.json @@ -0,0 +1,19 @@ +[ + { + "decoder_type": "Xspress", + "decoder_path": "/odin/lib", + "rx_type": "zmq", + "rx_address": "127.0.0.1", + "rx_ports": "15150,", + "shared_buffer_name": "odin_buf_1", + "max_buffer_mem": 1048576000, + "frame_ready_endpoint": "tcp://127.0.0.1:10001", + "frame_release_endpoint": "tcp://127.0.0.1:10002" + }, + { + "decoder_config": { + "enable_packet_logging": false, + "frame_timeout_ms": 1000 + } + } +] \ No newline at end of file diff --git a/deploy/fr2.json b/deploy/fr2.json new file mode 100644 index 0000000..10fb689 --- /dev/null +++ b/deploy/fr2.json @@ -0,0 +1,19 @@ +[ + { + "decoder_type": "Xspress", + "decoder_path": "/odin/lib", + "rx_type": "zmq", + "rx_address": "127.0.0.1", + "rx_ports": "15151,", + "shared_buffer_name": "odin_buf_2", + "max_buffer_mem": 1048576000, + "frame_ready_endpoint": "tcp://127.0.0.1:10011", + "frame_release_endpoint": "tcp://127.0.0.1:10012" + }, + { + "decoder_config": { + "enable_packet_logging": false, + "frame_timeout_ms": 1000 + } + } +] \ No newline at end of file diff --git a/deploy/layout.kdl b/deploy/layout.kdl new file mode 100644 index 0000000..4586005 --- /dev/null +++ b/deploy/layout.kdl @@ -0,0 +1,25 @@ +layout { + pane size=1 borderless=true { + plugin location="zellij:tab-bar" + } + pane split_direction="vertical" { + pane split_direction="horizontal" { + pane command="./stFrameReceiver1.sh" + pane command="./stFrameProcessor1.sh" + pane command="./stFrameReceiver2.sh" + pane command="./stFrameProcessor2.sh" + } + pane split_direction="horizontal" { + pane command="./stControlServer.sh" + pane command="./stMetaWriter.sh" + pane command="bash" { + args "-c" "sleep 3; ./stOdinServer.sh" + } + } + } + pane size=2 borderless=true { + plugin location="zellij:status-bar" + } +} +session_name "XSPRESS" +attach_to_session true \ No newline at end of file diff --git a/deploy/odin_server.cfg b/deploy/odin_server.cfg new file mode 100644 index 0000000..b368b0a --- /dev/null +++ b/deploy/odin_server.cfg @@ -0,0 +1,40 @@ +[server] +debug_mode = 0 +http_port = 8888 +http_addr = 0.0.0.0 +static_path = ../example-config/static +adapters = fp, fr, meta_listener, xspress + +[tornado] +logging = error + +[adapter.xspress] +module = xspress_detector.control.adapter.XspressAdapter +endpoint = 127.0.0.1:12000 +num_cards = 2 +num_tf = 16384 +base_ip = 192.168.0.1 +max_channels = 8 +max_spectra = 4096 +settings_path = /your/settings_path +run_flags = 2 +debug = 1 +num_process = 2 +daq_endpoints = tcp://127.0.0.1:15150,tcp://127.0.0.1:15151 + +[adapter.fp] +# module = xspress_detector.control.fp_xspress_adapter.FPXspressAdapter +module = odin_data.control.odin_data_adapter.OdinDataAdapter +frame_processor = true +endpoints = 127.0.0.1:10004,127.0.0.1:10014 +update_interval = 0.2 + +[adapter.fr] +module = odin_data.control.frame_receiver_adapter.FrameReceiverAdapter +endpoints = 127.0.0.1:10000,127.0.0.1:10010 +update_interval = 0.2 + +[adapter.meta_listener] +module = odin_data.control.meta_listener_adapter.MetaListenerAdapter +endpoints = 127.0.0.1:5659 +update_interval = 0.5 diff --git a/deploy/stControlServer.sh b/deploy/stControlServer.sh new file mode 100755 index 0000000..2876544 --- /dev/null +++ b/deploy/stControlServer.sh @@ -0,0 +1,3 @@ +#!/bin/bash +SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )" +/odin/bin/xspressControl -j $SCRIPT_DIR/xspress.json -s 1 \ No newline at end of file diff --git a/deploy/stFrameProcessor1.sh b/deploy/stFrameProcessor1.sh new file mode 100755 index 0000000..7ff892f --- /dev/null +++ b/deploy/stFrameProcessor1.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )" + +export HDF5_PLUGIN_PATH=/odin/h5plugin + +/odin/bin/frameProcessor --ctrl=tcp://0.0.0.0:10004 --config=$SCRIPT_DIR/fp1.json --log-config $SCRIPT_DIR/log4cxx.xml diff --git a/deploy/stFrameProcessor2.sh b/deploy/stFrameProcessor2.sh new file mode 100755 index 0000000..f66f5ba --- /dev/null +++ b/deploy/stFrameProcessor2.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )" + +export HDF5_PLUGIN_PATH=/odin/h5plugin + +/odin/bin/frameProcessor --ctrl=tcp://0.0.0.0:10014 --config=$SCRIPT_DIR/fp2.json --log-config $SCRIPT_DIR/log4cxx.xml diff --git a/deploy/stFrameReceiver1.sh b/deploy/stFrameReceiver1.sh new file mode 100755 index 0000000..1f222cc --- /dev/null +++ b/deploy/stFrameReceiver1.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )" + +/odin/bin/frameReceiver --io-threads 1 --ctrl=tcp://0.0.0.0:10000 --config=$SCRIPT_DIR/fr1.json --log-config $SCRIPT_DIR/log4cxx.xml diff --git a/deploy/stFrameReceiver2.sh b/deploy/stFrameReceiver2.sh new file mode 100755 index 0000000..4cdb17d --- /dev/null +++ b/deploy/stFrameReceiver2.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )" + +/odin/bin/frameReceiver --io-threads 1 --ctrl=tcp://0.0.0.0:10010 --config=$SCRIPT_DIR/fr2.json --log-config $SCRIPT_DIR/log4cxx.xml diff --git a/deploy/stLiveViewMerge.sh b/deploy/stLiveViewMerge.sh new file mode 100755 index 0000000..47f197e --- /dev/null +++ b/deploy/stLiveViewMerge.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )" + +/venv/bin/xspress_live_merge --sub_ports 15500,15501 diff --git a/deploy/stMetaWriter.sh b/deploy/stMetaWriter.sh new file mode 100755 index 0000000..fa93d88 --- /dev/null +++ b/deploy/stMetaWriter.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +numactl --membind=0 --cpunodebind=0 /venv/bin/xspress_meta_writer -w xspress_detector.data.xspress_meta_writer.XspressMetaWriter --sensor-shape 8 4096 --data-endpoints tcp://127.0.0.1:10008,tcp://127.0.0.1:10018 --static-log-fields beamline=${BEAMLINE},detector="Xspress4" diff --git a/deploy/stOdinServer.sh b/deploy/stOdinServer.sh new file mode 100755 index 0000000..7ebca61 --- /dev/null +++ b/deploy/stOdinServer.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )" + +# Increase maximum fds available for ZeroMQ sockets +ulimit -n 2048 + +/venv/bin/xspress_control --config=$SCRIPT_DIR/odin_server.cfg --logging=info --access_logging=ERROR \ No newline at end of file diff --git a/deploy/xspress.json b/deploy/xspress.json new file mode 100644 index 0000000..73544c7 --- /dev/null +++ b/deploy/xspress.json @@ -0,0 +1,9 @@ +[ + { + "app": + { + "debug_level": 2, + "ctrl_endpoint": "tcp://127.0.0.1:12000" + } + } +]