Merge dev into main

.devcontainer/devcontainer.json

@@ -12,19 +12,12 @@
     "remoteEnv": {
         "DISPLAY": "${localEnv:DISPLAY}"
     },
-    // Add the URLs of features you want added when the container is built.
-    "features": {
-        "ghcr.io/devcontainers/features/common-utils:1": {
-            "username": "none",
-            "upgradePackages": false
-        }
-    },
     // Set *default* container specific settings.json values on container create.
-    "settings": {
-        "python.defaultInterpreterPath": "/venv/bin/python"
-    },
     "customizations": {
         "vscode": {
+            "settings": {
+                "python.defaultInterpreterPath": "/venv/bin/python"
+            },
             // Add the IDs of extensions you want installed when the container is created.
             "extensions": [
                 "ms-python.python",
@@ -51,4 +44,4 @@
     "workspaceFolder": "${localWorkspaceFolder}",
     // After the container is created, install the python project in editable form
     "postCreateCommand": "pip install -e '.[dev]'"
-}
+}

.github/workflows/code.yml

@@ -33,12 +33,12 @@ jobs:
       fail-fast: false
       matrix:
         os: ["ubuntu-latest"] # can add windows-latest, macos-latest
-        python: ["3.9", "3.10", "3.11"]
+        python: ["3.10", "3.11"]
         install: ["-e .[dev]"]
         # Make one version be non-editable to test both paths of version code
         include:
           - os: "ubuntu-latest"
-            python: "3.8"
+            python: "3.10"
             install: ".[dev]"
 
     runs-on: ${{ matrix.os }}

.gitignore

@@ -66,4 +66,3 @@ venv*
 
 # further build artifacts
 lockfiles/
-

.vscode/launch.json

@@ -12,7 +12,8 @@
             "args": [
                 "softioc",
                 "172.23.252.201",
-                "SOME-PREFIX"
+                "SOME-PREFIX",
+                "./screens"
             ],
             "console": "integratedTerminal",
             "justMyCode": false

Dockerfile

@@ -3,7 +3,7 @@
 # The devcontainer should use the build target and run as root with podman
 # or docker with user namespaces.
 #
-FROM python:3.11 as build
+FROM python:3.10 as build
 
 ARG PIP_OPTIONS=.
 
@@ -24,7 +24,7 @@ WORKDIR /context
 # install python package into /venv
 RUN pip install ${PIP_OPTIONS}
 
-FROM python:3.11-slim as runtime
+FROM python:3.10-slim as runtime
 
 # Add apt-get system dependecies for runtime here if needed
 

pyproject.toml

@@ -7,20 +7,18 @@ name = "PandABlocks-ioc"
 classifiers = [
     "Development Status :: 3 - Alpha",
     "License :: OSI Approved :: Apache Software License",
-    "Programming Language :: Python :: 3.8",
-    "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
 ]
 description = "One line description of your module"
 dependencies = [
+    "setuptools>=64",
     "numpy",
     "click",
     "h5py",
-    "softioc>=4.1.0",
+    "softioc>=4.4.0",
     "pandablocks>=0.3.1",
-    "pvi[cli]>=0.4",
-    "aiohttp",
+    "pvi>=0.5",
 ] # Add project dependencies here, e.g. ["click", "numpy"]
 dynamic = ["version"]
 license.file = "LICENSE"
@@ -37,8 +35,9 @@ dev = [
     "Flake8-pyproject",
     "pipdeptree",
     "pre-commit",
+    "p4p",
     "pydata-sphinx-theme>=0.12",
-    "pytest-asyncio",
+    "pytest-asyncio>=0.20",
     "pytest-cov",
     "sphinx-autobuild",
     "sphinx-copybutton",
@@ -88,22 +87,24 @@ addopts = """
     --cov=pandablocks_ioc --cov-report term --cov-report xml:cov.xml
     """
 # https://iscinumpy.gitlab.io/post/bound-version-constraints/#watch-for-warnings
-# The ignores are all related to the test DummyServer, both async and in_thread variants, 
-# which appear to have issues cleanly shutting down and raise exceptions in their destructors.
-# The issue seems like all we need is to add await asyncio.sleep(0) to allow asyncio to
-# clean up its connections, but that doesn't seem to behave as expected inside pytest.
+# First ignore is due to p4p staticly linking an old numpy version and us installing
+# Next is something that needs to be fixed in PandABlocks-client asyncio.py's write_and_drain function
+# which triggers a deprecation warning on Python 3.9+. See https://github.com/PandABlocks/PandABlocks-client/issues/47.
+# a more recent version with a different C API. See https://github.com/mdavidsaver/p4p/issues/102.
 filterwarnings = """
     error
-    ignore:unclosed transport <_SelectorSocketTransport:
-    ignore:unclosed <socket.socket:
-    ignore:unclosed event loop <_UnixSelectorEventLoop:
+    ignore:numpy.ufunc size changed
+    ignore:The explicit passing of coroutine objects to asyncio.wait()
 """
-# ignore::ResourceWarning
 # Doctest python code in docs, python code in src docstrings, test functions in tests
 testpaths = "docs src tests"
+asyncio_mode = "auto"
 
 [tool.coverage.run]
+concurrency = ["thread", "multiprocessing"]
 data_file = "/tmp/pandablocks_ioc.coverage"
+branch = true
+omit = ["tests/*"]
 
 [tool.coverage.paths]
 # Tests are run from installed location, map back to the src directory
@@ -120,8 +121,8 @@ skipsdist=True
 # Don't create a virtualenv for the command, requires tox-direct plugin
 direct = True
 passenv = *
-allowlist_externals = 
-    pytest 
+allowlist_externals =
+    pytest
     pre-commit
     mypy
     sphinx-build

src/pandablocks_ioc/__main__.py

@@ -1,6 +1,7 @@
 import logging
 
 import click
+from pandablocks.asyncio import AsyncioClient
 
 from pandablocks_ioc.ioc import create_softioc
 
@@ -31,11 +32,15 @@ def cli(ctx, log_level: str):
 @cli.command()
 @click.argument("host")
 @click.argument("prefix")
-def softioc(host: str, prefix: str):
+@click.argument("screens_dir")
+def softioc(host: str, prefix: str, screens_dir: str):
     """
-    Create a soft IOC, using "prefix" for the namespace of the records.
+    Connect to the given HOST and create an IOC with the given PREFIX.
+    Create .bob files for screens in the SCREENS_DIR. Directory must exist.
     """
-    create_softioc(host, prefix)
+    create_softioc(
+        client=AsyncioClient(host), record_prefix=prefix, screens_dir=screens_dir
+    )
 
 
 # test with: python -m pandablocks_ioc

src/pandablocks_ioc/_hdf_ioc.py

@@ -18,7 +18,8 @@
 from softioc import alarm, builder
 from softioc.pythonSoftIoc import RecordWrapper
 
-from ._types import ONAM_STR, ZNAM_STR
+from ._pvi import PviGroup, add_pvi_info
+from ._types import ONAM_STR, ZNAM_STR, EpicsName
 
 
 class HDF5RecordController:
@@ -52,51 +53,76 @@ def __init__(self, client: AsyncioClient, record_prefix: str):
 
         # Create the records, including an uppercase alias for each
         # Naming convention and settings (mostly) copied from FSCN2 HDF5 records
-        file_path_record_name = self._HDF5_PREFIX + ":FilePath"
+        file_path_record_name = EpicsName(self._HDF5_PREFIX + ":FilePath")
         self._file_path_record = builder.longStringOut(
             file_path_record_name,
             length=path_length,
             DESC="File path for HDF5 files",
             validate=self._parameter_validate,
         )
+        add_pvi_info(
+            PviGroup.INPUTS,
+            self._file_path_record,
+            file_path_record_name,
+            builder.longStringOut,
+        )
         self._file_path_record.add_alias(
             record_prefix + ":" + file_path_record_name.upper()
         )
 
-        file_name_record_name = self._HDF5_PREFIX + ":FileName"
+        file_name_record_name = EpicsName(self._HDF5_PREFIX + ":FileName")
         self._file_name_record = builder.longStringOut(
             file_name_record_name,
             length=filename_length,
             DESC="File name prefix for HDF5 files",
             validate=self._parameter_validate,
         )
+        add_pvi_info(
+            PviGroup.INPUTS,
+            self._file_name_record,
+            file_name_record_name,
+            builder.longStringOut,
+        )
         self._file_name_record.add_alias(
             record_prefix + ":" + file_name_record_name.upper()
         )
 
-        num_capture_record_name = self._HDF5_PREFIX + ":NumCapture"
+        num_capture_record_name = EpicsName(self._HDF5_PREFIX + ":NumCapture")
         self._num_capture_record = builder.longOut(
             num_capture_record_name,
             initial_value=0,  # Infinite capture
             DESC="Number of frames to capture. 0=infinite",
             DRVL=0,
         )
+
+        add_pvi_info(
+            PviGroup.INPUTS,
+            self._num_capture_record,
+            num_capture_record_name,
+            builder.longOut,
+        )
         # No validate - users are allowed to change this at any time
         self._num_capture_record.add_alias(
             record_prefix + ":" + num_capture_record_name.upper()
         )
 
-        flush_period_record_name = self._HDF5_PREFIX + ":FlushPeriod"
+        flush_period_record_name = EpicsName(self._HDF5_PREFIX + ":FlushPeriod")
         self._flush_period_record = builder.aOut(
             flush_period_record_name,
             initial_value=1.0,
             DESC="Frequency that data is flushed (seconds)",
         )
+        add_pvi_info(
+            PviGroup.INPUTS,
+            self._flush_period_record,
+            flush_period_record_name,
+            builder.aOut,
+        )
         self._flush_period_record.add_alias(
             record_prefix + ":" + flush_period_record_name.upper()
         )
 
-        capture_control_record_name = self._HDF5_PREFIX + ":Capture"
+        capture_control_record_name = EpicsName(self._HDF5_PREFIX + ":Capture")
         self._capture_control_record = builder.boolOut(
             capture_control_record_name,
             ZNAM=ZNAM_STR,
@@ -105,27 +131,45 @@ def __init__(self, client: AsyncioClient, record_prefix: str):
             validate=self._capture_validate,
             DESC="Start/stop HDF5 capture",
         )
+        add_pvi_info(
+            PviGroup.INPUTS,
+            self._capture_control_record,
+            capture_control_record_name,
+            builder.boolOut,
+        )
         self._capture_control_record.add_alias(
             record_prefix + ":" + capture_control_record_name.upper()
         )
 
-        status_message_record_name = self._HDF5_PREFIX + ":Status"
+        status_message_record_name = EpicsName(self._HDF5_PREFIX + ":Status")
         self._status_message_record = builder.stringIn(
             status_message_record_name,
             initial_value="OK",
             DESC="Reports current status of HDF5 capture",
         )
+        add_pvi_info(
+            PviGroup.OUTPUTS,
+            self._status_message_record,
+            status_message_record_name,
+            builder.stringIn,
+        )
         self._status_message_record.add_alias(
             record_prefix + ":" + status_message_record_name.upper()
         )
 
-        currently_capturing_record_name = self._HDF5_PREFIX + ":Capturing"
+        currently_capturing_record_name = EpicsName(self._HDF5_PREFIX + ":Capturing")
         self._currently_capturing_record = builder.boolIn(
             currently_capturing_record_name,
             ZNAM=ZNAM_STR,
             ONAM=ONAM_STR,
             DESC="If HDF5 file is currently being written",
         )
+        add_pvi_info(
+            PviGroup.OUTPUTS,
+            self._currently_capturing_record,
+            currently_capturing_record_name,
+            builder.boolIn,
+        )
         self._currently_capturing_record.add_alias(
             record_prefix + ":" + currently_capturing_record_name.upper()
         )
@@ -212,8 +256,12 @@ async def _handle_hdf5_data(self) -> None:
                         pipeline[0].queue.put_nowait(
                             EndData(captured_frames, EndReason.OK)
                         )
-
                         break
+                elif not isinstance(data, EndData):
+                    raise RuntimeError(
+                        f"Data was recieved that was of type {type(data)}, not"
+                        "StartData, EndData, ReadyData or FrameData"
+                    )
                 # Ignore EndData - handle terminating capture with the Capture
                 # record or when we capture the requested number of frames