diff --git a/pyproject.toml b/pyproject.toml index 851f2a1fbe..2e0da3a2b6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,15 +15,15 @@ description = "Ophyd devices and other utils that could be used across DLS beaml dependencies = [ "click", "ophyd", - "ophyd-async<0.4.0", # Need to pin to <0.4.0 as this requires pydantic>2.0 see https://github.com/DiamondLightSource/dodal/issues/679 + "ophyd-async>=0.5.2", "bluesky", "pyepics", "dataclasses-json", "pillow", - "zocalo>=0.32.0,<1.0.0", # TODO remove pin against <1.0.0, see #679 + "zocalo>=1.0.0", "requests", "graypy", - "pydantic", + "pydantic>=2.0", "opencv-python-headless", # For pin-tip detection. "aioca", # Required for CA support with ophyd-async. "p4p", # Required for PVA support with ophyd-async. diff --git a/src/dodal/beamlines/i03.py b/src/dodal/beamlines/i03.py index 807b3f98f2..5ae42ea043 100644 --- a/src/dodal/beamlines/i03.py +++ b/src/dodal/beamlines/i03.py @@ -1,13 +1,13 @@ -from ophyd_async.panda import HDFPanda +from ophyd_async.fastcs.panda import HDFPanda from dodal.common.beamlines.beamline_parameters import get_beamline_parameters from dodal.common.beamlines.beamline_utils import ( device_instantiation, - get_directory_provider, - set_directory_provider, + get_path_provider, + set_path_provider, ) from dodal.common.beamlines.beamline_utils import set_beamline as set_utils_beamline -from dodal.common.udc_directory_provider import PandASubdirectoryProvider +from dodal.common.udc_directory_provider import PandASubpathProvider from dodal.devices.aperturescatterguard import ( AperturePosition, ApertureScatterguard, @@ -52,7 +52,7 @@ set_log_beamline(BL) set_utils_beamline(BL) -set_directory_provider(PandASubdirectoryProvider()) +set_path_provider(PandASubpathProvider()) def aperture_scatterguard( @@ -378,7 +378,7 @@ def panda( "-EA-PANDA-01:", wait_for_connection, fake_with_ophyd_sim, - directory_provider=get_directory_provider(), + path_provider=get_path_provider(), ) diff --git a/src/dodal/beamlines/i13_1.py b/src/dodal/beamlines/i13_1.py index 05f3779676..fcd354a721 100644 --- a/src/dodal/beamlines/i13_1.py +++ b/src/dodal/beamlines/i13_1.py @@ -1,14 +1,14 @@ from pathlib import Path -from ophyd_async.epics.areadetector.aravis import AravisDetector +from ophyd_async.epics.adaravis import AravisDetector from dodal.common.beamlines.beamline_utils import ( device_instantiation, - get_directory_provider, - set_directory_provider, + get_path_provider, + set_path_provider, ) from dodal.common.beamlines.beamline_utils import set_beamline as set_utils_beamline -from dodal.common.visit import StaticVisitDirectoryProvider +from dodal.common.visit import StaticVisitPathProvider from dodal.devices.motors import XYZPositioner from dodal.log import set_beamline as set_log_beamline from dodal.utils import get_beamline_name @@ -16,8 +16,8 @@ BL = get_beamline_name("i13-1") set_log_beamline(BL) set_utils_beamline(BL) -set_directory_provider( - StaticVisitDirectoryProvider( +set_path_provider( + StaticVisitPathProvider( BL, Path("/data/2024/cm37257-4/"), # latest commissioning visit ) @@ -60,7 +60,7 @@ def side_camera( bl_prefix=False, drv_suffix="CAM:", hdf_suffix="HDF5:", - directory_provider=get_directory_provider(), + path_provider=get_path_provider(), wait=wait_for_connection, fake=fake_with_ophyd_sim, ) diff --git a/src/dodal/beamlines/i22.py b/src/dodal/beamlines/i22.py index 5984dcba87..b9f07bc692 100644 --- a/src/dodal/beamlines/i22.py +++ b/src/dodal/beamlines/i22.py @@ -1,16 +1,17 @@ from pathlib import Path -from ophyd_async.epics.areadetector import AravisDetector, PilatusDetector -from ophyd_async.panda import HDFPanda +from ophyd_async.epics.adaravis import AravisDetector +from ophyd_async.epics.adpilatus import PilatusDetector +from ophyd_async.fastcs.panda import HDFPanda from dodal.common.beamlines.beamline_utils import ( device_instantiation, - get_directory_provider, - set_directory_provider, + get_path_provider, + set_path_provider, ) from dodal.common.beamlines.beamline_utils import set_beamline as set_utils_beamline from dodal.common.beamlines.device_helpers import numbered_slits -from dodal.common.visit import DirectoryServiceClient, StaticVisitDirectoryProvider +from dodal.common.visit import RemoteDirectoryServiceClient, StaticVisitPathProvider from dodal.devices.focusing_mirror import FocusingMirror from dodal.devices.i22.dcm import CrystalMetadata, DoubleCrystalMonochromator from dodal.devices.i22.fswitch import FSwitch @@ -32,11 +33,11 @@ # Communication with GDA is also WIP so for now we determine an arbitrary scan number # locally and write the commissioning directory. The scan number is not guaranteed to # be unique and the data is at risk - this configuration is for testing only. -set_directory_provider( - StaticVisitDirectoryProvider( +set_path_provider( + StaticVisitPathProvider( BL, Path("/dls/i22/data/2024/cm37271-2/bluesky"), - client=DirectoryServiceClient("http://i22-control:8088/api"), + client=RemoteDirectoryServiceClient("http://i22-control:8088/api"), ) ) @@ -61,7 +62,7 @@ def saxs( sensor_thickness=(0.45, "mm"), distance=(4711.833684146172, "mm"), ), - directory_provider=get_directory_provider(), + path_provider=get_path_provider(), ) @@ -97,7 +98,7 @@ def waxs( sensor_thickness=(0.45, "mm"), distance=(175.4199417092314, "mm"), ), - directory_provider=get_directory_provider(), + path_provider=get_path_provider(), ) @@ -112,7 +113,7 @@ def i0( wait_for_connection, fake_with_ophyd_sim, type="Cividec Diamond XBPM", - directory_provider=get_directory_provider(), + path_provider=get_path_provider(), ) @@ -127,7 +128,7 @@ def it( wait_for_connection, fake_with_ophyd_sim, type="PIN Diode", - directory_provider=get_directory_provider(), + path_provider=get_path_provider(), ) @@ -295,7 +296,7 @@ def panda1( "-EA-PANDA-01:", wait_for_connection, fake_with_ophyd_sim, - directory_provider=get_directory_provider(), + path_provider=get_path_provider(), ) @@ -310,7 +311,7 @@ def panda2( "-EA-PANDA-02:", wait_for_connection, fake_with_ophyd_sim, - directory_provider=get_directory_provider(), + path_provider=get_path_provider(), ) @@ -325,7 +326,7 @@ def panda3( "-EA-PANDA-03:", wait_for_connection, fake_with_ophyd_sim, - directory_provider=get_directory_provider(), + path_provider=get_path_provider(), ) @@ -340,7 +341,7 @@ def panda4( "-EA-PANDA-04:", wait_for_connection, fake_with_ophyd_sim, - directory_provider=get_directory_provider(), + path_provider=get_path_provider(), ) @@ -361,7 +362,7 @@ def oav( description="AVT Mako G-507B", distance=(-1.0, "m"), ), - directory_provider=get_directory_provider(), + path_provider=get_path_provider(), ) diff --git a/src/dodal/beamlines/p38.py b/src/dodal/beamlines/p38.py index bca78b1e79..11da31b0da 100644 --- a/src/dodal/beamlines/p38.py +++ b/src/dodal/beamlines/p38.py @@ -1,16 +1,16 @@ from pathlib import Path -from ophyd_async.epics.areadetector import AravisDetector -from ophyd_async.panda import HDFPanda +from ophyd_async.epics.adaravis import AravisDetector +from ophyd_async.fastcs.panda import HDFPanda from dodal.common.beamlines.beamline_utils import ( device_instantiation, - get_directory_provider, - set_directory_provider, + get_path_provider, + set_path_provider, ) from dodal.common.beamlines.beamline_utils import set_beamline as set_utils_beamline from dodal.common.beamlines.device_helpers import numbered_slits -from dodal.common.visit import LocalDirectoryServiceClient, StaticVisitDirectoryProvider +from dodal.common.visit import LocalDirectoryServiceClient, StaticVisitPathProvider from dodal.devices.focusing_mirror import FocusingMirror from dodal.devices.i22.dcm import CrystalMetadata, DoubleCrystalMonochromator from dodal.devices.i22.fswitch import FSwitch @@ -30,8 +30,8 @@ # Communication with GDA is also WIP so for now we determine an arbitrary scan number # locally and write the commissioning directory. The scan number is not guaranteed to # be unique and the data is at risk - this configuration is for testing only. -set_directory_provider( - StaticVisitDirectoryProvider( +set_path_provider( + StaticVisitPathProvider( BL, Path("/dls/p38/data/2024/cm37282-2/bluesky"), client=LocalDirectoryServiceClient(), @@ -50,7 +50,7 @@ def d3( fake_with_ophyd_sim, drv_suffix="DET:", hdf_suffix="HDF5:", - directory_provider=get_directory_provider(), + path_provider=get_path_provider(), ) @@ -67,7 +67,7 @@ def d11( fake_with_ophyd_sim, drv_suffix="DET:", hdf_suffix="HDF5:", - directory_provider=get_directory_provider(), + path_provider=get_path_provider(), ) @@ -82,7 +82,7 @@ def d12( fake_with_ophyd_sim, drv_suffix="DET:", hdf_suffix="HDF5:", - directory_provider=get_directory_provider(), + path_provider=get_path_provider(), ) @@ -96,7 +96,7 @@ def i0( "-EA-XBPM-01:", wait_for_connection, fake_with_ophyd_sim, - directory_provider=get_directory_provider(), + path_provider=get_path_provider(), ) @@ -273,7 +273,7 @@ def panda1( "-EA-PANDA-01:", wait_for_connection, fake_with_ophyd_sim, - directory_provider=get_directory_provider(), + path_provider=get_path_provider(), ) @@ -288,7 +288,7 @@ def panda2( "-EA-PANDA-02:", wait_for_connection, fake_with_ophyd_sim, - directory_provider=get_directory_provider(), + path_provider=get_path_provider(), ) @@ -303,7 +303,7 @@ def panda3( "-EA-PANDA-03:", wait_for_connection, fake_with_ophyd_sim, - directory_provider=get_directory_provider(), + path_provider=get_path_provider(), ) diff --git a/src/dodal/beamlines/p45.py b/src/dodal/beamlines/p45.py index 3b105ccd9b..4a55060f38 100644 --- a/src/dodal/beamlines/p45.py +++ b/src/dodal/beamlines/p45.py @@ -1,15 +1,15 @@ from pathlib import Path -from ophyd_async.epics.areadetector import AravisDetector -from ophyd_async.panda import HDFPanda +from ophyd_async.epics.adaravis import AravisDetector +from ophyd_async.fastcs.panda import HDFPanda from dodal.common.beamlines.beamline_utils import ( device_instantiation, - get_directory_provider, - set_directory_provider, + get_path_provider, + set_path_provider, ) from dodal.common.beamlines.beamline_utils import set_beamline as set_utils_beamline -from dodal.common.visit import StaticVisitDirectoryProvider +from dodal.common.visit import StaticVisitPathProvider from dodal.devices.p45 import Choppers, TomoStageWithStretchAndSkew from dodal.log import set_beamline as set_log_beamline from dodal.utils import get_beamline_name, skip_device @@ -17,8 +17,8 @@ BL = get_beamline_name("p45") set_log_beamline(BL) set_utils_beamline(BL) -set_directory_provider( - StaticVisitDirectoryProvider( +set_path_provider( + StaticVisitPathProvider( BL, Path("/data/2024/cm37283-2/"), # latest commissioning visit ) @@ -62,7 +62,7 @@ def det( fake_with_ophyd_sim, drv_suffix="DET:", hdf_suffix="HDF5:", - directory_provider=get_directory_provider(), + path_provider=get_path_provider(), ) @@ -79,7 +79,7 @@ def diff( fake_with_ophyd_sim, drv_suffix="DET:", hdf_suffix="HDF5:", - directory_provider=get_directory_provider(), + path_provider=get_path_provider(), ) @@ -97,7 +97,7 @@ def panda1( "-MO-PANDA-01:", wait_for_connection, fake_with_ophyd_sim, - directory_provider=get_directory_provider(), + path_provider=get_path_provider(), ) @@ -112,5 +112,5 @@ def panda2( "-MO-PANDA-02:", wait_for_connection, fake_with_ophyd_sim, - directory_provider=get_directory_provider(), + path_provider=get_path_provider(), ) diff --git a/src/dodal/beamlines/training_rig.py b/src/dodal/beamlines/training_rig.py index 3d31794444..3c026c6576 100644 --- a/src/dodal/beamlines/training_rig.py +++ b/src/dodal/beamlines/training_rig.py @@ -1,10 +1,14 @@ from pathlib import Path -from ophyd_async.core import StaticDirectoryProvider -from ophyd_async.epics.areadetector.aravis import AravisDetector +from ophyd_async.epics.adaravis import AravisDetector -from dodal.common.beamlines.beamline_utils import device_instantiation +from dodal.common.beamlines.beamline_utils import ( + device_instantiation, + get_path_provider, + set_path_provider, +) from dodal.common.beamlines.beamline_utils import set_beamline as set_utils_beamline +from dodal.common.visit import LocalDirectoryServiceClient, StaticVisitPathProvider from dodal.devices.training_rig.sample_stage import TrainingRigSampleStage from dodal.log import set_beamline as set_log_beamline from dodal.utils import get_beamline_name @@ -24,6 +28,14 @@ set_log_beamline(BL) set_utils_beamline(BL) +set_path_provider( + StaticVisitPathProvider( + BL, + Path("/exports/mybeamline/data"), + client=LocalDirectoryServiceClient(), + ) +) + def sample_stage( wait_for_connection: bool = True, fake_with_ophyd_sim: bool = False @@ -40,7 +52,6 @@ def sample_stage( def det( wait_for_connection: bool = True, fake_with_ophyd_sim: bool = False ) -> AravisDetector: - directory_provider = StaticDirectoryProvider(Path("/exports/mybeamline/data")) return device_instantiation( AravisDetector, "det", @@ -49,5 +60,5 @@ def det( fake_with_ophyd_sim, drv_suffix="DET:", hdf_suffix="HDF5:", - directory_provider=directory_provider, + path_provider=get_path_provider(), ) diff --git a/src/dodal/common/beamlines/beamline_utils.py b/src/dodal/common/beamlines/beamline_utils.py index 4ff68ca5d9..ac1418e44d 100644 --- a/src/dodal/common/beamlines/beamline_utils.py +++ b/src/dodal/common/beamlines/beamline_utils.py @@ -8,14 +8,14 @@ from ophyd_async.core import Device as OphydV2Device from ophyd_async.core import wait_for_connection as v2_device_wait_for_connection -from dodal.common.types import UpdatingDirectoryProvider +from dodal.common.types import UpdatingPathProvider from dodal.utils import AnyDevice, BeamlinePrefix, skip_device DEFAULT_CONNECTION_TIMEOUT: Final[float] = 5.0 ACTIVE_DEVICES: dict[str, AnyDevice] = {} BL = "" -DIRECTORY_PROVIDER: UpdatingDirectoryProvider | None = None +PATH_PROVIDER: UpdatingPathProvider | None = None def set_beamline(beamline: str): @@ -125,15 +125,15 @@ def device_instantiation( return device_instance -def set_directory_provider(provider: UpdatingDirectoryProvider): - global DIRECTORY_PROVIDER +def set_path_provider(provider: UpdatingPathProvider): + global PATH_PROVIDER - DIRECTORY_PROVIDER = provider + PATH_PROVIDER = provider -def get_directory_provider() -> UpdatingDirectoryProvider: - if DIRECTORY_PROVIDER is None: +def get_path_provider() -> UpdatingPathProvider: + if PATH_PROVIDER is None: raise ValueError( - "DirectoryProvider has not been set! Ophyd-async StandardDetectors will not be able to write!" + "PathProvider has not been set! Ophyd-async StandardDetectors will not be able to write!" ) - return DIRECTORY_PROVIDER + return PATH_PROVIDER diff --git a/src/dodal/common/types.py b/src/dodal/common/types.py index 05c62ff37a..22c546c655 100644 --- a/src/dodal/common/types.py +++ b/src/dodal/common/types.py @@ -5,7 +5,7 @@ ) from bluesky.utils import Msg -from ophyd_async.core import DirectoryProvider +from ophyd_async.core import PathProvider # String identifier used by 'wait' or stubs that await Group = str @@ -15,6 +15,8 @@ PlanGenerator = Callable[..., MsgGenerator] -class UpdatingDirectoryProvider(DirectoryProvider, ABC): +class UpdatingPathProvider(PathProvider, ABC): + @abstractmethod + async def data_session(self) -> str: ... @abstractmethod async def update(self, **kwargs) -> None: ... diff --git a/src/dodal/common/udc_directory_provider.py b/src/dodal/common/udc_directory_provider.py index 3e8f79a4bb..9f5ea707ac 100644 --- a/src/dodal/common/udc_directory_provider.py +++ b/src/dodal/common/udc_directory_provider.py @@ -1,27 +1,37 @@ from pathlib import Path -from ophyd_async.core import DirectoryInfo +from ophyd_async.core import FilenameProvider, PathInfo -from dodal.common.types import UpdatingDirectoryProvider +from dodal.common.types import UpdatingPathProvider from dodal.log import LOGGER -class PandASubdirectoryProvider(UpdatingDirectoryProvider): +class PandAFilenameProvider(FilenameProvider): + def __init__(self, suffix: str | None = None): + self.suffix = suffix + + def __call__(self, device_name: str | None = None): + return f"{device_name}-{self.suffix}" + + +class PandASubpathProvider(UpdatingPathProvider): """Directory provider for the HDFPanda. Points to a panda subdirectory within the directory path provided""" resource_dir = Path("panda") - def __init__(self, directory: Path | None = None): - if directory is None: + def __init__(self, root_directory: Path | None = None, suffix: str = ""): + self._output_directory: Path | None = ( + root_directory / self.resource_dir if root_directory else None + ) + self._filename_provider = PandAFilenameProvider(suffix=suffix) + if self._output_directory is None: LOGGER.debug( f"{self.__class__.__name__} instantiated with no root path, update() must be called before writing data!" ) - self._directory_info = ( - DirectoryInfo(root=directory, resource_dir=self.resource_dir) - if directory - else None - ) + + async def data_session(self) -> str: + return self._filename_provider.suffix or "" async def update(self, *, directory: Path, suffix: str = "", **kwargs): """Update the root directory into which panda pcap files are written. This will result in the panda @@ -32,16 +42,13 @@ async def update(self, *, directory: Path, suffix: str = "", **kwargs): suffix: Optional str that will be appended to the panda device name along with the file type extension to construct the output filename """ - output_directory = directory / self.resource_dir - output_directory.mkdir(exist_ok=True) - - self._directory_info = DirectoryInfo( - root=directory, resource_dir=self.resource_dir, suffix=suffix + self._output_directory = directory / self.resource_dir + self._filename_provider.suffix = suffix + + def __call__(self, device_name: str | None = None) -> PathInfo: + assert self._output_directory, "Directory unknown for PandA to write into, update() needs to be called at least once" + return PathInfo( + directory_path=self._output_directory, + filename=self._filename_provider(device_name), + create_dir_depth=-1, # allows PandA HDFWriter to make any number of dirs ) - - def __call__(self) -> DirectoryInfo: - if self._directory_info is None: - raise ValueError( - "Directory unknown for PandA to write into, update() needs to be called at least once" - ) - return self._directory_info diff --git a/src/dodal/common/visit.py b/src/dodal/common/visit.py index ed97e1ef4f..ecfa23e1b3 100644 --- a/src/dodal/common/visit.py +++ b/src/dodal/common/visit.py @@ -1,11 +1,12 @@ from abc import ABC, abstractmethod from pathlib import Path +from typing import Literal from aiohttp import ClientSession -from ophyd_async.core import DirectoryInfo +from ophyd_async.core import FilenameProvider, PathInfo from pydantic import BaseModel -from dodal.common.types import UpdatingDirectoryProvider +from dodal.common.types import UpdatingPathProvider from dodal.log import LOGGER """ @@ -22,7 +23,7 @@ class DataCollectionIdentifier(BaseModel): collectionNumber: int -class DirectoryServiceClientBase(ABC): +class DirectoryServiceClient(ABC): """ Object responsible for I/O in determining collection number """ @@ -36,41 +37,53 @@ async def get_current_collection(self) -> DataCollectionIdentifier: """Get current collection""" -class DirectoryServiceClient(DirectoryServiceClientBase): +class DiamondFilenameProvider(FilenameProvider): + def __init__(self, beamline: str, client: DirectoryServiceClient): + self._beamline = beamline + self._client = client + self.collectionId: DataCollectionIdentifier | None = None + + def __call__(self, device_name: str | None = None): + assert device_name, "Diamond filename requires device_name to be passed" + assert self.collectionId is not None + return f"{self._beamline}-{self.collectionId.collectionNumber}-{device_name}" + + +class RemoteDirectoryServiceClient(DirectoryServiceClient): """Client for the VisitService REST API Currently exposed by the GDA Server to co-ordinate unique filenames. While VisitService is embedded in GDA, url is likely to be `ixx-control:8088/api` """ - _url: str - def __init__(self, url: str) -> None: self._url = url async def create_new_collection(self) -> DataCollectionIdentifier: - async with ClientSession() as session: - async with session.post(f"{self._url}/numtracker") as response: - response.raise_for_status() - json = await response.json() - new_collection = DataCollectionIdentifier.parse_obj(json) - LOGGER.debug("New DataCollection: %s", new_collection) - return new_collection + new_collection = await self._identifier_from_response("POST") + LOGGER.debug("New DataCollection: %s", new_collection) + return new_collection async def get_current_collection(self) -> DataCollectionIdentifier: - async with ClientSession() as session: - async with session.get(f"{self._url}/numtracker") as response: - response.raise_for_status() - json = await response.json() - current_collection = DataCollectionIdentifier.parse_obj(json) - LOGGER.debug("Current DataCollection: %s", current_collection) - return current_collection - + current_collection = await self._identifier_from_response("GET") + LOGGER.debug("Current DataCollection: %s", current_collection) + return current_collection -class LocalDirectoryServiceClient(DirectoryServiceClientBase): + async def _identifier_from_response( + self, + method: Literal["GET", "POST"], + ) -> DataCollectionIdentifier: + async with ( + ClientSession() as session, + session.request(method, f"{self._url}/numtracker") as response, + ): + response.raise_for_status() + json = await response.json() + return DataCollectionIdentifier.model_validate_json(json) + + +class LocalDirectoryServiceClient(DirectoryServiceClient): """Local or dummy impl of VisitService client to co-ordinate unique filenames.""" - _count: int - def __init__(self) -> None: self._count = 0 @@ -84,33 +97,30 @@ async def get_current_collection(self) -> DataCollectionIdentifier: return DataCollectionIdentifier(collectionNumber=self._count) -class StaticVisitDirectoryProvider(UpdatingDirectoryProvider): +class StaticVisitPathProvider(UpdatingPathProvider): """ - Static (single visit) implementation of DirectoryProvider whilst awaiting auth infrastructure to generate necessary information per-scan. + Static (single visit) implementation of PathProvider whilst awaiting auth infrastructure to generate necessary information per-scan. Allows setting a singular visit into which all run files will be saved. update() queries a visit service to get the next DataCollectionIdentifier to increment the suffix of all file writers' next files. Requires that all detectors are running with a mutual view on the filesystem. Supports a single Visit which should be passed as a Path relative to the root of the Detector IOC mounting. - i.e. to write to visit /dls/ixx/data/YYYY/cm12345-1, assuming all detectors are mounted with /data -> /dls/ixx/data, root=/data/YYYY/cm12345-1/ + i.e. to write to visit /dls/ixx/data/YYYY/cm12345-1 """ - _beamline: str - _root: Path - _client: DirectoryServiceClientBase - _current_collection: DirectoryInfo | None - _session: ClientSession | None - def __init__( self, beamline: str, root: Path, - client: DirectoryServiceClientBase | None = None, + client: DirectoryServiceClient | None = None, ): self._beamline = beamline - self._client = client or DirectoryServiceClient(f"{beamline}-control:8088/api") + self._client = client or RemoteDirectoryServiceClient( + f"{beamline}-control:8088/api" + ) + self._filename_provider = DiamondFilenameProvider(self._beamline, self._client) self._root = root - self._current_collection = None - self._session = None + self.current_collection: PathInfo | None + self._session: ClientSession | None async def update(self, **kwargs) -> None: """ @@ -121,33 +131,22 @@ async def update(self, **kwargs) -> None: # TODO: DAQ-4827: Pass AuthN information as part of request try: - collection_id_info = await self._client.create_new_collection() - self._current_collection = self._generate_directory_info(collection_id_info) + self._filename_provider.collectionId = ( + await self._client.create_new_collection() + ) except Exception: LOGGER.error( "Exception while updating data collection, preventing overwriting data by setting current_collection to None" ) - self._current_collection = None + self._collection_id_info = None raise - def _generate_directory_info( - self, - collection_id_info: DataCollectionIdentifier, - ) -> DirectoryInfo: - return DirectoryInfo( - # See DocString of DirectoryInfo. At DLS, root = visit directory, resource_dir is relative to it. - root=self._root, - # https://github.com/DiamondLightSource/dodal/issues/452 - # Currently all h5 files written to visit/ directory, as no guarantee that visit/dataCollection/ directory will have been produced. If it is as part of #452, append the resource_dir - resource_dir=Path("."), - # Diamond standard file naming - prefix=f"{self._beamline}-{collection_id_info.collectionNumber}-", - ) + async def data_session(self) -> str: + collection = await self._client.get_current_collection() + return f"{self._beamline}-{collection.collectionNumber}" - def __call__(self) -> DirectoryInfo: - if self._current_collection is not None: - return self._current_collection - else: - raise ValueError( - "No current collection, update() needs to be called at least once" - ) + def __call__(self, device_name: str | None = None) -> PathInfo: + assert device_name, "Must call PathProvider with device_name" + return PathInfo( + directory_path=self._root, filename=self._filename_provider(device_name) + ) diff --git a/src/dodal/devices/CTAB.py b/src/dodal/devices/CTAB.py index 6e2a3511f0..f8dacc0897 100644 --- a/src/dodal/devices/CTAB.py +++ b/src/dodal/devices/CTAB.py @@ -1,5 +1,5 @@ from ophyd_async.core import StandardReadable -from ophyd_async.epics.motion import Motor +from ophyd_async.epics.motor import Motor from ophyd_async.epics.signal import epics_signal_r diff --git a/src/dodal/devices/aperture.py b/src/dodal/devices/aperture.py index 4388c3777d..b007c402f9 100644 --- a/src/dodal/devices/aperture.py +++ b/src/dodal/devices/aperture.py @@ -1,5 +1,5 @@ from ophyd_async.core import StandardReadable -from ophyd_async.epics.motion import Motor +from ophyd_async.epics.motor import Motor from ophyd_async.epics.signal import epics_signal_r diff --git a/src/dodal/devices/dcm.py b/src/dodal/devices/dcm.py index 9b6c3bed71..cb303ce617 100644 --- a/src/dodal/devices/dcm.py +++ b/src/dodal/devices/dcm.py @@ -1,5 +1,5 @@ from ophyd_async.core import StandardReadable -from ophyd_async.epics.motion import Motor +from ophyd_async.epics.motor import Motor from ophyd_async.epics.signal import epics_signal_r diff --git a/src/dodal/devices/detector/detector.py b/src/dodal/devices/detector/detector.py index 7b4019b89c..8d0cd51e95 100644 --- a/src/dodal/devices/detector/detector.py +++ b/src/dodal/devices/detector/detector.py @@ -1,7 +1,7 @@ from enum import Enum, auto -from typing import Any +from pathlib import Path -from pydantic import BaseModel, root_validator, validator +from pydantic import BaseModel, Field, field_serializer, field_validator from dodal.devices.detector.det_dim_constants import ( EIGER2_X_16M_SIZE, @@ -28,9 +28,12 @@ class DetectorParams(BaseModel): """Holds parameters for the detector. Provides access to a list of Dectris detector sizes and a converter for distance to beam centre.""" + # https://github.com/pydantic/pydantic/issues/8379 + # Must use model_dump(by_alias=True) if serialising! + expected_energy_ev: float | None = None exposure_time: float - directory: str + directory: str # : Path https://github.com/DiamondLightSource/dodal/issues/774 prefix: str detector_distance: float omega_start: float @@ -39,46 +42,44 @@ class DetectorParams(BaseModel): num_triggers: int use_roi_mode: bool det_dist_to_beam_converter_path: str + override_run_number: int | None = Field(default=None, alias="run_number") trigger_mode: TriggerMode = TriggerMode.SET_FRAMES detector_size_constants: DetectorSizeConstants = EIGER2_X_16M_SIZE - beam_xy_converter: DetectorDistanceToBeamXYConverter = None # type: ignore # Filled in by validator - run_number: int = None # type: ignore # Filled in by validator enable_dev_shm: bool = ( False # Remove in https://github.com/DiamondLightSource/hyperion/issues/1395 ) - class Config: - arbitrary_types_allowed = True - json_encoders = { - DetectorDistanceToBeamXYConverter: lambda _: None, - DetectorSizeConstants: lambda d: d.det_type_string, - } - - # should be replaced with model_validator once move to pydantic 2 is complete - @root_validator(pre=True) - def create_beamxy_and_runnumber(cls, values: dict[str, Any]) -> dict[str, Any]: - values["beam_xy_converter"] = DetectorDistanceToBeamXYConverter( - values["det_dist_to_beam_converter_path"] + @property + def beam_xy_converter(self) -> DetectorDistanceToBeamXYConverter: + return DetectorDistanceToBeamXYConverter(self.det_dist_to_beam_converter_path) + + @property + def run_number(self) -> int: + return ( + get_run_number(self.directory, self.prefix) + if self.override_run_number is None + else self.override_run_number ) - if values.get("run_number") is None: - values["run_number"] = get_run_number(values["directory"], values["prefix"]) - return values - - @validator("detector_size_constants", pre=True) - def _parse_detector_size_constants( - cls, det_type: str, values: dict[str, Any] - ) -> DetectorSizeConstants: + + @field_serializer("detector_size_constants") + def serialize_detector_size_constants(self, size: DetectorSizeConstants): + return size.det_type_string + + @field_validator("detector_size_constants", mode="before") + @classmethod + def _parse_detector_size_constants(cls, det_type: str) -> DetectorSizeConstants: return ( det_type if isinstance(det_type, DetectorSizeConstants) else constants_from_type(det_type) ) - @validator("directory", pre=True) - def _parse_directory(cls, directory: str, values: dict[str, Any]) -> str: - if not directory.endswith("/"): - directory += "/" - return directory + @field_validator("directory", mode="before") + @classmethod + def _parse_directory(cls, directory: str | Path) -> str: + path = Path(directory) + assert path.is_dir() + return str(path) def get_beam_position_mm(self, detector_distance: float) -> tuple[float, float]: x_beam_mm = self.beam_xy_converter.get_beam_xy_from_det_dist( diff --git a/src/dodal/devices/detector/detector_motion.py b/src/dodal/devices/detector/detector_motion.py index 05306e504d..c200a6ab8f 100644 --- a/src/dodal/devices/detector/detector_motion.py +++ b/src/dodal/devices/detector/detector_motion.py @@ -1,7 +1,7 @@ from enum import Enum from ophyd_async.core import Device -from ophyd_async.epics.motion import Motor +from ophyd_async.epics.motor import Motor from ophyd_async.epics.signal import epics_signal_r, epics_signal_rw diff --git a/src/dodal/devices/fast_grid_scan.py b/src/dodal/devices/fast_grid_scan.py index a7706e0535..2842c23734 100644 --- a/src/dodal/devices/fast_grid_scan.py +++ b/src/dodal/devices/fast_grid_scan.py @@ -1,5 +1,5 @@ from abc import ABC, abstractmethod -from typing import Any, Generic, TypeVar +from typing import Generic, TypeVar import numpy as np from bluesky.plan_stubs import mv @@ -20,7 +20,7 @@ epics_signal_rw_rbv, epics_signal_x, ) -from pydantic import validator +from pydantic import field_validator from pydantic.dataclasses import dataclass from dodal.log import LOGGER @@ -69,9 +69,6 @@ class GridScanParamsCommon(AbstractExperimentWithBeamParams): y2_start: float = 0.1 z1_start: float = 0.1 z2_start: float = 0.1 - x_axis: GridAxis = GridAxis(0, 0, 0) - y_axis: GridAxis = GridAxis(0, 0, 0) - z_axis: GridAxis = GridAxis(0, 0, 0) # Whether to set the stub offsets after centering set_stub_offsets: bool = False @@ -91,28 +88,20 @@ def get_param_positions(self) -> dict: "z2_start": self.z2_start, } - class Config: - arbitrary_types_allowed = True - fields = { - "x_axis": {"exclude": True}, - "y_axis": {"exclude": True}, - "z_axis": {"exclude": True}, - } - - @validator("x_axis", always=True) - def _get_x_axis(cls, x_axis: GridAxis, values: dict[str, Any]) -> GridAxis: - return GridAxis(values["x_start"], values["x_step_size"], values["x_steps"]) + @property + def x_axis(self) -> GridAxis: + return GridAxis(self.x_start, self.x_step_size, self.x_steps) - @validator("y_axis", always=True) - def _get_y_axis(cls, y_axis: GridAxis, values: dict[str, Any]) -> GridAxis: - return GridAxis(values["y1_start"], values["y_step_size"], values["y_steps"]) + @property + def y_axis(self) -> GridAxis: + return GridAxis(self.y1_start, self.y_step_size, self.y_steps) - @validator("z_axis", always=True) - def _get_z_axis(cls, z_axis: GridAxis, values: dict[str, Any]) -> GridAxis: - return GridAxis(values["z2_start"], values["z_step_size"], values["z_steps"]) + @property + def z_axis(self) -> GridAxis: + return GridAxis(self.z2_start, self.z_step_size, self.z_steps) def get_num_images(self): - return self.x_steps * self.y_steps + self.x_steps * self.z_steps + return self.x_steps * (self.y_steps + self.z_steps) @property def is_3d_grid_scan(self): @@ -155,7 +144,8 @@ def get_param_positions(self): param_positions["dwell_time_ms"] = self.dwell_time_ms return param_positions - @validator("dwell_time_ms", always=True, check_fields=True) + @field_validator("dwell_time_ms") + @classmethod def non_integer_dwell_time(cls, dwell_time_ms: float) -> float: dwell_time_floor_rounded = np.floor(dwell_time_ms) dwell_time_is_close = np.isclose( diff --git a/src/dodal/devices/focusing_mirror.py b/src/dodal/devices/focusing_mirror.py index 0d693be039..c8fe794e98 100644 --- a/src/dodal/devices/focusing_mirror.py +++ b/src/dodal/devices/focusing_mirror.py @@ -8,9 +8,9 @@ HintedSignal, StandardReadable, observe_value, + soft_signal_r_and_setter, ) -from ophyd_async.core.signal import soft_signal_r_and_setter -from ophyd_async.epics.motion import Motor +from ophyd_async.epics.motor import Motor from ophyd_async.epics.signal import ( epics_signal_r, epics_signal_rw, diff --git a/src/dodal/devices/i22/dcm.py b/src/dodal/devices/i22/dcm.py index 6ff5f8430e..6424dfc604 100644 --- a/src/dodal/devices/i22/dcm.py +++ b/src/dodal/devices/i22/dcm.py @@ -6,7 +6,7 @@ from bluesky.protocols import Reading from event_model.documents.event_descriptor import DataKey from ophyd_async.core import ConfigSignal, StandardReadable, soft_signal_r_and_setter -from ophyd_async.epics.motion import Motor +from ophyd_async.epics.motor import Motor from ophyd_async.epics.signal import epics_signal_r # Conversion constant for energy and wavelength, taken from the X-Ray data booklet diff --git a/src/dodal/devices/i22/fswitch.py b/src/dodal/devices/i22/fswitch.py index b4c7ff763b..1bde0ffbda 100644 --- a/src/dodal/devices/i22/fswitch.py +++ b/src/dodal/devices/i22/fswitch.py @@ -4,8 +4,12 @@ from bluesky.protocols import Reading from event_model import DataKey -from ophyd_async.core import ConfigSignal, StandardReadable, soft_signal_r_and_setter -from ophyd_async.core.device import DeviceVector +from ophyd_async.core import ( + ConfigSignal, + DeviceVector, + StandardReadable, + soft_signal_r_and_setter, +) from ophyd_async.epics.signal import epics_signal_r diff --git a/src/dodal/devices/i22/nxsas.py b/src/dodal/devices/i22/nxsas.py index 2ed1565a83..8520a85cf3 100644 --- a/src/dodal/devices/i22/nxsas.py +++ b/src/dodal/devices/i22/nxsas.py @@ -1,12 +1,33 @@ +import asyncio +from collections.abc import Awaitable, Iterable from dataclasses import dataclass, fields +from typing import TypeVar from bluesky.protocols import Reading from event_model.documents.event_descriptor import DataKey -from ophyd_async.core import DirectoryProvider, merge_gathered_dicts -from ophyd_async.epics.areadetector import AravisDetector, PilatusDetector -from ophyd_async.epics.areadetector.aravis import AravisController +from ophyd_async.core import PathProvider +from ophyd_async.epics.adaravis import AravisController, AravisDetector +from ophyd_async.epics.adpilatus import PilatusDetector ValueAndUnits = tuple[float, str] +T = TypeVar("T") + + +# TODO: Remove this file as part of github.com/DiamondLightSource/dodal/issues/595 +# Until which, temporarily duplicated non-public method from ophyd_async +async def _merge_gathered_dicts( + coros: Iterable[Awaitable[dict[str, T]]], +) -> dict[str, T]: + """Merge dictionaries produced by a sequence of coroutines. + + Can be used for merging ``read()`` or ``describe``. For instance:: + + combined_read = await merge_gathered_dicts(s.read() for s in signals) + """ + ret: dict[str, T] = {} + for result in await asyncio.gather(*coros): + ret.update(result) + return ret @dataclass @@ -80,7 +101,7 @@ class NXSasPilatus(PilatusDetector): def __init__( self, prefix: str, - directory_provider: DirectoryProvider, + path_provider: PathProvider, drv_suffix: str, hdf_suffix: str, metadata_holder: NXSasMetadataHolder, @@ -93,7 +114,7 @@ def __init__( Writes hdf5 files.""" super().__init__( prefix, - directory_provider, + path_provider, drv_suffix=drv_suffix, hdf_suffix=hdf_suffix, name=name, @@ -101,7 +122,7 @@ def __init__( self._metadata_holder = metadata_holder async def read_configuration(self) -> dict[str, Reading]: - return await merge_gathered_dicts( + return await _merge_gathered_dicts( r for r in ( super().read_configuration(), @@ -110,7 +131,7 @@ async def read_configuration(self) -> dict[str, Reading]: ) async def describe_configuration(self) -> dict[str, DataKey]: - return await merge_gathered_dicts( + return await _merge_gathered_dicts( r for r in ( super().describe_configuration(), @@ -123,7 +144,7 @@ class NXSasOAV(AravisDetector): def __init__( self, prefix: str, - directory_provider: DirectoryProvider, + path_provider: PathProvider, drv_suffix: str, hdf_suffix: str, metadata_holder: NXSasMetadataHolder, @@ -137,7 +158,7 @@ def __init__( Writes hdf5 files.""" super().__init__( prefix, - directory_provider, + path_provider, drv_suffix=drv_suffix, hdf_suffix=hdf_suffix, name=name, @@ -146,7 +167,7 @@ def __init__( self._metadata_holder = metadata_holder async def read_configuration(self) -> dict[str, Reading]: - return await merge_gathered_dicts( + return await _merge_gathered_dicts( r for r in ( super().read_configuration(), @@ -155,7 +176,7 @@ async def read_configuration(self) -> dict[str, Reading]: ) async def describe_configuration(self) -> dict[str, DataKey]: - return await merge_gathered_dicts( + return await _merge_gathered_dicts( r for r in ( super().describe_configuration(), diff --git a/src/dodal/devices/i24/aperture.py b/src/dodal/devices/i24/aperture.py index 5a855e344b..55df736f72 100644 --- a/src/dodal/devices/i24/aperture.py +++ b/src/dodal/devices/i24/aperture.py @@ -1,7 +1,7 @@ from enum import Enum from ophyd_async.core import StandardReadable -from ophyd_async.epics.motion import Motor +from ophyd_async.epics.motor import Motor from ophyd_async.epics.signal import epics_signal_rw diff --git a/src/dodal/devices/i24/beamstop.py b/src/dodal/devices/i24/beamstop.py index 1d8cc4c337..a5c2ec52b6 100644 --- a/src/dodal/devices/i24/beamstop.py +++ b/src/dodal/devices/i24/beamstop.py @@ -1,7 +1,7 @@ from enum import Enum from ophyd_async.core import StandardReadable -from ophyd_async.epics.motion import Motor +from ophyd_async.epics.motor import Motor from ophyd_async.epics.signal import epics_signal_rw diff --git a/src/dodal/devices/i24/dcm.py b/src/dodal/devices/i24/dcm.py index 45182ea1dc..e40357f5e3 100644 --- a/src/dodal/devices/i24/dcm.py +++ b/src/dodal/devices/i24/dcm.py @@ -1,5 +1,5 @@ from ophyd_async.core import StandardReadable -from ophyd_async.epics.motion import Motor +from ophyd_async.epics.motor import Motor from ophyd_async.epics.signal import epics_signal_r diff --git a/src/dodal/devices/i24/i24_detector_motion.py b/src/dodal/devices/i24/i24_detector_motion.py index 9dc0ff0fe6..b0a3fb9350 100644 --- a/src/dodal/devices/i24/i24_detector_motion.py +++ b/src/dodal/devices/i24/i24_detector_motion.py @@ -1,5 +1,5 @@ from ophyd_async.core import StandardReadable -from ophyd_async.epics.motion import Motor +from ophyd_async.epics.motor import Motor class DetectorMotion(StandardReadable): diff --git a/src/dodal/devices/i24/pmac.py b/src/dodal/devices/i24/pmac.py index 706d4c5fd2..8afe49ad2a 100644 --- a/src/dodal/devices/i24/pmac.py +++ b/src/dodal/devices/i24/pmac.py @@ -2,12 +2,18 @@ from typing import SupportsFloat from bluesky.protocols import Triggerable -from ophyd_async.core import AsyncStatus, StandardReadable, wait_for_value -from ophyd_async.core.signal import CalculateTimeout, SignalR, SignalRW -from ophyd_async.core.signal_backend import SignalBackend -from ophyd_async.core.soft_signal_backend import SoftSignalBackend -from ophyd_async.core.utils import DEFAULT_TIMEOUT -from ophyd_async.epics.motion import Motor +from ophyd_async.core import ( + DEFAULT_TIMEOUT, + AsyncStatus, + CalculateTimeout, + SignalBackend, + SignalR, + SignalRW, + SoftSignalBackend, + StandardReadable, + wait_for_value, +) +from ophyd_async.epics.motor import Motor from ophyd_async.epics.signal import epics_signal_r, epics_signal_rw HOME_STR = r"\#1hmz\#2hmz\#3hmz" # Command to home the PMAC motors @@ -79,7 +85,12 @@ def __init__( super().__init__(backend, timeout, name) @AsyncStatus.wrap - async def set(self, value: LaserSettings, wait=True, timeout=CalculateTimeout): + async def set( + self, + value: LaserSettings, + wait=True, + timeout=CalculateTimeout, + ): await self.signal.set(value.value, wait, timeout) @@ -97,7 +108,12 @@ def __init__( super().__init__(backend, timeout, name) @AsyncStatus.wrap - async def set(self, value: EncReset, wait=True, timeout=CalculateTimeout): + async def set( + self, + value: EncReset, + wait=True, + timeout=CalculateTimeout, + ): await self.signal.set(value.value, wait, timeout) diff --git a/src/dodal/devices/linkam3.py b/src/dodal/devices/linkam3.py index 31621b9747..7f1c0b77c9 100644 --- a/src/dodal/devices/linkam3.py +++ b/src/dodal/devices/linkam3.py @@ -8,9 +8,9 @@ HintedSignal, StandardReadable, WatchableAsyncStatus, + WatcherUpdate, observe_value, ) -from ophyd_async.core.utils import WatcherUpdate from ophyd_async.epics.signal import epics_signal_r, epics_signal_rw diff --git a/src/dodal/devices/motors.py b/src/dodal/devices/motors.py index 5d7de2b388..4237082598 100644 --- a/src/dodal/devices/motors.py +++ b/src/dodal/devices/motors.py @@ -1,5 +1,5 @@ from ophyd_async.core import Device -from ophyd_async.epics.motion import Motor +from ophyd_async.epics.motor import Motor class XYZPositioner(Device): diff --git a/src/dodal/devices/oav/oav_to_redis_forwarder.py b/src/dodal/devices/oav/oav_to_redis_forwarder.py index 22a890efa4..525d8d5590 100644 --- a/src/dodal/devices/oav/oav_to_redis_forwarder.py +++ b/src/dodal/devices/oav/oav_to_redis_forwarder.py @@ -8,8 +8,12 @@ import numpy as np from aiohttp import ClientResponse, ClientSession from bluesky.protocols import Flyable, Stoppable -from ophyd_async.core import AsyncStatus, StandardReadable -from ophyd_async.core.signal import soft_signal_r_and_setter, soft_signal_rw +from ophyd_async.core import ( + AsyncStatus, + StandardReadable, + soft_signal_r_and_setter, + soft_signal_rw, +) from ophyd_async.epics.signal import epics_signal_r from PIL import Image from redis.asyncio import StrictRedis diff --git a/src/dodal/devices/robot.py b/src/dodal/devices/robot.py index f941306239..e871624807 100644 --- a/src/dodal/devices/robot.py +++ b/src/dodal/devices/robot.py @@ -10,8 +10,7 @@ set_and_wait_for_value, wait_for_value, ) -from ophyd_async.epics.signal import epics_signal_r, epics_signal_x -from ophyd_async.epics.signal.signal import epics_signal_rw_rbv +from ophyd_async.epics.signal import epics_signal_r, epics_signal_rw_rbv, epics_signal_x from dodal.log import LOGGER diff --git a/src/dodal/devices/scatterguard.py b/src/dodal/devices/scatterguard.py index 3df94b9ba4..3d77ee5fb9 100644 --- a/src/dodal/devices/scatterguard.py +++ b/src/dodal/devices/scatterguard.py @@ -1,5 +1,5 @@ from ophyd_async.core import StandardReadable -from ophyd_async.epics.motion import Motor +from ophyd_async.epics.motor import Motor class Scatterguard(StandardReadable): diff --git a/src/dodal/devices/scintillator.py b/src/dodal/devices/scintillator.py index 75a6b0b902..f595306fd2 100644 --- a/src/dodal/devices/scintillator.py +++ b/src/dodal/devices/scintillator.py @@ -1,5 +1,5 @@ from ophyd_async.core import StandardReadable -from ophyd_async.epics.motion import Motor +from ophyd_async.epics.motor import Motor class Scintillator(StandardReadable): diff --git a/src/dodal/devices/slits.py b/src/dodal/devices/slits.py index 2733b9379c..830c6dbd8c 100644 --- a/src/dodal/devices/slits.py +++ b/src/dodal/devices/slits.py @@ -1,5 +1,5 @@ from ophyd_async.core import StandardReadable -from ophyd_async.epics.motion import Motor +from ophyd_async.epics.motor import Motor class Slits(StandardReadable): diff --git a/src/dodal/devices/smargon.py b/src/dodal/devices/smargon.py index 27ea6f2a7a..50ddc944cf 100644 --- a/src/dodal/devices/smargon.py +++ b/src/dodal/devices/smargon.py @@ -7,7 +7,7 @@ from bluesky import plan_stubs as bps from bluesky.utils import Msg from ophyd_async.core import AsyncStatus, Device, StandardReadable, wait_for_value -from ophyd_async.epics.motion import Motor +from ophyd_async.epics.motor import Motor from ophyd_async.epics.signal import epics_signal_r from dodal.devices.util.epics_util import SetWhenEnabled diff --git a/src/dodal/devices/tetramm.py b/src/dodal/devices/tetramm.py index 501f450f79..4c7d86076b 100644 --- a/src/dodal/devices/tetramm.py +++ b/src/dodal/devices/tetramm.py @@ -1,23 +1,24 @@ import asyncio -from collections.abc import Sequence from enum import Enum from bluesky.protocols import Hints from ophyd_async.core import ( AsyncStatus, + DatasetDescriber, DetectorControl, DetectorTrigger, Device, - DirectoryProvider, - ShapeProvider, + PathProvider, StandardDetector, set_and_wait_for_value, soft_signal_r_and_setter, ) -from ophyd_async.epics.areadetector.utils import stop_busy_record -from ophyd_async.epics.areadetector.writers import HDFWriter, NDFileHDF -from ophyd_async.epics.signal import epics_signal_r, epics_signal_rw -from ophyd_async.epics.signal.signal import epics_signal_rw_rbv +from ophyd_async.epics.adcore import ADHDFWriter, NDFileHDFIO, stop_busy_record +from ophyd_async.epics.signal import ( + epics_signal_r, + epics_signal_rw, + epics_signal_rw_rbv, +) class TetrammRange(str, Enum): @@ -108,7 +109,7 @@ def __init__( self.minimum_values_per_reading = minimum_values_per_reading self.readings_per_frame = readings_per_frame - def get_deadtime(self, exposure: float) -> float: + def get_deadtime(self, exposure: float | None) -> float: # 2 internal clock cycles. Best effort approximation return 2 / self.base_sample_rate @@ -204,14 +205,17 @@ def _set_minimum_exposure(self, exposure: float): ) -class TetrammShapeProvider(ShapeProvider): +class TetrammDatasetDescriber(DatasetDescriber): max_channels = 11 def __init__(self, controller: TetrammController) -> None: self.controller = controller - async def __call__(self) -> Sequence[int]: - return [self.max_channels, self.controller.readings_per_frame] + async def np_datatype(self) -> str: + return " tuple[int, int]: + return (self.max_channels, self.controller.readings_per_frame) # TODO: Support MeanValue signals https://github.com/DiamondLightSource/dodal/issues/337 @@ -219,13 +223,13 @@ class TetrammDetector(StandardDetector): def __init__( self, prefix: str, - directory_provider: DirectoryProvider, + path_provider: PathProvider, name: str, type: str | None = None, **scalar_sigs: str, ) -> None: self.drv = TetrammDriver(prefix + "DRV:") - self.hdf = NDFileHDF(prefix + "HDF5:") + self.hdf = NDFileHDFIO(prefix + "HDF5:") controller = TetrammController(self.drv) config_signals = [ self.drv.values_per_reading, @@ -239,11 +243,11 @@ def __init__( self.type = None super().__init__( controller, - HDFWriter( + ADHDFWriter( self.hdf, - directory_provider, + path_provider, lambda: self.name, - TetrammShapeProvider(controller), + TetrammDatasetDescriber(controller), **scalar_sigs, ), config_signals, diff --git a/src/dodal/devices/training_rig/sample_stage.py b/src/dodal/devices/training_rig/sample_stage.py index bb5faddb7f..31230caa40 100644 --- a/src/dodal/devices/training_rig/sample_stage.py +++ b/src/dodal/devices/training_rig/sample_stage.py @@ -1,5 +1,5 @@ from ophyd_async.core import StandardReadable -from ophyd_async.epics.motion import Motor +from ophyd_async.epics.motor import Motor class TrainingRigSampleStage(StandardReadable): diff --git a/src/dodal/devices/turbo_slit.py b/src/dodal/devices/turbo_slit.py index c0e6358c44..7bb8b28f48 100644 --- a/src/dodal/devices/turbo_slit.py +++ b/src/dodal/devices/turbo_slit.py @@ -1,5 +1,5 @@ from ophyd_async.core import Device -from ophyd_async.epics.motion.motor import Motor +from ophyd_async.epics.motor import Motor class TurboSlit(Device): diff --git a/src/dodal/devices/undulator.py b/src/dodal/devices/undulator.py index d43e98a4ea..0e085d4d33 100644 --- a/src/dodal/devices/undulator.py +++ b/src/dodal/devices/undulator.py @@ -1,7 +1,7 @@ from enum import Enum from ophyd_async.core import ConfigSignal, StandardReadable, soft_signal_r_and_setter -from ophyd_async.epics.motion import Motor +from ophyd_async.epics.motor import Motor from ophyd_async.epics.signal import epics_signal_r # The acceptable difference, in mm, between the undulator gap and the DCM diff --git a/src/dodal/devices/util/adjuster_plans.py b/src/dodal/devices/util/adjuster_plans.py index 99b7e7b5e3..f136f827f2 100644 --- a/src/dodal/devices/util/adjuster_plans.py +++ b/src/dodal/devices/util/adjuster_plans.py @@ -8,7 +8,7 @@ from bluesky import plan_stubs as bps from bluesky.utils import Msg from ophyd.epics_motor import EpicsMotor -from ophyd_async.epics.motion import Motor +from ophyd_async.epics.motor import Motor from dodal.log import LOGGER diff --git a/src/dodal/devices/util/save_panda.py b/src/dodal/devices/util/save_panda.py index 31fd018821..af0ffaa8df 100644 --- a/src/dodal/devices/util/save_panda.py +++ b/src/dodal/devices/util/save_panda.py @@ -7,7 +7,7 @@ from bluesky.run_engine import RunEngine from ophyd_async.core import Device, save_device -from ophyd_async.panda import phase_sorter +from ophyd_async.fastcs.panda import phase_sorter from dodal.beamlines import module_name_for_beamline from dodal.utils import make_device diff --git a/src/dodal/devices/util/test_utils.py b/src/dodal/devices/util/test_utils.py index 45df1a23ac..2947718072 100644 --- a/src/dodal/devices/util/test_utils.py +++ b/src/dodal/devices/util/test_utils.py @@ -2,7 +2,7 @@ callback_on_mock_put, set_mock_value, ) -from ophyd_async.epics.motion import Motor +from ophyd_async.epics.motor import Motor def patch_motor(motor: Motor, initial_position=0): diff --git a/src/dodal/devices/xbpm_feedback.py b/src/dodal/devices/xbpm_feedback.py index 738c3c0d5f..a61804704a 100644 --- a/src/dodal/devices/xbpm_feedback.py +++ b/src/dodal/devices/xbpm_feedback.py @@ -1,8 +1,7 @@ from enum import Enum from bluesky.protocols import Triggerable -from ophyd_async.core import Device, observe_value -from ophyd_async.core.async_status import AsyncStatus +from ophyd_async.core import AsyncStatus, Device, observe_value from ophyd_async.epics.signal import epics_signal_r, epics_signal_rw diff --git a/src/dodal/devices/xspress3/xspress3.py b/src/dodal/devices/xspress3/xspress3.py index f8ae574f0a..3f2f855607 100644 --- a/src/dodal/devices/xspress3/xspress3.py +++ b/src/dodal/devices/xspress3/xspress3.py @@ -9,7 +9,7 @@ DeviceVector, wait_for_value, ) -from ophyd_async.epics.signal.signal import ( +from ophyd_async.epics.signal import ( epics_signal_r, epics_signal_rw, epics_signal_rw_rbv, diff --git a/src/dodal/devices/zocalo/zocalo_results.py b/src/dodal/devices/zocalo/zocalo_results.py index 1aa74c7432..8d029f1714 100644 --- a/src/dodal/devices/zocalo/zocalo_results.py +++ b/src/dodal/devices/zocalo/zocalo_results.py @@ -11,8 +11,12 @@ import workflows.transport from bluesky.protocols import Descriptor, Triggerable from numpy.typing import NDArray -from ophyd_async.core import HintedSignal, StandardReadable, soft_signal_r_and_setter -from ophyd_async.core.async_status import AsyncStatus +from ophyd_async.core import ( + AsyncStatus, + HintedSignal, + StandardReadable, + soft_signal_r_and_setter, +) from workflows.transport.common_transport import CommonTransport from dodal.devices.zocalo.zocalo_interaction import _get_zocalo_connection diff --git a/src/dodal/log.py b/src/dodal/log.py index 64745c9f8a..1d67f25f2b 100644 --- a/src/dodal/log.py +++ b/src/dodal/log.py @@ -8,28 +8,50 @@ from pathlib import Path from typing import TypedDict +import colorlog from bluesky.log import logger as bluesky_logger from graypy import GELFTCPHandler from ophyd.log import logger as ophyd_logger -from ophyd_async.log import ( - DEFAULT_DATE_FORMAT, - DEFAULT_FORMAT, - DEFAULT_LOG_COLORS, - ColoredFormatterWithDeviceName, -) -from ophyd_async.log import logger as ophyd_async_logger LOGGER = logging.getLogger("Dodal") +# Temporarily duplicated https://github.com/bluesky/ophyd-async/issues/550 +ophyd_async_logger = logging.getLogger("ophyd_async") LOGGER.setLevel(logging.DEBUG) -DEFAULT_FORMATTER = ColoredFormatterWithDeviceName( - fmt=DEFAULT_FORMAT, datefmt=DEFAULT_DATE_FORMAT, log_colors=DEFAULT_LOG_COLORS -) ERROR_LOG_BUFFER_LINES = 20000 INFO_LOG_DAYS = 30 DEBUG_LOG_FILES_TO_KEEP = 7 DEFAULT_GRAYLOG_PORT = 12231 +# Temporarily duplicated https://github.com/bluesky/ophyd-async/issues/550 +DEFAULT_FORMAT = ( + "%(log_color)s[%(levelname)1.1s %(asctime)s.%(msecs)03d " + "%(module)s:%(lineno)d] %(message)s" +) + +DEFAULT_DATE_FORMAT = "%y%m%d %H:%M:%S" + +DEFAULT_LOG_COLORS = { + "DEBUG": "cyan", + "INFO": "green", + "WARNING": "yellow", + "ERROR": "red", + "CRITICAL": "red,bg_white", +} + + +class ColoredFormatterWithDeviceName(colorlog.ColoredFormatter): + def format(self, record): + message = super().format(record) + if device_name := getattr(record, "ophyd_async_device_name", None): + message = f"[{device_name}]{message}" + return message + + +DEFAULT_FORMATTER = ColoredFormatterWithDeviceName( + fmt=DEFAULT_FORMAT, datefmt=DEFAULT_DATE_FORMAT, log_colors=DEFAULT_LOG_COLORS +) + class CircularMemoryHandler(logging.Handler): """Loosely based on the MemoryHandler, which keeps a buffer and writes it when full diff --git a/src/dodal/plans/data_session_metadata.py b/src/dodal/plans/data_session_metadata.py index 3b93ffe235..46b7edc59e 100644 --- a/src/dodal/plans/data_session_metadata.py +++ b/src/dodal/plans/data_session_metadata.py @@ -1,29 +1,28 @@ from bluesky import plan_stubs as bps from bluesky import preprocessors as bpp from bluesky.utils import make_decorator -from ophyd_async.core import DirectoryInfo from dodal.common.beamlines import beamline_utils -from dodal.common.types import MsgGenerator, UpdatingDirectoryProvider +from dodal.common.types import MsgGenerator, UpdatingPathProvider DATA_SESSION = "data_session" DATA_GROUPS = "data_groups" def attach_data_session_metadata_wrapper( - plan: MsgGenerator, provider: UpdatingDirectoryProvider | None = None + plan: MsgGenerator, provider: UpdatingPathProvider | None = None ) -> MsgGenerator: """ Attach data session metadata to the runs within a plan and make it correlate - with an ophyd-async DirectoryProvider. + with an ophyd-async PathProvider. - This updates the directory provider (which in turn makes a call to to a service + This updates the path provider (which in turn makes a call to to a service to figure out which scan number we are using for such a scan), and ensures the start document contains the correct data session. Args: plan: The plan to preprocess - provider: The directory provider that participating detectors are aware of. + provider: The path provider that participating detectors are aware of. Returns: MsgGenerator: A plan @@ -32,13 +31,12 @@ def attach_data_session_metadata_wrapper( Iterator[Msg]: Plan messages """ if provider is None: - provider = beamline_utils.get_directory_provider() + provider = beamline_utils.get_path_provider() yield from bps.wait_for([provider.update]) - directory_info: DirectoryInfo = provider() + ress = yield from bps.wait_for([provider.data_session]) + data_session = ress[0].result() # https://github.com/DiamondLightSource/dodal/issues/452 # As part of 452, write each dataCollection into their own folder, then can use resource_dir directly - assert directory_info.prefix is not None - data_session = directory_info.prefix.removesuffix("-") yield from bpp.inject_md_wrapper(plan, md={DATA_SESSION: data_session}) diff --git a/src/dodal/plans/motor_util_plans.py b/src/dodal/plans/motor_util_plans.py index 54cda5f79c..80d0e93611 100644 --- a/src/dodal/plans/motor_util_plans.py +++ b/src/dodal/plans/motor_util_plans.py @@ -6,7 +6,7 @@ from bluesky.preprocessors import finalize_wrapper, pchain from bluesky.utils import Msg, make_decorator from ophyd_async.core import Device -from ophyd_async.epics.motion import Motor +from ophyd_async.epics.motor import Motor from dodal.common import MsgGenerator diff --git a/tests/common/test_coordination.py b/tests/common/test_coordination.py index 67d8ed0771..3e24552786 100644 --- a/tests/common/test_coordination.py +++ b/tests/common/test_coordination.py @@ -1,4 +1,3 @@ -import uuid from inspect import Parameter, signature import pytest @@ -7,8 +6,10 @@ from dodal.common.coordination import group_uuid, inject from dodal.common.types import MsgGenerator +static_uuid = "51aef931-33b4-4b33-b7ad-a8287f541202" -@pytest.mark.parametrize("group", ["foo", "bar", "baz", str(uuid.uuid4())]) + +@pytest.mark.parametrize("group", ["foo", "bar", "baz", static_uuid]) def test_group_uid(group: str): gid = group_uuid(group) assert gid.startswith(f"{group}-") diff --git a/tests/common/test_udc_directory_provider.py b/tests/common/test_udc_directory_provider.py index 5e47eaac6c..6f31a5b514 100644 --- a/tests/common/test_udc_directory_provider.py +++ b/tests/common/test_udc_directory_provider.py @@ -3,7 +3,7 @@ import pytest -from dodal.common.udc_directory_provider import PandASubdirectoryProvider +from dodal.common.udc_directory_provider import PandASubpathProvider @pytest.mark.parametrize( @@ -16,17 +16,16 @@ ], ], ) -def test_udc_directory_provider_get_and_set(root, expected): - provider = PandASubdirectoryProvider(root) +def test_udc_path_provider_get_and_set(root, expected): + provider = PandASubpathProvider(root) directory_info = provider() - assert directory_info.root == root - assert directory_info.root.joinpath(directory_info.resource_dir) == expected + assert directory_info.directory_path == expected -def test_udc_directory_provider_excepts_before_update(): - provider = PandASubdirectoryProvider() +def test_udc_path_provider_excepts_before_update(): + provider = PandASubpathProvider() with pytest.raises( - ValueError, + AssertionError, match=re.escape( "Directory unknown for PandA to write into, update() needs to be called at least once" ), @@ -38,42 +37,29 @@ def test_udc_directory_provider_excepts_before_update(): "initial", [Path("."), None], ) -async def test_udc_directory_provider_after_update(initial, tmp_path): - provider = PandASubdirectoryProvider(initial) +async def test_udc_path_provider_after_update(initial, tmp_path): + provider = PandASubpathProvider(initial) await provider.update(directory=tmp_path) directory_info = provider() - assert directory_info.root == tmp_path - assert directory_info.resource_dir == Path("panda") + assert directory_info.directory_path == tmp_path / "panda" -async def test_udc_directory_provider_no_suffix(tmp_path): +async def test_udc_path_provider_no_suffix(tmp_path): initial = Path("initial") - provider = PandASubdirectoryProvider(initial) + provider = PandASubpathProvider(initial) root_path = tmp_path / "my_data" root_path.mkdir() await provider.update(directory=root_path) directory_info = provider() - assert directory_info.root == root_path - assert directory_info.resource_dir == Path("panda") - assert directory_info.suffix == "" + assert directory_info.directory_path == root_path / "panda" -async def test_udc_directory_provider_with_suffix(tmp_path): +async def test_udc_path_provider_with_suffix(tmp_path): initial = Path("initial") - provider = PandASubdirectoryProvider(initial) + provider = PandASubpathProvider(initial) root_path = tmp_path / "my_data" root_path.mkdir() await provider.update(directory=root_path, suffix="_123") directory_info = provider() - assert directory_info.root == root_path - assert directory_info.resource_dir == Path("panda") - assert directory_info.suffix == "_123" - - -async def test_udc_directory_provider_creates_subdirectory_if_not_exists(tmp_path): - root = tmp_path - subdir = root / Path("panda") - assert not subdir.exists() - provider = PandASubdirectoryProvider(Path("initial")) - await provider.update(directory=root) - assert subdir.exists() + assert directory_info.directory_path == root_path / "panda" + assert directory_info.filename.endswith("_123") diff --git a/tests/common/test_visit.py b/tests/common/test_visit.py new file mode 100644 index 0000000000..e9ccc3895b --- /dev/null +++ b/tests/common/test_visit.py @@ -0,0 +1,32 @@ +from unittest.mock import AsyncMock, MagicMock, patch + +from dodal.common.visit import RemoteDirectoryServiceClient + + +def create_valid_response(mock_request): + mock_request.return_value.__aenter__.return_value = (mock_response := MagicMock()) + mock_response.json = AsyncMock(return_value='{"collectionNumber": 1}') + + +@patch("dodal.common.visit.ClientSession.request") +async def test_when_create_new_collection_called_on_remote_directory_service_client_then_url_posted_to( + mock_request: MagicMock, +): + create_valid_response(mock_request) + test_url = "test.com" + client = RemoteDirectoryServiceClient(test_url) + collection = await client.create_new_collection() + assert collection.collectionNumber == 1 + mock_request.assert_called_with("POST", f"{test_url}/numtracker") + + +@patch("dodal.common.visit.ClientSession.request") +async def test_when_get_current_collection_called_on_remote_directory_service_client_then_url_got_from( + mock_request: MagicMock, +): + create_valid_response(mock_request) + test_url = "test.com" + client = RemoteDirectoryServiceClient(test_url) + collection = await client.get_current_collection() + assert collection.collectionNumber == 1 + mock_request.assert_called_with("GET", f"{test_url}/numtracker") diff --git a/tests/conftest.py b/tests/conftest.py index cfa44723fb..0fd442216c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -12,9 +12,18 @@ import pytest from bluesky.run_engine import RunEngine from ophyd.status import Status +from ophyd_async.core import ( + PathInfo, + PathProvider, +) from dodal.beamlines import i03 from dodal.common.beamlines import beamline_utils +from dodal.common.visit import ( + DirectoryServiceClient, + LocalDirectoryServiceClient, + StaticVisitPathProvider, +) from dodal.log import LOGGER, GELFTCPHandler, set_up_all_logging_handlers from dodal.utils import make_all_devices @@ -101,6 +110,27 @@ def vfm_mirror_voltages(RE: RunEngine): environ["EPICS_CA_REPEATER_PORT"] = s03_epics_repeater_port print(f"[EPICS_CA_REPEATER_PORT] = {s03_epics_repeater_port}") +PATH_INFO_FOR_TESTING: PathInfo = PathInfo( + directory_path=Path("/does/not/exist"), + filename="on_this_filesystem", +) + + +@pytest.fixture +def dummy_visit_client() -> DirectoryServiceClient: + return LocalDirectoryServiceClient() + + +@pytest.fixture +async def static_path_provider( + tmp_path: Path, dummy_visit_client: DirectoryServiceClient +) -> PathProvider: + svpp = StaticVisitPathProvider( + beamline="ixx", root=tmp_path, client=dummy_visit_client + ) + await svpp.update() + return svpp + @pytest.fixture async def RE(): diff --git a/tests/devices/i22/test_metadataholder.py b/tests/devices/i22/test_metadataholder.py index 7d083396d4..74fcf87ebb 100644 --- a/tests/devices/i22/test_metadataholder.py +++ b/tests/devices/i22/test_metadataholder.py @@ -1,14 +1,12 @@ -from pathlib import Path - import pytest -from ophyd_async.core import DeviceCollector, StaticDirectoryProvider -from ophyd_async.epics.areadetector import PilatusDetector +from ophyd_async.core import DeviceCollector, PathProvider +from ophyd_async.epics.adpilatus import PilatusDetector from dodal.devices.i22.nxsas import NXSasMetadataHolder, NXSasPilatus @pytest.fixture -def saxs(tmp_path: Path, RE) -> PilatusDetector: +def saxs(static_path_provider: PathProvider, RE) -> PilatusDetector: with DeviceCollector(mock=True): saxs = NXSasPilatus( prefix="-EA-PILAT-01:", @@ -22,7 +20,7 @@ def saxs(tmp_path: Path, RE) -> PilatusDetector: "m", ), # To get from configuration data after visit begins ), - directory_provider=StaticDirectoryProvider(tmp_path), + path_provider=static_path_provider, ) return saxs diff --git a/tests/devices/system_tests/test_eiger_system.py b/tests/devices/system_tests/test_eiger_system.py index a92f7f44fd..b60c860a41 100644 --- a/tests/devices/system_tests/test_eiger_system.py +++ b/tests/devices/system_tests/test_eiger_system.py @@ -1,15 +1,17 @@ # type: ignore # Eiger will soon be ophyd-async https://github.com/DiamondLightSource/dodal/issues/700 +from pathlib import Path + import pytest from dodal.devices.eiger import DetectorParams, EigerDetector @pytest.fixture() -def eiger(): +def eiger(tmp_path: Path): detector_params: DetectorParams = DetectorParams( expected_energy_ev=100, exposure_time=0.1, - directory="/tmp", + directory=str(tmp_path), prefix="file_name", detector_distance=100.0, omega_start=0.0, diff --git a/tests/devices/unit_tests/conftest.py b/tests/devices/unit_tests/conftest.py index ee8e39144f..7bf4544fcd 100644 --- a/tests/devices/unit_tests/conftest.py +++ b/tests/devices/unit_tests/conftest.py @@ -1,27 +1,10 @@ -from pathlib import Path - import pytest from bluesky.run_engine import RunEngine -from ophyd_async.core import ( - DirectoryInfo, - DirectoryProvider, - StaticDirectoryProvider, -) from dodal.beamlines import i03 from dodal.common.beamlines.beamline_utils import clear_devices from dodal.devices.util.test_utils import patch_motor -DIRECTORY_INFO_FOR_TESTING: DirectoryInfo = DirectoryInfo( - root=Path("/does/not/exist"), - resource_dir=Path("/on/this/filesystem"), -) - - -@pytest.fixture -def static_directory_provider(tmp_path: Path) -> DirectoryProvider: - return StaticDirectoryProvider(tmp_path) - @pytest.fixture def smargon(RE: RunEngine): diff --git a/tests/devices/unit_tests/detector/test_det_resolution.py b/tests/devices/unit_tests/detector/test_det_resolution.py index f0be9857dc..f451a9848f 100644 --- a/tests/devices/unit_tests/detector/test_det_resolution.py +++ b/tests/devices/unit_tests/detector/test_det_resolution.py @@ -1,4 +1,6 @@ -from unittest.mock import MagicMock, patch +from pathlib import Path +from unittest import mock +from unittest.mock import MagicMock, PropertyMock, patch import pytest from numpy import isclose @@ -10,11 +12,11 @@ @pytest.fixture(scope="function") -def detector_params(request): +def detector_params(request, tmp_path: Path): return DetectorParams( expected_energy_ev=100, exposure_time=1.0, - directory="test/dir", + directory=str(tmp_path), prefix="test", run_number=0, detector_distance=1.0, @@ -42,13 +44,17 @@ def test_resolution( det_distance_mm, expected_res, ): - detector_params.detector_distance = det_distance_mm - detector_params.beam_xy_converter.get_beam_xy_from_det_dist = MagicMock( - side_effect=[212.51, 219.98] - ) - detector_params.use_roi_mode = roi - get_detector_max_size.return_value = 434.6 - actual_res = resolution(detector_params, wavelength_angstroms, det_distance_mm) + with mock.patch( + "dodal.devices.detector.DetectorParams.beam_xy_converter", + new_callable=PropertyMock, + ) as prop_patch: + beam_xy_converter = MagicMock() + beam_xy_converter.get_beam_xy_from_det_dist.side_effect = [212.51, 219.98] + prop_patch.return_value = beam_xy_converter + detector_params.detector_distance = det_distance_mm + detector_params.use_roi_mode = roi + get_detector_max_size.return_value = 434.6 + actual_res = resolution(detector_params, wavelength_angstroms, det_distance_mm) assert isclose( expected_res, actual_res ), f"expected={expected_res}, actual={actual_res}" @@ -70,14 +76,17 @@ def test_resolution_with_roi( det_distance_mm, expected_res, ): - detector_params.use_roi_mode = roi - detector_params.detector_distance = det_distance_mm - detector_params.beam_xy_converter.get_beam_xy_from_det_dist = MagicMock( - side_effect=[212.51, 219.98] - ) - get_detector_max_size.return_value = 434.6 - - actual_res = resolution(detector_params, wavelength_angstroms, det_distance_mm) + with mock.patch( + "dodal.devices.detector.DetectorParams.beam_xy_converter", + new_callable=PropertyMock, + ) as prop_patch: + beam_xy_converter = MagicMock() + beam_xy_converter.get_beam_xy_from_det_dist.side_effect = [212.51, 219.98] + prop_patch.return_value = beam_xy_converter + detector_params.detector_distance = det_distance_mm + detector_params.use_roi_mode = roi + get_detector_max_size.return_value = 434.6 + actual_res = resolution(detector_params, wavelength_angstroms, det_distance_mm) assert isclose( actual_res, diff --git a/tests/devices/unit_tests/detector/test_detector.py b/tests/devices/unit_tests/detector/test_detector.py index 357b9c7e7c..f6ba3ddc3c 100644 --- a/tests/devices/unit_tests/detector/test_detector.py +++ b/tests/devices/unit_tests/detector/test_detector.py @@ -1,14 +1,18 @@ +from pathlib import Path from unittest.mock import MagicMock, patch +import pytest +from pydantic import ValidationError + from dodal.devices.detector import DetectorParams from dodal.devices.detector.det_dim_constants import EIGER2_X_16M_SIZE -def create_det_params_with_dir_and_prefix(directory, prefix="test"): +def create_det_params_with_dir_and_prefix(directory: str | Path, prefix="test"): return DetectorParams( expected_energy_ev=100, exposure_time=1.0, - directory=directory, + directory=directory, # type: ignore prefix=prefix, detector_distance=1.0, omega_start=0.0, @@ -21,27 +25,35 @@ def create_det_params_with_dir_and_prefix(directory, prefix="test"): ) -def test_if_trailing_slash_not_provided_then_appended(tmp_path): +def test_if_string_provided_check_is_dir(tmp_path: Path): assert not (_dir := str(tmp_path)).endswith("/") params = create_det_params_with_dir_and_prefix(_dir) - assert params.directory == _dir + "/" + assert params.directory == str(tmp_path) + file_path = tmp_path / "foo.h5" + file_path.touch() + with pytest.raises(ValidationError): + create_det_params_with_dir_and_prefix(str(file_path)) -def test_if_trailing_slash_provided_then_not_appended(tmp_path): - assert not (_dir := str(tmp_path)).endswith("/") - params = create_det_params_with_dir_and_prefix(_dir + "/") - assert params.directory == _dir + "/" - assert not params.directory.endswith("//") +def test_if_path_provided_check_is_dir(tmp_path: Path): + params = create_det_params_with_dir_and_prefix(tmp_path) + assert params.directory == str(tmp_path) + file_path = tmp_path / "foo.h5" + file_path.touch() + with pytest.raises(ValidationError): + create_det_params_with_dir_and_prefix(file_path) @patch( "src.dodal.devices.detector.DetectorDistanceToBeamXYConverter.parse_table", ) -def test_correct_det_dist_to_beam_converter_path_passed_in(mocked_parse_table): +def test_correct_det_dist_to_beam_converter_path_passed_in( + mocked_parse_table, tmp_path: Path +): params = DetectorParams( expected_energy_ev=100, exposure_time=1.0, - directory="directory", + directory=str(tmp_path), prefix="test", run_number=0, detector_distance=1.0, @@ -53,18 +65,17 @@ def test_correct_det_dist_to_beam_converter_path_passed_in(mocked_parse_table): det_dist_to_beam_converter_path="a fake directory", detector_size_constants=EIGER2_X_16M_SIZE, ) - params.json() assert params.beam_xy_converter.lookup_file == "a fake directory" @patch( "src.dodal.devices.detector.DetectorDistanceToBeamXYConverter.parse_table", ) -def test_run_number_correct_when_not_specified(mocked_parse_table, tmpdir): +def test_run_number_correct_when_not_specified(mocked_parse_table, tmp_path): params = DetectorParams( expected_energy_ev=100, exposure_time=1.0, - directory=str(tmpdir), + directory=str(tmp_path), prefix="test", detector_distance=1.0, omega_start=0.0, @@ -81,11 +92,11 @@ def test_run_number_correct_when_not_specified(mocked_parse_table, tmpdir): @patch( "src.dodal.devices.detector.DetectorDistanceToBeamXYConverter.parse_table", ) -def test_run_number_correct_when_specified(mocked_parse_table, tmpdir): +def test_run_number_correct_when_specified(mocked_parse_table, tmp_path): params = DetectorParams( expected_energy_ev=100, exposure_time=1.0, - directory=str(tmpdir), + directory=str(tmp_path), run_number=6, prefix="test", detector_distance=1.0, @@ -100,15 +111,65 @@ def test_run_number_correct_when_specified(mocked_parse_table, tmpdir): assert params.run_number == 6 +def test_detector_params_is_serialisable(tmp_path): + params = DetectorParams( + expected_energy_ev=100, + exposure_time=1.0, + directory=str(tmp_path), + prefix="test", + detector_distance=1.0, + omega_start=0.0, + omega_increment=0.0, + num_images_per_trigger=1, + num_triggers=1, + use_roi_mode=False, + det_dist_to_beam_converter_path="a fake directory", + detector_size_constants=EIGER2_X_16M_SIZE, + ) + json = params.model_dump_json() + assert '"run_number"' not in json + new_params = DetectorParams.model_validate_json(json) + assert new_params == params + + +# Until https://github.com/DiamondLightSource/dodal/issues/775 +def test_detector_params_deserialisation_unchanged(tmp_path: Path): + # The `directory` parameter in the `create_det_params_with_dir_and_prefix` function is used to + # specify the directory path where the detector data will be saved. This function creates an + # instance of `DetectorParams` with the provided directory path and other default parameters. The + # `directory` parameter can accept either a string or a `Path` object, and it is used to set the + # `directory` attribute of the `DetectorParams` instance. + json = f'{{"expected_energy_ev": 100.0, \ + "exposure_time": 1.0, \ + "directory": "{tmp_path}", \ + "prefix": "test", \ + "detector_distance": 1.0, \ + "omega_start": 0.0, \ + "omega_increment": 0.0, \ + "num_images_per_trigger": 1, \ + "num_triggers": 1, \ + "use_roi_mode": false, \ + "det_dist_to_beam_converter_path": "a fake directory", \ + "run_number": 17, \ + "trigger_mode": 1, \ + "detector_size_constants": "EIGER2_X_16M", \ + "enable_dev_shm": false}}' + assert '"run_number": 17' in json + new_params = DetectorParams.model_validate_json(json) + assert new_params.run_number == 17 + + @patch("os.listdir") -def test_prefix_is_used_to_determine_run_number(mock_listdir: MagicMock): +def test_prefix_is_used_to_determine_run_number( + mock_listdir: MagicMock, tmp_path: Path +): foos = (f"foo_{i}.nxs" for i in range(4)) bars = (f"bar_{i}.nxs" for i in range(7)) bazs = (f"baz_{i}.nxs" for i in range(23, 29)) files = [*foos, *bars, *bazs] mock_listdir.return_value = files - assert create_det_params_with_dir_and_prefix("dir", "foo").run_number == 4 - assert create_det_params_with_dir_and_prefix("dir", "bar").run_number == 7 - assert create_det_params_with_dir_and_prefix("dir", "baz").run_number == 29 - assert create_det_params_with_dir_and_prefix("dir", "qux").run_number == 1 + assert create_det_params_with_dir_and_prefix(tmp_path, "foo").run_number == 4 + assert create_det_params_with_dir_and_prefix(tmp_path, "bar").run_number == 7 + assert create_det_params_with_dir_and_prefix(tmp_path, "baz").run_number == 29 + assert create_det_params_with_dir_and_prefix(tmp_path, "qux").run_number == 1 diff --git a/tests/devices/unit_tests/test_eiger.py b/tests/devices/unit_tests/test_eiger.py index ac019191b3..d5e36a7557 100644 --- a/tests/devices/unit_tests/test_eiger.py +++ b/tests/devices/unit_tests/test_eiger.py @@ -1,5 +1,6 @@ # type: ignore # Eiger will soon be ophyd-async https://github.com/DiamondLightSource/dodal/issues/700 import threading +from pathlib import Path from unittest.mock import ANY, MagicMock, Mock, call, create_autospec, patch import pytest @@ -20,7 +21,6 @@ TEST_EXPECTED_ENERGY = 100.0 TEST_EXPOSURE_TIME = 1.0 -TEST_DIR = "/test/dir" TEST_PREFIX = "test" TEST_RUN_NUMBER = 0 TEST_DETECTOR_DISTANCE = 1.0 @@ -36,11 +36,12 @@ class StatusException(Exception): pass -def create_new_params() -> DetectorParams: +@pytest.fixture +def params(tmp_path: Path) -> DetectorParams: return DetectorParams( expected_energy_ev=TEST_EXPECTED_ENERGY, exposure_time=TEST_EXPOSURE_TIME, - directory=TEST_DIR, + directory=str(tmp_path), prefix=TEST_PREFIX, run_number=TEST_RUN_NUMBER, detector_distance=TEST_DETECTOR_DISTANCE, @@ -55,10 +56,10 @@ def create_new_params() -> DetectorParams: @pytest.fixture -def fake_eiger(request): +def fake_eiger(request, params: DetectorParams): FakeEigerDetector: EigerDetector = make_fake_device(EigerDetector) fake_eiger: EigerDetector = FakeEigerDetector.with_params( - params=create_new_params(), name=f"test fake Eiger: {request.node.name}" + params=params, name=f"test fake Eiger: {request.node.name}" ) return fake_eiger diff --git a/tests/devices/unit_tests/test_tetramm.py b/tests/devices/unit_tests/test_tetramm.py index 94f2e77d1d..9ee73dfab0 100644 --- a/tests/devices/unit_tests/test_tetramm.py +++ b/tests/devices/unit_tests/test_tetramm.py @@ -3,11 +3,11 @@ from ophyd_async.core import ( DetectorTrigger, DeviceCollector, - DirectoryProvider, + PathProvider, + TriggerInfo, set_mock_value, ) -from ophyd_async.core.detector import TriggerInfo -from ophyd_async.epics.areadetector import FileWriteMode +from ophyd_async.epics.adcore import FileWriteMode from dodal.devices.tetramm import ( TetrammController, @@ -41,17 +41,28 @@ async def tetramm_controller( @pytest.fixture -async def tetramm(static_directory_provider: DirectoryProvider) -> TetrammDetector: +async def tetramm(static_path_provider: PathProvider) -> TetrammDetector: async with DeviceCollector(mock=True): tetramm = TetrammDetector( "MY-TETRAMM:", - static_directory_provider, + static_path_provider, name=TEST_TETRAMM_NAME, ) return tetramm +@pytest.fixture +def supported_trigger_info() -> TriggerInfo: + return TriggerInfo( + number=1, + trigger=DetectorTrigger.constant_gate, + deadtime=1.0, + livetime=0.02, + frame_timeout=None, + ) + + async def test_max_frame_rate_is_calculated_correctly( tetramm_controller: TetrammController, ): @@ -164,12 +175,15 @@ async def test_sample_rate_scales_with_exposure_time( exposure: float, expected_values_per_reading: int, ): + set_mock_value(tetramm.hdf.file_path_exists, True) + await tetramm.prepare( TriggerInfo( - 100, - DetectorTrigger.edge_trigger, - 2e-5, - exposure, + number=100, + trigger=DetectorTrigger.edge_trigger, + deadtime=2e-5, + livetime=exposure, + frame_timeout=None, ) ) values_per_reading = await tetramm.drv.values_per_reading.get_value() @@ -252,10 +266,11 @@ async def test_prepare_with_too_low_a_deadtime_raises_error( ): await tetramm.prepare( TriggerInfo( - 5, - DetectorTrigger.edge_trigger, - 1.0 / 100_000.0, - VALID_TEST_EXPOSURE_TIME, + number=5, + trigger=DetectorTrigger.edge_trigger, + deadtime=1.0 / 100_000.0, + livetime=VALID_TEST_EXPOSURE_TIME, + frame_timeout=None, ) ) @@ -263,22 +278,25 @@ async def test_prepare_with_too_low_a_deadtime_raises_error( async def test_prepare_arms_tetramm( tetramm: TetrammDetector, ): + set_mock_value(tetramm.hdf.file_path_exists, True) await tetramm.prepare( TriggerInfo( - 5, - DetectorTrigger.edge_trigger, - 0.1, - VALID_TEST_EXPOSURE_TIME, + number=5, + trigger=DetectorTrigger.edge_trigger, + deadtime=0.1, + livetime=VALID_TEST_EXPOSURE_TIME, + frame_timeout=None, ) ) await assert_armed(tetramm.drv) -async def test_stage_sets_up_writer( - tetramm: TetrammDetector, +async def test_prepare_sets_up_writer( + tetramm: TetrammDetector, supported_trigger_info: TriggerInfo ): set_mock_value(tetramm.hdf.file_path_exists, True) await tetramm.stage() + await tetramm.prepare(supported_trigger_info) assert (await tetramm.hdf.num_capture.get_value()) == 0 assert (await tetramm.hdf.num_extra_dims.get_value()) == 0 @@ -289,17 +307,19 @@ async def test_stage_sets_up_writer( async def test_stage_sets_up_accurate_describe_output( - tetramm: TetrammDetector, + tetramm: TetrammDetector, supported_trigger_info: TriggerInfo ): assert await tetramm.describe() == {} set_mock_value(tetramm.hdf.file_path_exists, True) await tetramm.stage() + await tetramm.prepare(supported_trigger_info) assert await tetramm.describe() == { TEST_TETRAMM_NAME: { "source": "mock+ca://MY-TETRAMM:HDF5:FullFileName_RBV", - "shape": (11, 1000), + "shape": (11, 400), + "dtype_numpy": " None: self._name = name self._provider = provider @@ -53,12 +52,8 @@ async def read(self) -> dict[str, Reading]: } async def describe(self) -> dict[str, DataKey]: - directory_info = self._provider() - source = str( - directory_info.root - / directory_info.resource_dir - / f"{directory_info.prefix}{self.name}{directory_info.suffix}.h5" - ) + directory_info = self._provider(self.name) + source = str(directory_info.directory_path / f"{directory_info.filename}.h5") return { f"{self.name}_data": { "dtype": "string", @@ -87,13 +82,13 @@ def __init__(self): async def create_new_collection(self) -> DataCollectionIdentifier: if self.fail: - raise ClientResponseError(None, ()) # type: ignore + raise ValueError() return await super().create_new_collection() async def get_current_collection(self) -> DataCollectionIdentifier: if self.fail: - raise ClientResponseError(None, ()) # type: ignore + raise ValueError() return await super().get_current_collection() @@ -104,19 +99,17 @@ class DataEvent(BaseModel): @pytest.fixture -def client() -> DirectoryServiceClientBase: +def client() -> DirectoryServiceClient: return MockDirectoryServiceClient() @pytest.fixture -def provider( - client: DirectoryServiceClientBase, tmp_path: Path -) -> UpdatingDirectoryProvider: - return StaticVisitDirectoryProvider("example", tmp_path, client=client) +def provider(client: DirectoryServiceClient, tmp_path: Path) -> UpdatingPathProvider: + return StaticVisitPathProvider("example", tmp_path, client=client) @pytest.fixture(params=[1, 2]) -def detectors(request, provider: UpdatingDirectoryProvider) -> list[Readable]: +def detectors(request, provider: UpdatingPathProvider) -> list[Readable]: number_of_detectors = request.param with DeviceCollector(mock=True): dets: list[Readable] = [ @@ -185,7 +178,7 @@ def nested_run_without_metadata( def test_simple_run_gets_scan_number( RE: RunEngine, detectors: list[Readable], - provider: UpdatingDirectoryProvider, + provider: UpdatingPathProvider, tmp_path: Path, ) -> None: docs = collect_docs( @@ -203,7 +196,7 @@ def test_multi_run_gets_scan_numbers( RE: RunEngine, detectors: list[Readable], plan: Callable[[list[Readable]], MsgGenerator], - provider: UpdatingDirectoryProvider, + provider: UpdatingPathProvider, tmp_path: Path, ) -> None: """Test is here to demonstrate that multi run plans will overwrite files.""" @@ -222,7 +215,7 @@ def test_multi_run_gets_scan_numbers( def test_multi_run_single_stage( RE: RunEngine, detectors: list[Readable], - provider: UpdatingDirectoryProvider, + provider: UpdatingPathProvider, tmp_path: Path, ) -> None: docs = collect_docs( @@ -248,7 +241,7 @@ def test_multi_run_single_stage( def test_multi_run_single_stage_multi_group( RE: RunEngine, detectors: list[Readable], - provider: UpdatingDirectoryProvider, + provider: UpdatingPathProvider, tmp_path: Path, ) -> None: docs = collect_docs( @@ -273,7 +266,7 @@ def test_multi_run_single_stage_multi_group( def test_nested_run_with_metadata( RE: RunEngine, detectors: list[Readable], - provider: UpdatingDirectoryProvider, + provider: UpdatingPathProvider, tmp_path: Path, ) -> None: """Test is here to demonstrate that nested runs will be treated as a single run. @@ -296,7 +289,7 @@ def test_nested_run_with_metadata( def test_nested_run_without_metadata( RE: RunEngine, detectors: list[Readable], - provider: UpdatingDirectoryProvider, + provider: UpdatingPathProvider, tmp_path: Path, ) -> None: """Test is here to demonstrate that nested runs will be treated as a single run. @@ -316,10 +309,10 @@ def test_nested_run_without_metadata( assert_all_detectors_used_collection_numbers(tmp_path, docs, detectors, ["1", "1"]) -def test_visit_directory_provider_fails( +def test_visit_path_provider_fails( RE: RunEngine, detectors: list[Readable], - provider: UpdatingDirectoryProvider, + provider: UpdatingPathProvider, client: MockDirectoryServiceClient, ) -> None: client.fail = True @@ -331,10 +324,10 @@ def test_visit_directory_provider_fails( ) -def test_visit_directory_provider_fails_after_one_sucess( +def test_visit_path_provider_fails_after_one_sucess( RE: RunEngine, detectors: list[Readable], - provider: UpdatingDirectoryProvider, + provider: UpdatingPathProvider, client: MockDirectoryServiceClient, ) -> None: collect_docs( @@ -354,7 +347,7 @@ def test_visit_directory_provider_fails_after_one_sucess( def collect_docs( RE: RunEngine, plan: MsgGenerator, - provider: UpdatingDirectoryProvider, + provider: UpdatingPathProvider, ) -> list[DataEvent]: events = []