From be6148671dc6a63c803bc30db9eb07dbaf5d326b Mon Sep 17 00:00:00 2001 From: Jean-Philippe Lenain Date: Tue, 11 Jun 2024 17:57:53 +0200 Subject: [PATCH 1/5] Adapt to support EvB v6 data format --- .../makers/component/PedestalComponent.py | 287 +++++++++++------- src/nectarchain/makers/component/core.py | 106 ++++--- src/nectarchain/makers/core.py | 103 ++++--- 3 files changed, 312 insertions(+), 184 deletions(-) diff --git a/src/nectarchain/makers/component/PedestalComponent.py b/src/nectarchain/makers/component/PedestalComponent.py index bc879f40..6df6fea0 100644 --- a/src/nectarchain/makers/component/PedestalComponent.py +++ b/src/nectarchain/makers/component/PedestalComponent.py @@ -1,25 +1,26 @@ +import copy import logging -logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") -log = logging.getLogger(__name__) -log.handlers = logging.getLogger("__main__").handlers - -import copy +import numpy as np import numpy.ma as ma - -from .core import NectarCAMComponent from ctapipe.containers import EventType +from ctapipe.core.traits import Dict, Float, Integer, Unicode +from ctapipe_io_nectarcam.constants import HIGH_GAIN, LOW_GAIN, N_GAINS from ctapipe_io_nectarcam.containers import NectarCAMDataContainer -from ctapipe_io_nectarcam.constants import N_GAINS, HIGH_GAIN, LOW_GAIN -from ctapipe.core.traits import Integer, Unicode, Float, Dict + from ...data.container import NectarCAMPedestalContainer from ...utils import ComponentUtils -from .waveformsComponent import WaveformsComponent from .chargesComponent import ChargesComponent +from .core import NectarCAMComponent +from .waveformsComponent import WaveformsComponent -import numpy as np +logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") +log = logging.getLogger(__name__) +log.handlers = logging.getLogger("__main__").handlers -__all__ = ["PedestalEstimationComponent", ] +__all__ = [ + "PedestalEstimationComponent", +] class PedestalEstimationComponent(NectarCAMComponent): @@ -40,46 +41,53 @@ class PedestalEstimationComponent(NectarCAMComponent): Inplemented methods: WaveformsStdFilter (standard deviation of waveforms), ChargeDistributionFilter (charge distribution). wfs_std_threshold : float - Threshold of waveforms standard deviation in ADC counts above which a waveform is - excluded from pedestal computation. + Threshold of waveforms standard deviation in ADC counts above which a + waveform is excluded from pedestal computation. charge_sigma_high_thr : float - Threshold in charge distribution (number of sigmas above mean) beyond which a waveform - is excluded from pedestal computation. + Threshold in charge distribution (number of sigmas above mean) beyond which a + waveform is excluded from pedestal computation. charge_sigma_low_thr : float - Threshold in charge distribution (number of sigmas below mean) beyond which a waveform - is excluded from pedestal computation. + Threshold in charge distribution (number of sigmas below mean) beyond which a + waveform is excluded from pedestal computation. """ - ucts_tmin = Integer(None, - help="Minimum UCTS timestamp for events used in pedestal estimation", - allow_none=True, ).tag(config=True) + ucts_tmin = Integer( + None, + help="Minimum UCTS timestamp for events used in pedestal estimation", + allow_none=True, + ).tag(config=True) - ucts_tmax = Integer(None, - help="Maximum UCTS timestamp for events used in pedestal estimation", - allow_none=True, ).tag(config=True) + ucts_tmax = Integer( + None, + help="Maximum UCTS timestamp for events used in pedestal estimation", + allow_none=True, + ).tag(config=True) filter_method = Unicode( None, help="The waveforms filter method to be used.\n" - "Inplemented methods: WaveformsStdFilter (standard deviation of waveforms),\n" - " ChargeDistributionFilter (charge distribution).", + "Inplemented methods: WaveformsStdFilter (standard deviation of waveforms),\n" + " ChargeDistributionFilter (charge distribution).", read_only=False, allow_none=True, ).tag(config=True) wfs_std_threshold = Float( - 4., - help="Threshold of waveforms standard deviation in ADC counts above which a waveform is excluded from pedestal computation.", + 4.0, + help="Threshold of waveforms standard deviation in ADC counts above which a " + "waveform is excluded from pedestal computation.", ).tag(config=True) charge_sigma_high_thr = Float( - 3., - help="Threshold in charge distribution (number of sigmas above mean) beyond which a waveform is excluded from pedestal computation.", + 3.0, + help="Threshold in charge distribution (number of sigmas above mean) beyond " + "which a waveform is excluded from pedestal computation.", ).tag(config=True) charge_sigma_low_thr = Float( - 3., - help="Threshold in charge distribution (number of sigmas below mean) beyond which a waveform is excluded from pedestal computation.", + 3.0, + help="Threshold in charge distribution (number of sigmas below mean) beyond " + "which a waveform is excluded from pedestal computation.", ).tag(config=True) # I do not understand why but the ChargesComponents traits are not loaded @@ -95,8 +103,7 @@ class PedestalEstimationComponent(NectarCAMComponent): ).tag(config=True) SubComponents = copy.deepcopy(NectarCAMComponent.SubComponents) - SubComponents.default_value = ["WaveformsComponent", - "ChargesComponent"] + SubComponents.default_value = ["WaveformsComponent", "ChargesComponent"] SubComponents.read_only = True def __init__(self, subarray, config=None, parent=None, *args, **kwargs): @@ -104,7 +111,8 @@ def __init__(self, subarray, config=None, parent=None, *args, **kwargs): Component that computes calibration pedestal coefficients from raw data. Waveforms can be filtered based on time, standard deviation of the waveforms or charge distribution within the sample. - Use the `events_per_slice' parameter of `NectarCAMComponent' to reduce memory load. + Use the `events_per_slice' parameter of `NectarCAMComponent' to + reduce memory load. Parameters ---------- @@ -117,17 +125,19 @@ def __init__(self, subarray, config=None, parent=None, *args, **kwargs): Inplemented methods: WaveformsStdFilter (standard deviation of waveforms), ChargeDistributionFilter (charge distribution). wfs_std_threshold : float - Threshold of waveforms standard deviation in ADC counts above which a waveform is - excluded from pedestal computation. + Threshold of waveforms standard deviation in ADC counts above which a + waveform is excluded from pedestal computation. charge_sigma_high_thr : float - Threshold in charge distribution (number of sigmas above mean) beyond which a waveform - is excluded from pedestal computation. + Threshold in charge distribution (number of sigmas above mean) beyond which + a waveform is excluded from pedestal computation. charge_sigma_low_thr : float - Threshold in charge distribution (number of sigmas below mean) beyond which a waveform - is excluded from pedestal computation. + Threshold in charge distribution (number of sigmas below mean) beyond which + a waveform is excluded from pedestal computation. """ - super().__init__(subarray=subarray, config=config, parent=parent, *args, **kwargs) + super().__init__( + subarray=subarray, config=config, parent=parent, *args, **kwargs + ) # initialize members self._waveformsContainers = None @@ -138,15 +148,18 @@ def __init__(self, subarray, config=None, parent=None, *args, **kwargs): # initialize waveforms component waveformsComponent_kwargs = {} waveformsComponent_configurable_traits = ComponentUtils.get_configurable_traits( - WaveformsComponent) + WaveformsComponent + ) for key in kwargs.keys(): if key in waveformsComponent_configurable_traits.keys(): waveformsComponent_kwargs[key] = kwargs[key] self.waveformsComponent = WaveformsComponent( subarray=subarray, config=config, - parent=parent, *args, - **waveformsComponent_kwargs, ) + parent=parent, + *args, + **waveformsComponent_kwargs, + ) @staticmethod def calculate_stats(waveformsContainers, wfs_mask, statistics): @@ -165,7 +178,8 @@ def calculate_stats(waveformsContainers, wfs_mask, statistics): Returns ---------- ped_stats : `dict` - A dictionary containing 3D (n_chan,n_pixels,n_samples) arrays for each statistic + A dictionary containing 3D (n_chan,n_pixels,n_samples) arrays for each + statistic. """ ped_stats = {} @@ -173,9 +187,11 @@ def calculate_stats(waveformsContainers, wfs_mask, statistics): for stat in statistics: # Calculate the statistic along axis = 0, that is over events ped_stat_hg = getattr(ma, stat)( - ma.masked_array(waveformsContainers.wfs_hg, wfs_mask), axis=0) + ma.masked_array(waveformsContainers.wfs_hg, wfs_mask), axis=0 + ) ped_stat_lg = getattr(ma, stat)( - ma.masked_array(waveformsContainers.wfs_lg, wfs_mask), axis=0) + ma.masked_array(waveformsContainers.wfs_lg, wfs_mask), axis=0 + ) # Create a 3D array for the statistic array_shape = np.append([N_GAINS], np.shape(waveformsContainers.wfs_hg[0])) @@ -212,23 +228,34 @@ def timestamp_mask(self, tmin, tmax): Returns ---------- mask : `~numpy.ndarray` - A boolean array of shape (n_events,n_pixels,n_samples) that identifies waveforms to be masked + A boolean array of shape (n_events,n_pixels,n_samples) that identifies + waveforms to be masked. """ # Check if time filtering affects the data - if tmin > self._waveformsContainers.ucts_timestamp.min() or tmax < self._waveformsContainers.ucts_timestamp.max(): + if ( + tmin > self._waveformsContainers.ucts_timestamp.min() + or tmax < self._waveformsContainers.ucts_timestamp.max() + ): log.info( - f"Apply time interval selection: UCTS timestamps in interval {tmin}-{tmax}") + f"Apply time interval selection: UCTS timestamps in " + f"interval {tmin}-{tmax}" + ) # Define the mask on UCTS timestamps - t_mask = ((self._waveformsContainers.ucts_timestamp < tmin) | ( - self._waveformsContainers.ucts_timestamp > tmax)) + t_mask = (self._waveformsContainers.ucts_timestamp < tmin) | ( + self._waveformsContainers.ucts_timestamp > tmax + ) # Log information nonzero = np.count_nonzero(t_mask) - log.info(f"{len(t_mask) - nonzero}/{len(t_mask)} waveforms pass time selection.") + log.info( + f"{len(t_mask) - nonzero}/{len(t_mask)} waveforms pass time selection." + ) # Create waveforms mask to apply time selection new_mask = np.logical_or(self._wfs_mask, t_mask[:, np.newaxis, np.newaxis]) else: log.info( - f"The entire time interval will be used: UCTS timestamps in {tmin}-{tmax}") + f"The entire time interval will be used: UCTS timestamps " + f"in {tmin}-{tmax}" + ) new_mask = self._wfs_mask # Return waveforms mask @@ -236,28 +263,33 @@ def timestamp_mask(self, tmin, tmax): def waveformsStdFilter_mask(self, threshold): """ - Generates a mask to filter waveforms that have a standard deviation above a threshold. + Generates a mask to filter waveforms that have a standard deviation above + a threshold. This option is effective for dark room verification data. Parameters ---------- threshold : float - Waveform standard deviation (in ADC counts) above which the waveform is filtered out + Waveform standard deviation (in ADC counts) above which the waveform is + filtered out Returns ---------- mask : `~numpy.ndarray` - A boolean array of shape (n_events,n_pixels,n_samples) that identifies waveforms to be masked + A boolean array of shape (n_events,n_pixels,n_samples) that identifies + waveforms to be masked """ # Log - log.info(f"apply {self.filter_method} method with threshold = {threshold} ADC counts") + log.info( + f"apply {self.filter_method} method with threshold = {threshold} ADC counts" + ) # For each event and pixel calculate the waveform std wfs_std = np.std(self._waveformsContainers.wfs_hg, axis=2) # Mask events/pixels that exceed the threshold - std_mask = (wfs_std > threshold) + std_mask = wfs_std > threshold # Log information # number of masked waveforms nonzero = np.count_nonzero(std_mask) @@ -265,7 +297,9 @@ def waveformsStdFilter_mask(self, threshold): tot_wfs = self._waveformsContainers.nevents * self._waveformsContainers.npixels # fraction of waveforms masked (%) frac = 100 * nonzero / tot_wfs - log.info(f"{frac:.2f}% of the waveforms filtered out based on standard deviation.") + log.info( + f"{frac:.2f}% of the waveforms filtered out based on standard deviation." + ) # Create waveforms mask to apply time selection new_mask = np.logical_or(self._wfs_mask, std_mask[:, :, np.newaxis]) @@ -274,80 +308,102 @@ def waveformsStdFilter_mask(self, threshold): def chargeDistributionFilter_mask(self, sigma_low, sigma_high): """ - Generates a mask to filter waveforms that have a charge in the tails of the distribution. + Generates a mask to filter waveforms that have a charge in the tails of the + distribution. This option is useful for data with NSB. Parameters ---------- sigma_low : float - Number of standard deviation below mean charge beyond which waveforms are filtered out + Number of standard deviation below mean charge beyond which waveforms are + filtered out sigma_high : float - Number of standard deviation above mean charge beyond which waveforms are filtered out + Number of standard deviation above mean charge beyond which waveforms are + filtered out Returns ---------- mask : `~numpy.ndarray` - A boolean array of shape (n_events,n_pixels,n_samples) that identifies waveforms to be masked + A boolean array of shape (n_events,n_pixels,n_samples) that identifies + waveforms to be masked """ # Log - log.info(f"apply {self.filter_method} method to filter waveforms with charge outside " - f"the interval -{sigma_low}-{sigma_high} standard deviations around the mean value") + log.info( + f"apply {self.filter_method} method to filter waveforms with charge " + f"outside the interval -{sigma_low}-{sigma_high} standard deviations " + f"around the mean value" + ) # Check if waveforms and charges have the same shape wfs_shape = np.shape(self._waveformsContainers.wfs_hg) charges_shape = np.shape(self._chargesContainers.charges_hg) if wfs_shape[0] == charges_shape[0] and wfs_shape[1] == charges_shape[1]: - # For each event and pixel calculate the charge mean and std over all events # taking into account the mask already calculated - charge_array = ma.masked_array(self._chargesContainers.charges_hg, - np.any(self._wfs_mask, axis=2)) + charge_array = ma.masked_array( + self._chargesContainers.charges_hg, np.any(self._wfs_mask, axis=2) + ) charge_mean = ma.mean(charge_array, axis=0) charge_std = ma.std(charge_array, axis=0) # Mask events/pixels that are outside the core of the distribution low_threshold = charge_mean - sigma_low * charge_std - low_mask = (self._chargesContainers.charges_hg < low_threshold[np.newaxis, :]) + low_mask = self._chargesContainers.charges_hg < low_threshold[np.newaxis, :] high_threshold = charge_mean + sigma_high * charge_std - high_mask = (self._chargesContainers.charges_hg > high_threshold[np.newaxis, :]) + high_mask = ( + self._chargesContainers.charges_hg > high_threshold[np.newaxis, :] + ) charge_mask = np.logical_or(low_mask, high_mask) # Log information # number of masked waveforms nonzero = np.count_nonzero(charge_mask) # number of total waveforms - tot_wfs = self._waveformsContainers.nevents * self._waveformsContainers.npixels + tot_wfs = ( + self._waveformsContainers.nevents * self._waveformsContainers.npixels + ) # fraction of waveforms masked (%) frac = 100 * nonzero / tot_wfs log.info( - f"{frac:.2f}% of the waveforms filtered out based on charge distribution.") + f"{frac:.2f}% of the waveforms filtered out based on charge " + f"distribution." + ) # Create waveforms mask to apply time selection new_mask = np.logical_or(self._wfs_mask, charge_mask[:, :, np.newaxis]) else: log.warning( - "Waveforms and charges have incompatible shapes. No filtering applied.") + "Waveforms and charges have incompatible shapes. No filtering applied." + ) new_mask = self._wfs_mask return new_mask - def finish(self,*args,**kwargs): + def finish(self, *args, **kwargs): """ - Finish the component by filtering the waveforms and calculating the pedestal quantities + Finish the component by filtering the waveforms and calculating the pedestal + quantities. """ # Use only pedestal type events waveformsContainers = self.waveformsComponent.finish() - self._waveformsContainers = waveformsContainers.containers[EventType.SKY_PEDESTAL] + self._waveformsContainers = waveformsContainers.containers[ + EventType.SKY_PEDESTAL + ] + # log.info('JPL: waveformsContainers=',waveformsContainers.containers[ + # EventType.SKY_PEDESTAL].nsamples) # If we want to filter based on charges distribution # make sure that the charge distribution container is filled - if self.filter_method == "ChargeDistributionFilter" and \ - self._chargesContainers is None: + if ( + self.filter_method == "ChargeDistributionFilter" + and self._chargesContainers is None + ): log.debug("Compute charges from waveforms") chargesComponent_kwargs = {} - chargesComponent_configurable_traits = ComponentUtils.get_configurable_traits( - ChargesComponent) + chargesComponent_configurable_traits = ( + ComponentUtils.get_configurable_traits(ChargesComponent) + ) for key in kwargs.keys(): if key in chargesComponent_configurable_traits.keys(): chargesComponent_kwargs[key] = kwargs[key] @@ -355,15 +411,20 @@ def finish(self,*args,**kwargs): waveformsContainer=self._waveformsContainers, subarray=self.subarray, config=self.config, - parent=self.parent, *args, - **chargesComponent_kwargs, ) + parent=self.parent, + *args, + **chargesComponent_kwargs, + ) # Check if waveforms container is empty if self._waveformsContainers is None: log.warning("Waveforms container is none, pedestals cannot be evaluated") # container with no results return None - elif self._waveformsContainers.nevents is None or self._waveformsContainers.nevents == 0: + elif ( + self._waveformsContainers.nevents is None + or self._waveformsContainers.nevents == 0 + ): log.warning("Waveforms container is empty, pedestals cannot be evaluated") # container with no results return None @@ -371,42 +432,57 @@ def finish(self,*args,**kwargs): # Build mask to filter the waveforms # Mask based on the high gain channel that is most sensitive to signals # Initialize empty mask - self._wfs_mask = np.zeros(np.shape(self._waveformsContainers.wfs_hg), - dtype=bool) + self._wfs_mask = np.zeros( + np.shape(self._waveformsContainers.wfs_hg), dtype=bool + ) # Time selection # set the minimum time tmin = np.maximum( self.ucts_tmin or self._waveformsContainers.ucts_timestamp.min(), - self._waveformsContainers.ucts_timestamp.min()) - print('TMIN',self.ucts_tmin,self._waveformsContainers.ucts_timestamp.min(),tmin) + self._waveformsContainers.ucts_timestamp.min(), + ) + print( + "TMIN", + self.ucts_tmin, + self._waveformsContainers.ucts_timestamp.min(), + tmin, + ) # set the maximum time tmax = np.minimum( self.ucts_tmax or self._waveformsContainers.ucts_timestamp.max(), - self._waveformsContainers.ucts_timestamp.max()) - print('TMAX', self.ucts_tmax, self._waveformsContainers.ucts_timestamp.max(), tmax) + self._waveformsContainers.ucts_timestamp.max(), + ) + print( + "TMAX", + self.ucts_tmax, + self._waveformsContainers.ucts_timestamp.max(), + tmax, + ) # Add time selection to mask self._wfs_mask = self.timestamp_mask(tmin, tmax) # Filter Waveforms if self.filter_method is None: - log.info('no filtering applied to waveforms') - elif self.filter_method == 'WaveformsStdFilter': + log.info("no filtering applied to waveforms") + elif self.filter_method == "WaveformsStdFilter": self._wfs_mask = self.waveformsStdFilter_mask(self.wfs_std_threshold) - elif self.filter_method == 'ChargeDistributionFilter': - self._wfs_mask = self.chargeDistributionFilter_mask(self.charge_sigma_low_thr, - self.charge_sigma_high_thr) + elif self.filter_method == "ChargeDistributionFilter": + self._wfs_mask = self.chargeDistributionFilter_mask( + self.charge_sigma_low_thr, self.charge_sigma_high_thr + ) else: log.warning( - f"required filtering method {self.filter_method} not available") + f"required filtering method {self.filter_method} not available" + ) log.warning("no filtering applied to waveforms") # compute statistics for the pedestals # the statistic names must be valid numpy.ma attributes - statistics = ['mean', 'std'] - self._ped_stats = self.calculate_stats(self._waveformsContainers, - self._wfs_mask, - statistics) + statistics = ["mean", "std"] + self._ped_stats = self.calculate_stats( + self._waveformsContainers, self._wfs_mask, statistics + ) # calculate the number of events per pixel used to compute te quantitites # start wit total number of events @@ -424,9 +500,10 @@ def finish(self,*args,**kwargs): pixels_id=self._waveformsContainers.pixels_id, ucts_timestamp_min=np.uint64(tmin), ucts_timestamp_max=np.uint64(tmax), - pedestal_mean_hg=self._ped_stats['mean'][HIGH_GAIN], - pedestal_mean_lg=self._ped_stats['mean'][LOW_GAIN], - pedestal_std_hg=self._ped_stats['std'][HIGH_GAIN], - pedestal_std_lg=self._ped_stats['std'][LOW_GAIN], ) + pedestal_mean_hg=self._ped_stats["mean"][HIGH_GAIN], + pedestal_mean_lg=self._ped_stats["mean"][LOW_GAIN], + pedestal_std_hg=self._ped_stats["std"][HIGH_GAIN], + pedestal_std_lg=self._ped_stats["std"][LOW_GAIN], + ) return output diff --git a/src/nectarchain/makers/component/core.py b/src/nectarchain/makers/component/core.py index ee5b1470..ba7a7b40 100644 --- a/src/nectarchain/makers/component/core.py +++ b/src/nectarchain/makers/component/core.py @@ -1,10 +1,5 @@ -import logging - -logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") -log = logging.getLogger(__name__) -log.handlers = logging.getLogger("__main__").handlers - import copy +import logging from abc import abstractmethod from collections.abc import Iterable @@ -13,21 +8,15 @@ from ctapipe.core import Component, TelescopeComponent from ctapipe.core.traits import ComponentNameList, Integer, Unicode from ctapipe.instrument import CameraGeometry -from ctapipe.io import HDF5TableReader from ctapipe_io_nectarcam import constants from ctapipe_io_nectarcam.containers import NectarCAMDataContainer -from ...data.container import ( - ArrayDataContainer, - ChargesContainer, - ChargesContainers, - NectarCAMContainer, - TriggerMapContainer, - WaveformsContainer, - WaveformsContainers, -) from ...data.container.core import ArrayDataContainer +logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") +log = logging.getLogger(__name__) +log.handlers = logging.getLogger("__main__").handlers + __all__ = [ "ArrayDataComponent", "NectarCAMComponent", @@ -47,14 +36,19 @@ class NectarCAMComponent(TelescopeComponent): default_value=None, allow_none=True, read_only=True, - help="List of Component names that are used insite current component, this is used to resolve recusively the configurable traits defined in sub-components", + help="List of Component names that are used inside current component, this is " + "used to resolve recursively the " + "configurable traits defined in sub-components", ).tag(config=True) def __init__(self, subarray, config=None, parent=None, *args, **kwargs): super().__init__( subarray=subarray, config=config, parent=parent, *args, **kwargs ) - self.__pixels_id = parent._event_source.camera_config.expected_pixels_id + try: + self.__pixels_id = parent._event_source.camera_config.expected_pixels_id + except AttributeError: + self.__pixels_id = parent._event_source.camera_config.pixel_id_map self.__run_number = parent.run_number self.__npixels = parent.npixels @@ -111,7 +105,10 @@ def __init__(self, subarray, config=None, parent=None, *args, **kwargs): super().__init__( subarray=subarray, config=config, parent=parent, *args, **kwargs ) - self.__nsamples = parent._event_source.camera_config.num_samples + try: + self.__nsamples = parent._event_source.camera_config.num_samples + except AttributeError: + self.__nsamples = parent._event_source.camera_config.num_samples_nominal self.trigger_list = [] @@ -130,7 +127,8 @@ def _init_trigger_type(self, trigger: EventType, **kwargs): Initializes empty lists for different trigger types in the ArrayDataMaker class. Args: - trigger (EventType): The trigger type for which the lists are being initialized. + trigger (EventType): The trigger type for which the lists are being + initialized. Returns: None. The method only initializes the empty lists for the trigger type. @@ -155,7 +153,8 @@ def _compute_broken_pixels(wfs_hg, wfs_lg, **kwargs): wfs_lg (ndarray): Low gain waveforms. **kwargs: Additional keyword arguments. Returns: - tuple: Two arrays of zeros with the same shape as `wfs_hg` (or `wfs_lg`) but without the last dimension. + tuple: Two arrays of zeros with the same shape as `wfs_hg` (or `wfs_lg`) but + without the last dimension. """ log.debug("computation of broken pixels is not yet implemented") return np.zeros((wfs_hg.shape[:-1]), dtype=bool), np.zeros( @@ -206,7 +205,8 @@ def __call__(self, event: NectarCAMDataContainer, *args, **kwargs): **kwargs: Additional keyword arguments that can be passed to the method. Returns: - If the return_wfs keyword argument is True, the method returns the high and low gain waveforms from the event. + If the return_wfs keyword argument is True, the method returns the high and + low gain waveforms from the event. """ name = __class__._get_name_trigger(event.trigger.event_type) @@ -242,11 +242,14 @@ def select_container_array_field( container: ArrayDataContainer, pixel_id: np.ndarray, field: str ) -> np.ndarray: """ - Selects specific fields from an ArrayDataContainer object based on a given list of pixel IDs. + Selects specific fields from an ArrayDataContainer object based on a given list + of pixel IDs. Args: - container (ArrayDataContainer): An object of type ArrayDataContainer that contains the data. - pixel_id (ndarray): An array of pixel IDs for which the data needs to be selected. + container (ArrayDataContainer): An object of type ArrayDataContainer that + contains the data. + pixel_id (ndarray): An array of pixel IDs for which the data needs to be + selected. field (str): The name of the field to be selected from the container. Returns: @@ -257,7 +260,8 @@ def select_container_array_field( ) for pixel in pixel_id[~mask_contain_pixels_id]: log.warning( - f"You asked for pixel_id {pixel} but it is not present in this container, skip this one" + f"You asked for pixel_id {pixel} but it is not present in this " + f"container, skip this one" ) res = np.array( [ @@ -269,7 +273,10 @@ def select_container_array_field( for pixel in pixel_id[mask_contain_pixels_id] ] ) - ####could be nice to return np.ma.masked_array(data = res, mask = container.broken_pixels_hg.transpose(res.shape[1],res.shape[0],res.shape[2])) + # could be nice to return np.ma.masked_array(data=res, + # mask = container.broken_pixels_hg.transpose( + # res.shape[1],res.shape[0],res.shape[2]) + # ) return res @staticmethod @@ -313,7 +320,8 @@ def merge( if not (isinstance(container_a[field], np.ndarray)): if field != "nevents" and (container_a[field] != container_b[field]): raise Exception( - f"merge impossible because of {field} filed (values are {container_a[field]} and {container_b[field]}" + f"merge impossible because of {field} filed (values are " + f"{container_a[field]} and {container_b[field]}" ) for field in container_a.keys(): @@ -357,7 +365,8 @@ def nevents(self, trigger: EventType): Returns the number of events for the specified trigger type. Args: - trigger (EventType): The trigger type for which the number of events is requested. + trigger (EventType): The trigger type for which the number of events is + requested. Returns: int: The number of events for the specified trigger type. @@ -381,10 +390,12 @@ def broken_pixels_hg(self, trigger: EventType): Returns an array of broken pixels for high gain for the specified trigger type. Args: - trigger (EventType): The trigger type for which the broken pixels for high gain are requested. + trigger (EventType): The trigger type for which the broken pixels for high + gain are requested. Returns: - np.ndarray: An array of broken pixels for high gain for the specified trigger type. + np.ndarray: An array of broken pixels for high gain for the specified + trigger type. """ return np.array( self.__broken_pixels_hg[__class__._get_name_trigger(trigger)], @@ -406,10 +417,12 @@ def broken_pixels_lg(self, trigger: EventType): Returns an array of broken pixels for low gain for the specified trigger type. Args: - trigger (EventType): The trigger type for which the broken pixels for low gain are requested. + trigger (EventType): The trigger type for which the broken pixels for low + gain are requested. Returns: - np.ndarray: An array of broken pixels for low gain for the specified trigger type. + np.ndarray: An array of broken pixels for low gain for the specified + trigger type. """ return np.array( self.__broken_pixels_lg[__class__._get_name_trigger(trigger)], @@ -421,7 +434,8 @@ def ucts_timestamp(self, trigger: EventType): Returns an array of UCTS timestamps for the specified trigger type. Args: - trigger (EventType): The trigger type for which the UCTS timestamps are requested. + trigger (EventType): The trigger type for which the UCTS timestamps are + requested. Returns: np.ndarray: An array of UCTS timestamps for the specified trigger type. @@ -436,7 +450,8 @@ def ucts_busy_counter(self, trigger: EventType): Returns an array of UCTS busy counters for the specified trigger type. Args: - trigger (EventType): The trigger type for which the UCTS busy counters are requested. + trigger (EventType): The trigger type for which the UCTS busy counters are + requested. Returns: np.ndarray: An array of UCTS busy counters for the specified trigger type. @@ -451,7 +466,8 @@ def ucts_event_counter(self, trigger: EventType): Returns an array of UCTS event counters for the specified trigger type. Args: - trigger (EventType): The trigger type for which the UCTS event counters are requested. + trigger (EventType): The trigger type for which the UCTS event counters are + requested. Returns: np.ndarray: An array of UCTS event counters for the specified trigger type. @@ -466,7 +482,8 @@ def event_type(self, trigger: EventType): Returns an array of event types for the specified trigger type. Args: - trigger (EventType): The trigger type for which the event types are requested. + trigger (EventType): The trigger type for which the event types are + requested. Returns: np.ndarray: An array of event types for the specified trigger type. @@ -496,7 +513,8 @@ def multiplicity(self, trigger: EventType): Returns an array of multiplicities for the specified trigger type. Args: - trigger (EventType): The trigger type for which the multiplicities are requested. + trigger (EventType): The trigger type for which the multiplicities are + requested. Returns: np.ndarray: An array of multiplicities for the specified trigger type. @@ -514,7 +532,8 @@ def trig_pattern(self, trigger: EventType): Returns an array of trigger patterns for the specified trigger type. Args: - trigger (EventType): The trigger type for which the trigger patterns are requested. + trigger (EventType): The trigger type for which the trigger patterns are + requested. Returns: np.ndarray: An array of trigger patterns for the specified trigger type. @@ -527,13 +546,16 @@ def trig_pattern(self, trigger: EventType): def trig_pattern_all(self, trigger: EventType): """ - Returns an array of trigger patterns for all events for the specified trigger type. + Returns an array of trigger patterns for all events for the specified trigger + type. Args: - trigger (EventType): The trigger type for which the trigger patterns for all events are requested. + trigger (EventType): The trigger type for which the trigger patterns for all + events are requested. Returns: - np.ndarray: An array of trigger patterns for all events for the specified trigger type. + np.ndarray: An array of trigger patterns for all events for the specified + trigger type. """ return np.array( self.__trig_patter_all[f"{__class__._get_name_trigger(trigger)}"], diff --git a/src/nectarchain/makers/core.py b/src/nectarchain/makers/core.py index 08a18fe3..15996244 100644 --- a/src/nectarchain/makers/core.py +++ b/src/nectarchain/makers/core.py @@ -1,10 +1,5 @@ -import logging - -logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") -log = logging.getLogger(__name__) -log.handlers = logging.getLogger("__main__").handlers - import copy +import logging import os import pathlib from datetime import datetime @@ -31,18 +26,23 @@ from ..data.container import LightNectarCAMEventSource from ..data.container.core import NectarCAMContainer, TriggerMapContainer from ..utils import ComponentUtils -from .component import * from .component import NectarCAMComponent, get_valid_component +logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") +log = logging.getLogger(__name__) +log.handlers = logging.getLogger("__main__").handlers + __all__ = ["EventsLoopNectarCAMCalibrationTool"] -"""The code snippet is a part of a class hierarchy for data processing. -It includes the `BaseMaker` abstract class, the `EventsLoopMaker` and `ArrayDataMaker` subclasses. +"""The code snippet is a part of a class hierarchy for data processing. +It includes the `BaseMaker` abstract class, the `EventsLoopMaker` and `ArrayDataMaker` +subclasses. These classes are used to perform computations on data from a specific run.""" class BaseNectarCAMCalibrationTool(Tool): - """Mother class for all the makers, the role of makers is to do computation on the data.""" + """Mother class for all the makers, the role of makers is to do computation on the + data.""" name = "BaseNectarCAMCalibration" @@ -52,24 +52,33 @@ class BaseNectarCAMCalibrationTool(Tool): @default("provenance_log") def _default_provenance_log(self): - return f"{os.environ.get('NECTARCHAIN_LOG', '/tmp')}/{self.name}_{os.getpid()}_{datetime.now()}.provenance.log" + return ( + f"{os.environ.get('NECTARCHAIN_LOG', '/tmp')}/" + f"{self.name}_{os.getpid()}_{datetime.now()}.provenance.log" + ) @default("log_file") def _default_log_file(self): - return f"{os.environ.get('NECTARCHAIN_LOG', '/tmp')}/{self.name}_{os.getpid()}_{datetime.now()}.log" + return ( + f"{os.environ.get('NECTARCHAIN_LOG', '/tmp')}/" + f"{self.name}_{os.getpid()}_{datetime.now()}.log" + ) @staticmethod def load_run( run_number: int, max_events: int = None, run_file: str = None ) -> LightNectarCAMEventSource: - """Static method to load from $NECTARCAMDATA directory data for specified run with max_events + """Static method to load from $NECTARCAMDATA directory data for specified run + with max_events. Args:self.__run_number = run_number run_number (int): run_id - maxevents (int, optional): max of events to be loaded. Defaults to -1, to load everythings. + maxevents (int, optional): max of events to be loaded. Defaults to -1, to + load everything. run_file (optional) : if provided, will load this run file Returns: - List[ctapipe_io_nectarcam.LightNectarCAMEventSource]: List of EventSource for each run files + List[ctapipe_io_nectarcam.LightNectarCAMEventSource]: List of EventSource + for each run files. """ # Load the data from the run file. if run_file is None: @@ -92,7 +101,8 @@ class EventsLoopNectarCAMCalibrationTool(BaseNectarCAMCalibrationTool): Args: run_number (int): The ID of the run to be processed. - max_events (int, optional): The maximum number of events to be loaded. Defaults to None. + max_events (int, optional): The maximum number of events to be loaded. + Defaults to None. run_file (optional): The specific run file to be loaded. Example Usage: @@ -145,7 +155,8 @@ class EventsLoopNectarCAMCalibrationTool(BaseNectarCAMCalibrationTool): output_path = Path( help="output filename", default_value=pathlib.Path( - f"{os.environ.get('NECTARCAMDATA','/tmp')}/runs/{name}_run{run_number.default_value}.h5" + f"{os.environ.get('NECTARCAMDATA','/tmp')}/runs/" + f"{name}_run{run_number.default_value}.h5" ), ).tag(config=True) @@ -167,21 +178,25 @@ class EventsLoopNectarCAMCalibrationTool(BaseNectarCAMCalibrationTool): ).tag(config=True) events_per_slice = Integer( - help="number of events that will be treat before to pull the buffer and write to disk, if None, all the events will be loaded", + help="number of events that will be treat before to pull the buffer and write" + "to disk, if None, all the events will be loaded", default_value=None, allow_none=True, ).tag(config=True) def __new__(cls, *args, **kwargs): - """This method is used to pass to the current instance of Tool the traits defined - in the components provided in the componentsList trait. - WARNING : This method is maybe not the best way to do it, need to discuss with ctapipe developpers. + """This method is used to pass to the current instance of Tool the traits + defined in the components provided in the componentsList trait. + WARNING : This method is maybe not the best way to do it, need to discuss with + ctapipe developers. """ _cls = super(EventsLoopNectarCAMCalibrationTool, cls).__new__( cls, *args, **kwargs ) log.warning( - "the componentName in componentsList must be defined in the nectarchain.makers.component module, otherwise the import of the componentName will raise an error" + "the componentName in componentsList must be defined in the " + "nectarchain.makers.component module, otherwise the import of the " + "componentName will raise an error" ) for componentName in _cls.componentsList: class_name = ComponentUtils.get_class_name_from_ComponentName(componentName) @@ -199,7 +214,8 @@ def __init__(self, *args, **kwargs): def _init_output_path(self): self.output_path = pathlib.Path( - f"{os.environ.get('NECTARCAMDATA','/tmp')}/runs/{self.name}_run{self.run_number}.h5" + f"{os.environ.get('NECTARCAMDATA','/tmp')}/" + f"runs/{self.name}_run{self.run_number}.h5" ) def _load_eventsource(self, *args, **kwargs): @@ -217,7 +233,7 @@ def _get_provided_component_kwargs(self, componentName: str): output_component_kwargs[key] = getattr(self, key) return output_component_kwargs - def _init_writer(self, sliced: bool = False, slice_index: int = 0, group_name = None): + def _init_writer(self, sliced: bool = False, slice_index: int = 0, group_name=None): if hasattr(self, "writer"): self.writer.close() @@ -227,7 +243,8 @@ def _init_writer(self, sliced: bool = False, slice_index: int = 0, group_name = if self.overwrite: try: log.info( - f"overwrite set to true, trying to remove file {self.output_path}" + f"overwrite set to true, trying to " + f"remove file {self.output_path}" ) os.remove(self.output_path) log.info(f"{self.output_path} removed") @@ -236,11 +253,13 @@ def _init_writer(self, sliced: bool = False, slice_index: int = 0, group_name = else: if os.path.exists(self.output_path): raise Exception( - f"file {self.output_path} does exist,\n set overwrite to True if you want to overwrite" + f"file {self.output_path} does exist,\n set overwrite " + f"to True if you want to overwrite" ) group_name = f"data_{slice_index}" self.log.info( - f"initilization of writer in sliced mode (output written to {group_name})" + f"initilization of writer in sliced mode (output written " + f"to {group_name})" ) mode = "a" else: @@ -248,7 +267,8 @@ def _init_writer(self, sliced: bool = False, slice_index: int = 0, group_name = if self.overwrite: try: log.info( - f"overwrite set to true, trying to remove file {self.output_path}" + f"overwrite set to true, trying to remove " + f"file {self.output_path}" ) os.remove(self.output_path) log.info(f"{self.output_path} removed") @@ -257,7 +277,8 @@ def _init_writer(self, sliced: bool = False, slice_index: int = 0, group_name = else: if os.path.exists(self.output_path): raise Exception( - f"file {self.output_path} does exist,\n set overwrite to True if you want to overwrite" + f"file {self.output_path} does exist,\n set overwrite to True " + f"if you want to overwrite" ) if group_name is None: group_name = "data" @@ -266,7 +287,7 @@ def _init_writer(self, sliced: bool = False, slice_index: int = 0, group_name = os.makedirs(self.output_path.parent, exist_ok=True) self.writer = self.enter_context( HDF5TableWriter( - filename=self.output_path, # pathlib.Path(f"{os.environ.get('NECTARCAMDATA',self.output_path.parent)}/{self.output_path.stem}_run{self.run_number}{self.output_path.suffix}"), + filename=self.output_path, parent=self, mode=mode, group_name=group_name, @@ -277,7 +298,7 @@ def _init_writer(self, sliced: bool = False, slice_index: int = 0, group_name = self.log.warning("retry with w mode instead of a") self.writer = self.enter_context( HDF5TableWriter( - filename=self.output_path, # pathlib.Path(f"{os.environ.get('NECTARCAMDATA',self.output_path.parent)}/{self.output_path.stem}_run{self.run_number}{self.output_path.suffix}"), + filename=self.output_path, parent=self, mode="w", group_name=group_name, @@ -310,7 +331,10 @@ def setup(self, *args, **kwargs): def _setup_eventsource(self, *args, **kwargs): self._load_eventsource(*args, **kwargs) self.__npixels = self._event_source.camera_config.num_pixels - self.__pixels_id = self._event_source.camera_config.expected_pixels_id + try: + self.__pixels_id = self._event_source.camera_config.expected_pixels_id + except AttributeError: + self.__pixels_id = self._event_source.camera_config.pixel_id_map def _setup_components(self, *args, **kwargs): self.log.info("setup of components") @@ -341,8 +365,10 @@ def start( Method to extract data from the EventSource. Args: - n_events (int, optional): The maximum number of events to process. Default is np.inf. - restart_from_begining (bool, optional): Whether to restart the event source reader. Default is False. + n_events (int, optional): The maximum number of events to process. + Default is np.inf. + restart_from_begining (bool, optional): Whether to restart the event source + reader. Default is False. *args: Additional arguments that can be passed to the method. **kwargs: Additional keyword arguments that can be passed to the method. @@ -351,7 +377,8 @@ def start( """ if ~np.isfinite(n_events) and (self.events_per_slice is None): self.log.warning( - "neither needed events number specified or events per slice, it may cause a memory error" + "neither needed events number specified or events per slice, it may " + "cause a memory error" ) # if isinstance(trigger_type, EventType) or trigger_type is None: # trigger_type = [trigger_type] @@ -428,12 +455,14 @@ def _write_container(self, container: Container, index_component: int = 0) -> No elif isinstance(container, TriggerMapContainer): for key in container.containers.keys(): self.writer.write( - table_name=f"{container.containers[key].__class__.__name__}_{index_component}/{key.name}", + table_name=f"{container.containers[key].__class__.__name__}_" + f"{index_component}/{key.name}", containers=container.containers[key], ) else: raise TypeError( - "component output must be an instance of TriggerMapContainer or NectarCAMContainer" + "component output must be an instance of TriggerMapContainer or " + "NectarCAMContainer" ) except FieldValidationError as e: log.warning(e, exc_info=True) From 09dbf26f44d5b78294026503fae72ec3c1fc9832 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Lenain Date: Wed, 19 Jun 2024 15:20:04 +0200 Subject: [PATCH 2/5] Updated unit tests for test runs for both EvB verisons 4 and 6 --- .../calibration/tests/test_pedestal_tool.py | 439 ++++++++++-------- 1 file changed, 250 insertions(+), 189 deletions(-) diff --git a/src/nectarchain/makers/calibration/tests/test_pedestal_tool.py b/src/nectarchain/makers/calibration/tests/test_pedestal_tool.py index 2172f864..308d532f 100644 --- a/src/nectarchain/makers/calibration/tests/test_pedestal_tool.py +++ b/src/nectarchain/makers/calibration/tests/test_pedestal_tool.py @@ -4,93 +4,132 @@ import tables from ctapipe.utils import get_dataset_path from ctapipe_io_nectarcam.constants import N_SAMPLES + from nectarchain.data.container import NectarCAMPedestalContainer from nectarchain.makers.calibration import PedestalNectarCAMCalibrationTool -run_number = 3938 -run_file = get_dataset_path('NectarCAM.Run3938.30events.fits.fz') -# number of working pixels in run -n_pixels = 1834 +runs = { + "Run number": [3938, 5288], + "Run file": [ + get_dataset_path("NectarCAM.Run3938.30events.fits.fz"), + get_dataset_path("NectarCAM.Run5288.0001.fits.fz"), + ], + "N pixels": [1834, 1848], +} class TestPedestalCalibrationTool: - def test_base(self): """ Test basic functionality, including IO on disk """ # setup - n_slices = 3 + n_slices = [3, 2] events_per_slice = 10 - max_events = n_slices * events_per_slice - with tempfile.TemporaryDirectory() as tmpdirname: - outfile = tmpdirname + "/pedestal.h5" - - # run tool - tool = PedestalNectarCAMCalibrationTool( - run_number=run_number, - run_file=run_file, - max_events=max_events, - events_per_slice=events_per_slice, - log_level=0, - output_path=outfile, - overwrite=True, - filter_method=None, - ) - - tool.initialize() - tool.setup() - - tool.start() - output = tool.finish(return_output_component=True) - - # Check output in memory - assert output.nsamples == N_SAMPLES - assert np.all(output.nevents == max_events) - assert np.shape(output.pixels_id) == (n_pixels,) - assert output.ucts_timestamp_min == np.uint64(1674462932637854793) - assert output.ucts_timestamp_max == np.uint64(1674462932695877994) - assert np.shape(output.pedestal_mean_hg) == (n_pixels, N_SAMPLES) - assert np.shape(output.pedestal_mean_lg) == (n_pixels, N_SAMPLES) - assert np.shape(output.pedestal_std_hg) == (n_pixels, N_SAMPLES) - assert np.shape(output.pedestal_std_lg) == (n_pixels, N_SAMPLES) - assert np.allclose(output.pedestal_mean_hg, 245., atol=20.) - assert np.allclose(output.pedestal_mean_lg, 245., atol=20.) - assert np.allclose(output.pedestal_std_hg, 10, atol=10) - assert np.allclose(output.pedestal_std_lg, 2.5, atol=2.) - - # Check output on disk - # FIXME: use tables for the moment, update when h5 reader in nectarchain is working - with tables.open_file(outfile) as h5file: - for s in range(n_slices): - # Check individual groups - group_name = "data_{}".format(s + 1) - assert group_name in h5file.root.__members__ - table = h5file.root[group_name][NectarCAMPedestalContainer.__name__][0] - assert table['nsamples'] == N_SAMPLES - assert np.all(table['nevents'] == events_per_slice) - assert np.shape(table['pixels_id']) == (n_pixels,) - assert np.shape(table['pedestal_mean_hg']) == (n_pixels, N_SAMPLES) - assert np.shape(table['pedestal_mean_lg']) == (n_pixels, N_SAMPLES) - assert np.shape(table['pedestal_std_hg']) == (n_pixels, N_SAMPLES) - assert np.shape(table['pedestal_std_lg']) == (n_pixels, N_SAMPLES) - # Check combined results - group_name = "data_combined" - table = h5file.root[group_name][NectarCAMPedestalContainer.__name__][0] - assert table['nsamples'] == N_SAMPLES - assert np.all(table['nevents'] == max_events) - assert np.shape(table['pixels_id']) == (n_pixels,) - assert table['ucts_timestamp_min'] == np.uint64(1674462932637854793) - assert table['ucts_timestamp_max'] == np.uint64(1674462932695877994) - assert np.shape(table['pedestal_mean_hg']) == (n_pixels, N_SAMPLES) - assert np.shape(table['pedestal_mean_lg']) == (n_pixels, N_SAMPLES) - assert np.shape(table['pedestal_std_hg']) == (n_pixels, N_SAMPLES) - assert np.shape(table['pedestal_std_lg']) == (n_pixels, N_SAMPLES) - assert np.allclose(table['pedestal_mean_hg'], 245., atol=20.) - assert np.allclose(table['pedestal_mean_lg'], 245., atol=20.) - assert np.allclose(table['pedestal_std_hg'], 10, atol=10) - assert np.allclose(table['pedestal_std_lg'], 2.5, atol=2.) + max_events = [n_slices[0] * events_per_slice, 13] + + expected_ucts_timestamp_min = [1674462932637854793, 1715007113924900896] + expected_ucts_timestamp_max = [1674462932695877994, 1715007123524920096] + + for i, run in enumerate(runs["Run number"]): + run_number = runs["Run number"][i] + run_file = runs["Run file"][i] + n_pixels = runs["N pixels"][i] + with tempfile.TemporaryDirectory() as tmpdirname: + outfile = tmpdirname + "/pedestal.h5" + + # run tool + tool = PedestalNectarCAMCalibrationTool( + run_number=run_number, + run_file=run_file, + max_events=max_events[i], + events_per_slice=events_per_slice, + log_level=0, + output_path=outfile, + overwrite=True, + filter_method=None, + ) + + tool.initialize() + tool.setup() + + tool.start() + output = tool.finish(return_output_component=True) + + # Check output in memory + assert output.nsamples == N_SAMPLES + assert np.all(output.nevents == max_events[i]) + assert np.shape(output.pixels_id) == (n_pixels,) + assert output.ucts_timestamp_min == np.uint64( + expected_ucts_timestamp_min[i] + ) + assert output.ucts_timestamp_max == np.uint64( + expected_ucts_timestamp_max[i] + ) + assert np.shape(output.pedestal_mean_hg) == (n_pixels, N_SAMPLES) + assert np.shape(output.pedestal_mean_lg) == (n_pixels, N_SAMPLES) + assert np.shape(output.pedestal_std_hg) == (n_pixels, N_SAMPLES) + assert np.shape(output.pedestal_std_lg) == (n_pixels, N_SAMPLES) + assert np.allclose(output.pedestal_mean_hg, 245.0, atol=20.0) + assert np.allclose(output.pedestal_mean_lg, 245.0, atol=20.0) + assert np.allclose(output.pedestal_std_hg, 10, atol=10) + assert np.allclose(output.pedestal_std_lg, 2.5, atol=2.3) + + # Check output on disk + # FIXME: use tables for the moment, update when h5 reader in nectarchain + # is working + with tables.open_file(outfile) as h5file: + for s in range(n_slices[i]): + # Check individual groups + group_name = "data_{}".format(s + 1) + assert group_name in h5file.root.__members__ + table = h5file.root[group_name][ + NectarCAMPedestalContainer.__name__ + ][0] + assert table["nsamples"] == N_SAMPLES + assert np.allclose(table["nevents"], events_per_slice, atol=7) + assert np.shape(table["pixels_id"]) == (n_pixels,) + assert np.shape(table["pedestal_mean_hg"]) == ( + n_pixels, + N_SAMPLES, + ) + assert np.shape(table["pedestal_mean_lg"]) == ( + n_pixels, + N_SAMPLES, + ) + assert np.shape(table["pedestal_std_hg"]) == ( + n_pixels, + N_SAMPLES, + ) + assert np.shape(table["pedestal_std_lg"]) == ( + n_pixels, + N_SAMPLES, + ) + # Check combined results + group_name = "data_combined" + table = h5file.root[group_name][ + NectarCAMPedestalContainer.__name__ + ][0] + assert table["nsamples"] == N_SAMPLES + assert np.all(table["nevents"] == max_events[i]) + assert np.shape(table["pixels_id"]) == (n_pixels,) + assert table["ucts_timestamp_min"] == np.uint64( + expected_ucts_timestamp_min[i] + ) + assert table["ucts_timestamp_max"] == np.uint64( + expected_ucts_timestamp_max[i] + ) + assert np.shape(table["pedestal_mean_hg"]) == (n_pixels, N_SAMPLES) + assert np.shape(table["pedestal_mean_lg"]) == (n_pixels, N_SAMPLES) + assert np.shape(table["pedestal_std_hg"]) == (n_pixels, N_SAMPLES) + assert np.shape(table["pedestal_std_lg"]) == (n_pixels, N_SAMPLES) + assert np.allclose(table["pedestal_mean_hg"], 245.0, atol=20.0) + assert np.allclose(table["pedestal_mean_lg"], 245.0, atol=20.0) + assert np.allclose(table["pedestal_std_hg"], 10, atol=10) + assert np.allclose( + table["pedestal_std_lg"], 2.5, atol=2.0 if i == 0 else 2.3 + ) def test_timesel(self): """ @@ -98,48 +137,55 @@ def test_timesel(self): """ # setup - n_slices = 3 + n_slices = [3, 2] events_per_slice = 10 - max_events = n_slices * events_per_slice - tmin = 1674462932637860000 - tmax = 1674462932695700000 - with tempfile.TemporaryDirectory() as tmpdirname: - outfile = tmpdirname + "/pedestal.h5" - - # run tool - tool = PedestalNectarCAMCalibrationTool( - run_number=run_number, - run_file=run_file, - max_events=max_events, - events_per_slice=events_per_slice, - log_level=0, - output_path=outfile, - ucts_tmin=tmin, - ucts_tmax=tmax, - overwrite=True, - filter_method=None, - ) - - tool.initialize() - tool.setup() - - tool.start() - output = tool.finish(return_output_component=True) - - # check output - assert output.nsamples == N_SAMPLES - assert np.all(output.nevents <= max_events) - assert np.shape(output.pixels_id) == (n_pixels,) - assert output.ucts_timestamp_min >= tmin - assert output.ucts_timestamp_max <= tmax - assert np.shape(output.pedestal_mean_hg) == (n_pixels, N_SAMPLES) - assert np.shape(output.pedestal_mean_lg) == (n_pixels, N_SAMPLES) - assert np.shape(output.pedestal_std_hg) == (n_pixels, N_SAMPLES) - assert np.shape(output.pedestal_std_lg) == (n_pixels, N_SAMPLES) - assert np.allclose(output.pedestal_mean_hg, 245., atol=20.) - assert np.allclose(output.pedestal_mean_lg, 245., atol=20.) - assert np.allclose(output.pedestal_std_hg, 10, atol=10) - assert np.allclose(output.pedestal_std_lg, 2.5, atol=2.) + max_events = [n_slices[0] * events_per_slice, 13] + tmin = [1674462932637860000, 1715007113924900000] + tmax = [1674462932695700000, 1715007123524921000] + for i, run in enumerate(runs["Run number"]): + run_number = runs["Run number"][i] + run_file = runs["Run file"][i] + n_pixels = runs["N pixels"][i] + + with tempfile.TemporaryDirectory() as tmpdirname: + outfile = tmpdirname + "/pedestal.h5" + + # run tool + tool = PedestalNectarCAMCalibrationTool( + run_number=run_number, + run_file=run_file, + max_events=max_events[i], + events_per_slice=events_per_slice, + log_level=0, + output_path=outfile, + ucts_tmin=tmin[i], + ucts_tmax=tmax[i], + overwrite=True, + filter_method=None, + ) + + tool.initialize() + tool.setup() + + tool.start() + output = tool.finish(return_output_component=True) + + # check output + assert output.nsamples == N_SAMPLES + assert np.all(output.nevents <= max_events[i]) + assert np.shape(output.pixels_id) == (n_pixels,) + assert output.ucts_timestamp_min >= tmin[i] + assert output.ucts_timestamp_max <= tmax[i] + assert np.shape(output.pedestal_mean_hg) == (n_pixels, N_SAMPLES) + assert np.shape(output.pedestal_mean_lg) == (n_pixels, N_SAMPLES) + assert np.shape(output.pedestal_std_hg) == (n_pixels, N_SAMPLES) + assert np.shape(output.pedestal_std_lg) == (n_pixels, N_SAMPLES) + assert np.allclose(output.pedestal_mean_hg, 245.0, atol=20.0) + assert np.allclose(output.pedestal_mean_lg, 245.0, atol=20.0) + assert np.allclose(output.pedestal_std_hg, 10, atol=10) + assert np.allclose( + output.pedestal_std_lg, 2.5, atol=2.0 if i == 0 else 2.3 + ) def test_WaveformsStdFilter(self): """ @@ -147,44 +193,53 @@ def test_WaveformsStdFilter(self): """ # setup - n_slices = 3 + n_slices = [3, 2] events_per_slice = 10 - max_events = n_slices * events_per_slice - with tempfile.TemporaryDirectory() as tmpdirname: - outfile = tmpdirname + "/pedestal.h5" - - # run tool - tool = PedestalNectarCAMCalibrationTool( - run_number=run_number, - run_file=run_file, - max_events=max_events, - events_per_slice=events_per_slice, - log_level=0, - output_path=outfile, - overwrite=True, - filter_method='WaveformsStdFilter', - wfs_std_threshold=4., - ) - - tool.initialize() - tool.setup() - - tool.start() - output = tool.finish(return_output_component=True) - - # check output - assert output.nsamples == N_SAMPLES - assert np.all(output.nevents <= max_events) - assert np.shape(output.pixels_id) == (n_pixels,) - assert np.shape(output.pedestal_mean_hg) == (n_pixels, N_SAMPLES) - assert np.shape(output.pedestal_mean_lg) == (n_pixels, N_SAMPLES) - assert np.shape(output.pedestal_std_hg) == (n_pixels, N_SAMPLES) - assert np.shape(output.pedestal_std_lg) == (n_pixels, N_SAMPLES) - assert np.allclose(output.pedestal_mean_hg, 245., atol=20.) - assert np.allclose(output.pedestal_mean_lg, 245., atol=20.) - # verify that fluctuations are reduced - assert np.allclose(output.pedestal_std_hg, 3., atol=2.) - assert np.allclose(output.pedestal_std_lg, 2.5, atol=2.) + max_events = [n_slices[0] * events_per_slice, 13] + for i, run in enumerate(runs["Run number"]): + run_number = runs["Run number"][i] + run_file = runs["Run file"][i] + n_pixels = runs["N pixels"][i] + + with tempfile.TemporaryDirectory() as tmpdirname: + outfile = tmpdirname + "/pedestal.h5" + + # run tool + tool = PedestalNectarCAMCalibrationTool( + run_number=run_number, + run_file=run_file, + max_events=max_events[i], + events_per_slice=events_per_slice, + log_level=0, + output_path=outfile, + overwrite=True, + filter_method="WaveformsStdFilter", + wfs_std_threshold=4.0, + ) + + tool.initialize() + tool.setup() + + tool.start() + output = tool.finish(return_output_component=True) + + # check output + assert output.nsamples == N_SAMPLES + assert np.all(output.nevents <= max_events[i]) + assert np.shape(output.pixels_id) == (n_pixels,) + assert np.shape(output.pedestal_mean_hg) == (n_pixels, N_SAMPLES) + assert np.shape(output.pedestal_mean_lg) == (n_pixels, N_SAMPLES) + assert np.shape(output.pedestal_std_hg) == (n_pixels, N_SAMPLES) + assert np.shape(output.pedestal_std_lg) == (n_pixels, N_SAMPLES) + assert np.allclose(output.pedestal_mean_hg, 245.0, atol=20.0) + assert np.allclose(output.pedestal_mean_lg, 245.0, atol=20.0) + # verify that fluctuations are reduced + assert np.allclose( + output.pedestal_std_hg, 3.0, atol=2.0 if i == 0 else 4.0 + ) + assert np.allclose( + output.pedestal_std_lg, 2.5, atol=2.0 if i == 0 else 2.6 + ) def test_ChargeDistributionFilter(self): """ @@ -192,41 +247,47 @@ def test_ChargeDistributionFilter(self): """ # setup - n_slices = 2 + n_slices = [2, 1] events_per_slice = 10 - max_events = n_slices * events_per_slice - 1 - with tempfile.TemporaryDirectory() as tmpdirname: - outfile = tmpdirname + "/pedestal.h5" - - # run tool - tool = PedestalNectarCAMCalibrationTool( - run_number=run_number, - run_file=run_file, - max_events=max_events, - events_per_slice=events_per_slice, - log_level=0, - output_path=outfile, - overwrite=True, - filter_method='ChargeDistributionFilter', - charge_sigma_low_thr=1., - charge_sigma_high_thr=2., - ) - - tool.initialize() - tool.setup() - - tool.start() - output = tool.finish(return_output_component=True) - - # check output - assert output.nsamples == N_SAMPLES - assert np.all(output.nevents <= max_events) - assert np.shape(output.pixels_id) == (n_pixels,) - assert np.shape(output.pedestal_mean_hg) == (n_pixels, N_SAMPLES) - assert np.shape(output.pedestal_mean_lg) == (n_pixels, N_SAMPLES) - assert np.shape(output.pedestal_std_hg) == (n_pixels, N_SAMPLES) - assert np.shape(output.pedestal_std_lg) == (n_pixels, N_SAMPLES) - assert np.allclose(output.pedestal_mean_hg, 245., atol=20.) - assert np.allclose(output.pedestal_mean_lg, 245., atol=20.) - assert np.allclose(output.pedestal_std_hg, 10., atol=10.) - assert np.allclose(output.pedestal_std_lg, 2.5, atol=3.) + max_events = [n_slices[0] * events_per_slice - 1, 12] + for i, run in enumerate(runs["Run number"]): + run_number = runs["Run number"][i] + run_file = runs["Run file"][i] + n_pixels = runs["N pixels"][i] + with tempfile.TemporaryDirectory() as tmpdirname: + outfile = tmpdirname + "/pedestal.h5" + + # run tool + tool = PedestalNectarCAMCalibrationTool( + run_number=run_number, + run_file=run_file, + max_events=max_events[i], + events_per_slice=events_per_slice, + log_level=0, + output_path=outfile, + overwrite=True, + filter_method="ChargeDistributionFilter", + charge_sigma_low_thr=1.0, + charge_sigma_high_thr=2.0, + ) + + tool.initialize() + tool.setup() + + tool.start() + output = tool.finish(return_output_component=True) + + # check output + assert output.nsamples == N_SAMPLES + assert np.all(output.nevents <= max_events[i]) + assert np.shape(output.pixels_id) == (n_pixels,) + assert np.shape(output.pedestal_mean_hg) == (n_pixels, N_SAMPLES) + assert np.shape(output.pedestal_mean_lg) == (n_pixels, N_SAMPLES) + assert np.shape(output.pedestal_std_hg) == (n_pixels, N_SAMPLES) + assert np.shape(output.pedestal_std_lg) == (n_pixels, N_SAMPLES) + assert np.allclose(output.pedestal_mean_hg, 245.0, atol=20.0) + assert np.allclose(output.pedestal_mean_lg, 245.0, atol=20.0) + assert np.allclose(output.pedestal_std_hg, 10.0, atol=10.0) + assert np.allclose( + output.pedestal_std_lg, 2.5 if i == 0 else 2.2, atol=3.0 + ) From de334c15ea19a15da58863165155bf9275df1284 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Lenain Date: Thu, 4 Jul 2024 11:07:48 +0200 Subject: [PATCH 3/5] Don't access EventSource attributes directly through camera_config, but through nectarcam_service instead. --- src/nectarchain/makers/component/core.py | 10 ++-------- src/nectarchain/makers/core.py | 7 ++----- 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/src/nectarchain/makers/component/core.py b/src/nectarchain/makers/component/core.py index ba7a7b40..10e846dd 100644 --- a/src/nectarchain/makers/component/core.py +++ b/src/nectarchain/makers/component/core.py @@ -45,10 +45,7 @@ def __init__(self, subarray, config=None, parent=None, *args, **kwargs): super().__init__( subarray=subarray, config=config, parent=parent, *args, **kwargs ) - try: - self.__pixels_id = parent._event_source.camera_config.expected_pixels_id - except AttributeError: - self.__pixels_id = parent._event_source.camera_config.pixel_id_map + self.__pixels_id = parent._event_source.nectarcam_service.pixel_ids self.__run_number = parent.run_number self.__npixels = parent.npixels @@ -105,10 +102,7 @@ def __init__(self, subarray, config=None, parent=None, *args, **kwargs): super().__init__( subarray=subarray, config=config, parent=parent, *args, **kwargs ) - try: - self.__nsamples = parent._event_source.camera_config.num_samples - except AttributeError: - self.__nsamples = parent._event_source.camera_config.num_samples_nominal + self.__nsamples = parent._event_source.nectarcam_service.num_samples self.trigger_list = [] diff --git a/src/nectarchain/makers/core.py b/src/nectarchain/makers/core.py index 15996244..ad423411 100644 --- a/src/nectarchain/makers/core.py +++ b/src/nectarchain/makers/core.py @@ -330,11 +330,8 @@ def setup(self, *args, **kwargs): def _setup_eventsource(self, *args, **kwargs): self._load_eventsource(*args, **kwargs) - self.__npixels = self._event_source.camera_config.num_pixels - try: - self.__pixels_id = self._event_source.camera_config.expected_pixels_id - except AttributeError: - self.__pixels_id = self._event_source.camera_config.pixel_id_map + self.__npixels = self._event_source.nectarcam_service.num_pixels + self.__pixels_id = self._event_source.nectarcam_service.pixel_ids def _setup_components(self, *args, **kwargs): self.log.info("setup of components") From d5dc64509c65edc49b4f931ea6406edb5c505588 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Lenain Date: Thu, 4 Jul 2024 11:51:27 +0200 Subject: [PATCH 4/5] Pin scipy version to 1.11, needed for CI tests. (#125) * Pin scipy version to 1.11, needed for CI tests. * Pin scipy version to 1.11, needed for CI tests. --------- Co-authored-by: Jean-Philippe Lenain --- environment.yml | 1 + pyproject.toml | 1 + 2 files changed, 2 insertions(+) diff --git a/environment.yml b/environment.yml index d17f63b5..90164d34 100644 --- a/environment.yml +++ b/environment.yml @@ -13,6 +13,7 @@ dependencies: - pytest-runner - pip - pandas + - scipy=1.11 - zeo - zodb - pip: diff --git a/pyproject.toml b/pyproject.toml index c0833017..f4d196ad 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,6 +24,7 @@ dependencies = [ "ctapipe~=0.19", "ctapipe-io-nectarcam", "pandas", + "scipy==1.11", "zodb", "zeo", ] From ecc29eaa5d73acec3e419029713ac53a734a2cff Mon Sep 17 00:00:00 2001 From: Jean-Philippe Lenain Date: Thu, 4 Jul 2024 15:23:55 +0200 Subject: [PATCH 5/5] Get rid of direct EventSource.Camera_config calls throughout nectarchain, use of EventSource.nectarcam_service instead. --- src/nectarchain/data/container/eventSource.py | 153 ++++++------------ 1 file changed, 45 insertions(+), 108 deletions(-) diff --git a/src/nectarchain/data/container/eventSource.py b/src/nectarchain/data/container/eventSource.py index 50b24192..e8f84414 100644 --- a/src/nectarchain/data/container/eventSource.py +++ b/src/nectarchain/data/container/eventSource.py @@ -1,11 +1,4 @@ import logging -import sys - -logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") -log = logging.getLogger(__name__) -log.handlers = logging.getLogger("__main__").handlers - - import struct import numpy as np @@ -21,9 +14,14 @@ CDTS_BEFORE_37201_DTYPE, TIB_DTYPE, ) -from ctapipe_io_nectarcam.constants import N_GAINS, N_PIXELS, N_SAMPLES +from ctapipe_io_nectarcam.constants import N_PIXELS from ctapipe_io_nectarcam.containers import NectarCAMEventContainer +logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") +log = logging.getLogger(__name__) +log.handlers = logging.getLogger("__main__").handlers + + __all__ = ["LightNectarCAMEventSource"] @@ -38,62 +36,48 @@ def fill_nectarcam_event_container_from_zfile(self, array_event, event): Returns: - None - This function fills the NectarCAM event container in the NectarCAMDataContainer object with the event data from the zfile. It unpacks the necessary data from the event and assigns it to the corresponding fields in the event container. + This function fills the NectarCAM event container in the NectarCAMDataContainer + object with the event data from the zfile. It unpacks the necessary data from the + event and assigns it to the corresponding fields in the event container. The function performs the following steps: 1. Assigns the tel_id to the local variable. - 2. Creates a new NectarCAMEventContainer object and assigns it to the event_container field of the NectarCAMDataContainer object. - 3. Assigns the extdevices_presence field of the event to the extdevices_presence field of the event_container. - 4. Assigns the counters field of the event to the counters field of the event_container. - 5. Unpacks the TIB data from the event and assigns it to the corresponding fields in the event_container. - 6. Unpacks the CDTS data from the event and assigns it to the corresponding fields in the event_container. - 7. Calls the unpack_feb_data function to unpack the FEB counters and trigger pattern from the event and assign them to the corresponding fields in the event_container. + 2. Creates a new NectarCAMEventContainer object and assigns it to the + event_container field of the NectarCAMDataContainer object. + 3. Assigns the extdevices_presence field of the event to the extdevices_presence + field of the event_container. + 4. Assigns the counters field of the event to the counters field of the + event_container. + 5. Unpacks the TIB data from the event and assigns it to the corresponding fields in + the event_container. + 6. Unpacks the CDTS data from the event and assigns it to the corresponding fields + in the event_container. + 7. Calls the unpack_feb_data function to unpack the FEB counters and trigger pattern + from the event and assign them to the corresponding fields in the event_container. """ tel_id = self.tel_id event_container = NectarCAMEventContainer() array_event.nectarcam.tel[tel_id].evt = event_container - # event_container.configuration_id = event.configuration_id - # event_container.event_id = event.event_id - # event_container.tel_event_id = event.tel_event_id - # event_container.pixel_status = event.pixel_status - # event_container.ped_id = event.ped_id - # event_container.module_status = event.nectarcam.module_status event_container.extdevices_presence = event.nectarcam.extdevices_presence - # event_container.swat_data = event.nectarcam.swat_data event_container.counters = event.nectarcam.counters - ## unpack TIB data + # unpack TIB data unpacked_tib = event.nectarcam.tib_data.view(TIB_DTYPE)[0] - # event_container.tib_event_counter = unpacked_tib[0] - # event_container.tib_pps_counter = unpacked_tib[1] - # event_container.tib_tenMHz_counter = unpacked_tib[2] - # event_container.tib_stereo_pattern = unpacked_tib[3] event_container.tib_masked_trigger = unpacked_tib[4] # unpack CDTS data is_old_cdts = len(event.nectarcam.cdts_data) < 36 if is_old_cdts: unpacked_cdts = event.nectarcam.cdts_data.view(CDTS_BEFORE_37201_DTYPE)[0] event_container.ucts_event_counter = unpacked_cdts[0] - # event_container.ucts_pps_counter = unpacked_cdts[1] - # event_container.ucts_clock_counter = unpacked_cdts[2] event_container.ucts_timestamp = unpacked_cdts[3] - # event_container.ucts_camera_timestamp = unpacked_cdts[4] event_container.ucts_trigger_type = unpacked_cdts[5] - # event_container.ucts_white_rabbit_status = unpacked_cdts[6] else: unpacked_cdts = event.nectarcam.cdts_data.view(CDTS_AFTER_37201_DTYPE)[0] event_container.ucts_timestamp = unpacked_cdts[0] - # event_container.ucts_address = unpacked_cdts[1] # new event_container.ucts_event_counter = unpacked_cdts[2] - event_container.ucts_busy_counter = unpacked_cdts[3] # new - # event_container.ucts_pps_counter = unpacked_cdts[4] - # event_container.ucts_clock_counter = unpacked_cdts[5] + event_container.ucts_busy_counter = unpacked_cdts[3] event_container.ucts_trigger_type = unpacked_cdts[6] - # event_container.ucts_white_rabbit_status = unpacked_cdts[7] - # event_container.ucts_stereo_pattern = unpacked_cdts[8] # new - # event_container.ucts_num_in_bunch = unpacked_cdts[9] # new - # event_container.cdts_version = unpacked_cdts[10] # new # Unpack FEB counters and trigger pattern self.unpack_feb_data(event_container, event) @@ -102,53 +86,22 @@ def unpack_feb_data(self, event_container, event): """Unpack FEB counters and trigger pattern""" # Deduce data format version bytes_per_module = ( - len(event.nectarcam.counters) // self.camera_config.nectarcam.num_modules + len(event.nectarcam.counters) // self.nectarcam_service.num_modules ) # Remain compatible with data before addition of trigger pattern module_fmt = "IHHIBBBBBBBB" if bytes_per_module > 16 else "IHHIBBBB" n_fields = len(module_fmt) - rec_fmt = "=" + module_fmt * self.camera_config.nectarcam.num_modules + rec_fmt = "=" + module_fmt * self.nectarcam_service.num_modules # Unpack unpacked_feb = struct.unpack(rec_fmt, event.nectarcam.counters) # Initialize field containers - # n_camera_modules = N_PIXELS // 7 - # event_container.feb_abs_event_id = np.zeros(shape=(n_camera_modules,), dtype=np.uint32) - # event_container.feb_event_id = np.zeros(shape=(n_camera_modules,), dtype=np.uint16) - # event_container.feb_pps_cnt = np.zeros(shape=(n_camera_modules,), dtype=np.uint16) - # event_container.feb_ts1 = np.zeros(shape=(n_camera_modules,), dtype=np.uint32) - # event_container.feb_ts2_trig = np.zeros(shape=(n_camera_modules,), dtype=np.int16) - # event_container.feb_ts2_pps = np.zeros(shape=(n_camera_modules,), dtype=np.int16) if bytes_per_module > 16: n_patterns = 4 event_container.trigger_pattern = np.zeros( shape=(n_patterns, N_PIXELS), dtype=bool ) - # Unpack absolute event ID - # event_container.feb_abs_event_id[ - # self.camera_config.nectarcam.expected_modules_id] = unpacked_feb[0::n_fields] - ## Unpack PPS counter - # event_container.feb_pps_cnt[ - # self.camera_config.nectarcam.expected_modules_id] = unpacked_feb[1::n_fields] - ## Unpack relative event ID - # event_container.feb_event_id[ - # self.camera_config.nectarcam.expected_modules_id] = unpacked_feb[2::n_fields] - ## Unpack TS1 counter - # event_container.feb_ts1[ - # self.camera_config.nectarcam.expected_modules_id] = unpacked_feb[3::n_fields] - ## Unpack TS2 counters - # ts2_decimal = lambda bits: bits - (1 << 8) if bits & 0x80 != 0 else bits - # ts2_decimal_vec = np.vectorize(ts2_decimal) - # event_container.feb_ts2_trig[ - # self.camera_config.nectarcam.expected_modules_id] = ts2_decimal_vec( - # unpacked_feb[4::n_fields]) - # event_container.feb_ts2_pps[ - # self.camera_config.nectarcam.expected_modules_id] = ts2_decimal_vec( - # unpacked_feb[5::n_fields]) - # Loop over modules - - for module_idx, module_id in enumerate( - self.camera_config.nectarcam.expected_modules_id - ): + + for module_idx, module_id in enumerate(self.nectarcam_service.module_ids): offset = module_id * 7 if bytes_per_module > 16: field_id = 8 @@ -162,24 +115,14 @@ def unpack_feb_data(self, event_container, event): pattern_id, offset : offset + 7 ] = module_pattern - # Unpack native charge - # if len(event.nectarcam.charges_gain1) > 0: - # event_container.native_charge = np.zeros(shape=(N_GAINS, N_PIXELS), - # dtype=np.uint16) - # rec_fmt = '=' + 'H' * self.camera_config.num_pixels - # for gain_id in range(N_GAINS): - # unpacked_charge = struct.unpack(rec_fmt, getattr(event.nectarcam, - # f'charges_gain{gain_id + 1}')) - # event_container.native_charge[ - # gain_id, self.camera_config.expected_pixels_id] = unpacked_charge - def fill_trigger_info(self, array_event): """ Fill the trigger information for a given event. Parameters: - array_event (NectarCAMEventContainer): The NectarCAMEventContainer object to fill with trigger information. + array_event (NectarCAMEventContainer): The NectarCAMEventContainer object to + fill with trigger information. Returns: None @@ -238,16 +181,19 @@ def fill_trigger_info(self, array_event): trigger.event_type = EventType.SINGLE_PE else: self.log.warning( - f"Event {array_event.index.event_id} has unknown event type, trigger: {trigger_bits:08b}" + f"Event {array_event.index.event_id} has unknown event type, trigger: " + f"{trigger_bits:08b}" ) trigger.event_type = EventType.UNKNOWN class LightNectarCAMEventSource(NectarCAMEventSource): """ - LightNectarCAMEventSource is a subclass of NectarCAMEventSource that provides a generator for iterating over NectarCAM events. + LightNectarCAMEventSource is a subclass of NectarCAMEventSource that provides a + generator for iterating over NectarCAM events. - This implementation of the NectarCAMEventSource is mucvh lighter than the one within ctapipe_io_nectarcam, only the fileds interesting + This implementation of the NectarCAMEventSource is mucvh lighter than the one within + ctapipe_io_nectarcam, only the fileds interesting for nectachain are kept. Attributes: @@ -255,23 +201,28 @@ class LightNectarCAMEventSource(NectarCAMEventSource): max_events (int): The maximum number of events to process. tel_id (int): The telescope ID. nectarcam_service (NectarCAMService): The service container for NectarCAM. - trigger_information (bool): Flag indicating whether to fill trigger information in the event container. + trigger_information (bool): Flag indicating whether to fill trigger information + in the event container. obs_ids (list): The list of observation IDs. multi_file (MultiFileReader): The multi-file reader for reading the data source. r0_r1_calibrator (R0R1Calibrator): The calibrator for R0 to R1 conversion. - calibrate_flatfields_and_pedestals (bool): Flag indicating whether to calibrate flatfield and pedestal events. + calibrate_flatfields_and_pedestals (bool): Flag indicating whether to calibrate + flatfield and pedestal events. Methods: - _generator: The generator function that yields NectarCAMDataContainer objects representing each event. + _generator: The generator function that yields NectarCAMDataContainer objects + representing each event. """ def _generator(self): """ - The generator function that yields NectarCAMDataContainer objects representing each event. + The generator function that yields NectarCAMDataContainer objects representing + each event. Yields: - NectarCAMDataContainer: The NectarCAMDataContainer object representing each event. + NectarCAMDataContainer: The NectarCAMDataContainer object representing + each event. Raises: None @@ -297,23 +248,9 @@ def _generator(self): # fill R0/R1 data self.fill_r0r1_container(array_event, event) # fill specific NectarCAM event data - # fill specific NectarCAM event data self.fill_nectarcam_event_container_from_zfile(array_event, event) if self.trigger_information: self.fill_trigger_info(array_event) - # fill general monitoring data - # self.fill_mon_container_from_zfile(array_event, event) - # - ## gain select and calibrate to pe - # if self.r0_r1_calibrator.calibration_path is not None: - # # skip flatfield and pedestal events if asked - # if ( - # self.calibrate_flatfields_and_pedestals - # or array_event.trigger.event_type not in {EventType.FLATFIELD, - # EventType.SKY_PEDESTAL} - # ): - # self.r0_r1_calibrator.calibrate(array_event) - yield array_event