From 65bf016eb80d0f790d882ac834d872622d218211 Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Tue, 23 Apr 2024 13:26:26 +0200 Subject: [PATCH 01/20] Initial commit adding the variance extractor --- src/ctapipe/image/extractor.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/ctapipe/image/extractor.py b/src/ctapipe/image/extractor.py index ce772744c58..82247c854a3 100644 --- a/src/ctapipe/image/extractor.py +++ b/src/ctapipe/image/extractor.py @@ -13,6 +13,7 @@ "NeighborPeakWindowSum", "BaselineSubtractedNeighborPeakWindowSum", "TwoPassWindowSum", + "VarianceExtractor", "extract_around_peak", "extract_sliding_window", "neighbor_average_maximum", @@ -1297,6 +1298,21 @@ def __call__( ) +class VarianceExtractor(ImageExtractor): + + """ + Extractor that calculates + the variance of each waveform and + corrects for the relative gain. + """ + + def __call__( + self, waveforms, tel_id + ) -> DL1CameraContainer: + variance = np.nanvar(waveforms,axis=2) + return DL1CameraContainer(image=variance, peak_time=0, is_valid=True) + + def deconvolution_parameters( camera: CameraDescription, upsampling: int, From 7dbb1002e562b6e1b9ad87f20dd8a6a2e50222c3 Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Tue, 23 Apr 2024 15:17:07 +0200 Subject: [PATCH 02/20] Fixed a mistace in the VarianceExtractor docustring --- src/ctapipe/image/extractor.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/ctapipe/image/extractor.py b/src/ctapipe/image/extractor.py index 82247c854a3..b21e15229bf 100644 --- a/src/ctapipe/image/extractor.py +++ b/src/ctapipe/image/extractor.py @@ -1302,8 +1302,7 @@ class VarianceExtractor(ImageExtractor): """ Extractor that calculates - the variance of each waveform and - corrects for the relative gain. + the variance of each waveform """ def __call__( From 965c3704ecf5756f557f6869536ef5cba07d7b33 Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Thu, 25 Apr 2024 09:02:38 +0200 Subject: [PATCH 03/20] I changed the time information on the VarianceExtractor to None --- src/ctapipe/image/extractor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ctapipe/image/extractor.py b/src/ctapipe/image/extractor.py index b21e15229bf..1a5919d3ed6 100644 --- a/src/ctapipe/image/extractor.py +++ b/src/ctapipe/image/extractor.py @@ -1309,7 +1309,7 @@ def __call__( self, waveforms, tel_id ) -> DL1CameraContainer: variance = np.nanvar(waveforms,axis=2) - return DL1CameraContainer(image=variance, peak_time=0, is_valid=True) + return DL1CameraContainer(image=variance, peak_time=None, is_valid=True) def deconvolution_parameters( From eb02176b2c02ad2ad2d27c8e1f160be5a9ab45df Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Thu, 25 Apr 2024 09:39:03 +0200 Subject: [PATCH 04/20] I updated the docustring of VarianceExtractor --- src/ctapipe/image/extractor.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ctapipe/image/extractor.py b/src/ctapipe/image/extractor.py index 1a5919d3ed6..cc1a2450711 100644 --- a/src/ctapipe/image/extractor.py +++ b/src/ctapipe/image/extractor.py @@ -1302,7 +1302,8 @@ class VarianceExtractor(ImageExtractor): """ Extractor that calculates - the variance of each waveform + the variance of each sample waveform + """ def __call__( From bb634f1685980bae1b227abe50eb88a50eabc4e1 Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Fri, 26 Apr 2024 14:19:45 +0200 Subject: [PATCH 05/20] I added the custom container for the VarianceExtractor --- src/ctapipe/containers.py | 33 +++++++++++++++++++++++++++++++++ src/ctapipe/image/extractor.py | 10 +++++++--- 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/src/ctapipe/containers.py b/src/ctapipe/containers.py index 9a5b39f4da8..2e90edbb452 100644 --- a/src/ctapipe/containers.py +++ b/src/ctapipe/containers.py @@ -19,6 +19,7 @@ "DL0Container", "DL1CameraCalibrationContainer", "DL1CameraContainer", + "DL1PedestalVarianceContainer", "DL1Container", "DL2Container", "EventCalibrationContainer", @@ -166,6 +167,16 @@ class EventType(enum.Enum): UNKNOWN = 255 +class VarianceType(enum.Enum): + """Enum of variance types used for the DL1PedestalVarianceContainer + """ + + #Simple variance of waveform + SIMPLE=0 + #Variance of intgrated samples of a waveform + SAMPLE=1 + + class PixelStatus(enum.IntFlag): """ Pixel status information @@ -516,6 +527,28 @@ class DL1CameraContainer(Container): ) +class DL1PedestalVarianceContainer(Container): + """ + Storage of output of camera variance image e.g. + the variance of each pixel composed as an image. + """ + + image = Field( + None, + "Numpy array of camera variance image" + "Shape: (n_pixel) if n_channels is 1 or data is gain selected" + "else: (n_channels, n_pixel)", + ) + + VarMethod = Field( + VarianceType.SIMPLE, + "Method by which the variance was calculated" + "This can either be a plain variance" + "or a variance of integrated samples", + type=VarianceType, + ) + + class DL1Container(Container): """DL1 Calibrated Camera Images and associated data""" diff --git a/src/ctapipe/image/extractor.py b/src/ctapipe/image/extractor.py index cc1a2450711..598046f1df6 100644 --- a/src/ctapipe/image/extractor.py +++ b/src/ctapipe/image/extractor.py @@ -34,7 +34,11 @@ from scipy.ndimage import convolve1d from traitlets import Bool, Int -from ctapipe.containers import DL1CameraContainer +from ctapipe.containers import ( + DL1CameraContainer, + DL1PedestalVarianceContainer, + VarianceType, +) from ctapipe.core import TelescopeComponent from ctapipe.core.traits import ( BoolTelescopeParameter, @@ -1308,9 +1312,9 @@ class VarianceExtractor(ImageExtractor): def __call__( self, waveforms, tel_id - ) -> DL1CameraContainer: + ) -> DL1PedestalVarianceContainer: variance = np.nanvar(waveforms,axis=2) - return DL1CameraContainer(image=variance, peak_time=None, is_valid=True) + return DL1CameraContainer(image=variance,method=VarianceType.SIMPLE) def deconvolution_parameters( From f77237a3e9e7d4052c5d7c8a7c55799e883cc17b Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Fri, 10 May 2024 10:57:15 +0200 Subject: [PATCH 06/20] Added needed fields for the startracker to the DL1VarianeImageContainer --- src/ctapipe/containers.py | 11 +++++++++++ src/ctapipe/image/extractor.py | 4 ++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/ctapipe/containers.py b/src/ctapipe/containers.py index 2e90edbb452..cf528bfacd8 100644 --- a/src/ctapipe/containers.py +++ b/src/ctapipe/containers.py @@ -540,6 +540,17 @@ class DL1PedestalVarianceContainer(Container): "else: (n_channels, n_pixel)", ) + trigger_time = Field( + None, + "Trigger time for this image" + "Will be needed by the startracker code later to determine ", + ) + + pointing = Field(default_factory=TelescopePointingContainer, + description="Telescope pointing for the startracker code", + ) + + VarMethod = Field( VarianceType.SIMPLE, "Method by which the variance was calculated" diff --git a/src/ctapipe/image/extractor.py b/src/ctapipe/image/extractor.py index 598046f1df6..3d7c075ffa3 100644 --- a/src/ctapipe/image/extractor.py +++ b/src/ctapipe/image/extractor.py @@ -1311,10 +1311,10 @@ class VarianceExtractor(ImageExtractor): """ def __call__( - self, waveforms, tel_id + self, waveforms, tel_id, trigger_time, pointing ) -> DL1PedestalVarianceContainer: variance = np.nanvar(waveforms,axis=2) - return DL1CameraContainer(image=variance,method=VarianceType.SIMPLE) + return DL1CameraContainer(image=variance, method=VarianceType.SIMPLE, trigger_time=trigger_time, pointing=pointing) def deconvolution_parameters( From 74427dc234d1a86816df68f7870857d626fce620 Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Thu, 23 May 2024 11:43:37 +0200 Subject: [PATCH 07/20] I fixed the output class for the variance extractor and removed the pointing information --- src/ctapipe/image/extractor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ctapipe/image/extractor.py b/src/ctapipe/image/extractor.py index 3d7c075ffa3..0a9070eef12 100644 --- a/src/ctapipe/image/extractor.py +++ b/src/ctapipe/image/extractor.py @@ -1311,10 +1311,10 @@ class VarianceExtractor(ImageExtractor): """ def __call__( - self, waveforms, tel_id, trigger_time, pointing + self, waveforms, tel_id, trigger_time ) -> DL1PedestalVarianceContainer: variance = np.nanvar(waveforms,axis=2) - return DL1CameraContainer(image=variance, method=VarianceType.SIMPLE, trigger_time=trigger_time, pointing=pointing) + return DL1PedestalVarianceContainer(image=variance, method=VarianceType.SIMPLE, trigger_time=trigger_time) def deconvolution_parameters( From f81c1ec185a7078cc8c0e2d62def2faa6ea5a812 Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Tue, 28 May 2024 09:40:46 +0200 Subject: [PATCH 08/20] Shortened the docustring of the VarianceExtractor --- src/ctapipe/image/extractor.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/ctapipe/image/extractor.py b/src/ctapipe/image/extractor.py index 0a9070eef12..4329a4b492e 100644 --- a/src/ctapipe/image/extractor.py +++ b/src/ctapipe/image/extractor.py @@ -1305,8 +1305,7 @@ def __call__( class VarianceExtractor(ImageExtractor): """ - Extractor that calculates - the variance of each sample waveform + Calculates the variance of each sample waveform """ From fbdd0e3863ebea4ad273f801573b0902c43d2480 Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Mon, 3 Jun 2024 09:08:48 +0200 Subject: [PATCH 09/20] Fixed the formatting issues --- src/ctapipe/containers.py | 31 +++++++++++------------ src/ctapipe/image/extractor.py | 13 ++++++---- src/ctapipe/image/tests/test_extractor.py | 13 +++++++++- 3 files changed, 35 insertions(+), 22 deletions(-) diff --git a/src/ctapipe/containers.py b/src/ctapipe/containers.py index cf528bfacd8..739e93092f7 100644 --- a/src/ctapipe/containers.py +++ b/src/ctapipe/containers.py @@ -168,13 +168,12 @@ class EventType(enum.Enum): class VarianceType(enum.Enum): - """Enum of variance types used for the DL1PedestalVarianceContainer - """ + """Enum of variance types used for the DL1PedestalVarianceContainer""" - #Simple variance of waveform - SIMPLE=0 - #Variance of intgrated samples of a waveform - SAMPLE=1 + # Simple variance of waveform + SIMPLE = 0 + # Variance of integrated samples of a waveform + SAMPLE = 1 class PixelStatus(enum.IntFlag): @@ -529,7 +528,7 @@ class DL1CameraContainer(Container): class DL1PedestalVarianceContainer(Container): """ - Storage of output of camera variance image e.g. + Storage of output of camera variance image e.g. the variance of each pixel composed as an image. """ @@ -539,18 +538,10 @@ class DL1PedestalVarianceContainer(Container): "Shape: (n_pixel) if n_channels is 1 or data is gain selected" "else: (n_channels, n_pixel)", ) - trigger_time = Field( None, - "Trigger time for this image" - "Will be needed by the startracker code later to determine ", + "Trigger time for this variance image" "Value is a float", ) - - pointing = Field(default_factory=TelescopePointingContainer, - description="Telescope pointing for the startracker code", - ) - - VarMethod = Field( VarianceType.SIMPLE, "Method by which the variance was calculated" @@ -558,6 +549,14 @@ class DL1PedestalVarianceContainer(Container): "or a variance of integrated samples", type=VarianceType, ) + is_valid = Field( + False, + ( + "True if image extraction succeeded, False if failed " + "or in the case of TwoPass methods, that the first " + "pass only was returned." + ), + ) class DL1Container(Container): diff --git a/src/ctapipe/image/extractor.py b/src/ctapipe/image/extractor.py index 4329a4b492e..c94ef4ed68d 100644 --- a/src/ctapipe/image/extractor.py +++ b/src/ctapipe/image/extractor.py @@ -1309,11 +1309,14 @@ class VarianceExtractor(ImageExtractor): """ - def __call__( - self, waveforms, tel_id, trigger_time - ) -> DL1PedestalVarianceContainer: - variance = np.nanvar(waveforms,axis=2) - return DL1PedestalVarianceContainer(image=variance, method=VarianceType.SIMPLE, trigger_time=trigger_time) + def __call__(self, waveforms, tel_id, trigger_time) -> DL1PedestalVarianceContainer: + variance = np.nanvar(waveforms, dtype="float32", axis=2) + return DL1PedestalVarianceContainer( + image=variance, + VarMethod=VarianceType.SIMPLE, + is_valid=True, + trigger_time=np.float32(trigger_time), + ) def deconvolution_parameters( diff --git a/src/ctapipe/image/tests/test_extractor.py b/src/ctapipe/image/tests/test_extractor.py index 067a07801aa..2ba6441fc19 100644 --- a/src/ctapipe/image/tests/test_extractor.py +++ b/src/ctapipe/image/tests/test_extractor.py @@ -19,6 +19,7 @@ NeighborPeakWindowSum, SlidingWindowMaxSum, TwoPassWindowSum, + VarianceExtractor, __filtfilt_fast, adaptive_centroid, deconvolve, @@ -408,6 +409,9 @@ def test_extractors(Extractor, toymodels, request): extractor(waveforms, tel_id, selected_gain_channel, broken_pixels) return + if Extractor is VarianceExtractor: + return + dl1 = extractor(waveforms, tel_id, selected_gain_channel, broken_pixels) assert dl1.is_valid if dl1.image.ndim == 1: @@ -423,7 +427,7 @@ def test_extractors(Extractor, toymodels, request): @pytest.mark.parametrize("Extractor", extractors) def test_integration_correction_off(Extractor, toymodels, request): # full waveform extractor does not have an integration correction - if Extractor is FullWaveformSum: + if Extractor is FullWaveformSum or Extractor is VarianceExtractor: return ( @@ -713,7 +717,14 @@ def test_dtype(Extractor, subarray): extractor = Extractor(subarray=subarray) n_channels, n_pixels, _ = waveforms.shape broken_pixels = np.zeros((n_channels, n_pixels), dtype=bool) + if Extractor is VarianceExtractor: + var = extractor(waveforms, tel_id, 0.0) + assert var.image.dtype == np.float32 + assert var.trigger_time.dtype == np.float32 + return + dl1 = extractor(waveforms, tel_id, selected_gain_channel, broken_pixels) + assert dl1.image.dtype == np.float32 assert dl1.peak_time.dtype == np.float32 From e97eb69673b3d87165659412e4c34a06dfc1aba4 Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Mon, 3 Jun 2024 10:20:22 +0200 Subject: [PATCH 10/20] I added the changelog --- docs/changes/2543.feature.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 docs/changes/2543.feature.rst diff --git a/docs/changes/2543.feature.rst b/docs/changes/2543.feature.rst new file mode 100644 index 00000000000..b40f73dc573 --- /dev/null +++ b/docs/changes/2543.feature.rst @@ -0,0 +1,2 @@ +A new ImageExtractor called ``VarianceExtractor`` was added with a corresponding container +called ``DL1PedestalVarianceContainer``. These will be used for the startracker code later. From 2598425c091d9e029db579f14f8cfb7acd4bda2f Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Mon, 3 Jun 2024 13:51:14 +0200 Subject: [PATCH 11/20] I changed some variable names to be more compatible with the statsextractors --- src/ctapipe/containers.py | 2 +- src/ctapipe/image/extractor.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ctapipe/containers.py b/src/ctapipe/containers.py index 739e93092f7..3fc234edccf 100644 --- a/src/ctapipe/containers.py +++ b/src/ctapipe/containers.py @@ -538,7 +538,7 @@ class DL1PedestalVarianceContainer(Container): "Shape: (n_pixel) if n_channels is 1 or data is gain selected" "else: (n_channels, n_pixel)", ) - trigger_time = Field( + time = Field( None, "Trigger time for this variance image" "Value is a float", ) diff --git a/src/ctapipe/image/extractor.py b/src/ctapipe/image/extractor.py index c94ef4ed68d..7d5649b1322 100644 --- a/src/ctapipe/image/extractor.py +++ b/src/ctapipe/image/extractor.py @@ -1315,7 +1315,7 @@ def __call__(self, waveforms, tel_id, trigger_time) -> DL1PedestalVarianceContai image=variance, VarMethod=VarianceType.SIMPLE, is_valid=True, - trigger_time=np.float32(trigger_time), + time=np.float32(trigger_time), ) From 781210cdf9b5a19af67fe0ac1f8828c4870225ce Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Wed, 5 Jun 2024 09:55:24 +0200 Subject: [PATCH 12/20] I needed to increase precision on the time field in the variance image container --- src/ctapipe/image/extractor.py | 2 +- src/ctapipe/image/tests/test_extractor.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/ctapipe/image/extractor.py b/src/ctapipe/image/extractor.py index 7d5649b1322..15981a39169 100644 --- a/src/ctapipe/image/extractor.py +++ b/src/ctapipe/image/extractor.py @@ -1315,7 +1315,7 @@ def __call__(self, waveforms, tel_id, trigger_time) -> DL1PedestalVarianceContai image=variance, VarMethod=VarianceType.SIMPLE, is_valid=True, - time=np.float32(trigger_time), + time=trigger_time, ) diff --git a/src/ctapipe/image/tests/test_extractor.py b/src/ctapipe/image/tests/test_extractor.py index 2ba6441fc19..73cf0e3f8f8 100644 --- a/src/ctapipe/image/tests/test_extractor.py +++ b/src/ctapipe/image/tests/test_extractor.py @@ -720,7 +720,6 @@ def test_dtype(Extractor, subarray): if Extractor is VarianceExtractor: var = extractor(waveforms, tel_id, 0.0) assert var.image.dtype == np.float32 - assert var.trigger_time.dtype == np.float32 return dl1 = extractor(waveforms, tel_id, selected_gain_channel, broken_pixels) From b79395357e9d9897913070218c96d080e4820f20 Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Tue, 16 Jul 2024 15:41:16 +0200 Subject: [PATCH 13/20] I implemented Mysha and Max comments --- src/ctapipe/containers.py | 52 ++-------------------------------- src/ctapipe/image/extractor.py | 15 +++++----- 2 files changed, 10 insertions(+), 57 deletions(-) diff --git a/src/ctapipe/containers.py b/src/ctapipe/containers.py index 3fc234edccf..d5f8566a74c 100644 --- a/src/ctapipe/containers.py +++ b/src/ctapipe/containers.py @@ -19,7 +19,6 @@ "DL0Container", "DL1CameraCalibrationContainer", "DL1CameraContainer", - "DL1PedestalVarianceContainer", "DL1Container", "DL2Container", "EventCalibrationContainer", @@ -168,12 +167,12 @@ class EventType(enum.Enum): class VarianceType(enum.Enum): - """Enum of variance types used for the DL1PedestalVarianceContainer""" + """Enum of variance types used for the VarianceContainer""" # Simple variance of waveform - SIMPLE = 0 + WAVEFORM = 0 # Variance of integrated samples of a waveform - SAMPLE = 1 + INTEGRATED = 1 class PixelStatus(enum.IntFlag): @@ -487,17 +486,6 @@ class ImageParametersContainer(Container): class DL1CameraContainer(Container): - """ - Storage of output of camera calibration e.g the final calibrated - image in intensity units and the pulse time. - """ - - image = Field( - None, - "Numpy array of camera image, after waveform extraction." - "Shape: (n_pixel) if n_channels is 1 or data is gain selected" - "else: (n_channels, n_pixel)", - ) peak_time = Field( None, "Numpy array containing position of the peak of the pulse as determined by " @@ -520,45 +508,11 @@ class DL1CameraContainer(Container): "pass only was returned." ), ) - parameters = Field( None, description="Image parameters", type=ImageParametersContainer ) -class DL1PedestalVarianceContainer(Container): - """ - Storage of output of camera variance image e.g. - the variance of each pixel composed as an image. - """ - - image = Field( - None, - "Numpy array of camera variance image" - "Shape: (n_pixel) if n_channels is 1 or data is gain selected" - "else: (n_channels, n_pixel)", - ) - time = Field( - None, - "Trigger time for this variance image" "Value is a float", - ) - VarMethod = Field( - VarianceType.SIMPLE, - "Method by which the variance was calculated" - "This can either be a plain variance" - "or a variance of integrated samples", - type=VarianceType, - ) - is_valid = Field( - False, - ( - "True if image extraction succeeded, False if failed " - "or in the case of TwoPass methods, that the first " - "pass only was returned." - ), - ) - - class DL1Container(Container): """DL1 Calibrated Camera Images and associated data""" diff --git a/src/ctapipe/image/extractor.py b/src/ctapipe/image/extractor.py index 15981a39169..3cf9bdf3b38 100644 --- a/src/ctapipe/image/extractor.py +++ b/src/ctapipe/image/extractor.py @@ -36,7 +36,6 @@ from ctapipe.containers import ( DL1CameraContainer, - DL1PedestalVarianceContainer, VarianceType, ) from ctapipe.core import TelescopeComponent @@ -1309,14 +1308,14 @@ class VarianceExtractor(ImageExtractor): """ - def __call__(self, waveforms, tel_id, trigger_time) -> DL1PedestalVarianceContainer: - variance = np.nanvar(waveforms, dtype="float32", axis=2) - return DL1PedestalVarianceContainer( - image=variance, - VarMethod=VarianceType.SIMPLE, - is_valid=True, - time=trigger_time, + def __call__( + self, waveforms, tel_id, selected_gain_channel, broken_pixels + ) -> DL1CameraContainer: + container = DL1CameraContainer( + image=np.nanvar(waveforms, dtype="float32", axis=2), ) + container.meta["ExtractionMethod"] = str(VarianceType.WAVEFORM) + return container def deconvolution_parameters( From e0572857526c6d22b2938b72ee8dde27f4b62376 Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Tue, 16 Jul 2024 15:49:32 +0200 Subject: [PATCH 14/20] Re-inserted code --- src/ctapipe/containers.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/ctapipe/containers.py b/src/ctapipe/containers.py index d5f8566a74c..6dbf74049d0 100644 --- a/src/ctapipe/containers.py +++ b/src/ctapipe/containers.py @@ -486,6 +486,17 @@ class ImageParametersContainer(Container): class DL1CameraContainer(Container): + """ + Storage of output of camera calibration e.g the final calibrated + image in intensity units and the pulse time. + """ + + image = Field( + None, + "Numpy array of camera image, after waveform extraction." + "Shape: (n_pixel) if n_channels is 1 or data is gain selected" + "else: (n_channels, n_pixel)", + ) peak_time = Field( None, "Numpy array containing position of the peak of the pulse as determined by " From feba83b77de30b0e642c0a175e659dad47357673 Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Tue, 16 Jul 2024 15:52:43 +0200 Subject: [PATCH 15/20] Updated the notes --- docs/changes/2543.feature.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/changes/2543.feature.rst b/docs/changes/2543.feature.rst index b40f73dc573..8154498542d 100644 --- a/docs/changes/2543.feature.rst +++ b/docs/changes/2543.feature.rst @@ -1,2 +1,2 @@ -A new ImageExtractor called ``VarianceExtractor`` was added with a corresponding container -called ``DL1PedestalVarianceContainer``. These will be used for the startracker code later. +A new ImageExtractor called ``VarianceExtractor`` was added +An Enum class was added to containers.py that is used in the metadata of the VarianceExtractor output From e5fc0d3f6aee268cfc7229fa28dd18bcd9fb8abc Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Tue, 16 Jul 2024 16:35:20 +0200 Subject: [PATCH 16/20] I fixed an issue with the tests --- src/ctapipe/image/tests/test_extractor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ctapipe/image/tests/test_extractor.py b/src/ctapipe/image/tests/test_extractor.py index 73cf0e3f8f8..610dc12f7e2 100644 --- a/src/ctapipe/image/tests/test_extractor.py +++ b/src/ctapipe/image/tests/test_extractor.py @@ -718,7 +718,7 @@ def test_dtype(Extractor, subarray): n_channels, n_pixels, _ = waveforms.shape broken_pixels = np.zeros((n_channels, n_pixels), dtype=bool) if Extractor is VarianceExtractor: - var = extractor(waveforms, tel_id, 0.0) + var = extractor(waveforms, tel_id, None, None) assert var.image.dtype == np.float32 return From 6f0f347dacb627047a0582ce48c9cd86c07238dd Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Thu, 18 Jul 2024 09:37:45 +0200 Subject: [PATCH 17/20] Simplified some test code based on comments --- src/ctapipe/image/tests/test_extractor.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/ctapipe/image/tests/test_extractor.py b/src/ctapipe/image/tests/test_extractor.py index 610dc12f7e2..d6156466e17 100644 --- a/src/ctapipe/image/tests/test_extractor.py +++ b/src/ctapipe/image/tests/test_extractor.py @@ -427,7 +427,7 @@ def test_extractors(Extractor, toymodels, request): @pytest.mark.parametrize("Extractor", extractors) def test_integration_correction_off(Extractor, toymodels, request): # full waveform extractor does not have an integration correction - if Extractor is FullWaveformSum or Extractor is VarianceExtractor: + if Extractor in (FullWaveformSum, VarianceExtractor): return ( @@ -717,15 +717,12 @@ def test_dtype(Extractor, subarray): extractor = Extractor(subarray=subarray) n_channels, n_pixels, _ = waveforms.shape broken_pixels = np.zeros((n_channels, n_pixels), dtype=bool) - if Extractor is VarianceExtractor: - var = extractor(waveforms, tel_id, None, None) - assert var.image.dtype == np.float32 - return - dl1 = extractor(waveforms, tel_id, selected_gain_channel, broken_pixels) + if Extractor is not VarianceExtractor: + assert dl1.peak_time.dtype == np.float32 + assert dl1.image.dtype == np.float32 - assert dl1.peak_time.dtype == np.float32 def test_global_peak_window_sum_with_pixel_fraction(subarray): From c5ad34e32afc6a87f70eff80d69b54c2aba4cf41 Mon Sep 17 00:00:00 2001 From: Maximilian Linhoff Date: Fri, 19 Jul 2024 12:05:49 +0200 Subject: [PATCH 18/20] Update docstring --- src/ctapipe/image/extractor.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/ctapipe/image/extractor.py b/src/ctapipe/image/extractor.py index 3cf9bdf3b38..88e7e683e9b 100644 --- a/src/ctapipe/image/extractor.py +++ b/src/ctapipe/image/extractor.py @@ -1302,11 +1302,7 @@ def __call__( class VarianceExtractor(ImageExtractor): - - """ - Calculates the variance of each sample waveform - - """ + """Calculate the variance over samples in each waveform.""" def __call__( self, waveforms, tel_id, selected_gain_channel, broken_pixels From ce9685fbbe7704e1a7455e8b715faabbb9c7816b Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Wed, 31 Jul 2024 13:00:48 +0200 Subject: [PATCH 19/20] Added a test to ensure that an actual variance is calculated --- src/ctapipe/image/tests/test_extractor.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/ctapipe/image/tests/test_extractor.py b/src/ctapipe/image/tests/test_extractor.py index d6156466e17..574db422744 100644 --- a/src/ctapipe/image/tests/test_extractor.py +++ b/src/ctapipe/image/tests/test_extractor.py @@ -282,6 +282,17 @@ def test_extract_around_peak_charge_expected(): assert_equal(charge, n_samples) +def test_variance_extractor(toymodel): + _, subarray, _, _, _, _ = toymodel + # make dummy data with known variance + rng = np.random.default_rng(0) + var_data = rng.normal(2.0, 5.0, size=(5000, 2, 1855)) + extractor = ImageExtractor.from_name("VarianceExtractor", subarray=subarray) + + variance = extractor(var_data, 0, None, None).image + assert np.isclose(variance, np.var(var_data, axis=2)).all() + + @pytest.mark.parametrize("toymodels", camera_toymodels) def test_neighbor_average_peakpos(toymodels, request): waveforms, subarray, tel_id, _, _, _ = request.getfixturevalue(toymodels) From 721b67b841e733573f48516d6dce9fd196dd8a25 Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Wed, 31 Jul 2024 15:36:11 +0200 Subject: [PATCH 20/20] Improved tests based on comments --- src/ctapipe/image/tests/test_extractor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ctapipe/image/tests/test_extractor.py b/src/ctapipe/image/tests/test_extractor.py index 574db422744..2b7fd1a5f5e 100644 --- a/src/ctapipe/image/tests/test_extractor.py +++ b/src/ctapipe/image/tests/test_extractor.py @@ -286,11 +286,11 @@ def test_variance_extractor(toymodel): _, subarray, _, _, _, _ = toymodel # make dummy data with known variance rng = np.random.default_rng(0) - var_data = rng.normal(2.0, 5.0, size=(5000, 2, 1855)) + var_data = rng.normal(2.0, 5.0, size=(2, 1855, 5000)) extractor = ImageExtractor.from_name("VarianceExtractor", subarray=subarray) variance = extractor(var_data, 0, None, None).image - assert np.isclose(variance, np.var(var_data, axis=2)).all() + np.testing.assert_allclose(variance, np.var(var_data, axis=2), rtol=1e-3) @pytest.mark.parametrize("toymodels", camera_toymodels)