Fixing the random seed (#38)

* Fix random seed in S1PhotonHits

* Add random seed logic to more plugins

* replace np.random.rand with np.random.random

* Clip negative fields

* Fixing the nestpy random seed

* BugFix

* Move logging level definition into setup functions

* rename 'fixed_seed' to 'deterministic_seed'

.gitignore

@@ -4,3 +4,4 @@ fuse.egg-info/*
 .eggs/*
 build/*
 .DS_Store
+.vscode/*

fuse/context.py

@@ -153,7 +153,6 @@ def full_chain_context(out_dir, config):
         "s2_time_model": config["s2_time_model"],
         "singlet_fraction_gas": config["singlet_fraction_gas"],
         "s2_luminescence_model": config["s2_luminescence_model"],
-        "s2_luminescence_model": config["s2_luminescence_model"],
         "tpc_radius": config["tpc_radius"],
         "diffusion_constant_transverse": config["diffusion_constant_transverse"],
         "s2_aft_skewness": config["s2_aft_skewness"],

fuse/plugins/detector_physics/csv_input.py

@@ -1,7 +1,5 @@
 import strax
 import straxen
-import os
-import numba
 import logging
 
 import pandas as pd
@@ -13,16 +11,13 @@
 
 logging.basicConfig(handlers=[logging.StreamHandler()])
 log = logging.getLogger('fuse.detector_physics.csv_input')
-log.setLevel('WARNING')
 
 @export
 class ChunkCsvInput(strax.Plugin):
     """
     Plugin which reads a CSV file containing instructions for the detector physics simulation
     and returns the data in chunks
     """
-
-
     __version__ = "0.0.0"
 
     depends_on = tuple()
@@ -75,15 +70,31 @@ class ChunkCsvInput(strax.Plugin):
         help='n_interactions_per_chunk',
     )
 
+    deterministic_seed = straxen.URLConfig(
+        default=True, type=bool,
+        help='Set the random seed from lineage and run_id, or pull the seed from the OS.',
+    )
+
     def setup(self):
 
         if self.debug:
             log.setLevel('DEBUG')
             log.debug("Running ChunkCsvInput in debug mode")
+        else: 
+            log.setLevel('WARNING')
 
+        if self.deterministic_seed:
+            hash_string = strax.deterministic_hash((self.run_id, self.lineage))
+            seed = int(hash_string.encode().hex(), 16)
+            self.rng = np.random.default_rng(seed = seed)
+            log.debug(f"Generating random numbers from seed {seed}")
+        else: 
+            self.rng = np.random.default_rng()
+            log.debug(f"Generating random numbers with seed pulled from OS")
 
         self.file_reader = csv_file_loader(
             input_file = self.input_file,
+            random_number_generator = self.rng,
             event_rate = self.source_rate,
             separation_scale = self.separation_scale,
             n_interactions_per_chunk = self.n_interactions_per_chunk,
@@ -131,6 +142,7 @@ class csv_file_loader():
 
     def __init__(self,
                  input_file,
+                 random_number_generator,
                  event_rate,
                  separation_scale,
                  n_interactions_per_chunk,
@@ -141,6 +153,7 @@ def __init__(self,
                  ):
 
         self.input_file = input_file
+        self.rng = random_number_generator
         self.event_rate = event_rate/ 1e9 #Conversion to ns 
         self.separation_scale = separation_scale
         self.n_interactions_per_chunk = n_interactions_per_chunk
@@ -174,7 +187,7 @@ def output_chunk(self):
         instructions, n_simulated_events = self.__load_csv_file()
 
         #Assign event times and dynamic chunking
-        event_times = np.random.uniform(low = 0,
+        event_times = self.rng.uniform(low = 0,
                                         high = n_simulated_events/self.event_rate,
                                         size = n_simulated_events
                                         ).astype(np.int64)

fuse/plugins/detector_physics/electron_drift.py

@@ -1,14 +1,12 @@
 import strax
 import numpy as np
 import straxen
-import os
 import logging
 
 export, __all__ = strax.exporter()
 
 logging.basicConfig(handlers=[logging.StreamHandler()])
 log = logging.getLogger('fuse.detector_physics.electron_drift')
-log.setLevel('WARNING')
 
 @export
 class ElectronDrift(strax.Plugin):
@@ -89,6 +87,8 @@ def setup(self):
         if self.debug:
             log.setLevel('DEBUG')
             log.debug("Running ElectronDrift in debug mode")
+        else: 
+            log.setLevel('WARNING')
 
         #Can i do this scaling in the url config?
         if self.field_distortion_model == "inverse_fdc":

fuse/plugins/detector_physics/electron_extraction.py

@@ -8,7 +8,6 @@
 
 logging.basicConfig(handlers=[logging.StreamHandler()])
 log = logging.getLogger('fuse.detector_physics.electron_extraction')
-log.setLevel('WARNING')
 
 @export
 class ElectronExtraction(strax.Plugin):
@@ -94,13 +93,28 @@ class ElectronExtraction(strax.Plugin):
         help='s2_pattern_map',
     )
 
+    deterministic_seed = straxen.URLConfig(
+        default=True, type=bool,
+        help='Set the random seed from lineage and run_id, or pull the seed from the OS.',
+    )
 
     def setup(self):
 
         if self.debug:
             log.setLevel('DEBUG')
             log.debug("Running ElectronExtraction in debug mode")
+        else: 
+            log.setLevel('WARNING')
 
+        if self.deterministic_seed:
+            hash_string = strax.deterministic_hash((self.run_id, self.lineage))
+            seed = int(hash_string.encode().hex(), 16)
+            self.rng = np.random.default_rng(seed = seed)
+            log.debug(f"Generating random numbers from seed {seed}")
+        else: 
+            self.rng = np.random.default_rng()
+            log.debug(f"Generating random numbers with seed pulled from OS")
+
         self.pmt_mask = np.array(self.gains) > 0  # Converted from to pe (from cmt by default)
 
         #Is this else case ever used? if no -> remove
@@ -129,7 +143,6 @@ def compute(self, interactions_in_roi):
 
         xy_int = np.array([x, y]).T # maps are in R_true, so orginal position should be here
 
-
         if self.ext_eff_from_map:
             # Extraction efficiency is g2(x,y)/SE_gain(x,y)
             rel_s2_cor=self.s2_correction_map(xy_int)
@@ -146,7 +159,7 @@ def compute(self, interactions_in_roi):
         else:
             cy = self.electron_extraction_yield
 
-        n_electron = np.random.binomial(n=interactions_in_roi[mask]["n_electron_interface"], p=cy)
+        n_electron = self.rng.binomial(n=interactions_in_roi[mask]["n_electron_interface"], p=cy)
 
         result = np.zeros(len(interactions_in_roi), dtype=self.dtype)
         result["n_electron_extracted"][mask] = n_electron

fuse/plugins/detector_physics/electron_timing.py

@@ -1,14 +1,12 @@
 import strax
 import numpy as np
 import straxen
-import os
 import logging
 
 export, __all__ = strax.exporter()
 
 logging.basicConfig(handlers=[logging.StreamHandler()])
 log = logging.getLogger('fuse.detector_physics.electron_timing')
-log.setLevel('WARNING')
 
 @export
 class ElectronTiming(strax.Plugin):
@@ -40,13 +38,29 @@ class ElectronTiming(strax.Plugin):
         type=(int, float),
         help='electron_trapping_time',
     )
+
+    deterministic_seed = straxen.URLConfig(
+        default=True, type=bool,
+        help='Set the random seed from lineage and run_id, or pull the seed from the OS.',
+    )
 
     def setup(self):
 
         if self.debug:
             log.setLevel('DEBUG')
             log.debug("Running ElectronTiming in debug mode")
-
+        else: 
+            log.setLevel('WARNING')
+
+        if self.deterministic_seed:
+            hash_string = strax.deterministic_hash((self.run_id, self.lineage))
+            seed = int(hash_string.encode().hex(), 16)
+            self.rng = np.random.default_rng(seed = seed)
+            log.debug(f"Generating random numbers from seed {seed}")
+        else: 
+            self.rng = np.random.default_rng()
+            log.debug(f"Generating random numbers with seed pulled from OS")
+
     def compute(self, interactions_in_roi):
 
         #Just apply this to clusters with photons
@@ -85,7 +99,7 @@ def electron_timing(self,
         drift_time_mean_r = np.repeat(drift_time_mean, n_electron.astype(np.int64))
         drift_time_spread_r = np.repeat(drift_time_spread, n_electron.astype(np.int64))
 
-        timing = np.random.exponential(self.electron_trapping_time, size = time_r.shape[0])
-        timing += np.random.normal(drift_time_mean_r, drift_time_spread_r, size = time_r.shape[0])
+        timing = self.rng.exponential(self.electron_trapping_time, size = time_r.shape[0])
+        timing += self.rng.normal(drift_time_mean_r, drift_time_spread_r, size = time_r.shape[0])
 
         return time_r + timing.astype(np.int64)

fuse/plugins/detector_physics/s1_photon_hits.py

@@ -8,7 +8,6 @@
 
 logging.basicConfig(handlers=[logging.StreamHandler()])
 log = logging.getLogger('fuse.detector_physics.s1_photon_hits')
-log.setLevel('WARNING')
 
 @export
 class S1PhotonHits(strax.Plugin):
@@ -46,12 +45,28 @@ class S1PhotonHits(strax.Plugin):
         type=(int, float),
         help='Some placeholder for s1_detection_efficiency',
     )
+
+    deterministic_seed = straxen.URLConfig(
+        default=True, type=bool,
+        help='Set the random seed from lineage and run_id, or pull the seed from the OS.',
+    )
 
     def setup(self):
-
+        
         if self.debug:
             log.setLevel('DEBUG')
             log.debug("Running S1PhotonHits in debug mode")
+        else: 
+            log.setLevel('WARNING')
+
+        if self.deterministic_seed:
+            hash_string = strax.deterministic_hash((self.run_id, self.lineage))
+            seed = int(hash_string.encode().hex(), 16)
+            self.rng = np.random.default_rng(seed = seed)
+            log.debug(f"Generating random numbers from seed {seed}")
+        else: 
+            self.rng = np.random.default_rng()
+            log.debug(f"Generating random numbers with seed pulled from OS")
 
     def compute(self, interactions_in_roi):
 
@@ -99,6 +114,6 @@ def get_n_photons(self, n_photons, positions):
         ly /= 1 + self.p_double_pe_emision
         ly *= self.s1_detection_efficiency
 
-        n_photon_hits = np.random.binomial(n=n_photons, p=ly)
+        n_photon_hits = self.rng.binomial(n=n_photons, p=ly)
 
         return n_photon_hits

fuse/plugins/detector_physics/s1_photon_propagation.py

@@ -13,7 +13,10 @@
 
 logging.basicConfig(handlers=[logging.StreamHandler()])
 log = logging.getLogger('fuse.detector_physics.S1_Signal')
-log.setLevel('WARNING')
+
+#Initialize the nestpy random generator
+#The seed will be set in the setup function
+nest_rng = nestpy.RandomGen.rndm()
 
 @export
 class S1PhotonPropagation(strax.Plugin):
@@ -121,12 +124,31 @@ class S1PhotonPropagation(strax.Plugin):
         cache=True,
         help='photon_area_distribution',
     )
+
+    deterministic_seed = straxen.URLConfig(
+        default=True, type=bool,
+        help='Set the random seed from lineage and run_id, or pull the seed from the OS.',
+    )
 
     def setup(self):
 
         if self.debug:
             log.setLevel('DEBUG')
             log.debug("Running S1PhotonPropagation in debug mode")
+        else: 
+            log.setLevel('WARNING')
+
+        if self.deterministic_seed:
+            hash_string = strax.deterministic_hash((self.run_id, self.lineage))
+            seed = int(hash_string.encode().hex(), 16)
+            #Dont know but nestpy seems to have a problem with large seeds
+            self.short_seed = int(repr(seed)[-8:])
+            nest_rng.set_seed(self.short_seed)
+            self.rng = np.random.default_rng(seed = seed)
+            log.debug(f"Generating random numbers from seed {seed}")
+            log.debug(f"Generating nestpy random numbers from seed {self.short_seed}")
+        else: 
+            log.debug(f"Generating random numbers with seed pulled from OS")
 
         self.turned_off_pmts = np.arange(len(self.gains))[np.array(self.gains) == 0]
 
@@ -183,23 +205,23 @@ def compute(self, interactions_in_roi):
         #Do i want to save both -> timings with and without pmt transition time spread?
         # Correct for PMT Transition Time Spread (skip for pmt after-pulses)
         # note that PMT datasheet provides FWHM TTS, so sigma = TTS/(2*sqrt(2*log(2)))=TTS/2.35482
-        _photon_timings += np.random.normal(self.pmt_transit_time_mean,
+        _photon_timings += self.rng.normal(self.pmt_transit_time_mean,
                                             self.pmt_transit_time_spread / 2.35482,
                                             len(_photon_timings)).astype(np.int64)
 
         #Why is this done here and additionally in the get_n_photons function of S1PhotonHits??
-        _photon_is_dpe = np.random.binomial(n=1,
+        _photon_is_dpe = self.rng.binomial(n=1,
                                             p=self.p_double_pe_emision,
                                             size=len(_photon_timings)).astype(np.bool_)
 
 
         _photon_gains = self.gains[_photon_channels] \
-            * loop_uniform_to_pe_arr(np.random.random(len(_photon_channels)), _photon_channels, self.__uniform_to_pe_arr)
+            * loop_uniform_to_pe_arr(self.rng.random(len(_photon_channels)), _photon_channels, self.__uniform_to_pe_arr)
 
         # Add some double photoelectron emission by adding another sampled gain
         n_double_pe = _photon_is_dpe.sum()
         _photon_gains[_photon_is_dpe] += self.gains[_photon_channels[_photon_is_dpe]] \
-            * loop_uniform_to_pe_arr(np.random.random(n_double_pe), _photon_channels[_photon_is_dpe], self.__uniform_to_pe_arr) 
+            * loop_uniform_to_pe_arr(self.rng.random(n_double_pe), _photon_channels[_photon_is_dpe], self.__uniform_to_pe_arr) 
 
 
         result = np.zeros(_photon_channels.shape[0], dtype = self.dtype)
@@ -227,7 +249,7 @@ def photon_channels(self, positions, n_photon_hits):
         _photon_channels = np.array([]).astype(np.int64)
         for ppc, n in zip(p_per_channel, n_photon_hits):
             _photon_channels = np.append(_photon_channels,
-                                         np.random.choice(
+                                         self.rng.choice(
                                              channels,
                                              size=n,
                                              p=ppc / np.sum(ppc),
@@ -272,8 +294,8 @@ def photon_timings(self,
 
         if 'simple' in self.s1_model_type:
             # Simple S1 model enabled: use it for ER and NR.
-            _photon_timings += np.random.exponential(self.s1_decay_time, _n_hits_total).astype(np.int64)
-            _photon_timings += np.random.normal(0, self.s1_decay_spread, _n_hits_total).astype(np.int64)
+            _photon_timings += self.rng.exponential(self.s1_decay_time, _n_hits_total).astype(np.int64)
+            _photon_timings += self.rng.normal(0, self.s1_decay_spread, _n_hits_total).astype(np.int64)
 
         if 'nest' in self.s1_model_type or 'custom' in self.s1_model_type:
             # Pulse model depends on recoil type
@@ -315,7 +337,7 @@ def photon_timings(self,
                     # The first part of the scint_time is from exciton only, see
                     # https://github.com/NESTCollaboration/nestpy/blob/fe3d5d7da5d9b33ac56fbea519e02ef55152bc1d/src/nestpy/NEST.cpp#L164-L179
                     _photon_timings[counts_start: counts_start + counts] += \
-                       np.random.choice(scint_time, counts, replace=False).astype(np.int64)
+                       self.rng.choice(scint_time, counts, replace=False).astype(np.int64)
 
                 counts_start += counts
 
@@ -331,7 +353,7 @@ def optical_propagation(self, channels, z_positions):
         assert len(z_positions) == len(channels), 'Give each photon a z position'
 
         prop_time = np.zeros_like(channels)
-        z_rand = np.array([z_positions, np.random.rand(len(channels))]).T
+        z_rand = np.array([z_positions, self.rng.random(len(channels))]).T
 
         is_top = channels < self.n_top_pmts
         prop_time[is_top] = self.s1_optical_propagation_spline(z_rand[is_top], map_name='top')