From 72ebcbee73ac47c46fb08de66277982c1704cb68 Mon Sep 17 00:00:00 2001 From: Veronica Villa Date: Wed, 12 Feb 2020 14:26:52 -0800 Subject: [PATCH 1/7] Add source classification for the rest of GraceDB uploads --- bin/pycbc_live | 8 +++++++- bin/pycbc_optimize_snr | 5 ++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/bin/pycbc_live b/bin/pycbc_live index 189681f208e..ee89fa2d4e4 100755 --- a/bin/pycbc_live +++ b/bin/pycbc_live @@ -28,6 +28,7 @@ from pycbc.events.ranking import newsnr from pycbc.events.coinc import LiveCoincTimeslideBackgroundEstimator as Coincer from pycbc.events.single import LiveSingle from pycbc.io.live import SingleCoincForGraceDB +from pycbc.io.hdf import recursively_save_dict_contents_to_group import pycbc.waveform.bank from pycbc.vetoes.sgchisq import SingleDetSGChisq from pycbc.waveform.waveform import props @@ -331,6 +332,10 @@ class LiveEventManager(object): hdfp['channel_names/{}'.format(ifo)] = \ args.channel_name[ifo] + recursively_save_dict_contents_to_group(hdfp, + 'mc_area_args/', + self.mc_area_args) + cmd += '--params-file {} '.format(curr_fname) cmd += '--approximant {} '.format(apr) cmd += '--gracedb-server {} '.format(args.gracedb_server) @@ -371,7 +376,8 @@ class LiveEventManager(object): event = SingleCoincForGraceDB([ifo], single, bank=bank, psds=psds, followup_data=fud, low_frequency_cutoff=f_low, - channel_names=args.channel_name) + channel_names=args.channel_name, + mc_area_args=self.mc_area_args) end_time = int(single['foreground/%s/end_time' % ifo]) fname = 'single-%s-%s.xml.gz' % (ifo, end_time) diff --git a/bin/pycbc_optimize_snr b/bin/pycbc_optimize_snr index c5618e7f947..dc34c45832d 100755 --- a/bin/pycbc_optimize_snr +++ b/bin/pycbc_optimize_snr @@ -20,6 +20,7 @@ from pycbc.conversions import ( ) from scipy.optimize import differential_evolution from pycbc.io import live +from pycbc.io.hdf import recursively_load_hdf5_group_to_dict from pycbc.detector import Detector from pycbc.psd import interpolate @@ -319,7 +320,9 @@ for ifo in fp['channel_names'].keys(): kwargs = {'psds': {ifo: followup_data[ifo]['psd'] for ifo in ifos}, 'low_frequency_cutoff': flow, 'followup_data': followup_data, - 'channel_names': channel_names} + 'channel_names': channel_names, + 'mc_area_args': recursively_load_hdf5_group_to_dict(fp, + 'mc_area_args/')} doc = live.SingleCoincForGraceDB(ifos, coinc_results, upload_snr_series=True, **kwargs) From 05dbe5602b45501fd038c943724f93334e151404 Mon Sep 17 00:00:00 2001 From: Veronica Villa Date: Wed, 12 Feb 2020 15:09:58 -0800 Subject: [PATCH 2/7] Add hdf5 to dict functions --- pycbc/io/hdf.py | 42 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/pycbc/io/hdf.py b/pycbc/io/hdf.py index 0c1fe9e4065..74b34c46671 100644 --- a/pycbc/io/hdf.py +++ b/pycbc/io/hdf.py @@ -960,13 +960,53 @@ def recursively_save_dict_contents_to_group(h5file, path, dic): python dictionary to be converted to hdf5 format """ for key, item in dic.items(): - if isinstance(item, (np.ndarray, np.int64, np.float64, str, bytes, tuple, list)): + if isinstance(item, (np.ndarray, np.int64, np.float64, str, int, float, + bytes, tuple, list)): h5file[path + str(key)] = item elif isinstance(item, dict): recursively_save_dict_contents_to_group(h5file, path + key + '/', item) else: raise ValueError('Cannot save %s type' % type(item)) +def load_hdf5_to_dict(filename): + """ + Parameters + ---------- + filename: + hdf5 file to be loaded as a dictionary + + Returns + ------- + dictionary with hdf5 file content + + """ + with h5py.File(filename, 'r') as h5file: + recursively_load_hdf5_group_to_dict(h5file, '/') + +def recursively_load_hdf5_group_to_dict(h5file, path): + """ + Parameters + ---------- + h5file: + h5py file to be loaded as a dictionary + path: + path within h5py file to load + + Returns + ------- + dic: + dictionary with hdf5 file group content + """ + dic = {} + for key, item in h5file[path].items(): + if isinstance(item, h5py._hl.dataset.Dataset): + dic[key] = item[()] + elif isinstance(item, h5py._hl.group.Group): + dic[key] = recursively_load_hdf5_group_to_dict(h5file, path + key + '/') + else: + raise ValueError('Cannot load %s type' % type(item)) + return dic + def combine_and_copy(f, files, group): """ Combine the same column from multiple files and save to a third""" # ensure that the files input is stable for iteration order From 3bbc4847d433f2fa681c78b5b5e409788427f6fd Mon Sep 17 00:00:00 2001 From: Veronica Villa Date: Wed, 12 Feb 2020 15:10:51 -0800 Subject: [PATCH 3/7] Small corrections --- pycbc/mchirp_area.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pycbc/mchirp_area.py b/pycbc/mchirp_area.py index 085c3b2f20a..5798e06d6e1 100644 --- a/pycbc/mchirp_area.py +++ b/pycbc/mchirp_area.py @@ -320,7 +320,7 @@ def calc_probabilities(mchirp, snr, eff_distance, src_args): # equal to the maximum mass, the probability for BBH is 100% mc_max = mass_limits['max_m1'] / (2 ** 0.2) if trig_mc_det['central'] > mc_max * (1 + z['central']): - if mass_gap is not False: + if mass_gap: probabilities = {"BNS": 0.0, "GNS": 0.0, "NSBH": 0.0, "GG": 0.0, "BHG": 0.0, "BBH": 1.0} else: From 077ad5affe55090e5229625388c01d7ac03403ff Mon Sep 17 00:00:00 2001 From: Veronica Villa Date: Thu, 13 Feb 2020 07:40:58 -0800 Subject: [PATCH 4/7] Simplify save_hdf5_to_dict functions --- bin/pycbc_optimize_snr | 7 ++++--- pycbc/io/hdf.py | 19 ++----------------- 2 files changed, 6 insertions(+), 20 deletions(-) diff --git a/bin/pycbc_optimize_snr b/bin/pycbc_optimize_snr index dc34c45832d..6b6c5907eea 100755 --- a/bin/pycbc_optimize_snr +++ b/bin/pycbc_optimize_snr @@ -20,7 +20,7 @@ from pycbc.conversions import ( ) from scipy.optimize import differential_evolution from pycbc.io import live -from pycbc.io.hdf import recursively_load_hdf5_group_to_dict +from pycbc.io.hdf import load_hdf5_to_dict from pycbc.detector import Detector from pycbc.psd import interpolate @@ -317,12 +317,13 @@ channel_names = {} for ifo in fp['channel_names'].keys(): channel_names[ifo] = fp['channel_names'][ifo][()] +mc_area_args = load_hdf5_to_dict(fp, 'mc_area_args/') + kwargs = {'psds': {ifo: followup_data[ifo]['psd'] for ifo in ifos}, 'low_frequency_cutoff': flow, 'followup_data': followup_data, 'channel_names': channel_names, - 'mc_area_args': recursively_load_hdf5_group_to_dict(fp, - 'mc_area_args/')} + 'mc_area_args': mc_area_args} doc = live.SingleCoincForGraceDB(ifos, coinc_results, upload_snr_series=True, **kwargs) diff --git a/pycbc/io/hdf.py b/pycbc/io/hdf.py index 74b34c46671..972c6ab8520 100644 --- a/pycbc/io/hdf.py +++ b/pycbc/io/hdf.py @@ -968,29 +968,14 @@ def recursively_save_dict_contents_to_group(h5file, path, dic): else: raise ValueError('Cannot save %s type' % type(item)) -def load_hdf5_to_dict(filename): - """ - Parameters - ---------- - filename: - hdf5 file to be loaded as a dictionary - - Returns - ------- - dictionary with hdf5 file content - - """ - with h5py.File(filename, 'r') as h5file: - recursively_load_hdf5_group_to_dict(h5file, '/') - -def recursively_load_hdf5_group_to_dict(h5file, path): +def load_hdf5_to_dict(h5file, path): """ Parameters ---------- h5file: h5py file to be loaded as a dictionary path: - path within h5py file to load + path within h5py file to load: '/' for the whole h5py file Returns ------- From 0845decbe32169b18d4f5b76c9a0f3e0f98de69c Mon Sep 17 00:00:00 2001 From: Veronica Villa Date: Thu, 13 Feb 2020 07:46:11 -0800 Subject: [PATCH 5/7] Fixing codeclimate issues --- pycbc/io/hdf.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pycbc/io/hdf.py b/pycbc/io/hdf.py index 972c6ab8520..92eb5ecde1f 100644 --- a/pycbc/io/hdf.py +++ b/pycbc/io/hdf.py @@ -976,7 +976,7 @@ def load_hdf5_to_dict(h5file, path): h5py file to be loaded as a dictionary path: path within h5py file to load: '/' for the whole h5py file - + Returns ------- dic: @@ -987,7 +987,8 @@ def load_hdf5_to_dict(h5file, path): if isinstance(item, h5py._hl.dataset.Dataset): dic[key] = item[()] elif isinstance(item, h5py._hl.group.Group): - dic[key] = recursively_load_hdf5_group_to_dict(h5file, path + key + '/') + dic[key] = recursively_load_hdf5_group_to_dict(h5file, + path + key + '/') else: raise ValueError('Cannot load %s type' % type(item)) return dic From 033c4c169fe802a1d76fca6c1739df6ec6ad2c6b Mon Sep 17 00:00:00 2001 From: Veronica Villa Date: Thu, 13 Feb 2020 07:53:40 -0800 Subject: [PATCH 6/7] Fixing codeclimate issues --- pycbc/io/hdf.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pycbc/io/hdf.py b/pycbc/io/hdf.py index 92eb5ecde1f..01abf18c5b0 100644 --- a/pycbc/io/hdf.py +++ b/pycbc/io/hdf.py @@ -987,8 +987,7 @@ def load_hdf5_to_dict(h5file, path): if isinstance(item, h5py._hl.dataset.Dataset): dic[key] = item[()] elif isinstance(item, h5py._hl.group.Group): - dic[key] = recursively_load_hdf5_group_to_dict(h5file, - path + key + '/') + dic[key] = load_hdf5_to_dict(h5file, path + key + '/') else: raise ValueError('Cannot load %s type' % type(item)) return dic From 69ce21637b8469aa12dd7c217bd842c0e796f5d9 Mon Sep 17 00:00:00 2001 From: Veronica Villa Date: Thu, 13 Feb 2020 08:21:48 -0800 Subject: [PATCH 7/7] Remove access to protected members --- pycbc/io/hdf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pycbc/io/hdf.py b/pycbc/io/hdf.py index 01abf18c5b0..9241dee3421 100644 --- a/pycbc/io/hdf.py +++ b/pycbc/io/hdf.py @@ -984,9 +984,9 @@ def load_hdf5_to_dict(h5file, path): """ dic = {} for key, item in h5file[path].items(): - if isinstance(item, h5py._hl.dataset.Dataset): + if isinstance(item, h5py.Dataset): dic[key] = item[()] - elif isinstance(item, h5py._hl.group.Group): + elif isinstance(item, h5py.Group): dic[key] = load_hdf5_to_dict(h5file, path + key + '/') else: raise ValueError('Cannot load %s type' % type(item))