diff --git a/.github/workflows/inference-workflow.yml b/.github/workflows/inference-workflow.yml index d31076da5e6..704351cdb7e 100644 --- a/.github/workflows/inference-workflow.yml +++ b/.github/workflows/inference-workflow.yml @@ -25,7 +25,7 @@ jobs: wget -qO - https://download.pegasus.isi.edu/pegasus/gpg.txt | sudo apt-key add - echo "deb https://download.pegasus.isi.edu/pegasus/ubuntu bionic main" | sudo tee -a /etc/apt/sources.list sudo apt-get -o Acquire::Retries=3 update - sudo apt-get -o Acquire::Retries=3 install pegasus=5.0.6-1+ubuntu18 + sudo apt-get -o Acquire::Retries=3 install pegasus=5.0.8-1+ubuntu18 - run: sudo apt-get -o Acquire::Retries=3 install *fftw3* intel-mkl* - name: Install pycbc run: | diff --git a/.github/workflows/search-workflow.yml b/.github/workflows/search-workflow.yml index f7f8c744ad5..d3cd392ebb9 100644 --- a/.github/workflows/search-workflow.yml +++ b/.github/workflows/search-workflow.yml @@ -30,7 +30,7 @@ jobs: wget -qO - https://download.pegasus.isi.edu/pegasus/gpg.txt | sudo apt-key add - echo "deb https://download.pegasus.isi.edu/pegasus/ubuntu bionic main" | sudo tee -a /etc/apt/sources.list sudo apt-get -o Acquire::Retries=3 update - sudo apt-get -o Acquire::Retries=3 install pegasus=5.0.6-1+ubuntu18 + sudo apt-get -o Acquire::Retries=3 install pegasus=5.0.8-1+ubuntu18 - run: sudo apt-get -o Acquire::Retries=3 install *fftw3* intel-mkl* - name: Install pycbc run: | diff --git a/.github/workflows/tmpltbank-workflow.yml b/.github/workflows/tmpltbank-workflow.yml index e971e97b2b7..c460bcc0637 100644 --- a/.github/workflows/tmpltbank-workflow.yml +++ b/.github/workflows/tmpltbank-workflow.yml @@ -29,7 +29,7 @@ jobs: wget -qO - https://download.pegasus.isi.edu/pegasus/gpg.txt | sudo apt-key add - echo "deb https://download.pegasus.isi.edu/pegasus/ubuntu bionic main" | sudo tee -a /etc/apt/sources.list sudo apt-get -o Acquire::Retries=3 update - sudo apt-get -o Acquire::Retries=3 install pegasus=5.0.6-1+ubuntu18 + sudo apt-get -o Acquire::Retries=3 install pegasus=5.0.8-1+ubuntu18 - run: sudo apt-get -o Acquire::Retries=3 install *fftw3* intel-mkl* - name: Install pycbc run: | diff --git a/.github/workflows/workflow-tests.yml b/.github/workflows/workflow-tests.yml index 79643cc8793..1b49d7427a5 100644 --- a/.github/workflows/workflow-tests.yml +++ b/.github/workflows/workflow-tests.yml @@ -34,7 +34,7 @@ jobs: wget -qO - https://download.pegasus.isi.edu/pegasus/gpg.txt | sudo apt-key add - echo "deb https://download.pegasus.isi.edu/pegasus/ubuntu bionic main" | sudo tee -a /etc/apt/sources.list sudo apt-get -o Acquire::Retries=3 update - sudo apt-get -o Acquire::Retries=3 install pegasus=5.0.6-1+ubuntu18 + sudo apt-get -o Acquire::Retries=3 install pegasus=5.0.8-1+ubuntu18 - run: sudo apt-get -o Acquire::Retries=3 install *fftw3* intel-mkl* - name: Install pycbc run: | diff --git a/.gitignore b/.gitignore index be6fd9999fb..405cf23fa4a 100644 --- a/.gitignore +++ b/.gitignore @@ -4,8 +4,18 @@ *.log dist/ html/ -pycbc_inspiralc build/ *.pyc docs/Makefile -PyCBC.egg-info +PyCBC.egg-info/ +pycbc/events/eventmgr_cython.cpp +pycbc/events/simd_threshold_cython.cpp +pycbc/fft/fftw_pruned_cython.cpp +pycbc/filter/matchedfilter_cpu.cpp +pycbc/filter/simd_correlate_cython.cpp +pycbc/inference/models/relbin_cpu.cpp +pycbc/types/array_cpu.cpp +pycbc/vetoes/chisq_cpu.cpp +pycbc/waveform/decompress_cpu_cython.cpp +pycbc/waveform/spa_tmplt_cpu.cpp +pycbc/waveform/utils_cpu.cpp diff --git a/bin/all_sky_search/pycbc_coinc_statmap b/bin/all_sky_search/pycbc_coinc_statmap index 8bdbca8bd0f..a0ff2eabcbf 100755 --- a/bin/all_sky_search/pycbc_coinc_statmap +++ b/bin/all_sky_search/pycbc_coinc_statmap @@ -508,9 +508,9 @@ f.attrs['hierarchical_removal_iterations'] = h_iterations # Write whether hierarchical removals were removed against the # inclusive background or the exclusive background. Have to use -# numpy.string_ datatype. +# numpy.bytes_ datatype. if h_iterations != 0: hrm_method = args.hierarchical_removal_against - f.attrs['hierarchical_removal_method'] = numpy.string_(hrm_method) + f.attrs['hierarchical_removal_method'] = numpy.bytes_(hrm_method) logging.info("Done") diff --git a/bin/all_sky_search/pycbc_sngls_statmap b/bin/all_sky_search/pycbc_sngls_statmap index 074214dbdda..57a1c5197ec 100755 --- a/bin/all_sky_search/pycbc_sngls_statmap +++ b/bin/all_sky_search/pycbc_sngls_statmap @@ -436,10 +436,10 @@ f.attrs['hierarchical_removal_iterations'] = h_iterations # Write whether hierarchical removals were removed against the # inclusive background or the exclusive background. Have to use -# numpy.string_ datatype. +# numpy.bytes_ datatype. if h_iterations != 0: hrm_method = args.hierarchical_removal_against - f.attrs['hierarchical_removal_method'] = numpy.string_(hrm_method) + f.attrs['hierarchical_removal_method'] = numpy.bytes_(hrm_method) logging.info("Done") diff --git a/bin/inference/pycbc_inference b/bin/inference/pycbc_inference index 701663a3550..442172eb967 100644 --- a/bin/inference/pycbc_inference +++ b/bin/inference/pycbc_inference @@ -108,7 +108,11 @@ fft.from_cli(opts) with ctx: # read configuration file - cp = configuration.WorkflowConfigParser.from_cli(opts) + cp_original = configuration.WorkflowConfigParser.from_cli(opts) + # some models will interally modify original cp for sampling, + # such as joint_primary_marginalized, we need to save original + # and let modify the copied one + cp = cp_original.__deepcopy__(cp_original) # create an empty checkpoint file, if needed condor_ckpt = cp.has_option('sampler', 'checkpoint-signal') @@ -138,7 +142,7 @@ with ctx: if pool.is_main_process(): for fn in [sampler.checkpoint_file, sampler.backup_file]: with loadfile(fn, 'a') as fp: - fp.write_config_file(cp) + fp.write_config_file(cp_original) # Run the sampler sampler.run() diff --git a/bin/live/pycbc_live_supervise_collated_trigger_fits b/bin/live/pycbc_live_supervise_collated_trigger_fits index 3206f549c44..087ae5d1339 100755 --- a/bin/live/pycbc_live_supervise_collated_trigger_fits +++ b/bin/live/pycbc_live_supervise_collated_trigger_fits @@ -226,6 +226,11 @@ def fit_over_multiparam( "specified parameters", len(daily_files) ) + logging.info( + "Smoothing fits using fit_over_multiparam with %d files and " + "specified parameters", + len(daily_files) + ) file_id_str = f'{first_date}-{end_date}' out_fname = fit_over_controls['fit-over-format'].format( dates=file_id_str, @@ -270,7 +275,10 @@ def plot_fits( ] fits_plot_arguments += sv.dict_to_args(plot_fit_options) - title = "Fit parameters for pycbc-live, triggers from " + day_title_str + title = "Fit parameters for pycbc-live, triggers from {}, {}".format( + ifo, + day_title_str + ) if smoothed == True: title += ', smoothed' fits_plot_arguments += ['--title', title] @@ -290,7 +298,6 @@ def single_significance_fits( day_str, day_dt, controls, - test_options, stat_files=None, ): """ @@ -299,7 +306,10 @@ def single_significance_fits( """ daily_options['output'] = os.path.join( output_dir, - daily_controls['sig-daily-format'].format(date=day_str), + daily_controls['sig-daily-format'].format( + ifos=''.join(sorted(controls['ifos'].split())), + date=day_str + ), ) daily_args = ['pycbc_live_single_significance_fits'] @@ -319,7 +329,10 @@ def plot_single_significance_fits(daily_output, daily_plot_options, controls): """ Plotting daily significance fits, and link to public directory if wanted """ - daily_plot_output = f'{daily_output[:-4]}_{{ifo}}.png' + daily_plot_output = daily_output[:-4].replace( + ''.join(sorted(controls['ifos'].split())), + '{ifo}' + ) + '.png' logging.info( "Plotting daily significance fits from %s to %s", daily_output, @@ -362,9 +375,14 @@ def combine_significance_fits( Supervise the smoothing of live trigger significance fits using pycbc_live_combine_single_significance_fits """ + # This has a trick to do partial formatting, get the IFOs into the + # string, but not the date daily_files, first_date, end_date = find_daily_fit_files( combined_controls, - combined_controls['daily-format'], + combined_controls['daily-format'].format( + ifos=''.join(sorted(controls['ifos'].split())), + date='{date}' + ), controls['output-directory'], ) logging.info( @@ -463,7 +481,6 @@ def supervise_collation_fits_dq(args, day_dt, day_str): combined_control_options = config_opts['significance_combined_fits_control'] combined_plot_options = config_opts['plot_significance_combined'] combined_plot_control_options = config_opts['plot_significance_combined_control'] - test_options = config_opts['test'] # The main output directory will have a date subdirectory which we # put the output into @@ -538,7 +555,6 @@ def supervise_collation_fits_dq(args, day_dt, day_str): day_str, day_dt, controls, - test_options, stat_files=stat_files, ) plot_single_significance_fits( @@ -569,7 +585,7 @@ def get_yesterday_date(): day_dt = datetime.utcnow() - timedelta(days=1) day_dt = datetime.combine(day_dt, datetime.min.time()) day_str = day_dt.strftime('%Y_%m_%d') - return date_dt, date_str + return day_dt, day_str parser = argparse.ArgumentParser(description=__doc__) pycbc.add_common_pycbc_options(parser) diff --git a/bin/minifollowups/pycbc_page_coincinfo b/bin/minifollowups/pycbc_page_coincinfo index 10f08bc2a01..b2dc9e363b7 100644 --- a/bin/minifollowups/pycbc_page_coincinfo +++ b/bin/minifollowups/pycbc_page_coincinfo @@ -30,7 +30,7 @@ from pycbc import add_common_pycbc_options import pycbc.results import pycbc.pnutils from pycbc.io.hdf import HFile -from pycbc.events import ranking +from pycbc.events import ranking, stat as pystat from pycbc.results import followup parser = argparse.ArgumentParser() @@ -69,11 +69,22 @@ parser.add_argument('--include-summary-page-link', action='store_true', parser.add_argument('--include-gracedb-link', action='store_true', help="If given, will provide a link to search GraceDB for events " "within a 3s window around the coincidence time.") - +parser.add_argument('--max-columns', type=int, + help="Maximum number of columns allowed in the table (not including detector names)") +pystat.insert_statistic_option_group(parser, + default_ranking_statistic='single_ranking_only') args = parser.parse_args() pycbc.init_logging(args.verbose) +if args.ranking_statistic not in ['quadsum', 'single_ranking_only']: + logging.warning( + "For the coincident info table, we only use single ranking, not %s, " + "this option will be ignored", + args.ranking_statistic + ) + args.ranking_statistic = 'quadsum' + # Get the nth loudest trigger from the output of pycbc_coinc_statmap f = HFile(args.statmap_file, 'r') d = f[args.statmap_file_subspace_name] @@ -146,12 +157,16 @@ statmapfile = d # table. Each entry in data corresponds to each row in the final table and # should be a list of values. So data is will be a list of lists. data = [] +row_labels = [] +rank_method = pystat.get_statistic_from_opts(args, list(files.keys())) + for ifo in files.keys(): # ignore ifo if coinc didn't participate (only for multi-ifo workflow) if (statmapfile['%s/time' % ifo][n] == -1.0): continue + row_labels.append(ifo) d = files[ifo] i = idx[ifo] tid = d['template_id'][i] @@ -161,7 +176,12 @@ for ifo in files.keys(): time = d['end_time'][i] utc = lal.GPSToUTC(int(time))[0:6] - + trig_dict = { + k: numpy.array([d[k][i]]) + for k in d.keys() + if not k.endswith('_template') + and k not in ['gating', 'search', 'template_boundaries'] + } # Headers will store the headers that will appear in the table. headers = [] data.append([]) @@ -170,9 +190,6 @@ for ifo in files.keys(): if args.include_summary_page_link: data[-1].append(pycbc.results.dq.get_summary_page_link(ifo, utc)) headers.append("Detector status") - else: - data[-1].append(ifo) - headers.append("Ifo") # End times data[-1].append(str(datetime.datetime(*utc))) @@ -180,14 +197,28 @@ for ifo in files.keys(): headers.append("UTC End Time") headers.append("GPS End time") + #headers.append("Stat") + # Determine statistic naming + if args.sngl_ranking == "newsnr": + sngl_stat_name = "Reweighted SNR" + elif args.sngl_ranking == "newsnr_sgveto": + sngl_stat_name = "Reweighted SNR (+sgveto)" + elif args.sngl_ranking == "newsnr_sgveto_psdvar": + sngl_stat_name = "Reweighted SNR (+sgveto+psdvar)" + elif args.sngl_ranking == "snr": + sngl_stat_name = "SNR" + else: + sngl_stat_name = args.sngl_ranking + + stat = rank_method.get_sngl_ranking(trig_dict) + headers.append(sngl_stat_name) + data[-1].append('%5.2f' % stat[0]) + # SNR and phase (not showing any single-det stat here) data[-1].append('%5.2f' % d['snr'][i]) data[-1].append('%5.2f' % d['coa_phase'][i]) - #data[-1].append('%5.2f' % ranking.newsnr(d['snr'][i], rchisq)) headers.append("ρ") headers.append("Phase") - #headers.append("Stat") - # Signal-glitch discrimators data[-1].append('%5.2f' % rchisq) data[-1].append('%i' % d['chisq_dof'][i]) @@ -218,7 +249,12 @@ for ifo in files.keys(): headers.append("s2z") headers.append("Duration") -html += str(pycbc.results.static_table(data, headers)) +html += str(pycbc.results.static_table( + data, + headers, + columns_max=args.max_columns, + row_labels=row_labels +)) ############################################################################### pycbc.results.save_fig_with_metadata(html, args.output_file, {}, diff --git a/bin/minifollowups/pycbc_page_injinfo b/bin/minifollowups/pycbc_page_injinfo index a09292ecc6e..e4bf1cfe8e8 100644 --- a/bin/minifollowups/pycbc_page_injinfo +++ b/bin/minifollowups/pycbc_page_injinfo @@ -20,6 +20,7 @@ import sys import numpy import pycbc.results +from pycbc import conversions as conv import pycbc.pnutils from pycbc import init_logging, add_common_pycbc_options from pycbc.detector import Detector @@ -34,6 +35,8 @@ parser.add_argument('--injection-index', type=int, required=True, help="The index of the injection to print out. Required") parser.add_argument('--n-nearest', type=int, help="Optional, used in the title") +parser.add_argument('--max-columns', type=int, + help="Optional, maximum number of columns used for the table") args = parser.parse_args() @@ -65,13 +68,24 @@ labels = { 'spin1z': 's1z', 'spin2x': 's2x', 'spin2y': 's2y', - 'spin2z': 's2z' + 'spin2z': 's2z', + 'chieff': 'χeff', + 'chip': 'χp', } params += ['tc'] m1, m2 = f['injections']['mass1'][iidx], f['injections']['mass2'][iidx] -mchirp, eta = pycbc.pnutils.mass1_mass2_to_mchirp_eta(m1, m2) +s1x, s2x = f['injections']['spin1x'][iidx], f['injections']['spin2x'][iidx] +s1y, s2y = f['injections']['spin1y'][iidx], f['injections']['spin2y'][iidx] +s1z, s2z = f['injections']['spin1z'][iidx], f['injections']['spin2z'][iidx] + +derived = {} +derived['mchirp'], derived['eta'] = \ + pycbc.pnutils.mass1_mass2_to_mchirp_eta(m1, m2) +derived['mtotal'] = conv.mtotal_from_mass1_mass2(m1, m2) +derived['chieff'] = conv.chi_eff(m1, m2, s1z, s2z) +derived['chip'] = conv.chi_p(m1, m2, s1x, s1y, s2x, s2y) if 'optimal_snr' in ' '.join(list(f['injections'].keys())): ifolist = f.attrs['ifos'].split(' ') @@ -82,32 +96,30 @@ else: eff_dist = {} for ifo in ['H1', 'L1', 'V1']: eff_dist[ifo] = Detector(ifo).effective_distance( - f['injections/distance'][iidx], - f['injections/ra'][iidx], - f['injections/dec'][iidx], - f['injections/polarization'][iidx], - f['injections/tc'][iidx], - f['injections/inclination'][iidx]) - + f['injections/distance'][iidx], + f['injections/ra'][iidx], + f['injections/dec'][iidx], + f['injections/polarization'][iidx], + f['injections/tc'][iidx], + f['injections/inclination'][iidx] + ) params += ['dec_chirp_dist', 'eff_dist_h', 'eff_dist_l', 'eff_dist_v'] dec_dist = max(eff_dist['H1'], eff_dist['L1']) dec_chirp_dist = pycbc.pnutils.chirp_distance(dec_dist, mchirp) params += ['mass1', 'mass2', 'mchirp', 'eta', 'ra', 'dec', 'inclination', 'spin1x', 'spin1y', 'spin1z', 'spin2x', 'spin2y', - 'spin2z'] + 'spin2z', 'chieff', 'chip'] for p in params: if p in f['injections']: data += ["%.2f" % f['injections'][p][iidx]] + elif p in derived.keys(): + data += [f'{derived[p]:.2f}'] elif 'eff_dist' in p: ifo = '%s1' % p.split('_')[-1] data += ["%.2f" % eff_dist[ifo.upper()]] - elif p == 'mchirp': - data += ["%.2f" % mchirp] - elif p == 'eta': - data += ["%.2f" % eta] elif p == 'dec_chirp_dist': data += ["%.2f" % dec_chirp_dist] else: @@ -117,7 +129,7 @@ for p in params: headers += [labels[p]] table = numpy.array([data], dtype=str) -html = str(pycbc.results.static_table(table, headers)) +html = str(pycbc.results.static_table(table, headers, columns_max=args.max_columns)) tag = '' if args.n_nearest is not None: diff --git a/bin/minifollowups/pycbc_page_snglinfo b/bin/minifollowups/pycbc_page_snglinfo index 99f3a1629d2..2ce04edbcc1 100644 --- a/bin/minifollowups/pycbc_page_snglinfo +++ b/bin/minifollowups/pycbc_page_snglinfo @@ -68,6 +68,8 @@ parser.add_argument('--include-gracedb-link', action='store_true', parser.add_argument('--significance-file', help="If given, will search for this trigger's id in the file to see if " "stat and p_astro values exists for this trigger.") +parser.add_argument('--max-columns', type=int, + help="Optional. Set a maximum number of columns to be used in the output table") pystat.insert_statistic_option_group(parser, default_ranking_statistic='single_ranking_only') @@ -117,7 +119,7 @@ if args.n_loudest is not None: sngl_file.apply_mask(l[0]) # make a table for the single detector information ############################ -time = sngl_file.end_time +time = sngl_file.end_time[0] utc = lal.GPSToUTC(int(time))[0:6] # Headers here will contain the list of headers that will appear in the @@ -129,6 +131,8 @@ headers = [] # single list that will hold the values to go into the table. data = [[]] +row_labels = [args.instrument] + # DQ summary link if args.include_summary_page_link: data[0].append(pycbc.results.dq.get_summary_page_link(args.instrument, utc)) @@ -141,11 +145,10 @@ headers.append("UTC") headers.append("End time") # SNR and statistic -data[0].append('%5.2f' % sngl_file.snr) -data[0].append('%5.2f' % sngl_file.get_column('coa_phase')) -data[0].append('%5.2f' % stat) headers.append("ρ") +data[0].append('%5.2f' % sngl_file.snr[0]) headers.append("Phase") +data[0].append('%5.2f' % sngl_file.get_column('coa_phase')[0]) # Determine statistic naming if args.sngl_ranking == "newsnr": sngl_stat_name = "Reweighted SNR" @@ -169,30 +172,31 @@ else: ) headers.append(stat_name) +data[0].append('%5.2f' % stat[0]) # Signal-glitch discrimators -data[0].append('%5.2f' % sngl_file.rchisq) -data[0].append('%i' % sngl_file.get_column('chisq_dof')) +data[0].append('%5.2f' % sngl_file.rchisq[0]) +data[0].append('%i' % sngl_file.get_column('chisq_dof')[0]) headers.append("χ2r") headers.append("χ2 bins") try: - data[0].append('%5.2f' % sngl_file.sgchisq) + data[0].append('%5.2f' % sngl_file.sgchisq[0]) headers.append("sgχ2") except: pass try: - data[0].append('%5.2f' % sngl_file.psd_var_val) + data[0].append('%5.2f' % sngl_file.psd_var_val[0]) headers.append("PSD var") except: pass # Template parameters -data[0].append('%5.2f' % sngl_file.mass1) -data[0].append('%5.2f' % sngl_file.mass2) -data[0].append('%5.2f' % sngl_file.mchirp) -data[0].append('%5.2f' % sngl_file.spin1z) -data[0].append('%5.2f' % sngl_file.spin2z) -data[0].append('%5.2f' % sngl_file.template_duration) +data[0].append('%5.2f' % sngl_file.mass1[0]) +data[0].append('%5.2f' % sngl_file.mass2[0]) +data[0].append('%5.2f' % sngl_file.mchirp[0]) +data[0].append('%5.2f' % sngl_file.spin1z[0]) +data[0].append('%5.2f' % sngl_file.spin2z[0]) +data[0].append('%5.2f' % sngl_file.template_duration[0]) headers.append("m1") headers.append("m2") headers.append("Mc") @@ -221,7 +225,7 @@ if args.include_gracedb_link: data[0].append(gdb_search_link) html = pycbc.results.dq.redirect_javascript + \ - str(pycbc.results.static_table(data, headers)) + str(pycbc.results.static_table(data, headers, row_labels=row_labels, columns_max=args.max_columns)) ############################################################################### # Set up default titles and the captions for the file diff --git a/bin/plotting/pycbc_plot_bank_corner b/bin/plotting/pycbc_plot_bank_corner index 01fdf9100cf..6d55b2fbcab 100644 --- a/bin/plotting/pycbc_plot_bank_corner +++ b/bin/plotting/pycbc_plot_bank_corner @@ -72,6 +72,13 @@ parser.add_argument("--parameters", "property of that parameter will be used. If not " "provided, will plot all of the parameters in the " "bank.") +parser.add_argument( + '--log-parameters', + nargs='+', + default=[], + help="Which parameters are to be plotted on a log scale? " + "Must be also given in parameters" +) parser.add_argument('--plot-histogram', action='store_true', help="Plot 1D histograms of parameters on the " @@ -103,6 +110,11 @@ parser.add_argument("--color-parameter", help="Color scatter points according to the parameter given. " "May optionally provide a label in the same way as for " "--parameters. Default=No scatter point coloring.") +parser.add_argument( + '--log-colormap', + action='store_true', + help="Should the colorbar be plotted on a log scale?" +) parser.add_argument('--dpi', type=int, default=200, @@ -117,6 +129,13 @@ parser.add_argument('--title', add_style_opt_to_parser(parser) args = parser.parse_args() +for lp in args.log_parameters: + if not lp in args.parameters: + parser.error( + "--log-parameters should be in --parameters. " + f"{lp} not in [{', '.join(args.parameters)}]" + ) + pycbc.init_logging(args.verbose) set_style_from_cli(args) @@ -146,7 +165,7 @@ if args.fits_file is not None: param = fits_f[p][:].astype(float) # We need to check for the cardinal '-1' value which means # that the fit is invalid - param[param <= 0] = 0 if 'count' in p and 'log' not in p else np.nan + param[param <= 0] = np.nan bank[p] = param logging.info("Got %d templates from the bank", banklen) @@ -227,12 +246,21 @@ if cpar: for p in required_minmax: minval = np.nanmin(bank_fa[p][bank_fa[p] != -np.inf]) maxval = np.nanmax(bank_fa[p][bank_fa[p] != np.inf]) - valrange = maxval - minval + if (p in args.log_parameters) or (p == cpar and args.log_colormap): + # Extend the range by 10% in log-space + logvalrange = np.log(maxval) - np.log(minval) + if p not in mins: + mins[p] = np.exp(np.log(minval) - 0.05 * logvalrange) + if p not in maxs: + maxs[p] = np.exp(np.log(maxval) + 0.05 * logvalrange) + else: + # Extend the range by 10% + valrange = maxval - minval + if p not in mins: + mins[p] = minval - 0.05 * valrange + if p not in maxs: + maxs[p] = maxval + 0.05 * valrange - if p not in mins: - mins[p] = minval - 0.05 * valrange - if p not in maxs: - maxs[p] = maxval + 0.05 * valrange # Deal with non-coloring case: zvals = bank_fa[cpar] if cpar else None @@ -247,6 +275,7 @@ fig, axis_dict = create_multidim_plot( plot_scatter=True, plot_contours=False, scatter_cmap="viridis", + scatter_log_cmap=args.log_colormap, marginal_title=False, marginal_percentiles=[], fill_color='g', @@ -258,6 +287,7 @@ fig, axis_dict = create_multidim_plot( hist_color=hist_color, mins=mins, maxs=maxs, + log_parameters=args.log_parameters, ) title_text = f"{os.path.basename(args.bank_file)}" @@ -293,6 +323,19 @@ for i in range(len(args.parameters)): for s0, s1 in zip(sharex_axes[:-1], sharex_axes[1:]): s0.sharex(s1) +for (p1, p2), ax in axis_dict.items(): + if p1 == p2 and p1 in args.log_parameters: + if p1 == args.parameters[-1] and len(args.parameters) == 2: + # This will be turned on its side, so set _y_ axis to log + ax[0].semilogy() + else: + ax[0].semilogx() + else: + if p1 in args.log_parameters: + ax[0].semilogx() + if p2 in args.log_parameters: + ax[0].semilogy() + logging.info("Plot generated") fig.set_dpi(args.dpi) diff --git a/bin/pycbc_live b/bin/pycbc_live index be649e6ae0e..52a0de2433a 100755 --- a/bin/pycbc_live +++ b/bin/pycbc_live @@ -848,6 +848,8 @@ parser.add_argument('--idq-state-channel', action=MultiDetMultiColonOptionAction parser.add_argument('--idq-threshold', type=float, help='Threshold used to veto triggers at times of ' 'low iDQ False Alarm Probability') +parser.add_argument('--idq-reweighting', action='store_true',default=False, + help='Reweight triggers based on iDQ False Alarm Probability') parser.add_argument('--data-quality-channel', action=MultiDetMultiColonOptionAction, help="Channel containing data quality information. Used " @@ -1311,17 +1313,35 @@ with ctx: if len(results[ifo][key]): results[ifo][key] = results[ifo][key][idx] if data_reader[ifo].idq is not None: - logging.info("Checking %s's iDQ information", ifo) + logging.info("Reading %s's iDQ information", ifo) start = data_reader[ifo].start_time times = results[ifo]['end_time'] - idx = data_reader[ifo].idq.indices_of_flag( + flag_active = data_reader[ifo].idq.flag_at_times( start, valid_pad, times, - padding=data_reader[ifo].dq_padding) - logging.info('Keeping %d/%d %s triggers after iDQ', - len(idx), len(times), ifo) - for key in results[ifo]: - if len(results[ifo][key]): - results[ifo][key] = results[ifo][key][idx] + padding=data_reader[ifo].dq_padding + ) + + if args.idq_reweighting: + logging.info( + 'iDQ flagged %d/%d %s triggers', + numpy.sum(flag_active), + len(times), + ifo + ) + results[ifo]['dq_state'] = flag_active.astype(int) + else: + # use idq as a veto + keep = numpy.logical_not(flag_active) + logging.info( + 'Keeping %d/%d %s triggers after iDQ', + numpy.sum(keep), + len(times), + ifo + ) + for key in results[ifo]: + if len(results[ifo][key]): + results[ifo][key] = \ + results[ifo][key][keep] # Calculate and add the psd variation for the results if args.psd_variation: diff --git a/bin/pycbc_make_banksim b/bin/pycbc_make_banksim index fa733e6d5d6..3198b1cf0dc 100644 --- a/bin/pycbc_make_banksim +++ b/bin/pycbc_make_banksim @@ -185,8 +185,19 @@ shutil.copy(banksim_prog, 'scripts/pycbc_banksim') os.chmod('scripts/pycbc_banksim', 0o0777) logging.info("Creating injection file") -inj_str = "lalapps_inspinj " + get_ini_opts(confs, "inspinj") + "--output inj.xml" -os.system(inj_str) +if confs.has_section("inspinj"): + logging.info("Using lalapps_inspinj to create injections") + inj_str = "lalapps_inspinj " + get_ini_opts(confs, "inspinj") + "--output inj.xml" + os.system(inj_str) +elif confs.has_section("external_injection"): + logging.info("Using external injection file. Please ensure the file is in sim_inspiral table (.xml) format.") + inj_file_path = confs.get("external_injection", "inj-file") + if inj_file_path == "inj.xml": + pass + else: + os.system("cp {} inj.xml".format(inj_file_path)) +else: + raise ValueError("Need to specify the injection method. Either provide [inspinj] section or [external_injection]") logging.info("Splitting template bank") subprocess.call(['pycbc_splitbank', diff --git a/companion.txt b/companion.txt index 769c574d315..d5295032928 100644 --- a/companion.txt +++ b/companion.txt @@ -6,7 +6,11 @@ healpy # Needed for GraceDB uploads and skymap generation ligo-gracedb>=2.10.0 -ligo.skymap!=1.1.0 +#ligo.skymap!=1.1.0 +# Temporarily use fork that doesn't force numpy<2.0 incompatibilities +# Remove when PR merged, or numpy > 2.0 is a hard requirement +# https://github.com/lpsinger/ligo.skymap/pull/28 +git+https://github.com/ahnitz/ligo.skymap@lreq # auxiliary samplers epsie>=1.0 @@ -18,6 +22,10 @@ https://github.com/willvousden/ptemcee/archive/master.tar.gz --extra-index-url https://download.pytorch.org/whl/cpu torch nessai>=0.11.0 + +# Needed to make sure numpy2 build works, remove when PR merged +# https://github.com/pypmc/pypmc/pull/105/files +git+https://github.com/ahnitz/pypmc@n2 snowline # useful to look at PyCBC Live with htop diff --git a/docs/banksim.rst b/docs/banksim.rst index 94204160796..3b886f31cd2 100644 --- a/docs/banksim.rst +++ b/docs/banksim.rst @@ -26,7 +26,7 @@ Below is an example. .. literalinclude:: ../examples/banksim/banksim_simple.ini -There are four sections that must be present [inspinj], [executables], [workflow], +There are four sections that must be present [inspinj]/[external_injection], [executables], [workflow], and [banksim]. #. inspinj @@ -41,6 +41,17 @@ and [banksim]. determine the actual approximants that will be compared. That is set in the [banksim] section. + If you want to use another method to create injections (Eg. ``pycbc_create_injections``), + instead of using [inspinj], you can name the section [external_injection] and specify the + path of the injection file. + + .. code-block:: bash + + [external_injection] + inj-file = /path/to/inj.xml + + Note: The injection file should be in the sim_inspiral table (.xml) format. + #. executables This section lists the location of the pycbc_banksim script. Make note @@ -61,7 +72,7 @@ and [banksim]. has to calculate fitting factors for. The injection - file generated from the [inspinj] section is split + file generated from the [inspinj] section or provided externally is split into smaller pieces to satisfy this requirement. Note that this option has a direct effect on the memory requirements of each banksim job, as each injection diff --git a/examples/search/plotting.ini b/examples/search/plotting.ini index 0b1ab2a2cbe..5a7d4f55837 100644 --- a/examples/search/plotting.ini +++ b/examples/search/plotting.ini @@ -61,6 +61,8 @@ window = 0.1 [html_snippet] [page_coincinfo] +sngl-ranking = newsnr_sgveto_psdvar + [page_coincinfo-background] statmap-file-subspace-name=background_exc diff --git a/pycbc/__init__.py b/pycbc/__init__.py index d6db0026e0a..99f4cc6ef37 100644 --- a/pycbc/__init__.py +++ b/pycbc/__init__.py @@ -212,8 +212,13 @@ def makedir(path): # preserve common state information which we have relied on when using # multiprocessing based pools. import multiprocessing - if hasattr(multiprocessing, 'set_start_method'): - multiprocessing.set_start_method('fork') + if multiprocessing.get_start_method(allow_none=True) is None: + if hasattr(multiprocessing, 'set_start_method'): + multiprocessing.set_start_method('fork') + elif multiprocessing.get_start_method() != 'fork': + warnings.warn("PyCBC requires the use of the 'fork' start method" + " for multiprocessing, it is currently set to {}" + .format(multiprocessing.get_start_method())) else: HAVE_OMP = True diff --git a/pycbc/conversions.py b/pycbc/conversions.py index 3daf5c8a1b0..d494e6f24fa 100644 --- a/pycbc/conversions.py +++ b/pycbc/conversions.py @@ -77,9 +77,9 @@ def ensurearray(*args): inputs was an array. """ input_is_array = any(isinstance(arg, numpy.ndarray) for arg in args) - args = numpy.broadcast_arrays(*args) + args = list(numpy.broadcast_arrays(*args)) args.append(input_is_array) - return args + return tuple(args) def formatreturn(arg, input_is_array=False): @@ -431,10 +431,79 @@ def lambda_tilde(mass1, mass2, lambda1, lambda2): mask = m1 < m2 ldiff[mask] = -ldiff[mask] eta = eta_from_mass1_mass2(m1, m2) - p1 = (lsum) * (1 + 7. * eta - 31 * eta ** 2.0) - p2 = (1 - 4 * eta)**0.5 * (1 + 9 * eta - 11 * eta ** 2.0) * (ldiff) + p1 = lsum * (1 + 7. * eta - 31 * eta ** 2.0) + p2 = (1 - 4 * eta)**0.5 * (1 + 9 * eta - 11 * eta ** 2.0) * ldiff return formatreturn(8.0 / 13.0 * (p1 + p2), input_is_array) +def delta_lambda_tilde(mass1, mass2, lambda1, lambda2): + """ Delta lambda tilde parameter defined as + equation 15 in + https://journals.aps.org/prd/pdf/10.1103/PhysRevD.91.043002 + """ + m1, m2, lambda1, lambda2, input_is_array = ensurearray( + mass1, mass2, lambda1, lambda2) + lsum = lambda1 + lambda2 + ldiff, _ = ensurearray(lambda1 - lambda2) + mask = m1 < m2 + ldiff[mask] = -ldiff[mask] + eta = eta_from_mass1_mass2(m1, m2) + p1 = numpy.sqrt(1 - 4 * eta) * ( + 1 - (13272 / 1319) * eta + + (8944 / 1319) * eta ** 2 + ) * lsum + p2 = ( + 1 - (15910 / 1319) * eta + + (32850 / 1319) * eta ** 2 + + (3380 / 1319) * eta ** 3 + ) * ldiff + return formatreturn(1 / 2 * (p1 + p2), input_is_array) + +def lambda1_from_delta_lambda_tilde_lambda_tilde(delta_lambda_tilde, + lambda_tilde, + mass1, + mass2): + """ Returns lambda1 parameter by using delta lambda tilde, + lambda tilde, mass1, and mass2. + """ + m1, m2, delta_lambda_tilde, lambda_tilde, input_is_array = ensurearray( + mass1, mass2, delta_lambda_tilde, lambda_tilde) + eta = eta_from_mass1_mass2(m1, m2) + p1 = 1 + 7.0*eta - 31*eta**2.0 + p2 = (1 - 4*eta)**0.5 * (1 + 9*eta - 11*eta**2.0) + p3 = (1 - 4*eta)**0.5 * (1 - 13272/1319*eta + 8944/1319*eta**2) + p4 = 1 - (15910/1319)*eta + (32850/1319)*eta**2 + (3380/1319)*eta**3 + amp = 1/((p1*p4)-(p2*p3)) + l_tilde_lambda1 = 13/16 * (p3-p4) * lambda_tilde + l_delta_tilde_lambda1 = (p1-p2) * delta_lambda_tilde + lambda1 = formatreturn( + amp * (l_delta_tilde_lambda1 - l_tilde_lambda1), + input_is_array + ) + return lambda1 + +def lambda2_from_delta_lambda_tilde_lambda_tilde( + delta_lambda_tilde, + lambda_tilde, + mass1, + mass2): + """ Returns lambda2 parameter by using delta lambda tilde, + lambda tilde, mass1, and mass2. + """ + m1, m2, delta_lambda_tilde, lambda_tilde, input_is_array = ensurearray( + mass1, mass2, delta_lambda_tilde, lambda_tilde) + eta = eta_from_mass1_mass2(m1, m2) + p1 = 1 + 7.0*eta - 31*eta**2.0 + p2 = (1 - 4*eta)**0.5 * (1 + 9*eta - 11*eta**2.0) + p3 = (1 - 4*eta)**0.5 * (1 - 13272/1319*eta + 8944/1319*eta**2) + p4 = 1 - (15910/1319)*eta + (32850/1319)*eta**2 + (3380/1319)*eta**3 + amp = 1/((p1*p4)-(p2*p3)) + l_tilde_lambda2 = 13/16 * (p3+p4) * lambda_tilde + l_delta_tilde_lambda2 = (p1+p2) * delta_lambda_tilde + lambda2 = formatreturn( + amp * (l_tilde_lambda2 - l_delta_tilde_lambda2), + input_is_array + ) + return lambda2 def lambda_from_mass_tov_file(mass, tov_file, distance=0.): """Return the lambda parameter(s) corresponding to the input mass(es) @@ -1774,5 +1843,8 @@ def nltides_gw_phase_diff_isco(f_low, f0, amplitude, n, m1, m2): 'nltides_gw_phase_diff_isco', 'spin_from_pulsar_freq', 'freqlmn_from_other_lmn', 'taulmn_from_other_lmn', 'remnant_mass_from_mass1_mass2_spherical_spin_eos', - 'remnant_mass_from_mass1_mass2_cartesian_spin_eos' + 'remnant_mass_from_mass1_mass2_cartesian_spin_eos', + 'lambda1_from_delta_lambda_tilde_lambda_tilde', + 'lambda2_from_delta_lambda_tilde_lambda_tilde', + 'delta_lambda_tilde' ] diff --git a/pycbc/events/eventmgr.py b/pycbc/events/eventmgr.py index d431bc9a54b..3cde84c3c4a 100644 --- a/pycbc/events/eventmgr.py +++ b/pycbc/events/eventmgr.py @@ -149,8 +149,8 @@ def findchirp_cluster_over_window(times, values, window_length): indices = numpy.zeros(len(times), dtype=numpy.int32) tlen = len(times) - absvalues = numpy.array(abs(values), copy=False) - times = numpy.array(times, dtype=numpy.int32, copy=False) + absvalues = numpy.asarray(abs(values)) + times = numpy.asarray(times, dtype=numpy.int32) k = findchirp_cluster_over_window_cython(times, absvalues, window_length, indices, tlen) diff --git a/pycbc/events/stat.py b/pycbc/events/stat.py index f61e7c55b66..e04ba732cad 100644 --- a/pycbc/events/stat.py +++ b/pycbc/events/stat.py @@ -506,7 +506,8 @@ def get_hist(self, ifos=None): # renormalise to PDF self.weights[ifo] = \ - weights[ifo] / (weights[ifo].sum() * bin_volume) + (weights[ifo] / (weights[ifo].sum() * bin_volume)) + self.weights[ifo] = self.weights[ifo].astype(numpy.float32) if param[ifo].dtype == numpy.int8: # Older style, incorrectly sorted histogram file @@ -641,13 +642,17 @@ def logsignalrate(self, stats, shift, to_shift): # Get reference ifo information rate = numpy.zeros(len(shift), dtype=numpy.float32) - ps = {ifo: numpy.array(stats[ifo]['coa_phase'], ndmin=1) + ps = {ifo: numpy.array(stats[ifo]['coa_phase'], + dtype=numpy.float32, ndmin=1) for ifo in self.ifos} - ts = {ifo: numpy.array(stats[ifo]['end_time'], ndmin=1) + ts = {ifo: numpy.array(stats[ifo]['end_time'], + dtype=numpy.float64, ndmin=1) for ifo in self.ifos} - ss = {ifo: numpy.array(stats[ifo]['snr'], ndmin=1) + ss = {ifo: numpy.array(stats[ifo]['snr'], + dtype=numpy.float32, ndmin=1) for ifo in self.ifos} - sigs = {ifo: numpy.array(stats[ifo]['sigmasq'], ndmin=1) + sigs = {ifo: numpy.array(stats[ifo]['sigmasq'], + dtype=numpy.float32, ndmin=1) for ifo in self.ifos} for ref_ifo in self.ifos: rtype = rtypes[ref_ifo] @@ -1670,6 +1675,12 @@ def single(self, trigs): numpy.ndarray The array of single detector values """ + try: + # exists if accessed via coinc_findtrigs + self.curr_tnum = trigs.template_num + except AttributeError: + # exists for SingleDetTriggers & pycbc_live get_coinc + self.curr_tnum = trigs['template_id'] # single-ifo stat = log of noise rate sngl_stat = self.lognoiserate(trigs) @@ -1681,12 +1692,6 @@ def single(self, trigs): singles['end_time'] = trigs['end_time'][:] singles['sigmasq'] = trigs['sigmasq'][:] singles['snr'] = trigs['snr'][:] - try: - # exists if accessed via coinc_findtrigs - self.curr_tnum = trigs.template_num - except AttributeError: - # exists for SingleDetTriggers & pycbc_live get_coinc - self.curr_tnum = trigs['template_id'] # Store benchmark log volume as single-ifo information since the coinc # method does not have access to template id @@ -2271,14 +2276,46 @@ def __init__(self, sngl_ranking, files=None, ifos=None, ifos=ifos, **kwargs) self.dq_rates_by_state = {} self.dq_bin_by_tid = {} - self.dq_state_segments = {} + self.dq_state_segments = None + self.low_latency = False + self.single_dtype.append(('dq_state', int)) for ifo in self.ifos: key = f'{ifo}-dq_stat_info' if key in self.files.keys(): self.dq_rates_by_state[ifo] = self.assign_dq_rates(key) self.dq_bin_by_tid[ifo] = self.assign_template_bins(key) - self.dq_state_segments[ifo] = self.setup_segments(key) + self.check_low_latency(key) + if not self.low_latency: + if self.dq_state_segments is None: + self.dq_state_segments = {} + self.dq_state_segments[ifo] = self.setup_segments(key) + + def check_low_latency(self, key): + """ + Check if the statistic file indicates low latency mode. + Parameters + ---------- + key: str + Statistic file key string. + Returns + ------- + None + """ + ifo = key.split('-')[0] + with h5py.File(self.files[key], 'r') as dq_file: + ifo_grp = dq_file[ifo] + if 'dq_segments' not in ifo_grp.keys(): + # if segs are not in file, we must be in LL + if self.dq_state_segments is not None: + raise ValueError( + 'Either all dq stat files must have segments or none' + ) + self.low_latency = True + elif self.low_latency: + raise ValueError( + 'Either all dq stat files must have segments or none' + ) def assign_template_bins(self, key): """ @@ -2337,9 +2374,7 @@ def assign_dq_rates(self, key): def setup_segments(self, key): """ - Check if segments definitions are in stat file - If they are, we are running offline and need to store them - If they aren't, we are running online + Store segments from stat file """ ifo = key.split('-')[0] with h5py.File(self.files[key], 'r') as dq_file: @@ -2368,35 +2403,32 @@ def update_file(self, key): return True # We also need to check if the DQ files have updated if key.endswith('dq_stat_info'): + ifo = key.split('-')[0] logger.info( "Updating %s statistic %s file", - ''.join(self.ifos), + ifo, key ) - self.assign_dq_rates(key) - self.assign_template_bins(key) - self.setup_segments(key) + self.dq_rates_by_state[ifo] = self.assign_dq_rates(key) + self.dq_bin_by_tid[ifo] = self.assign_template_bins(key) return True return False - def find_dq_noise_rate(self, trigs, dq_state): + def find_dq_noise_rate(self, trigs): """Get dq values for a specific ifo and dq states""" - try: - tnum = trigs.template_num - except AttributeError: - tnum = trigs['template_id'] - try: ifo = trigs.ifo except AttributeError: - ifo = trigs['ifo'] - assert len(numpy.unique(ifo)) == 1 - # Should be exactly one ifo provided - ifo = ifo[0] + ifo = trigs.get('ifo', None) + if ifo is None: + ifo = self.ifos[0] + assert ifo in self.ifos - dq_val = numpy.zeros(len(dq_state)) + dq_state = trigs['dq_state'] + dq_val = numpy.ones(len(dq_state)) + tnum = self.curr_tnum if ifo in self.dq_rates_by_state: for (i, st) in enumerate(dq_state): if isinstance(tnum, numpy.ndarray): @@ -2437,17 +2469,7 @@ def lognoiserate(self, trigs): Array of log noise rate density for each input trigger. """ - # make sure every trig has a dq state - try: - ifo = trigs.ifo - except AttributeError: - ifo = trigs['ifo'] - assert len(numpy.unique(ifo)) == 1 - # Should be exactly one ifo provided - ifo = ifo[0] - - dq_state = self.find_dq_state_by_time(ifo, trigs['end_time'][:]) - dq_rate = self.find_dq_noise_rate(trigs, dq_state) + dq_rate = self.find_dq_noise_rate(trigs) dq_rate = numpy.maximum(dq_rate, 1) logr_n = ExpFitFgBgNormStatistic.lognoiserate( @@ -2455,6 +2477,27 @@ def lognoiserate(self, trigs): logr_n += numpy.log(dq_rate) return logr_n + def single(self, trigs): + # make sure every trig has a dq state + try: + ifo = trigs.ifo + except AttributeError: + ifo = trigs.get('ifo', None) + if ifo is None: + ifo = self.ifos[0] + assert ifo in self.ifos + + singles = ExpFitFgBgNormStatistic.single(self, trigs) + + if self.low_latency: + # trigs should already have a dq state assigned + singles['dq_state'] = trigs['dq_state'][:] + else: + singles['dq_state'] = self.find_dq_state_by_time( + ifo, trigs['end_time'][:] + ) + return singles + class DQExpFitFgBgKDEStatistic(DQExpFitFgBgNormStatistic): """ diff --git a/pycbc/frame/frame.py b/pycbc/frame/frame.py index a67a3d090d9..bea7386418a 100644 --- a/pycbc/frame/frame.py +++ b/pycbc/frame/frame.py @@ -896,8 +896,8 @@ def __init__(self, frame_src, force_update_cache=force_update_cache, increment_update_cache=increment_update_cache) - def indices_of_flag(self, start_time, duration, times, padding=0): - """ Return the indices of the times lying in the flagged region + def flag_at_times(self, start_time, duration, times, padding=0): + """ Check whether the idq flag was on at given times Parameters ---------- @@ -905,32 +905,45 @@ def indices_of_flag(self, start_time, duration, times, padding=0): Beginning time to request for duration: int Number of seconds to check. + times: array of floats + Times to check for an active flag padding: float - Number of seconds to add around flag inactive times to be considered - inactive as well. + Amount of time in seconds to flag around samples + below the iDQ FAP threshold Returns ------- - indices: numpy.ndarray - Array of indices marking the location of triggers within valid - time. + flag_state: numpy.ndarray + Boolean array of whether flag was on at given times """ - from pycbc.events.veto import indices_outside_times + from pycbc.events.veto import indices_within_times + + # convert start and end times to buffer indices sr = self.idq.raw_buffer.sample_rate s = int((start_time - self.idq.raw_buffer.start_time - padding) * sr) - 1 e = s + int((duration + padding) * sr) + 1 + + # find samples when iDQ FAP is below threshold and state is valid idq_fap = self.idq.raw_buffer[s:e] - stamps = idq_fap.sample_times.numpy() low_fap = idq_fap.numpy() <= self.threshold idq_valid = self.idq_state.raw_buffer[s:e] idq_valid = idq_valid.numpy().astype(bool) valid_low_fap = numpy.logical_and(idq_valid, low_fap) + + # find times corresponding to the valid low FAP samples glitch_idx = numpy.flatnonzero(valid_low_fap) + stamps = idq_fap.sample_times.numpy() glitch_times = stamps[glitch_idx] + + # construct start and end times of flag segments starts = glitch_times - padding ends = starts + 1.0 / sr + padding * 2.0 - idx = indices_outside_times(times, starts, ends) - return idx + + # check if times were flagged + idx = indices_within_times(times, starts, ends) + flagged_bool = numpy.zeros(len(times), dtype=bool) + flagged_bool[idx] = True + return flagged_bool def advance(self, blocksize): """ Add blocksize seconds more to the buffer, push blocksize seconds diff --git a/pycbc/inference/models/hierarchical.py b/pycbc/inference/models/hierarchical.py index 81db28fa79f..d0507efa359 100644 --- a/pycbc/inference/models/hierarchical.py +++ b/pycbc/inference/models/hierarchical.py @@ -607,26 +607,14 @@ def _loglikelihood(self): class JointPrimaryMarginalizedModel(HierarchicalModel): - """ Hierarchical heterodyne likelihood for coherent multiband - parameter estimation which combines data from space-borne and - ground-based GW detectors coherently. Currently, this only - supports LISA as the space-borne GW detector. - - Sub models are treated as if the same GW source (such as a GW - from stellar-mass BBH) is observed in different frequency bands by - space-borne and ground-based GW detectors, then transform all - the parameters into the same frame in the sub model level, use - `HierarchicalModel` to get the joint likelihood, and marginalize - over all the extrinsic parameters supported by `RelativeTimeDom` - or its variants. Note that LISA submodel only supports the `Relative` - for now, for ground-based detectors, please use `RelativeTimeDom` - or its variants. - - Although this likelihood model is used for multiband parameter - estimation, users can still use it for other purposes, such as - GW + EM parameter estimation, in this case, please use `RelativeTimeDom` - or its variants for the GW data, for the likelihood of EM data, - there is no restrictions. + """This likelihood model can be used for cases when one of the submodels + can be marginalized to accelerate the total likelihood. This likelihood + model also allows for further acceleration of other models during + marginalization, if some extrinsic parameters can be tightly constrained + by the primary model. More specifically, such as the EM + GW parameter + estimation, the sky localization can be well measured. For LISA + 3G + multiband observation, SOBHB signals' (tc, ra, dec) can be tightly + constrained by 3G network, so this model is also useful for this case. """ name = 'joint_primary_marginalized' @@ -640,6 +628,11 @@ def __init__(self, variable_params, submodels, **kwargs): self.other_models.pop(kwargs['primary_lbl'][0]) self.other_models = list(self.other_models.values()) + # determine whether to accelerate total_loglr + from .tools import str_to_bool + self.static_margin_params_in_other_models = \ + str_to_bool(kwargs['static_margin_params_in_other_models'][0]) + def write_metadata(self, fp, group=None): """Adds metadata to the output files @@ -686,34 +679,43 @@ def total_loglr(self): """ # calculate = - 2 + up to a constant - # note that for SOBHB signals, ground-based detectors dominant SNR - # and accuracy of (tc, ra, dec) self.primary_model.return_sh_hh = True sh_primary, hh_primary = self.primary_model.loglr self.primary_model.return_sh_hh = False - - margin_names_vector = list( - self.primary_model.marginalize_vector_params.keys()) - if 'logw_partial' in margin_names_vector: - margin_names_vector.remove('logw_partial') + # set logr, otherwise it will store (sh, hh) + setattr(self.primary_model._current_stats, 'loglr', + self.primary_model.marginalize_loglr(sh_primary, hh_primary)) + if isinstance(sh_primary, numpy.ndarray): + nums = len(sh_primary) + else: + nums = 1 margin_params = {} - nums = 1 - for key, value in self.primary_model.current_params.items(): - # add marginalize_vector_params - if key in margin_names_vector: - margin_params[key] = value - if isinstance(value, numpy.ndarray): - nums = len(value) - # add distance if it has been marginalized, - # use numpy array for it is just let it has the same - # shape as marginalize_vector_params, here we assume - # self.primary_model.current_params['distance'] is a number - if self.primary_model.distance_marginalization: - margin_params['distance'] = numpy.full( - nums, self.primary_model.current_params['distance']) - - # add likelihood contribution from space-borne detectors, we + if self.static_margin_params_in_other_models: + # Due to the high precision of extrinsic parameters constrined + # by the primary model, the mismatch of wavefroms in others by + # varing those parameters is pretty small, so we can keep them + # static to accelerate total_loglr. Here, we use matched-filering + # SNR instead of lilkelihood, because luminosity distance and + # inclination has a very strong degeneracy, change of inclination + # will change best match distance, so change the amplitude of + # waveform. Using SNR will cancel out the effect of amplitude.err + i_max_extrinsic = numpy.argmax( + numpy.abs(sh_primary) / hh_primary**0.5) + for p in self.primary_model.marginalized_params_name: + if isinstance(self.primary_model.current_params[p], + numpy.ndarray): + margin_params[p] = \ + self.primary_model.current_params[p][i_max_extrinsic] + else: + margin_params[p] = self.primary_model.current_params[p] + else: + for key, value in self.primary_model.current_params.items(): + # add marginalize_vector_params + if key in self.primary_model.marginalized_params_name: + margin_params[key] = value + + # add likelihood contribution from other_models, we # calculate sh/hh for each marginalized parameter point sh_others = numpy.full(nums, 0 + 0.0j) hh_others = numpy.zeros(nums) @@ -723,24 +725,46 @@ def total_loglr(self): # not using self.primary_model.current_params, because others_model # may have its own static parameters current_params_other = other_model.current_params.copy() - for i in range(nums): + if not self.static_margin_params_in_other_models: + for i in range(nums): + current_params_other.update( + {key: value[i] if isinstance(value, numpy.ndarray) + else value for key, value in margin_params.items()}) + other_model.update(**current_params_other) + other_model.return_sh_hh = True + sh_other, hh_other = other_model.loglr + sh_others[i] += sh_other + hh_others[i] += hh_other + other_model.return_sh_hh = False + # set logr, otherwise it will store (sh, hh) + setattr(other_model._current_stats, 'loglr', + other_model.marginalize_loglr(sh_other, hh_other)) + else: + # use one margin point set to approximate all the others current_params_other.update( - {key: value[i] if isinstance(value, numpy.ndarray) else - value for key, value in margin_params.items()}) + {key: value[0] if isinstance(value, numpy.ndarray) + else value for key, value in margin_params.items()}) other_model.update(**current_params_other) other_model.return_sh_hh = True - sh_others[i], hh_others[i] = other_model.loglr + sh_other, hh_other = other_model.loglr other_model.return_sh_hh = False + # set logr, otherwise it will store (sh, hh) + setattr(other_model._current_stats, 'loglr', + other_model.marginalize_loglr(sh_other, hh_other)) + sh_others += sh_other + hh_others += hh_other if nums == 1: + # the type of the original sh/hh_others are numpy.array, + # might not the same as sh/hh_primary during reconstruct, + # during reconstruct of distance, sh/hh_others need to be scalar sh_others = sh_others[0] + hh_others = hh_others[0] sh_total = sh_primary + sh_others hh_total = hh_primary + hh_others - # calculate marginalize_vector_weights - self.primary_model.marginalize_vector_weights = \ - - numpy.log(self.primary_model.vsamples) loglr = self.primary_model.marginalize_loglr(sh_total, hh_total) + return loglr def others_lognl(self): @@ -805,6 +829,10 @@ def from_config(cls, cp, **kwargs): sparam_map = map_params(hpiter(cp.options('static_params'), submodel_lbls)) + # get the acceleration label + kwargs['static_margin_params_in_other_models'] = shlex.split( + cp.get('model', 'static_margin_params_in_other_models')) + # we'll need any waveform transforms for the initializing sub-models, # as the underlying models will receive the output of those transforms @@ -856,18 +884,21 @@ def from_config(cls, cp, **kwargs): cp.get('static_params', param.fullname)) # set the variable params: different from the standard - # hierarchical model, in this multiband model, all sub-models - # has the same variable parameters, so we don't need to worry - # about the unique variable issue. Besides, the primary model - # needs to do marginalization, so we must set variable_params - # and prior section before initializing it. + # hierarchical model, in this JointPrimaryMarginalizedModel model, + # all sub-models has the same variable parameters, so we don't + # need to worry about the unique variable issue. Besides, + # the primary model needs to do marginalization, so we must set + # variable_params and prior section before initializing it. subcp.add_section('variable_params') for param in vparam_map[lbl]: if lbl in kwargs['primary_lbl']: + # set variable_params for the primary model subcp.set('variable_params', param.subname, cp.get('variable_params', param.fullname)) else: + # all variable_params in other models will come + # from the primary model during sampling subcp.set('static_params', param.subname, 'REPLACE') for section in cp.sections(): @@ -875,13 +906,6 @@ def from_config(cls, cp, **kwargs): if 'prior-' in section and lbl in kwargs['primary_lbl']: prior_section = '%s' % section subcp[prior_section] = cp[prior_section] - # if `waveform_transforms` has a prefix, - # add it into sub-models' config - elif '%s_waveform_transforms' % lbl in section: - transforms_section = '%s' % section - subcp[transforms_section] = cp[transforms_section] - else: - pass # similar to the standard hierarchical model, # add the outputs from the waveform transforms if sub-model @@ -918,15 +942,8 @@ def from_config(cls, cp, **kwargs): # here we ignore `coa_phase`, because if it's been marginalized, # it will not be listed in `variable_params` and `prior` sections primary_model = submodels[kwargs['primary_lbl'][0]] - marginalized_params = primary_model.marginalize_vector_params.copy() - if 'logw_partial' in marginalized_params: - marginalized_params.pop('logw_partial') - marginalized_params = list(marginalized_params.keys()) - else: - marginalized_params = [] - # this may also include 'f_ref', 'f_lower', 'approximant', - # but doesn't matter - marginalized_params += list(primary_model.static_params.keys()) + marginalized_params = primary_model.marginalized_params_name.copy() + for p in primary_model.static_params.keys(): p_full = '%s__%s' % (kwargs['primary_lbl'][0], p) if p_full not in cp['static_params']: @@ -940,6 +957,10 @@ def from_config(cls, cp, **kwargs): cp['variable_params'].pop(p) cp.pop(section) + # save the vitual config file to disk for later check + with open('internal_top.ini', 'w', encoding='utf-8') as file: + cp.write(file) + # now load the model logging.info("Loading joint_primary_marginalized model") return super(HierarchicalModel, cls).from_config( @@ -956,6 +977,12 @@ def reconstruct(self, rec=None, seed=None): rec = {} def get_loglr(): + # make sure waveform transforms have been applied in + # the top-level model + if self.waveform_transforms is not None: + self._current_params = transforms.apply_transforms( + self._current_params, self.waveform_transforms, + inverse=False) self.update_all_models(**rec) return self.total_loglr() diff --git a/pycbc/inference/models/marginalized_gaussian_noise.py b/pycbc/inference/models/marginalized_gaussian_noise.py index 9052a5018ed..05a402aa8cf 100644 --- a/pycbc/inference/models/marginalized_gaussian_noise.py +++ b/pycbc/inference/models/marginalized_gaussian_noise.py @@ -211,6 +211,8 @@ def __init__(self, variable_params, sample_rate=None, **kwargs): + # the flag used in `_loglr` + self.return_sh_hh = False self.sample_rate = float(sample_rate) self.kwargs = kwargs variable_params, kwargs = self.setup_marginalization( @@ -259,6 +261,7 @@ def _nowaveform_loglr(self): def _loglr(self): r"""Computes the log likelihood ratio, + or inner product and if `self.return_sh_hh` is True. .. math:: @@ -369,7 +372,11 @@ def _loglr(self): hh_total += hh loglr = self.marginalize_loglr(sh_total, hh_total) - return loglr + if self.return_sh_hh: + results = (sh_total, hh_total) + else: + results = loglr + return results class MarginalizedPolarization(DistMarg, BaseGaussianNoise): diff --git a/pycbc/inference/models/relbin.py b/pycbc/inference/models/relbin.py index 4f574992fa3..8c6be79a1ec 100644 --- a/pycbc/inference/models/relbin.py +++ b/pycbc/inference/models/relbin.py @@ -596,10 +596,11 @@ def _loglr(self): filt += filter_i norm += norm_i + loglr = self.marginalize_loglr(filt, norm) if self.return_sh_hh: results = (filt, norm) else: - results = self.marginalize_loglr(filt, norm) + results = loglr return results def write_metadata(self, fp, group=None): diff --git a/pycbc/inference/models/tools.py b/pycbc/inference/models/tools.py index 044c716440f..325a8913658 100644 --- a/pycbc/inference/models/tools.py +++ b/pycbc/inference/models/tools.py @@ -228,6 +228,14 @@ def pop_prior(param): self.distance_interpolator = i kwargs['static_params']['distance'] = dist_ref + + # Save marginalized parameters' name into one place, + # coa_phase will be a static param if been marginalized + if marginalize_distance: + self.marginalized_params_name =\ + list(self.marginalize_vector_params.keys()) +\ + [marginalize_distance_param] + return variable_params, kwargs def reset_vector_params(self): diff --git a/pycbc/io/record.py b/pycbc/io/record.py index c287c2e5cc0..1fbc73aa176 100644 --- a/pycbc/io/record.py +++ b/pycbc/io/record.py @@ -1507,7 +1507,7 @@ def _isstring(dtype): """Given a numpy dtype, determines whether it is a string. Returns True if the dtype is string or unicode. """ - return dtype.type == numpy.unicode_ or dtype.type == numpy.string_ + return dtype.type == numpy.unicode_ or dtype.type == numpy.bytes_ def aliases_from_fields(fields): diff --git a/pycbc/results/dq.py b/pycbc/results/dq.py index ce3aeb43b8f..7be7bb1d4ff 100644 --- a/pycbc/results/dq.py +++ b/pycbc/results/dq.py @@ -22,8 +22,7 @@ """ -data_h1_string = """H1 -  +data_h1_string = """ Summary   @@ -31,8 +30,7 @@ 'https://alog.ligo-wa.caltech.edu/aLOG/includes/search.php?adminType=search'); return true;">aLOG""" -data_l1_string="""L1 -  +data_l1_string=""" Summary   diff --git a/pycbc/results/scatter_histograms.py b/pycbc/results/scatter_histograms.py index f89cc5a563f..dac56ab79cc 100644 --- a/pycbc/results/scatter_histograms.py +++ b/pycbc/results/scatter_histograms.py @@ -43,7 +43,7 @@ if 'matplotlib.backends' not in sys.modules: # nopep8 matplotlib.use('agg') -from matplotlib import (offsetbox, pyplot, gridspec) +from matplotlib import (offsetbox, pyplot, gridspec, colors) from pycbc.results import str_utils from pycbc.io import FieldArray @@ -337,7 +337,7 @@ def create_marginalized_hist(ax, values, label, percentiles=None, linestyle='-', plot_marginal_lines=True, title=True, expected_value=None, expected_color='red', rotated=False, - plot_min=None, plot_max=None): + plot_min=None, plot_max=None, log_scale=False): """Plots a 1D marginalized histogram of the given param from the given samples. @@ -380,6 +380,8 @@ def create_marginalized_hist(ax, values, label, percentiles=None, creates. scalefac : {1., float} Factor to scale the default font sizes by. Default is 1 (no scaling). + log_scale : boolean + Should the histogram bins be logarithmically spaced """ if fillcolor is None: htype = 'step' @@ -389,7 +391,19 @@ def create_marginalized_hist(ax, values, label, percentiles=None, orientation = 'horizontal' else: orientation = 'vertical' - ax.hist(values, bins=50, histtype=htype, orientation=orientation, + if log_scale: + bins = numpy.logspace( + numpy.log10(numpy.nanmin(values)), + numpy.log10(numpy.nanmax(values)), + 50 + ) + else: + bins = numpy.linspace( + numpy.nanmin(values), + numpy.nanmax(values), + 50, + ) + ax.hist(values, bins=bins, histtype=htype, orientation=orientation, facecolor=fillcolor, edgecolor=color, ls=linestyle, lw=2, density=True) if percentiles is None: @@ -545,6 +559,7 @@ def create_multidim_plot(parameters, samples, labels=None, marginal_title=True, marginal_linestyle='-', zvals=None, show_colorbar=True, cbar_label=None, vmin=None, vmax=None, scatter_cmap='plasma', + scatter_log_cmap=False, log_parameters=None, plot_density=False, plot_contours=True, density_cmap='viridis', contour_color=None, label_contours=True, @@ -614,6 +629,10 @@ def create_multidim_plot(parameters, samples, labels=None, zvals. scatter_cmap : {'plasma', string} The color map to use for the scatter points. Default is 'plasma'. + scatter_log_cmap : boolean + Should the scatter point coloring be on a log scale? Default False + log_parameters : list or None + Which parameters should be plotted on a log scale plot_density : {False, bool} Plot the density of points as a color map. plot_contours : {True, bool} @@ -649,6 +668,8 @@ def create_multidim_plot(parameters, samples, labels=None, """ if labels is None: labels = {p: p for p in parameters} + if log_parameters is None: + log_parameters = [] # set up the figure with a grid of axes # if only plotting 2 parameters, make the marginal plots smaller nparams = len(parameters) @@ -732,6 +753,7 @@ def create_multidim_plot(parameters, samples, labels=None, create_marginalized_hist( ax, samples[param], label=labels[param], color=hist_color, fillcolor=fill_color, + log_scale=param in log_parameters, plot_marginal_lines=plot_marginal_lines, linestyle=marginal_linestyle, linecolor=line_color, title=marginal_title, expected_value=expected_value, @@ -749,8 +771,13 @@ def create_multidim_plot(parameters, samples, labels=None, alpha = 0.3 else: alpha = 1. + if scatter_log_cmap: + cmap_norm = colors.LogNorm(vmin=vmin, vmax=vmax) + else: + cmap_norm = colors.Normalize(vmin=vmin, vmax=vmax) + plt = ax.scatter(x=samples[px], y=samples[py], c=zvals, s=5, - edgecolors='none', vmin=vmin, vmax=vmax, + edgecolors='none', norm=cmap_norm, cmap=scatter_cmap, alpha=alpha, zorder=2) if plot_contours or plot_density: diff --git a/pycbc/results/static/css/pycbc/orange.css b/pycbc/results/static/css/pycbc/orange.css index 8b2b44b0aea..1674f8ae1c2 100644 --- a/pycbc/results/static/css/pycbc/orange.css +++ b/pycbc/results/static/css/pycbc/orange.css @@ -92,3 +92,17 @@ font-size:16px; a { color:#000000; } + +table { + display: block; + overflow-x: auto; + white-space: nowrap; +} + +td { + text-align: center; +} + +th { + text-align: center; +} diff --git a/pycbc/results/table_utils.py b/pycbc/results/table_utils.py index dfdaa7297d2..aec7c62ffdb 100644 --- a/pycbc/results/table_utils.py +++ b/pycbc/results/table_utils.py @@ -23,7 +23,10 @@ # """ This module provides functions to generate sortable html tables """ -import mako.template, uuid +import mako.template +import uuid +import copy +import numpy google_table_template = mako.template.Template(""" @@ -103,42 +106,89 @@ def html_table(columns, names, page_size=None, format_strings=None): static_table_template = mako.template.Template(""" - % if titles is not None: - - % for i in range(len(titles)): - - % endfor - - % endif - - % for i in range(len(data)): - - % for j in range(len(data[i])): - + % for row in range(n_rows): + % if titles is not None: + + % if row_labels is not None: + + % endif + % for i in range(n_columns): + + % endfor + + % endif + + % for i in range(len(data)): + + % if row_labels is not None: + + % endif + % for j in range(n_columns): + + % endfor + % endfor - % endfor
- ${titles[i]} -
- ${data[i][j]} -
+ + ${titles[row * n_columns + i]} +
+ ${row_labels[i]} + + ${data[i][row * n_columns + j]} +
""") -def static_table(data, titles=None): - """ Return an html tableo of this data +def static_table(data, titles=None, columns_max=None, row_labels=None): + """ Return an html table of this data Parameters ---------- - data : two-dimensional numpy string array + data : two-dimensional string array Array containing the cell values titles : numpy array - Vector str of titles + Vector str of titles, must be the same length as data + columns_max : integer or None + If given, will restrict the number of columns in the table + row_labels : list of strings + Optional list of row labels to be given as the first cell in + each data row. Does not count towards columns_max Returns ------- html_table : str A string containing the html table. """ - return static_table_template.render(data=data, titles=titles) + data = copy.deepcopy(data) + titles = copy.deepcopy(titles) + row_labels = copy.deepcopy(row_labels) + drows, dcols = numpy.array(data).shape + if titles is not None and not len(titles) == dcols: + raise ValueError("titles and data lengths do not match") + + if row_labels is not None and not len(row_labels) == drows: + raise ValueError( + "row_labels must be the same number of rows supplied to data" + ) + + if columns_max is not None: + n_rows = int(numpy.ceil(len(data[0]) / columns_max)) + n_columns = min(columns_max, len(data[0])) + if len(data[0]) < n_rows * n_columns: + # Pad the data and titles with empty strings + n_missing = int(n_rows * n_columns - len(data[0])) + data = numpy.hstack((data, numpy.zeros((len(data), n_missing), dtype='U1'))) + if titles is not None: + titles += [' '] * n_missing + else: + n_rows = 1 + n_columns = len(data[0]) + + return static_table_template.render( + data=data, + titles=titles, + n_columns=n_columns, + n_rows=n_rows, + row_labels=row_labels, + ) diff --git a/pycbc/tmpltbank/coord_utils.py b/pycbc/tmpltbank/coord_utils.py index 429336d2153..4195ca728ce 100644 --- a/pycbc/tmpltbank/coord_utils.py +++ b/pycbc/tmpltbank/coord_utils.py @@ -440,7 +440,7 @@ def get_mu_params(lambdas, metricParams, fUpper): mus : list of floats or numpy.arrays Position of the system(s) in the mu coordinate system """ - lambdas = numpy.array(lambdas, copy=False) + lambdas = numpy.asarray(lambdas) # If original inputs were floats we need to make this a 2D array if len(lambdas.shape) == 1: resize_needed = True @@ -451,7 +451,7 @@ def get_mu_params(lambdas, metricParams, fUpper): evecs = metricParams.evecs[fUpper] evals = metricParams.evals[fUpper] - evecs = numpy.array(evecs, copy=False) + evecs = numpy.asarray(evecs) mus = ((lambdas.T).dot(evecs)).T mus = mus * numpy.sqrt(evals)[:,None] @@ -479,7 +479,7 @@ def get_covaried_params(mus, evecsCV): xis : list of floats or numpy.arrays Position of the system(s) in the xi coordinate system """ - mus = numpy.array(mus, copy=False) + mus = numpy.asarray(mus) # If original inputs were floats we need to make this a 2D array if len(mus.shape) == 1: resize_needed = True diff --git a/pycbc/types/config.py b/pycbc/types/config.py index c690c8f4906..cdbf67bff95 100644 --- a/pycbc/types/config.py +++ b/pycbc/types/config.py @@ -49,7 +49,7 @@ def __deepcopy__(self, memo): self.write(config_string) config_string.seek(0) new_config = self.__class__() - new_config.readfp(config_string) + new_config.read_file(config_string) return new_config diff --git a/pycbc/waveform/generator.py b/pycbc/waveform/generator.py index fc2b48e7fb9..ff669aede72 100644 --- a/pycbc/waveform/generator.py +++ b/pycbc/waveform/generator.py @@ -37,8 +37,8 @@ from pycbc import transforms from pycbc.types import TimeSeries from pycbc.waveform import parameters -from pycbc.waveform.utils import apply_fd_time_shift, taper_timeseries, \ - ceilpow2 +from pycbc.waveform.utils import apply_fseries_time_shift, taper_timeseries, \ + ceilpow2, apply_fd_time_shift from pycbc.detector import Detector from pycbc.pool import use_mpi import lal as _lal @@ -927,8 +927,8 @@ def generate(self, **kwargs): # happens at the end of the time series (as they are for f-domain), # so we add an additional shift to account for it tshift = 1./df - abs(hp._epoch) - hp = apply_fd_time_shift(hp, tshift, copy=True) - hc = apply_fd_time_shift(hc, tshift, copy=True) + hp = apply_fseries_time_shift(hp, tshift, copy=True) + hc = apply_fseries_time_shift(hc, tshift, copy=True) hp._epoch = hc._epoch = self._epoch h = {} diff --git a/requirements.txt b/requirements.txt index 25f99b30322..709649b789b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ astropy>=2.0.3,!=4.2.1,!=4.0.5 Mako>=1.0.1 scipy>=0.16.0 matplotlib>=2.0.0 -numpy>=1.16.0,!=1.19.0,<2.0.0 +numpy>=1.16.0,!=1.19.0 pillow h5py>=3.0.0,!=3.7.0 jinja2 @@ -17,9 +17,17 @@ tqdm gwdatafind>=1.1.3 # Requirements for full pegasus env -pegasus-wms.api >= 5.0.6 -# Need GitPython: See discussion in https://github.com/gwastro/pycbc/pull/4454 +# https://pegasus.isi.edu/documentation/user-guide/installation.html#mixing-environments-system-venv-conda +# six is listed, but is now not needed. +pegasus-wms.api >= 5.0.8 +boto3 +certifi GitPython +pyjwt +pyyaml +s3transfer +urllib3 + # need to pin until pegasus for further upstream # addresses incompatibility between old flask/jinja2 and latest markupsafe markupsafe <= 2.0.1 diff --git a/setup.py b/setup.py index 652544bbd0e..83e30f2b5ac 100755 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ setup_requires = ['numpy>=1.16.0'] install_requires = setup_requires + [ 'cython>=0.29', - 'numpy>=1.16.0,!=1.19.0,<2.0.0', + 'numpy>=1.16.0,!=1.19.0', 'scipy>=0.16.0', 'astropy>=2.0.3,!=4.2.1,!=4.0.5', 'matplotlib>=1.5.1', @@ -45,7 +45,7 @@ 'tqdm', 'setuptools', 'gwdatafind', - 'pegasus-wms.api >= 5.0.6', + 'pegasus-wms.api >= 5.0.8', 'python-ligo-lw >= 1.7.0', 'ligo-segments', 'lalsuite!=7.2', diff --git a/tools/docker_build_dist.sh b/tools/docker_build_dist.sh index 14d26c33f29..21f65b02aab 100755 --- a/tools/docker_build_dist.sh +++ b/tools/docker_build_dist.sh @@ -44,7 +44,7 @@ if [ "x${PYCBC_CONTAINER}" == "xpycbc_rhel_virtualenv" ]; then yum makecache yum -y install openssl-devel yum -y install python3-virtualenv - yum -y install hdf5-static libxml2-static zlib-static libstdc++-static cfitsio-static glibc-static swig fftw-static gsl-static --skip-broken + yum -y install hdf5-static libxml2-static zlib-static libstdc++-static cfitsio-static glibc-static swig fftw-static gsl-static gsl gsl-devel --skip-broken CVMFS_PATH=/cvmfs/software.igwn.org/pycbc/${ENV_OS}/virtualenv mkdir -p ${CVMFS_PATH} diff --git a/tox.ini b/tox.ini index 0baf40ef5cf..d3d96b2039b 100644 --- a/tox.ini +++ b/tox.ini @@ -38,6 +38,10 @@ conda_deps= gsl lapack==3.6.1 conda_channels=conda-forge +setenv = + ; Tell the linker to look for shared libs inside the temporary Conda env. + ; Needed to build BBHx's wheel, whick links to LAPACK. + LIBRARY_PATH={envdir}/lib:{env:LIBRARY_PATH:} commands = pytest # The following are long running or may require @@ -81,5 +85,9 @@ conda_deps= lapack==3.6.1 openmpi conda_channels=conda-forge -setenv = PYCBC_TEST_TYPE=docs +setenv = + PYCBC_TEST_TYPE=docs + ; Tell the linker to look for shared libs inside the temporary Conda env. + ; Needed to build BBHx's wheel, whick links to LAPACK. + LIBRARY_PATH={envdir}/lib:{env:LIBRARY_PATH:} commands = bash tools/pycbc_test_suite.sh