From bba7c0ef171d073fc5560e99f878b5be847de680 Mon Sep 17 00:00:00 2001 From: Aaron Date: Tue, 26 Apr 2022 16:07:48 -0500 Subject: [PATCH 1/8] Update empirical_distr.py --- enterprise_extensions/empirical_distr.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/enterprise_extensions/empirical_distr.py b/enterprise_extensions/empirical_distr.py index a3c76827..0ba002df 100644 --- a/enterprise_extensions/empirical_distr.py +++ b/enterprise_extensions/empirical_distr.py @@ -195,7 +195,7 @@ def logprob(self, params): return self._logpdf(*params)[0] -def make_empirical_distributions(pta, paramlist, params, chain, +def make_empirical_distributions(pta, paramlist, chain, burn=0, nbins=81, filename='distr.pkl', return_distribution=True, save_dists=True): @@ -227,6 +227,7 @@ def make_empirical_distributions(pta, paramlist, params, chain, if len(pl) == 1: idx = pta.param_names.index(pl[0]) + prior_min = pta.params[idx].prior._defaults['pmin'] prior_max = pta.params[idx].prior._defaults['pmax'] @@ -270,7 +271,7 @@ def make_empirical_distributions(pta, paramlist, params, chain, return distr -def make_empirical_distributions_KDE(pta, paramlist, params, chain, +def make_empirical_distributions_KDE(pta, paramlist, chain, burn=0, nbins=41, filename='distr.pkl', bandwidth=0.1, return_distribution=True, From 7aef51707977bc68c82208ff54cad6bb35f8cb95 Mon Sep 17 00:00:00 2001 From: Aaron Date: Wed, 27 Apr 2022 15:44:38 -0500 Subject: [PATCH 2/8] Update sampler.py --- enterprise_extensions/sampler.py | 37 +++++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/enterprise_extensions/sampler.py b/enterprise_extensions/sampler.py index b73f0a12..dc0aecff 100644 --- a/enterprise_extensions/sampler.py +++ b/enterprise_extensions/sampler.py @@ -16,7 +16,7 @@ EmpiricalDistribution2DKDE) -def extend_emp_dists(pta, emp_dists, npoints=100_000, save_ext_dists=False, outdir='chains'): +def extend_emp_dists(pta, emp_dists, npoints=100_000, save_ext_dists=False, outdir='./chains'): new_emp_dists = [] modified = False # check if anything was changed for emp_dist in emp_dists: @@ -29,8 +29,17 @@ def extend_emp_dists(pta, emp_dists, npoints=100_000, save_ext_dists=False, outd # check 2 conditions on both params to make sure that they cover their priors # skip if emp dist already covers the prior param_idx = pta.param_names.index(param) - prior_min = pta.params[param_idx].prior._defaults['pmin'] - prior_max = pta.params[param_idx].prior._defaults['pmax'] + if pta.params[param_idx].type not in ['uniform', 'normal']: + msg = 'This prior cannot be covered automatically by the empirical distribution\n' + msg += 'Please check that your prior is covered by the empirical distribution.\n' + print(msg) + continue + if pta.params[param_idx].type == 'uniform': + prior_min = pta.params[param_idx].prior._defaults['pmin'] + prior_max = pta.params[param_idx].prior._defaults['pmax'] + elif pta.params[param_idx].type == 'normal': + prior_min = pta.params[param_idx].prior._defaults['mu'] - 10 * pta.params[param_idx].prior._defaults['sigma'] + prior_max = pta.params[param_idx].prior._defaults['mu'] + 10 * pta.params[param_idx].prior._defaults['sigma'] # no need to extend if histogram edges are already prior min/max if isinstance(emp_dist, EmpiricalDistribution2D): @@ -54,8 +63,12 @@ def extend_emp_dists(pta, emp_dists, npoints=100_000, save_ext_dists=False, outd idxs_to_remove = [] for ii, (param, nbins) in enumerate(zip(emp_dist.param_names, emp_dist._Nbins)): param_idx = pta.param_names.index(param) - prior_min = pta.params[param_idx].prior._defaults['pmin'] - prior_max = pta.params[param_idx].prior._defaults['pmax'] + if pta.params[param_idx].type == 'uniform': + prior_min = pta.params[param_idx].prior._defaults['pmin'] + prior_max = pta.params[param_idx].prior._defaults['pmax'] + elif pta.params[param_idx].type == 'normal': + prior_min = pta.params[param_idx].prior._defaults['mu'] - 10 * pta.params[param_idx].prior._defaults['sigma'] + prior_max = pta.params[param_idx].prior._defaults['mu'] + 10 * pta.params[param_idx].prior._defaults['sigma'] # drop samples that are outside the prior range (in case prior is smaller than samples) if isinstance(emp_dist, EmpiricalDistribution2D): samples[(samples[:, ii] < prior_min) | (samples[:, ii] > prior_max), ii] = -np.inf @@ -77,8 +90,17 @@ def extend_emp_dists(pta, emp_dists, npoints=100_000, save_ext_dists=False, outd if emp_dist.param_name not in pta.param_names: continue param_idx = pta.param_names.index(emp_dist.param_name) - prior_min = pta.params[param_idx].prior._defaults['pmin'] - prior_max = pta.params[param_idx].prior._defaults['pmax'] + if pta.params[param_idx].type not in ['uniform', 'normal']: + msg = 'This prior cannot be covered automatically by the empirical distribution\n' + msg += 'Please check that your prior is covered by the empirical distribution.\n' + print(msg) + continue + if pta.params[param_idx].type == 'uniform': + prior_min = pta.params[param_idx].prior._defaults['pmin'] + prior_max = pta.params[param_idx].prior._defaults['pmax'] + elif pta.params[param_idx].type == 'uniform': + prior_min = pta.params[param_idx].prior._defaults['mu'] - 10 * pta.params[param_idx].prior._defaults['sigma'] + prior_max = pta.params[param_idx].prior._defaults['mu'] + 10 * pta.params[param_idx].prior._defaults['sigma'] # check 2 conditions on param to make sure that it covers the prior # skip if emp dist already covers the prior if isinstance(emp_dist, EmpiricalDistribution1D): @@ -96,7 +118,6 @@ def extend_emp_dists(pta, emp_dists, npoints=100_000, save_ext_dists=False, outd new_bins = [] idxs_to_remove = [] # drop samples that are outside the prior range (in case prior is smaller than samples) - if isinstance(emp_dist, EmpiricalDistribution1D): samples[(samples < prior_min) | (samples > prior_max)] = -np.inf elif isinstance(emp_dist, EmpiricalDistribution1DKDE): From 46724b82de9e031eb0510f67506222baf8431429 Mon Sep 17 00:00:00 2001 From: Aaron Date: Thu, 28 Apr 2022 12:57:49 -0500 Subject: [PATCH 3/8] cover priors with size!=None --- enterprise_extensions/sampler.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/enterprise_extensions/sampler.py b/enterprise_extensions/sampler.py index dc0aecff..a5238de2 100644 --- a/enterprise_extensions/sampler.py +++ b/enterprise_extensions/sampler.py @@ -25,16 +25,20 @@ def extend_emp_dists(pta, emp_dists, npoints=100_000, save_ext_dists=False, outd prior_ok=True for ii, (param, nbins) in enumerate(zip(emp_dist.param_names, emp_dist._Nbins)): if param not in pta.param_names: # skip if one of the parameters isn't in our PTA object - continue + short_par = '_'.join(param.split('_')[:-1]) # make sure we aren't skipping priors with size!=None + if short_par in pta.param_names: + param = short_par + else: + continue # check 2 conditions on both params to make sure that they cover their priors # skip if emp dist already covers the prior param_idx = pta.param_names.index(param) if pta.params[param_idx].type not in ['uniform', 'normal']: - msg = 'This prior cannot be covered automatically by the empirical distribution\n' + msg = '{} cannot be covered automatically by the empirical distribution\n'.format(pta.params[param_idx].prior) msg += 'Please check that your prior is covered by the empirical distribution.\n' print(msg) continue - if pta.params[param_idx].type == 'uniform': + elif pta.params[param_idx].type == 'uniform': prior_min = pta.params[param_idx].prior._defaults['pmin'] prior_max = pta.params[param_idx].prior._defaults['pmax'] elif pta.params[param_idx].type == 'normal': @@ -138,14 +142,15 @@ def extend_emp_dists(pta, emp_dists, npoints=100_000, save_ext_dists=False, outd new_emp_dists.append(emp_dist) continue - if save_ext_dists and modified: # if user wants to save them, and they have been modified... - pickle.dump(new_emp_dists, outdir + 'new_emp_dists.pkl') + if save_ext_dists and modified: # if user wants to save them, and they have been modified... + with open(outdir + '/new_emp_dists.pkl', 'wb') as f: + pickle.dump(new_emp_dists, f) return new_emp_dists class JumpProposal(object): - def __init__(self, pta, snames=None, empirical_distr=None, f_stat_file=None, save_ext_dists=False, outdir='chains'): + def __init__(self, pta, snames=None, empirical_distr=None, f_stat_file=None, save_ext_dists=False, outdir='./chains'): """Set up some custom jump proposals""" self.params = pta.params self.pnames = pta.param_names From b05191a7b149e0a9ed53d7ef2815dfee90244c6e Mon Sep 17 00:00:00 2001 From: Aaron Date: Thu, 28 Apr 2022 13:59:25 -0500 Subject: [PATCH 4/8] unremove unused parameter --- enterprise_extensions/empirical_distr.py | 4 ++-- enterprise_extensions/sampler.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/enterprise_extensions/empirical_distr.py b/enterprise_extensions/empirical_distr.py index 0ba002df..d8d14030 100644 --- a/enterprise_extensions/empirical_distr.py +++ b/enterprise_extensions/empirical_distr.py @@ -195,7 +195,7 @@ def logprob(self, params): return self._logpdf(*params)[0] -def make_empirical_distributions(pta, paramlist, chain, +def make_empirical_distributions(pta, paramlist, params, chain, burn=0, nbins=81, filename='distr.pkl', return_distribution=True, save_dists=True): @@ -271,7 +271,7 @@ def make_empirical_distributions(pta, paramlist, chain, return distr -def make_empirical_distributions_KDE(pta, paramlist, chain, +def make_empirical_distributions_KDE(pta, paramlist, params, chain, burn=0, nbins=41, filename='distr.pkl', bandwidth=0.1, return_distribution=True, diff --git a/enterprise_extensions/sampler.py b/enterprise_extensions/sampler.py index a5238de2..db55badc 100644 --- a/enterprise_extensions/sampler.py +++ b/enterprise_extensions/sampler.py @@ -136,6 +136,7 @@ def extend_emp_dists(pta, emp_dists, npoints=100_000, save_ext_dists=False, outd minval=prior_min, maxval=prior_max, bandwidth=emp_dist.bandwidth) new_emp_dists.append(new_emp) + print(new_emp_dists) else: print('Unable to extend class of unknown type to the edges of the priors.') From 3d47e4eae7586f25b5c4f0a848b9a7cf0836ae3e Mon Sep 17 00:00:00 2001 From: Aaron Date: Thu, 28 Apr 2022 14:01:42 -0500 Subject: [PATCH 5/8] Remove print statement --- enterprise_extensions/sampler.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/enterprise_extensions/sampler.py b/enterprise_extensions/sampler.py index db55badc..e47c668a 100644 --- a/enterprise_extensions/sampler.py +++ b/enterprise_extensions/sampler.py @@ -136,8 +136,6 @@ def extend_emp_dists(pta, emp_dists, npoints=100_000, save_ext_dists=False, outd minval=prior_min, maxval=prior_max, bandwidth=emp_dist.bandwidth) new_emp_dists.append(new_emp) - print(new_emp_dists) - else: print('Unable to extend class of unknown type to the edges of the priors.') new_emp_dists.append(emp_dist) From 150b78e862a61d6488b57da5dbf214cb034d250a Mon Sep 17 00:00:00 2001 From: Aaron Date: Thu, 28 Apr 2022 15:51:18 -0500 Subject: [PATCH 6/8] take param names from pta.params --- enterprise_extensions/sampler.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/enterprise_extensions/sampler.py b/enterprise_extensions/sampler.py index e47c668a..4b1fea21 100644 --- a/enterprise_extensions/sampler.py +++ b/enterprise_extensions/sampler.py @@ -24,15 +24,16 @@ def extend_emp_dists(pta, emp_dists, npoints=100_000, save_ext_dists=False, outd # check if we need to extend the distribution prior_ok=True for ii, (param, nbins) in enumerate(zip(emp_dist.param_names, emp_dist._Nbins)): - if param not in pta.param_names: # skip if one of the parameters isn't in our PTA object + param_names = [par.name for par in pta.params] + if param not in param_names: # skip if one of the parameters isn't in our PTA object short_par = '_'.join(param.split('_')[:-1]) # make sure we aren't skipping priors with size!=None - if short_par in pta.param_names: + if short_par in param_names: param = short_par else: continue # check 2 conditions on both params to make sure that they cover their priors # skip if emp dist already covers the prior - param_idx = pta.param_names.index(param) + param_idx = param_names.index(param) if pta.params[param_idx].type not in ['uniform', 'normal']: msg = '{} cannot be covered automatically by the empirical distribution\n'.format(pta.params[param_idx].prior) msg += 'Please check that your prior is covered by the empirical distribution.\n' @@ -66,7 +67,7 @@ def extend_emp_dists(pta, emp_dists, npoints=100_000, save_ext_dists=False, outd maxvals = [] idxs_to_remove = [] for ii, (param, nbins) in enumerate(zip(emp_dist.param_names, emp_dist._Nbins)): - param_idx = pta.param_names.index(param) + param_idx = param_names.index(param) if pta.params[param_idx].type == 'uniform': prior_min = pta.params[param_idx].prior._defaults['pmin'] prior_max = pta.params[param_idx].prior._defaults['pmax'] @@ -91,9 +92,14 @@ def extend_emp_dists(pta, emp_dists, npoints=100_000, save_ext_dists=False, outd new_emp_dists.append(new_emp) elif isinstance(emp_dist, EmpiricalDistribution1D) or isinstance(emp_dist, EmpiricalDistribution1DKDE): - if emp_dist.param_name not in pta.param_names: - continue - param_idx = pta.param_names.index(emp_dist.param_name) + param_names = [par.name for par in pta.params] + if param not in param_names: # skip if one of the parameters isn't in our PTA object + short_par = '_'.join(param.split('_')[:-1]) # make sure we aren't skipping priors with size!=None + if short_par in param_names: + param = short_par + else: + continue + param_idx = param_names.index(emp_dist.param_name) if pta.params[param_idx].type not in ['uniform', 'normal']: msg = 'This prior cannot be covered automatically by the empirical distribution\n' msg += 'Please check that your prior is covered by the empirical distribution.\n' From 77ccf1bbbfcc174a0b915966a054d7e4b6f43d84 Mon Sep 17 00:00:00 2001 From: Aaron Date: Fri, 29 Apr 2022 14:56:38 -0500 Subject: [PATCH 7/8] fix 1D param name --- enterprise_extensions/sampler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/enterprise_extensions/sampler.py b/enterprise_extensions/sampler.py index 4b1fea21..5e6f1e3a 100644 --- a/enterprise_extensions/sampler.py +++ b/enterprise_extensions/sampler.py @@ -93,8 +93,8 @@ def extend_emp_dists(pta, emp_dists, npoints=100_000, save_ext_dists=False, outd elif isinstance(emp_dist, EmpiricalDistribution1D) or isinstance(emp_dist, EmpiricalDistribution1DKDE): param_names = [par.name for par in pta.params] - if param not in param_names: # skip if one of the parameters isn't in our PTA object - short_par = '_'.join(param.split('_')[:-1]) # make sure we aren't skipping priors with size!=None + if emp_dist.param_name not in param_names: # skip if one of the parameters isn't in our PTA object + short_par = '_'.join(emp_dist.param_namem.split('_')[:-1]) # make sure we aren't skipping priors with size!=None if short_par in param_names: param = short_par else: From eb6bd2dc1ed65a8de1418f1716d3158d48d7e70d Mon Sep 17 00:00:00 2001 From: Hazboun6 Date: Fri, 29 Apr 2022 15:33:23 -0700 Subject: [PATCH 8/8] fixed typo and added param switch --- enterprise_extensions/sampler.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/enterprise_extensions/sampler.py b/enterprise_extensions/sampler.py index 5e6f1e3a..b9a4d3d9 100644 --- a/enterprise_extensions/sampler.py +++ b/enterprise_extensions/sampler.py @@ -94,12 +94,14 @@ def extend_emp_dists(pta, emp_dists, npoints=100_000, save_ext_dists=False, outd elif isinstance(emp_dist, EmpiricalDistribution1D) or isinstance(emp_dist, EmpiricalDistribution1DKDE): param_names = [par.name for par in pta.params] if emp_dist.param_name not in param_names: # skip if one of the parameters isn't in our PTA object - short_par = '_'.join(emp_dist.param_namem.split('_')[:-1]) # make sure we aren't skipping priors with size!=None + short_par = '_'.join(emp_dist.param_name.split('_')[:-1]) # make sure we aren't skipping priors with size!=None if short_par in param_names: param = short_par else: continue - param_idx = param_names.index(emp_dist.param_name) + else: + param = emp_dist.param_name + param_idx = param_names.index(param) if pta.params[param_idx].type not in ['uniform', 'normal']: msg = 'This prior cannot be covered automatically by the empirical distribution\n' msg += 'Please check that your prior is covered by the empirical distribution.\n'